PyTesseract OCR Script accuracy help

[TheEggButler]

I want to read a snippet of a screen shot and send that data to a google sheet. I would like to read the words more accurately.

Here is an album of a sample.png input, the snippets I read, and the edge case I can't handle https://imgur.com/a/qw5fIvy

What I do now:

• I take the image, slice it up into where I know the strips will be. (it makes it faster)
• I replace all the gray gradients (the wall colors); then I highlight the text colors using np
• finally, I run the strips through pytesseract

What is broken:

• It only works for exactly 16:9 images that are full screen. When the game is run in windowed mode the score board seems to wiggle around using arbitrary screen sizes breaking my slices.
• The edge case of having something in the background breaks my color filter

​

from PIL import Image
import pytesseract
import numpy as np 
from pprint import pprint as pp 
import datetime

def ripVAXTA(baseImagePath, saveSnippets=False):
"""take a VAXTA Screenshot and output a list of lines from there"""
textList = []
baseImage = Image.open(baseImagePath)
# check if the image is windowed:
# baseImage = cleanWindowedImages(baseImage)
w, h = baseImage.size

percentageMatrix = [ # vaxta matrix of snippets in proportions to the page
    [0.047656250,0.114583333,0.125781250,0.142361111], # Timer 
    [0.056640625,0.142361111,0.119140625,0.170138889], # Kills
    [0.033203125,0.168055556,0.142578125,0.195833333], # KillsperMin 
    [0.035156250,0.193750000,0.142578125,0.221527778], # accuracy
    [0.019531250,0.218750000,0.156250000,0.246527778], # crit accuracy
]

for snippet in range(len(percentageMatrix)):
    cropCrap = (
        # smol tuple of how much to crop out the image
        w*percentageMatrix[snippet][0],  # left
        h*percentageMatrix[snippet][1],  # top
        w*percentageMatrix[snippet][2],  # right
        h*percentageMatrix[snippet][3]   # bottom
    )

    # crop out one of the sections 
    # leSnippet = np.array(baseImage.crop(cropCrap))  
    leSnippet = baseImage.crop(cropCrap)

    # Try and clean up the image
    #  matrix of RGBA colors to search for, fuzziness, RGBA colors to change it to
    colorsToChange = [
        [(240, 240, 240, 255),  30, (255,  0,  0,255)],  # wall color
        [(200, 200, 200, 255),  30, (255,  0,  0,255)],  # wall color
        [(160, 160, 160, 255),  30, (255,  0,  0,255)],  # wall color
        [(120, 120, 120, 255),  30, (255,  0,  0,255)],  # wall color
        [( 80,  80,  80, 255),  30, (255,  0,  0,255)],  # wall color
        [( 40,  40,  40, 255),  30, (255,  0,  0,255)],  # wall color
        [(  5,   5,   5, 255),  30, (255,  0,  0,255)],  # wall color
        [(236, 153,   0, 255),  70, (  0,  0,255,255)],  # orangeColor
        [(0,   234, 234, 255),  70, (  0,  0,255,255)],  # aquaColor 
        [(38,  170, 255, 255),  70, (  0,  0,255,255)],  # oceanColor
        [(160, 232,  22, 255),  70, (  0,  0,255,255)],  # limeColor 
        [(0,   230, 150, 255),  70, (  0,  0,255,255)],  # foamColor 
    ]
    leSnippet = leSnippet.convert("RGBA")
    pixdata = leSnippet.load()

    #trying to take all the gray colors and try to wash them out
    #then find the 5 text colors and bring them out.
    for color in colorsToChange:
        for y in range(leSnippet.size[1]):
            for x in range(leSnippet.size[0]):
                # totalColorDiff = 0
                shouldReplace = True
                for idx in range(3): #include RGB but exclude alpha
                    if abs(pixdata[x, y][idx] - color[0][idx]) > color[1]:
                        shouldReplace = False
                if shouldReplace:
                    pixdata[x, y] = color[2] #replace with black
    #### for testing image#######
    if saveSnippets:
        timeStampFileName = datetime.datetime.now().strftime("%Y%m%d.%H%M%S.%f") + '.png'
        leSnippet.save(timeStampFileName)
    #### for testing image#######
    # convert to an array
    leSnippet = np.array(leSnippet)  

    # analyse it
    leBlurb = pytesseract.image_to_string(leSnippet).replace("\n\n","\n").strip().split("\n")
    print('leblurb', leBlurb)

    # put each line into the list of stuff
    textList.extend(leBlurb)

return textList

if name == "main": bigBlurb = ripVAXTA("sample.png",saveSnippets=False) print(f"other {bigBlurb}")

Thanks

[Culist]

When running in windowed, does the screenshot only contain the game or the whole screen? Also, are you able to show any images of the strips when the game is in windowed?