[Python automation with OCR] This script will move you move to line #18 on the text "2018-10-02" then double click on it. #Automation #OCR #Windows

Automator.py

import pyautogui as gui
import pytesseract
from PIL import Image

TESSERACT_EXE = "C:/Program Files (x86)/Tesseract-OCR/tesseract.exe"

pytesseract.pytesseract.tesseract_cmd = TESSERACT_EXE

# Path to screenshot
TMP_PATH = "C:/Users/Yukie Nanahara/Desktop/tmp.png"

# Take Screenshot
gui.screenshot(TMP_PATH)

# Identify Text from Image
data = pytesseract.image_to_data(Image.open(TMP_PATH), lang=r'eng')

# Today is 2018-10-02

# Process OCR output

# Convert output string to array table
rows = str.split(data, "\n")
table = [str.split(row, "\t") for row in rows]

# Find column number of identified text
colIdx = table[0].index("text")

# Drop rows without 'text' field
table = [row for row in table if len(row) == 12]

# Find target row
textColumn = [row[colIdx] for row in table]
rowIdx = textColumn.index("2018-10-02") # Target Text

# Move mouse to target text
x = int(table[rowIdx][6]) # left
y = int(table[rowIdx][7]) # top

gui.moveTo(x, y)
gui.doubleClick()