ShairozS

ShairozS

from arcgis.learn import prepare_data
from arcgis.learn import EntityRecognizer
import re
import os
import pandas as pd
from arcgis.gis import GIS
from arcgis.raster.functions import colormap
from arcgis.geocoding import batch_geocode
import zipfile,unicodedata
from itertools import repeat

ShairozS

# Initialize the EntityRecognizer model
ner = EntityRecognizer(data)

# Find an optimal learning rate
lr=ner.lr_find()

# Set how many epochs (passes over training data) we'd like the model to train for
epochs = 50

# Fit the model using the found learning rate and our specified epochs

ShairozS

from PIL import Image

img_folder = r'F:\Data\Imagery\OCR' # Folder containing topic folders (i.e "News", "Letters" ..etc.)

for subfol in os.listdir(img_folder): # For each of the topic folders
  sfpath = os.path.join(img_folder, subfol)
  for imgfile in os.listdir(sfpath): # Get all images in the topic
      imgpath = os.path.join(sfpath, imgfile)
      img = Image.open(imgpath) # Read in the image with Pillow
      img = img.resize((600,800)) # Resize the image

ShairozS

def image_to_text(imglist, ndocs=10):
    '''
    Take in a list of PIL images and return a list of extracted text using OCR
    '''
    headers = {
        # Request headers
        'Content-Type': 'application/octet-stream',
        'Ocp-Apim-Subscription-Key': 'YOUR_KEY_HERE',
    }

ShairozS

'''
Read in a list of scanned images (as .png files > 50x50px) and output a set of .txt files containing the text content of these scans
'''


from functions import preprocess, image_to_text
from PIL import Image
import os
from spellchecker import SpellChecker

ShairozS

def read_and_return(foldername, fileext='.txt'):
  '''
  Read all text files with fileext from foldername, and place them into a list of tuples as
  [(filename, text), ... , (filename, text)]
  '''
    allfiles = os.listdir(foldername)
    allfiles = [os.path.join(foldername, f) for f in allfiles if f.endswith(fileext)]
    alltext = []
    for filename in allfiles:
        with open(filename, 'r') as f:

ShairozS

from gensim import corpora, models, similarities
from gensim.parsing.preprocessing import remove_stopwords, preprocess_string

def preprocess(document):
    clean = remove_stopwords(document)
    clean = preprocess_string(document) 
    return(clean)
  
def run_lda(textlist, 
            num_topics=10,

ShairozS

def find_topic(textlist, dictionary, lda):
    '''
    https://stackoverflow.com/questions/16262016/how-to-predict-the-topic-of-a-new-query-using-a-trained-lda-model-using-gensim
    
     For each query ( document in the test file) , tokenize the 
     query, create a feature vector just like how it was done while training
     and create text_corpus
    '''
    
    text_corpus = []

ShairozS

def topic_label(ldamodel, topicnum):
    alltopics = ldamodel.show_topics(formatted=False)
    topic = alltopics[topicnum]
    topic = dict(topic[1])
    return(max(topic, key=lambda key: topic[key]))