Last active
November 28, 2016 09:11
-
-
Save blha303/6a1edd09cb3c7a354c30cd669045ef67 to your computer and use it in GitHub Desktop.
I wanted to try my hand at making a neural net kinda thingy
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from __future__ import print_function | |
import sys,random | |
# http://stackoverflow.com/a/31505798 | |
import re | |
caps = "([A-Z])" | |
prefixes = "(Mr|St|Mrs|Ms|Dr)[.]" | |
suffixes = "(Inc|Ltd|Jr|Sr|Co)" | |
starters = "(Mr|Mrs|Ms|Dr|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)" | |
acronyms = "([A-Z][.][A-Z][.](?:[A-Z][.])?)" | |
websites = "[.](com|net|org|io|gov)" | |
def split_into_sentences(text): | |
text = " " + text + " " | |
text = text.replace("\n"," ") | |
text = re.sub(prefixes,"\\1<prd>",text) | |
text = re.sub(websites,"<prd>\\1",text) | |
if "Ph.D" in text: text = text.replace("Ph.D.","Ph<prd>D<prd>") | |
text = re.sub("\s" + caps + "[.] "," \\1<prd> ",text) | |
text = re.sub(acronyms+" "+starters,"\\1<stop> \\2",text) | |
text = re.sub(caps + "[.]" + caps + "[.]" + caps + "[.]","\\1<prd>\\2<prd>\\3<prd>",text) | |
text = re.sub(caps + "[.]" + caps + "[.]","\\1<prd>\\2<prd>",text) | |
text = re.sub(" "+suffixes+"[.] "+starters," \\1<stop> \\2",text) | |
text = re.sub(" "+suffixes+"[.]"," \\1<prd>",text) | |
text = re.sub(" " + caps + "[.]"," \\1<prd>",text) | |
if "”" in text: text = text.replace(".”","”.") | |
if "\"" in text: text = text.replace(".\"","\".") | |
if "!" in text: text = text.replace("!\"","\"!") | |
if "?" in text: text = text.replace("?\"","\"?") | |
text = text.replace(".",".<stop>") | |
text = text.replace("?","?<stop>") | |
text = text.replace("!","!<stop>") | |
text = text.replace("<prd>",".") | |
sentences = text.split("<stop>") | |
sentences = sentences[:-1] | |
sentences = [s.strip() for s in sentences] | |
return sentences | |
# end stackoverflow | |
outp = {} | |
with open(sys.argv[1]) as f: | |
for sentence in split_into_sentences(f.read()): | |
words = sentence.split() | |
for i, word in enumerate(words): | |
if not word in outp: | |
outp[word] = [] | |
if i+1 < len(words): | |
outp[word].append(words[i+1]) | |
seed = random.choice(outp.keys()) | |
for i in xrange(1,10000 if not len(sys.argv) > 2 else int(sys.argv[2])): | |
print("{}".format(seed), end=" ") | |
if not outp[seed]: | |
seed = random.choice(outp.keys()) | |
else: | |
seed = random.choice(outp[seed]) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment