Skip to content

Instantly share code, notes, and snippets.

@blha303
Last active November 28, 2016 09:11
Show Gist options
  • Save blha303/6a1edd09cb3c7a354c30cd669045ef67 to your computer and use it in GitHub Desktop.
Save blha303/6a1edd09cb3c7a354c30cd669045ef67 to your computer and use it in GitHub Desktop.
I wanted to try my hand at making a neural net kinda thingy
from __future__ import print_function
import sys,random
# http://stackoverflow.com/a/31505798
import re
caps = "([A-Z])"
prefixes = "(Mr|St|Mrs|Ms|Dr)[.]"
suffixes = "(Inc|Ltd|Jr|Sr|Co)"
starters = "(Mr|Mrs|Ms|Dr|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)"
acronyms = "([A-Z][.][A-Z][.](?:[A-Z][.])?)"
websites = "[.](com|net|org|io|gov)"
def split_into_sentences(text):
text = " " + text + " "
text = text.replace("\n"," ")
text = re.sub(prefixes,"\\1<prd>",text)
text = re.sub(websites,"<prd>\\1",text)
if "Ph.D" in text: text = text.replace("Ph.D.","Ph<prd>D<prd>")
text = re.sub("\s" + caps + "[.] "," \\1<prd> ",text)
text = re.sub(acronyms+" "+starters,"\\1<stop> \\2",text)
text = re.sub(caps + "[.]" + caps + "[.]" + caps + "[.]","\\1<prd>\\2<prd>\\3<prd>",text)
text = re.sub(caps + "[.]" + caps + "[.]","\\1<prd>\\2<prd>",text)
text = re.sub(" "+suffixes+"[.] "+starters," \\1<stop> \\2",text)
text = re.sub(" "+suffixes+"[.]"," \\1<prd>",text)
text = re.sub(" " + caps + "[.]"," \\1<prd>",text)
if "”" in text: text = text.replace(".”","”.")
if "\"" in text: text = text.replace(".\"","\".")
if "!" in text: text = text.replace("!\"","\"!")
if "?" in text: text = text.replace("?\"","\"?")
text = text.replace(".",".<stop>")
text = text.replace("?","?<stop>")
text = text.replace("!","!<stop>")
text = text.replace("<prd>",".")
sentences = text.split("<stop>")
sentences = sentences[:-1]
sentences = [s.strip() for s in sentences]
return sentences
# end stackoverflow
outp = {}
with open(sys.argv[1]) as f:
for sentence in split_into_sentences(f.read()):
words = sentence.split()
for i, word in enumerate(words):
if not word in outp:
outp[word] = []
if i+1 < len(words):
outp[word].append(words[i+1])
seed = random.choice(outp.keys())
for i in xrange(1,10000 if not len(sys.argv) > 2 else int(sys.argv[2])):
print("{}".format(seed), end=" ")
if not outp[seed]:
seed = random.choice(outp.keys())
else:
seed = random.choice(outp[seed])
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment