markov.py

import markovify
import os
import json
import re

directory = os.path.join(os.getcwd(), "general")
filenames = os.listdir(directory)

data = []

for filename in filenames:
    fpath = os.path.join(directory, filename)
    with open(fpath) as f:
        contents = json.load(f)
    contents = [m['text'] for m in contents] #list of lists
    data.extend(contents)

url_pattern = re.compile(r'<.*>')
codeb_pattern = re.compile(r'`.*`')
text = '\n'.join(data)
text = re.sub(url_pattern, '', text)
text = re.sub(codeb_pattern, '', text)

# Build the model.
text_model = markovify.Text(text)

# Print five randomly-generated sentences
#for i in range(5):
#    print(text_model.make_sentence())

# Print three randomly-generated sentences of no more than 140 characters
for i in range(30):
    text_model.make_short_sentence(140)