extract_thai_tweets.py

"""
Script to extract tweets in Thai and do some cleaning up to prepare them for the Mozilla Common Voice Project.

Released under the MIT License
(c) 2021 Mishari Muqbil
"""
import re
import string

tweet_data = open("data/tweet.js").read()

text_re = r'"full_text" : "(.*?)",\n.*"lang" : "th"'
mention_re = r"^@.*?\s"

thai_tweets = re.findall(text_re, tweet_data, flags=re.M)


for t in thai_tweets:
  r = string.ascii_letters + string.digits + string.punctuation
  
  if t.startswith("RT") or t.startswith("rt"):
    continue
    
  t = re.sub(mention_re,"",t)
  
  t = ''.join([x for x in t if x not in r])
  
  t = re.sub('\s+', ' ', t)
  t = t.strip()
  print(t)