Thiago Marzagão thiagomarzagao

data scientist, OhioState Ph.D., Fulbright alumnus, researcher, amateur roboticist, former machine learning professor

thiagomarzagao / scrape_gdf.py

Created January 19, 2015 13:07

	# scrape e-Compras GDF (https://www.compras.df.gov.br/)

	import os
	import requests
	from bs4 import BeautifulSoup

	baseurl = 'https://www.compras.df.gov.br/publico/'
	basepath = '/Users/thiagomarzagao/Desktop/HTML/'
	primeiro_id = 0 # ID of the first auction
	ultimo_id = 48355 # ID of the last auction (as of 12/18/14)

thiagomarzagao / html_to_json.py

Created January 19, 2015 13:09

	'''
	parse (HTML -> JSON) e-Compras GDF content
	'''

	import os
	import re
	import json
	import socket
	from bs4 import BeautifulSoup

thiagomarzagao / index.html

Created February 1, 2015 20:04

plot Brazil

	<!DOCTYPE html>
	<meta charset="utf-8">
	<body>
	<script src="http://d3js.org/d3.v3.min.js"></script>
	<script src="http://d3js.org/topojson.v1.min.js"></script>
	<script>

	var width = 960,
	height = 1160;

thiagomarzagao / catmat_svm.py

Created December 5, 2015 20:19

código p/ treinar classificador SVM p/ CATMAT

	import os
	import pickle
	from sklearn.utils import shuffle
	from sklearn import linear_model
	from sklearn import cross_validation

	# carrega X
	with open('X.pkl', mode = 'rb') as fbuffer:
	X = pickle.load(fbuffer)

thiagomarzagao / parseX.py

Created December 7, 2015 16:18

cria X.pkl

	import os
	import re
	import pickle
	from nltk.stem import RSLPStemmer
	from sklearn.feature_extraction.text import TfidfVectorizer

	def pre_process():
	vanilla = u'[^\u0041-\u005A \
	\u0061-\u007A \
	\u00C0-\u00D6 \

thiagomarzagao / parseY.py

Created December 7, 2015 16:20

cria Y.pkl

	import os
	import pickle
	import numpy as np

	basepath = '/caminho/ate/CSVs/' # altere conforme necessario
	flist = [fname for fname in os.listdir(basepath) if '.csv' in fname]
	grupos = []
	classes = []
	counter = 0
	for fname in flist:

thiagomarzagao / wordscores.py

Created February 11, 2016 17:41

Wordscores in Python

	### WORDSCORES (LBG-2003)
	### author: Thiago Marzagao
	### contact: marzagao ddott 1 at osu ddott edu

	import os
	import numpy as np
	import pandas as pd

	ipath = '/Users/username/inputdata/' # folder containing the CSV files
	opath = '/Users/username/outputdata/' # folder where output will be saved

thiagomarzagao / fightinwords.py

Created February 11, 2016 17:45

Fighting' Words in Python

	### FIGHTIN' WORDS (MCQ-2008)
	### author: Thiago Marzagao
	### contact: marzagao ddott 1 at osu ddott edu

	import os
	import sys
	import pandas as pd
	import numpy as np
	from numpy import matrix as m

thiagomarzagao / mangled.html

Last active February 11, 2016 23:11

HTML that shows mangled code block

	<div class="highlight"><pre><code class="language-python" data-lang="python"><table style="border-spacing: 0"><tbody><tr><td class="gutter gl" style="text-align: right"><pre class="lineno">1
	2
	3
	4
	5
	6
	7
	8
	9
	10

thiagomarzagao / deepnet.R

Created May 31, 2016 01:07

	library(tm)
	library(Matrix)

	setwd('/Users/thiagomarzagao/Dropbox/dataScience/UnB-CIC/aulaText/')
	comprasnet <- read.table('subset.csv',
	stringsAsFactors = FALSE,
	sep = ',',
	nrows = 1000)
	corpus <- Corpus(VectorSource(comprasnet$V2))
	corpus <- tm_map(corpus, PlainTextDocument)