jtemporal

sudo apt-get install postgresql-client-common
sudo apt-get install postgresql-client
psql -U user_name -h host_name db_name

jtemporal

version: '2' 
services:
  redis:
    image: redis
    container_name: redis

  postgres:
    image: postgres
    environment:
      - POSTGRES_PASSWORD

jtemporal

Starting redis
Starting postgres
Starting sentry
Starting learndocker_sentry-cron_1
Starting learndocker_upgrade_1
Starting learndocker_sentry-worker_1
Attaching to redis, postgres, learndocker_sentry-cron_1, learndocker_sentry-worker_1, sentry, learndocker_upgrade_1
redis            |                 _._                                                  
redis            |            _.-``__ ''-._                                             
redis            |       _.-``    `.  `_.  ''-._           Redis 3.2.1 (00000000/0) 64 bit

jtemporal

$ python rosie.py run
2017-01-26 00:04:03 Creating the CSV file
2017-01-26 00:04:03 Reading the XML file
2017-01-26 00:04:04 Writing record #2,609 to the CSV
2017-01-26 00:04:04 Done!
2017-01-26 00:04:04 Creating the CSV file
2017-01-26 00:04:04 Reading the XML file
2017-01-26 00:05:36 Writing record #341,938 to the CSV
2017-01-26 00:05:36 Done!
2017-01-26 00:05:36 Creating the CSV file

jtemporal

root@rosie-staging:~/rosie# docker run --rm -v /tmp/serenata-data:/tmp/serenata-data rosie
2017-01-27 18:35:26 Creating the CSV file
2017-01-27 18:35:26 Reading the XML file
2017-01-27 18:35:27 Writing record #2,796 to the CSV
2017-01-27 18:35:27 Done!
2017-01-27 18:35:27 Creating the CSV file
2017-01-27 18:35:27 Reading the XML file
2017-01-27 18:37:41 Writing record #342,077 to the CSV
2017-01-27 18:37:41 Done!
2017-01-27 18:37:41 Creating the CSV file

jtemporal

(serenata_rosie) root@rosie-staging:~/rosie# python rosie.py run ../tmp/
2017-01-27 20:52:07 Creating the CSV file                                                                                          2017-01-27 20:52:07 Reading the XML file                                                                                           Traceback (most recent call last): #2 to the CSV                                                                                     File "rosie.py", line 36, in <module>                                                                                                command()                                                  
  File "rosie.py", line 23, in run
    rosie.main(target_directory)
  File "/root/rosie/rosie/__init__.py", line 64, in main
    dataset = Dataset(target_directory).get()
  File "/root/rosie/rosie/dataset.py", line 16, in get
    self.update_datasets()
  File "/root/rosie/rosie/dataset.py", line 28, in update_datasets
    ceap.convert_to_csv()

jtemporal

In [30]: import pandas as pd

In [31]: import numpy as np

In [32]: dataset = pd.read_csv('../serenata-de-amor/data/2016-09-03-companies.xz', dtype={'cnpj': np.str}, low_memory=False)

In [33]: dataset.loc[0]
Out[33]: 
situation_date                                                     03/11/2005
type                                                                   MATRIZ

jtemporal

$ docker run --rm -v /tmp/serenata-data:/tmp/serenata-data rosie
Traceback (most recent call last):
  File "rosie.py", line 36, in <module>
    command()
  File "rosie.py", line 23, in run
    rosie.main(target_directory)
  File "/rosie/__init__.py", line 64, in main
    dataset = Dataset(target_directory).get()
  File "/rosie/dataset.py", line 16, in get
    self.update_datasets()

jtemporal

In [1]: import serenata_toolbox

In [2]: ceap = serenata_toolbox.CEAPDataset('data/')

In [3]: ceap.fetch()
---------------------------------------------------------------------------
BadZipFile                                Traceback (most recent call last)
<ipython-input-5-04de461fb5d4> in <module>()
----> 1 ceap.fetch()

jtemporal

root@rosie-staging:~/rosie# docker run --rm -v /tmp/serenata-data:/tmp/serenata-data rosie
2017-01-30 13:34:50 Creating the CSV file
2017-01-30 13:34:50 Reading the XML file
2017-01-30 13:34:52 Writing record #3,200 to the CSV
2017-01-30 13:34:52 Done!
2017-01-30 13:34:52 Creating the CSV file
2017-01-30 13:34:52 Reading the XML file
2017-01-30 13:37:36 Writing record #342,225 to the CSV
2017-01-30 13:37:36 Done!
2017-01-30 13:37:36 Creating the CSV file