wilk

#! ./bin/python3

'''
Collector has the purpose to insert in a MongoDB collection all the cleaned data.
The model is set as follows:

Model {
  id: ObjectID,
  description: String,
  amount: Float,

wilk

# test the collection
collection_counter = 0
collection_amount = 0
cursor = db[DB_COLLECTION_NAME].find()
for document in cursor:
  print(document)
  collection_amount += document['amount']
  collection_counter += 1

assert total_counter == collection_counter

wilk

import csv
from datetime import datetime

EXPENSE_ACCOUNT = os.getenv('EXPENSE_ACCOUNT', 'expenses')
INCOME_ACCOUNT = os.getenv('INCOME_ACCOUNT', 'income')

total_counter = 0
total_amount = 0
for filename in csv_files:
  filepath = join(CLEANED_FOLDER, filename)

wilk

from os import listdir
from os.path import isfile, join
import json

TAGS_FILE = os.path.abspath(os.getenv('TAGS_FILE', 'config/sample-tags.json'))

# load tags mapping
with open(TAGS_FILE) as json_file:
  tags = json.load(json_file)

wilk

from pymongo import MongoClient

DB_NAME = os.getenv('DB_NAME', 'collector')
DB_COLLECTION_NAME = os.getenv('DB_COLLECTION_NAME', 'collected')

# mongodb client
client = MongoClient(host=os.getenv('DB_HOST'), port=int(os.getenv('DB_PORT')))
db = client[DB_NAME]

# empty the collected collection

wilk

import sys
import os

CLEANED_FOLDER = os.path.abspath(os.getenv('CLEANED_FOLDER', 'cleaned'))

if not os.path.isdir(CLEANED_FOLDER):
  print('Launch the cleaner before the collector')
  sys.exit()

wilk

collector:
  # activate the virtualenv before starting the cleaner
  command: bash -c "source ./bin/activate && python src/collector.py"
  image: data.python3:1
  environment:
    - CLEANED_FOLDER=cleaned
    - TAGS_FILE=config/sample-tags.json
    - EXPENSE_ACCOUNT=expenses
    - INCOME_ACCOUNT=income
    - DB_HOST=mongodb

wilk

collector:
    # activate the virtualenv before starting the cleaner
    command: bash -c "source ./bin/activate && python src/collector.py"
    image: data.python3:1
    environment:
      - CLEANED_FOLDER=cleaned
      - DB_HOST=mongodb
      - DB_PORT=27017
      - DB_NAME=collector
      - DB_COLLECTION_NAME=collected

wilk

mongodb:
  image: mongo:3.4
  ports:
    - "27017:27017"
  volumes:
   - db_mongo:/data/db:rw

wilk

volumes:
  db_mongo: