luminoso

import logging

# level=logging._nameToLevel[verbosity_level]
# level=logging.DEBUG

# format="%(asctime)s [%(module)-12.12s] [%(levelname)-5.5s]  %(message)s",

logging.basicConfig(
    level=logging.DEBUG,
    format="%(asctime)s [%(levelname)-5.5s]  %(message)s",

luminoso

# tests the avalability of cuda for pytorch

import torch

torch.cuda.current_device()
torch.cuda.device(0)
torch.cuda.device_count()
torch.cuda.get_device_name(0)
torch.cuda.is_available()

luminoso

# trick to make iJulia work on CentOS 7

# CentOS looks like to have some kind of ABI incompatibility
# ERROR: LoadError: InitError: could not load library "/home/centos/.julia/artifacts/8b67c92bc6eb60d96d3488bf4b48bd3a38f70c53/lib/libzmq.so"
# /usr/bin/../lib64/libstdc++.so.6: version `GLIBCXX_3.4.20' not found (required by /home/centos/.julia/artifacts/8b67c92bc6eb60d96d3488bf4b48bd3a38f70c53/lib/libzmq.so)

]
add [email protected]
add iJulia
using iJulia

luminoso

from pymongo import MongoClient

# pprint library is used to make the output look more pretty
from pprint import pprint

import urllib.parse
username = urllib.parse.quote_plus('xx')
password = urllib.parse.quote_plus('yy')

client = MongoClient(f'mongodb://{username}:{password}@172.31.8.226:27017/')

luminoso

# auto reload modules/files
%load_ext autoreload
%autoreload 2

# print execution times
%load_ext autotime

# interactive matplotlib outputs
%matplotlib inline

luminoso

# reads and writes compressed jsonlines to a file

# from bson import json_util
# writeall_jsonl_gz('filename.jsonl.gz', objects, dumps=json_util.dumps)


import gzip
# !python -m pip install jsonlines
from typing import List, Dict
import jsonlines

luminoso

# list magics
%lsmagic

#
# print all outputs of a cell
#
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"

#

luminoso

stats = {
      "errors": 0,
      "successes": 0,
      "min_offset": None,
      "max_offset": None,
      "errorTypes": {}
}

def errback_listener(error, **kargs):
      if type(error).__name__ not in kargs["stats"]["errorTypes"]:

luminoso

### Starting Zookeeper
./bin/zookeeper-server-start.sh config/zookeeper.properties

### Starting Kafka
./bin/kafka-server-start.sh config/server.properties 

#################################################
### Topic creation

./bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic ipma.3days_day

luminoso

Keybase proof

I hereby claim:

• I am luminoso on github.
• I am luminoso (https://keybase.io/luminoso) on keybase.
• I have a public key whose fingerprint is 69E8 E8A1 0F70 B513 1949 008B 2E09 50FE 5A90 AAD1

To claim this, I am signing this object: