Skip to content

Instantly share code, notes, and snippets.

View FanchenBao's full-sized avatar

Fanchen Bao FanchenBao

View GitHub Profile
@FanchenBao
FanchenBao / vanilla_version.py
Last active December 10, 2019 00:01
Expanded demo code for Python3 logging with multiprocessing
import logging
import logging.handlers
import multiprocessing
from time import sleep
from random import random, randint
# Almost the same as the demo code, but added `console_handler` to directly
# read logging info from the console
def listener_configurer():
@FanchenBao
FanchenBao / log_from_main.py
Last active December 10, 2019 00:01
Expanded demo code for Python3 logging with multiprocessing
def main():
queue = multiprocessing.Queue(-1)
# set up main logger following example from work_process
worker_configurer(queue)
main_logger = logging.getLogger('main')
listener = multiprocessing.Process(
target=listener_process, args=(queue,))
listener.start()
@FanchenBao
FanchenBao / main_logger_after_listener.py
Created December 10, 2019 00:00
Expanded demo code for Python3 logging with multiprocessing
def main():
queue = multiprocessing.Queue(-1)
listener = multiprocessing.Process(
target=listener_process, args=(queue,))
listener.start()
# set up main logger after listener
worker_configurer(queue)
main_logger = logging.getLogger('main')
main_logger.info('Logging from main')
@FanchenBao
FanchenBao / worker_process_no_log_config.py
Created December 10, 2019 00:14
Expanded demo code for Python3 logging with multiprocessing
# comment out logger configuration for worker process
def worker_process(queue):
# worker_configurer(queue)
for i in range(3):
sleep(random())
innerlogger = logging.getLogger('worker')
innerlogger.info(f'Logging a random number {randint(0, 10)}')
@FanchenBao
FanchenBao / no_log_config_in_main.py
Created December 10, 2019 00:33
Expanded demo code for Python3 logging with multiprocessing
# logger configuration uncommented in worker_process
def worker_process(queue):
worker_configurer(queue)
for i in range(3):
sleep(random())
innerlogger = logging.getLogger('worker')
innerlogger.info(f'Logging a random number {randint(0, 10)}')
# logger configuration commented out in main
@FanchenBao
FanchenBao / listener.py
Last active December 10, 2019 01:11
Expanded demo code for Python3 logging with multiprocessing
import logging
from logging import handlers
from time import sleep
def listener_configurer():
root = logging.getLogger()
file_handler = handlers.RotatingFileHandler('mptest.log', 'a', 300, 10)
console_handler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s %(processName)-10s %(name)s %(levelname)-8s %(message)s')
@FanchenBao
FanchenBao / accumulate_vnstat_output.py
Last active April 6, 2022 23:45
A quick script to accumulate the "total" column of `vnstat -5hdm` output
import re
from typing import List
"""
This scripts accumulates the 'total' column of vnstat output in this format:
wlan0 / hourly
hour rx | tx | total | avg. rate
@FanchenBao
FanchenBao / my_queue.py
Last active October 24, 2023 04:20
A custom MyQueue class that avoids `NotImplementedError` when calling queue.qsize() method.
from multiprocessing.queues import Queue
import multiprocessing
# The following implementation of custom MyQueue to avoid NotImplementedError
# when calling queue.qsize() in MacOS X comes almost entirely from this github
# discussion: https://github.com/keras-team/autokeras/issues/368
# Necessary modification is made to make the code compatible with Python3.
class SharedCounter(object):
@FanchenBao
FanchenBao / obtain_credentials.py
Created December 29, 2019 02:58
Use `requests` to obtain temporary credential for accessing AWS S3 from AWS IoT device certificate and private key
import requests
credential_provider_endpoint = 'https://<your_credentials_provider_endpoint>/role-aliases/iot-s3-access-role-alias/credentials'
device_cert_path = '<path_to_device_cert>'
device_private_key_path = '<path_to_device_private_key>'
resp = requests.get(
credential_provider_endpoint,
headers={'x-amzn-iot-thingname': 'TestThing'},
cert=(device_cert_path, device_private_key_path),
@FanchenBao
FanchenBao / obtain_credentials_and_download_from_s3.py
Last active December 29, 2019 03:11
Combine both obtaining credentials and downloading a file from S3, using AWS IoT Thing
import boto3
import requests
def obtain_temporary_credentials() -> Tuple[str, str, str]:
"""Obtain temporary credentials from AWS IoT device certificate and private key."""
credential_provider_endpoint = 'https://<your_credentials_provider_endpoint>/role-aliases/iot-s3-access-role-alias/credentials'
device_cert_path = '<path_to_device_cert>'
device_private_key_path = '<path_to_device_private_key>'