Skip to content

Instantly share code, notes, and snippets.

View 2minchul's full-sized avatar
😀

Colin (MinChul Lee) 2minchul

😀
View GitHub Profile
@2minchul
2minchul / main.go
Last active September 5, 2021 15:54
get jpg size without full read. written in Golang
package main
import (
"errors"
"io"
"os"
)
type ImageSize struct {
Width int
@2minchul
2minchul / large_csv_to_parquets.py
Last active August 31, 2021 13:14
Large csv to parquet files using dask
from contextlib import contextmanager
import dask.dataframe as dd
import pyarrow # noqa
from dask.distributed import Client, LocalCluster
"""
Don't let `total_memory_limit` exceed your memory.
This script will aborts, if runtime memory usage exceeds `total_memory_limit`
In my experience, it works for me:
import datetime
from operator import itemgetter
from pprint import pprint
import boto3
def chunks(lst, n):
"""Yield successive n-sized chunks from lst."""
for i in range(0, len(lst), n):
@2minchul
2minchul / ugly_json.py
Created December 24, 2020 09:21
fix truncated json string
import io
import json
class Ascii:
quote = ord('"')
backslash = ord('\\')
comma = ord(',')
colon = ord(':')
@2minchul
2minchul / exif_rotate.py
Last active December 28, 2020 04:45
transpose exif orientation with PIL
import io
import os
from PIL import Image
from PIL.JpegImagePlugin import JpegImageFile
class Orientation:
key = 0x0112 # see: PIL.ExifTags.TAGS
flip_horizontal = 2
@2minchul
2minchul / remove_exif.py
Last active December 23, 2020 09:27
remove exif from jpg without PIL
"""
This code is based on https://github.com/AzadKurt/pexif written by AzadKurt
"""
def remove_exif(img_bytes: bytes) -> bytes:
"""
returns the bytes of the image without exif
:param img_bytes: image bytes including exif
@2minchul
2minchul / pyahocorasick_example.py
Created October 20, 2020 09:24
pyahocorasick example
import ahocorasick # pip install pyahocorasick
"""
See: http://ieva.rocks/2016/11/24/keyword-matching-with-aho-corasick/
"""
def make_aho_automaton(keywords):
a = ahocorasick.Automaton() # initialize
for (key, cat) in keywords:
@2minchul
2minchul / logging.go
Created July 29, 2020 01:58
golang logging example
package logging
import (
"io"
"log"
"os"
)
type Logger struct {
Info *log.Logger
@2minchul
2minchul / readme.md
Last active April 14, 2020 06:30
install mysql5.7 on Amazon Linux 2

add MySQL Yum Repository

sudo amazon-linux-extras install epel
wget https://dev.mysql.com/get/mysql80-community-release-el7-3.noarch.rpm
yum localinstall mysql80-community-release-el7-3.noarch.rpm

yum repolist enabled | grep "mysql.*-community.*" to check default mysql version

enable mysql5.7

@2minchul
2minchul / main.go
Created December 4, 2019 05:06
Cancellation for http request using NewRequestWithContext in Golang
package main
import (
"context"
"net/http"
"time"
)
func main() {
ctx := context.Background()