Skip to content

Instantly share code, notes, and snippets.

@aydinemre
aydinemre / multi_process.py
Created August 13, 2018 15:28
Python Multiprocess shared variable example
import multiprocessing
from functools import partial
# Manager to create shared object.
manager = multiprocessing.Manager()
# Create a global variable.
dictionary = manager.dict()
@aydinemre
aydinemre / read_json_from_sql.py
Last active November 12, 2018 14:03
To read json column from postgres like sql database
# -*- coding: utf-8 -*-
import json
import pandas as pd
from db.oracle_client import get_oracle_conn
file_name = "data.csv"
# -*- coding: utf-8 -*-
"""
pip install scrapy
pip install pandas
scrapy runspider scraper/scraper.py -s LOG_ENABLED=False
"""
import scrapy
@aydinemre
aydinemre / pandas_to_datetime.py
Created December 11, 2018 11:21
If pandas data frame has multiple format time
import pandas as pd
df = pd.DataFrame({"purchase_time": ["19-09-1995 12:14:00.000","12/09/2018 15:34"]})
df['Timestamp'] = pd.to_datetime(df.purchase_time, format='%d/%m/%Y %H:%M', errors='coerce')
mask = df.Timestamp.isnull()
df.loc[mask, 'Timestamp'] = pd.to_datetime(df[mask]['purchase_time'], format='%d-%m-%Y %H:%M:%S.%f', errors='coerce')
@aydinemre
aydinemre / clean-up-boot-partition-ubuntu.md
Created August 31, 2019 06:09 — forked from ipbastola/clean-up-boot-partition-ubuntu.md
Safest way to clean up boot partition - Ubuntu 14.04LTS-x64, Ubuntu 16.04LTS-x64

Safest way to clean up boot partition - Ubuntu 14.04LTS-x64, Ubuntu 16.04LTS-x64

Reference

Case I: if /boot is not 100% full and apt is working

1. Check the current kernel version

$ uname -r 
@aydinemre
aydinemre / connection.py
Created May 9, 2020 16:18
Python 3.6 Multi Process bug fix. Allow sending more than 2 GB at once on a multiprocessing connection on non-Windows systems. https://stackoverflow.com/q/47776486/8289026
#
# A higher level module for using sockets (or Windows named pipes)
#
# multiprocessing/connection.py
#
# Copyright (c) 2006-2008, R Oudkerk
# Licensed to PSF under a Contributor Agreement.
#
__all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ]
@aydinemre
aydinemre / connection.py
Last active May 9, 2020 16:18
Python 3.6 Multi Process bug fix. Allow sending more than 2 GB at once on a multiprocessing connection on non-Windows systems. /usr/lib/Python3.6/multiprocessing/connection.py https://stackoverflow.com/q/47776486/8289026
#
# A higher level module for using sockets (or Windows named pipes)
#
# multiprocessing/connection.py
#
# Copyright (c) 2006-2008, R Oudkerk
# Licensed to PSF under a Contributor Agreement.
#
__all__ = [ 'Client', 'Listener', 'Pipe', 'wait' ]
@aydinemre
aydinemre / clarifamp.py
Created May 9, 2020 16:22
Multiprocess image annotation example with using clarifai object detection api.
#!/usr/bin/env python
# coding: utf-8
import json
import os
from datetime import datetime
from multiprocessing import Pool, cpu_count
from pathlib import Path
import pandas as pd
from PIL import Image
import itertools
import pandas as pd
from selenium import webdriver
from selenium.webdriver.support.select import Select
driver = webdriver.Chrome(executable_path='./chromedriver')
belediyeler = ['basaksehir-belediyesi-arsa-rayic']
import os
from urllib.request import urlretrieve
import pandas as pd
import requests
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from webdriver_manager.chrome import ChromeDriverManager