Santhin

#source stack
from scrapy import spiderloader
from scrapy.utils import project
from scrapy.crawler import CrawlerRunner
from twisted.internet import reactor, defer
from scrapy.utils.log import configure_logging
import logging

from datetime import datetime

Santhin

# asyncio reactor installation (CORRECT) - `reactor` must not be defined at this point
# https://docs.scrapy.org/en/latest/_modules/scrapy/utils/reactor.html?highlight=asyncio%20reactor#
import scrapy 
import asyncio
from twisted.internet import asyncioreactor
scrapy.utils.reactor.install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
is_asyncio_reactor_installed = scrapy.utils.reactor.is_asyncio_reactor_installed()
print(f"Is asyncio reactor installed: {is_asyncio_reactor_installed}")
from twisted.internet import reactor

Santhin

#!/bin/bash

# Stop and remove all containers
echo "Removing containers :"
if [ -n "$(docker container ls -aq)" ]; then
  docker container stop $(docker container ls -aq);
  docker container rm $(docker container ls -aq);
fi;

# Remove all images

Santhin

import sqlalchemy as sa
import urllib
import pandas as pd 
import pickle5 as pickle
from tqdm import tqdm


class PandasToSQL:
    """
    This class wrap to_sql function from pandas with tqdm progress bar

Santhin

#source: https://www.kaggle.com/gemartin/load-data-reduce-memory-usage


import pandas as pd
import numpy as np


def reduce_mem_usage(df):
    """ iterate through all the columns of a dataframe and modify the data type
        to reduce memory usage.        

Santhin

Install Windows Subsystem for Linux (WSL) [ubuntu]

Follow this guide to install WSL:

https://docs.microsoft.com/en-us/windows/wsl/install-win10

Prepare WSL (ubuntu)

0. Get your build tools and python required libraries installed:

Santhin

Update

sudo apt-get update

Install ZSH & Oh-My-ZSH

sudo apt-get install zsh

Santhin

Lol ale fajne api