Santhin

    async def fetch_url_with_pubsub(self, url: str, cache_ttl: int = 300) -> Dict:
        """Fetch URL with pub/sub pattern to prevent cache stampede"""
        cache_key = f"{CACHE_KEY_PREFIX}{url}"
        channel_key = f"{PUBSUB_CHANNEL_PREFIX}{url}"

        # Check cache first
        cached_result = await self.redis_client.get(cache_key)
        if cached_result:
            self.cache_stats.increment_hits()
            logger.info(f"Cache HIT for {url}")

Santhin

Lol ale fajne api

Santhin

Update

sudo apt-get update

Install ZSH & Oh-My-ZSH

sudo apt-get install zsh

Santhin

Install Windows Subsystem for Linux (WSL) [ubuntu]

Follow this guide to install WSL:

https://docs.microsoft.com/en-us/windows/wsl/install-win10

Prepare WSL (ubuntu)

0. Get your build tools and python required libraries installed:

Santhin

#source: https://www.kaggle.com/gemartin/load-data-reduce-memory-usage


import pandas as pd
import numpy as np


def reduce_mem_usage(df):
    """ iterate through all the columns of a dataframe and modify the data type
        to reduce memory usage.        

Santhin

import sqlalchemy as sa
import urllib
import pandas as pd 
import pickle5 as pickle
from tqdm import tqdm


class PandasToSQL:
    """
    This class wrap to_sql function from pandas with tqdm progress bar

Santhin

#!/bin/bash

# Stop and remove all containers
echo "Removing containers :"
if [ -n "$(docker container ls -aq)" ]; then
  docker container stop $(docker container ls -aq);
  docker container rm $(docker container ls -aq);
fi;

# Remove all images

Santhin

# asyncio reactor installation (CORRECT) - `reactor` must not be defined at this point
# https://docs.scrapy.org/en/latest/_modules/scrapy/utils/reactor.html?highlight=asyncio%20reactor#
import scrapy 
import asyncio
from twisted.internet import asyncioreactor
scrapy.utils.reactor.install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
is_asyncio_reactor_installed = scrapy.utils.reactor.is_asyncio_reactor_installed()
print(f"Is asyncio reactor installed: {is_asyncio_reactor_installed}")
from twisted.internet import reactor

Santhin

#source stack
from scrapy import spiderloader
from scrapy.utils import project
from scrapy.crawler import CrawlerRunner
from twisted.internet import reactor, defer
from scrapy.utils.log import configure_logging
import logging

from datetime import datetime