thangarajan8

import sys
from timeit import timeit
n = int(sys.argv[1])

test1 = f"""
a_list= []
for i in range({n}):
    a_list.append(i)
"""
print(timeit(test1))

thangarajan8

lines = sc.textFile('data.txt') #reading a text file

lines_filtered = lines.filter(lambda line : ('word1' in line)) #filtering line contain the word "word1"

lines_filtered.first() #took 1s to run
lines_filtered.collect() #took 100s to run

thangarajan8

Repatition
  1. create even number of records in resultant partitions so the resources are consumed equally
  2. Go for full shuffle so it will cost effective
  3. used to increase or decerase number of partitions

Coalesce:
  1. Create un-even number of records in resultant partitions due to this load will be un-balanced
  2. won't go for full shuffle so it will be fast
  3. used to decrease number of partitions
  

thangarajan8

#https://www.hackerrank.com/challenges/missing-numbers/problem?isFullScreen=false 
a = "11 4 11 7 13 4 12 11 10 14".split(" ")
b = "11 4 11 7 3 7 10 13 4 8 12 11 10 14 12".split(" ")

result = []
arr = list(map(int,a))

brr = list(map(int,b))
a_dict = {}
b_dict = {}

thangarajan8

import pandas as pd
import time
import numpy as np
#http://eforexcel.com/wp/wp-content/uploads/2020/09/5m-Sales-Records.zip
df = pd.read_csv("5m Sales Records.csv")

def filter1(df):
    start_time = time.time()
    
    for i in df.Country.unique():

thangarajan8

select date_parse('2021-12-31 00:00:00','%Y-%m-%d %H:%i:%s')

thangarajan8

multi_date_format
07/01/2020 13:01
03/01/2020
02/01/2020 13:01
01/01/2020 13:01
05/01/2020 13:01
04-Jan-20
06/01/2020 13:01

thangarajan8

SELECT *
FROM
(
  SELECT '2021-01-15 13:01:01' AS multi_date_format
  UNION ALL
  SELECT '2021/01/15 13:01:02'
  UNION ALL
  SELECT '2021/01/03'
  UNION ALL
  SELECT '04 JAN 2021'

thangarajan8

SELECT 
       Coalesce(
         try(date_parse(multi_date_format, '%Y-%m-%d %H:%i:%s')),
         try(date_parse(multi_date_format, '%Y/%m/%d %H:%i:%s')),
         try(date_parse(multi_date_format, '%Y/%m/%d')),
         try(date_parse(multi_date_format, '%d %M %Y')),
         try(date_parse(multi_date_format, '%d %M %Y %H:%i:%s')),
         try(date_parse(multi_date_format, '%d/%m/%Y %H:%i:%s')),
         try(date_parse(multi_date_format, '%d-%m-%Y %H:%i:%s'))
       )  as DateConvertedToTimestamp,

thangarajan8

import json
import javalang as jl
tree = jl.parse.parse(content)
def json_ast_encoder(o):
    if type(o) is set and len(o) == 0:
        return []
    if hasattr(o, "__dict__"):
        return o.__dict__
    return ""