j-thepac

'''
Calling Normal Function using Async Functions
    1. to_thread
    2. gather+to_thread
Calling ASync Function using Async Functions
    1. No need of to_thread
    2. only gather


'''

j-thepac

'''
find max_duration, avg_duration for each CPID using Duration , round numbers to 2 deciaml places 

ChargingEvent	CPID	StartDate	StartTime	EndDate	EndTime	Energy	PluginDuration
16673806	AN11719	31/12/17	14:46:00	31/12/17	18:00:00	2.4	3.2333333333333334
16670986	AN01706	31/12/17	11:25:00	31/12/17	13:14:00	6.1	1.8166666666666667
3174961	AN18584	31/12/17	11:26:11	01/01/18	12:54:11	24	25.466666666666665
16674334	AN00812	31/12/17	15:18:00	01/01/18	14:06:00	6.7	22.8
3176831	AN24139	31/12/17	18:25:18	01/01/18	13:09:18	6.1	18.733333333333334
16673920	AN03984	31/12/17	14:54:00	31/12/17	19:19:00	5.6	4.416666666666667

j-thepac

/*
id,name,salary,boss
10,k,1500,50
50,H,3000,10
10,J,4500,20
20,K,10000,NULL
70,J,1500,10
60,Oliver,2000,70
30,L,1501,50

j-thepac

/*
To create a table with group_id and winner_id based on below logic 

Expected Result
group_id,winner_id
1,45
2,20
3,40

In matches tables 

j-thepac

server_name = "jdbc:sqlserver://xxxx-dev-cca-weu-syn-01-ondemand.sql.azuresynapse.net"
database_name = "mydb"
url = server_name + ":1433;" + "databaseName=" + database_name + ";"
table_name = "T"  #View / Table 
accessToken = mssparkutils.credentials.getToken('DW')

df = (spark.read \
.format("com.microsoft.sqlserver.jdbc.spark") \
.option("url", url) \
.option("dbtable", table_name) \

j-thepac

Important Resources

Open Applications

• OSX - open is a universal opener in OSX
• Linux - has xdg-open (open bashrc > add "alias open = xdg-open" > save file >source ./bashrc)
• Windows - use start

Online Virtual terminals

j-thepac

"""
db=DedicatedSQLAuth()
res=db.ddlStatements("drop table my_schema.A")
"""
class DedicatedSQLAuth():
    def __init__(self):
        self.dbName= "myDb"
        self.serverName="myServer"
        self.token= mssparkutils.credentials.getToken("DW")

j-thepac

/*.  

supports to 4 parameter types: int, float, bool, string.



%run /<path>/Notebook1 { "parameterInt": 1, "parameterFloat": 2.5, "parameterBool": true, "parameterString": "abc" }.

*/

j-thepac

# %%
import os

# %%
os.system("pip3 install pyspark==3.3.2")

# %%
import findspark
findspark.init('/Users/deepakjayaprakash/Downloads/spark-3.3.2-bin-hadoop3/')

j-thepac

# %%
import httpx

# %%
url = "https://www.nseindia.com/api/marketStatus"
headers = {
    "Accept": "*/*",
    "Accept-Language": "en-US,en;q=0.9,hi;q=0.8",
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" 
}