j-thepac

server_name = "jdbc:sqlserver://xxxx-dev-cca-weu-syn-01-ondemand.sql.azuresynapse.net"
database_name = "mydb"
url = server_name + ":1433;" + "databaseName=" + database_name + ";"
table_name = "T"  #View / Table 
accessToken = mssparkutils.credentials.getToken('DW')

df = (spark.read \
.format("com.microsoft.sqlserver.jdbc.spark") \
.option("url", url) \
.option("dbtable", table_name) \

j-thepac

Important Resources

Open Applications

• OSX - open is a universal opener in OSX
• Linux - has xdg-open (open bashrc > add "alias open = xdg-open" > save file >source ./bashrc)
• Windows - use start

Online Virtual terminals

j-thepac

"""
db=DedicatedSQLAuth()
res=db.ddlStatements("drop table my_schema.A")
"""
class DedicatedSQLAuth():
    def __init__(self):
        self.dbName= "myDb"
        self.serverName="myServer"
        self.token= mssparkutils.credentials.getToken("DW")

j-thepac

/*.  

supports to 4 parameter types: int, float, bool, string.



%run /<path>/Notebook1 { "parameterInt": 1, "parameterFloat": 2.5, "parameterBool": true, "parameterString": "abc" }.

*/

j-thepac

# %%
import os

# %%
os.system("pip3 install pyspark==3.3.2")

# %%
import findspark
findspark.init('/Users/deepakjayaprakash/Downloads/spark-3.3.2-bin-hadoop3/')

j-thepac

# %%
import httpx

# %%
url = "https://www.nseindia.com/api/marketStatus"
headers = {
    "Accept": "*/*",
    "Accept-Language": "en-US,en;q=0.9,hi;q=0.8",
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" 
}

j-thepac

"""
PYSPARK 


Py4JJavaError: An error occurred while calling o560.save.
: java.lang.NoSuchMethodError: org.apache.spark.sql.AnalysisException.<init>(Ljava/lang/String;Lscala/Option;Lscala/Option;Lscala/Option;Lscala/Option;)V
	at org.apac

j-thepac

# Session Management with API 

import requests,httpx

def bse():
    reqUrl = "https://api.bseindia.com/RealTimeBseIndiaAPI/api/GetSensexData/w"
    headersList = {"Referer": "https://www.bseindia.com/" }
    response = requests.GET(reqUrl, headers=headersList)
    print(response.text)
    

j-thepac

from tenacity import (retry
                      ,stop_after_attempt
                      ,wait_fixed
                      ,retry_if_exception_type
                      ,stop_after_delay)
import httpx
import asyncio
import socket
@retry(stop=(stop_after_delay(3) | stop_after_attempt(5)),wait=wait_fixed(2),retry=retry_if_exception_type(Exception))
async def stop_after_7_attempts():  #stop_after_10s_or_5_retries

j-thepac

DBeaver

Window > Preferences > Editors > Text Editors "Background Color"

VS Code

VS Code > Settings > Settings > run code

"workbench.colorCustomizations": { "editor.background": "#D5D8DC"