Fabric Notes For ArcGIS Data Prep

fabric_arcgis.py

# Create an environment to ensure correct python packages are loaded
# Not sure what exactly is needed - I went with the arcgis package
# Create a notebook to hold the python code to filter/aggregate data and push up via the arcgis api
# Create a pipeline to exec the notebook on schedule with parameters eg credentials etc
# Set necessary security / access control on who can see the parameters
# Create a parameters cell in the notebook
pwd="og_value"

# Set up logging to the Lakehouse filesystem so we can track what's happening eg
import logging
import time

# datetime for the name format
datestr = time.strftime("_%Y%m%d_T_%H%M")

# datetime for the log
datestr_log = time.strftime("%Y-%m-%d - %H:%M:%S:%M")

log_path ='/lakehouse/default/Files/ArcGISLogs/'
file_name = 'AutomatedArcGIS.log_' + datestr
logging.basicConfig(filename=log_path + file_name + '.txt',  
                                           force = True,
                                           filemode='w',
                                           level=logging.INFO) ##or DEBUG

logging.info("This is an information message")
logging.debug("This is a debug message")
logging.warning("This is a warning msg")


# Check we can load arcgis
import arcgis
# print(arcgis.__version__)
logging.info(f"arcgis version is {arcgis.__version__}")

# connect to ArcGIS Online and define our item to upload - will error if it's already there
from arcgis.gis import GIS
from arcgis.features import FeatureLayerCollection

logging.info("Completed log msg")


gis = GIS("https://www.arcgis.com", "some_user_name", pwd)

file_path = "/lakehouse/default/Files/population.csv"

item_properties = {
    "title": "Population data",
    "tags": "example, feature layer",
    "description": "This is an example feature layer uploaded using ArcGIS Python API."
}

# upload
uploaded_item = gis.content.add(item_properties, data=file_path)

published_layer = uploaded_item.publish()

mssparkutils.notebook.exit(published_layer.url)

print(f"Feature Layer URL: {published_layer.url}")