R - Databricks - Write table to catalog

r_databricks_save_dataframe_catalog.r

#This program downloads some data from Statistics Canada and saves it in a databricks catalog
#F. Khurshed
#2024-07-16

#installl and load relevant packages
install.packages("cansim")
install.packages("sparklyr")

library(cansim)
library(sparklyr)

#download data from Statistics Canada
interest_rates <- get_cansim("10-10-0139-01")

#connect via spark to catalog
sc <- spark_connect(method = "databricks")

# Set the catalog where the table is located
tbl_change_db <- "statcan"

#convert to spark data frame
daily_interest_rates <- sdf_copy_to(sc, interest_rates, name = "daily_interest_rates", overwrite = TRUE)

# Use spark_write_table() function to write the table
daily_interest_rates %>% spark_write_table(name="daily_interest_rates", mode="overwrite", schema="main")