Snoop all the delta tables in your data lake

data_lake_delta_tables.py

# Databricks notebook source

# COMMAND ----------

%py

def recurse(path: str) -> [str]:
  def is_delta_path_valid(path: str) -> bool:
    try:
      return [] != [x for x in dbutils.fs.ls(path) if x.isDir() and x.name == "_delta_log/"]
    except:
      return False
  def sub_folders(path: str) -> [str]:
    try:
      return [x for x in dbutils.fs.ls(path) if x.isDir()]
    except:
      return []
  def get_folders(path: str) -> ([str], [str]):
    try:
      subs = set([x.path for x in sub_folders(path)])
      deltas = set([x for x in subs if is_delta_path_valid(x)])
      return list(subs - deltas), list(deltas)
    except:
      return None, None
  subs, deltas = get_folders(path)
  for sub in subs:
    deltas.extend(recurse(sub))
  return list(deltas)
  
path = 'abfss://yourdatalake@yourdatalake.dfs.core.windows.net/'
deltas = recurse(path)
for delta in deltas:
  print(delta)