pyarrow_reading

from pyarrow import fs as py_fs
import pyarrow.parquet as pq

fs = py_fs.AzureFileSystem(account_name="mittasblobforray2")
filePath = "testcontainer1/10G-data"


selector = py_fs.FileSelector(base_dir=filePath, recursive=True)
file_info = fs.get_file_info(selector)
for file in file_info:
    print(f"File Name: {file}")
    if file.is_file:
        print(f"Reading File: {file.path}")
        pq.read_table(file.path, filesystem=fs)
    
fs.create_dir("testcontainer1/10G_DIR_DATA2", recursive=False)
print("Successfully created directory!!")