Python program to download files from S3 to DBFS in Databricks

download_from_s3_to_dbfs.py

import os
import boto3

aws_access_key = ""
aws_secret_key = ""
bucket_name = ""
region_name = ""
# Path of the files in S3 bucket
prefix_path = ""
# Path to be written in DBFS
destination_path_prefix = ""

s3_client = boto3.client('s3', aws_access_key_id=aws_access_key, aws_secret_access_key=aws_secret_key, region_name= region_name)

response = s3_client.list_objects_v2(
    Bucket=bucket_name,
    Prefix=prefix_path
  )
print(response)
for obj in response['Contents']:
    key = obj['Key']
    print(key)
    local_path = f"/tmp/{os.path.basename(key)}"
    s3_client.download_file(bucket_name, key, local_path)
    move_status = dbutils.fs.mv(f"file://{local_path}", f"{destination_path_prefix}{os.path.basename(key)}", True)
    print("Source -->%s, Destination--> %s, Status=%s" %(key, os.path.join(destination_path_prefix,os.path.basename(key)), move_status))

Thanks so much .testing purpose