fsspec.py

# Import fsspec & alluxio fsspec implementationimport fsspec
from alluxiofs import AlluxioFileSystem
fsspec.register_implementation(“alluxio”, AlluxioFileSystem)
# Create Alluxio filesystem with S3 as the underlying storage system
alluxio = fsspec.filesystem(“alluxio”, target_protocol=“s3”, etcd_host=args.etcd_host)
# Ray reads data from Alluxio using S3 URL
ds = ray.data.read_images(“s3://datasets/imagenet-full/train”, filesystem=alluxio)