Investigate problems when scanning huge directory tree of DWD CDC HTTP server

fsspec-dwd.py

"""
Investigate problems when scanning huge directory
tree of DWD CDC HTTP server.

This repro will reveal that fsspec seems to randomly
include a single folder within its results list::

    python fsspec-dwd.py | grep -v zip

The result varies between single-line outputs of e.g.:

    https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/1_minute/precipitation/historical/1996
    https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/1_minute/precipitation/historical/2010
    https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/1_minute/precipitation/historical/2004
    https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/1_minute/precipitation/historical/1996
    https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/1_minute/precipitation/historical/2006
    https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/1_minute/precipitation/historical/2000
    https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/1_minute/precipitation/historical/1999

"""
import os
from fsspec.implementations.http import HTTPFileSystem


def process(url):
    fs = HTTPFileSystem()
    files = fs.find(url)
    for name in files:
        print(name)


if __name__ == "__main__":
    large_folder = "https://opendata.dwd.de/climate_environment/CDC/observations_germany/climate/1_minute/precipitation/historical/"
    url = os.environ.get("DWD_URL", large_folder)
    process(url)