Skip to content

Instantly share code, notes, and snippets.

@marklit
Last active March 3, 2025 15:23
Show Gist options
  • Save marklit/e5acbee9a761857ff95112af68c9fdbd to your computer and use it in GitHub Desktop.
Save marklit/e5acbee9a761857ff95112af68c9fdbd to your computer and use it in GitHub Desktop.
Satellogic's Open Satellite Feed

Satellogic's Open Satellite Feed

https://registry.opendata.aws/satellogic-earthview/

Footprints Heatmap

$ aws --no-sign-request \
      --output json \
      s3api \
      list-objects \
      --bucket satellogic-earthview \
      --max-items=100000000 \
    | jq -c '.Contents[]' \
    > satellogic.s3.json # 13 GB & 35,480,124 lines on 2025-03-03

$ python3 -m venv ~/.satl
$ source ~/.satl/bin/activate
$ python3 -m pip install \
    utm \
    rich
import json

from   rich.progress import track
import utm


with open('enriched.s3.json', 'w') as f:
    for line in track(open('satellogic.s3.json'),
                      total=35_480_124):
        rec = json.loads(line)
        if not rec['Key'].startswith('data/json/'):
            continue

        date_, \
        time_, \
        rec['sat'], \
        rec['zone'], \
        rec['region'], \
        rec['region2'], \
        _ = rec['Key'].split('/')[5].split('_')

        rec['captured_at'] = \
            date_[0:4] + '-' + date_[4:6] + '-' + date_[6:8] + 'T' + \
            time_[0:2] + ':' + time_[2:4] + ':' + time_[4:6] + 'Z'

        try:
            lat, lon = utm.to_latlon(float(rec['region']),
                                     float(rec['region2']),
                                     int(rec['zone'][0:2]),
                                     northern=rec['zone'][2] == 'N')
        # WIP:
        # OutOfRangeError: northing out of range
        # (must be between 0 m and 10,000,000 m)
        except Exception as exc:
            continue

        rec['lat'], rec['lon'] = float(lat), float(lon)

        f.write(json.dumps(rec, sort_keys=True) + '\n')
$ ~/duckdb satellogic.duckdb
CREATE OR REPLACE TABLE s3 AS
    SELECT * EXCLUDE(lat, lon),
           ST_POINT(lon, lat) AS geom
    FROM   READ_JSON('enriched.s3.json');

COPY (
    SELECT   h3_cell_to_boundary_wkt(
                  h3_latlng_to_cell(ST_Y(geom),
                                    ST_X(geom),
                                    5))::GEOMETRY geom,
             COUNT(*) num_recs
    FROM     s3
    WHERE    ST_X(geom) BETWEEN -175 AND 175
    AND      ST_Y(geom) BETWEEN -90  AND  90
    GROUP BY 1
) TO 'h3_5.gpkg'
    WITH (FORMAT GDAL,
          DRIVER 'GPKG',
          LAYER_CREATION_OPTIONS 'WRITE_BBOX=YES');

Images per satellite per month

All data was captured between July and December 2022:

WITH a AS (
    SELECT   sat,
             STRFTIME(captured_at,  '%m') month_,
             ROUND(COUNT(*) / 1000)::INT num_rec
    FROM     s3
    GROUP BY 1, 2
    ORDER BY 1, 2
)
PIVOT    a
ON       month_
USING    SUM(num_rec)
GROUP BY sat
ORDER BY sat[3:]::INT;
┌─────────┬────────┬────────┬────────┬────────┬────────┬────────┐
│   sat   │   07   │   08   │   09   │   10   │   11   │   12   │
│ varchar │ int128 │ int128 │ int128 │ int128 │ int128 │ int128 │
├─────────┼────────┼────────┼────────┼────────┼────────┼────────┤
│ SN8     │      3 │        │        │        │        │        │
│ SN9     │     54 │     66 │     68 │     69 │    100 │     58 │
│ SN11    │     55 │     52 │     82 │     51 │     80 │     78 │
│ SN12    │        │        │      1 │        │        │        │
│ SN13    │     66 │     15 │        │        │     13 │     41 │
│ SN14    │        │        │      7 │      4 │        │      1 │
│ SN15    │     86 │     52 │     95 │     86 │     49 │        │
│ SN16    │     71 │     53 │     37 │     72 │     67 │     96 │
│ SN18    │     80 │     75 │     60 │     63 │    118 │    153 │
│ SN20    │     29 │     71 │     56 │     94 │    130 │     73 │
│ SN21    │     70 │     51 │     60 │    117 │    101 │     82 │
│ SN22    │     61 │     33 │     63 │     95 │    119 │    111 │
│ SN23    │     66 │     72 │     70 │     16 │        │        │
│ SN24    │     68 │     75 │     66 │     95 │     78 │    131 │
│ SN25    │     93 │     39 │     88 │      9 │        │        │
│ SN27    │     71 │     77 │    106 │     99 │     86 │    100 │
│ SN28    │        │     52 │     94 │    107 │    129 │    110 │
│ SN29    │     18 │     44 │     71 │    104 │     75 │    159 │
│ SN30    │     39 │     45 │     70 │     79 │     95 │     60 │
│ SN31    │     51 │    100 │     64 │     81 │    146 │    107 │
├─────────┴────────┴────────┴────────┴────────┴────────┴────────┤
│ 20 rows                                             7 columns │
└───────────────────────────────────────────────────────────────┘

Example Metadata

$ aws s3 cp --no-sign-request \
    s3://satellogic-earthview/data/json/zone=04N/region=603411_2346301/date=2022-09-15/20220915_010014_SN20_04N_603411_2346301_metadata.json ./

$ jq -S . 20220915_010014_SN20_04N_603411_2346301_metadata.json
{
  "assets": {
    "analytic": {
      "eo:bands": [
        {
          "common_name": "red",
          "name": "Red"
        },
        {
          "common_name": "green",
          "name": "Green"
        },
        {
          "common_name": "blue",
          "name": "Blue"
        },
        {
          "common_name": "nir",
          "name": "NIR"
        }
      ],
      "href": "https://satellogic-earthview.s3.us-west-2.amazonaws.com/data/tif/zone=04N/region=603411_2346301/date=2022-09-15/20220915_010014_SN20_04N_603411_2346301_TOA.tif",
      "roles": [
        "data",
        "reflectance"
      ],
      "type": "image/tiff; application=geotiff; profile=cloud-optimized"
    },
    "preview": {
      "href": "https://satellogic-earthview.s3.us-west-2.amazonaws.com/data/png/zone=04N/region=603411_2346301/date=2022-09-15/20220915_010014_SN20_04N_603411_2346301_preview.png",
      "roles": [
        "overview"
      ],
      "type": "image/png"
    },
    "thumbnail": {
      "href": "https://satellogic-earthview.s3.us-west-2.amazonaws.com/data/png/zone=04N/region=603411_2346301/date=2022-09-15/20220915_010014_SN20_04N_603411_2346301_thumbnail.png",
      "roles": [
        "thumbnail"
      ],
      "type": "image/png"
    },
    "visual": {
      "eo:bands": [
        {
          "common_name": "red",
          "name": "Red"
        },
        {
          "common_name": "green",
          "name": "Green"
        },
        {
          "common_name": "blue",
          "name": "Blue"
        },
        {
          "common_name": "nir",
          "name": "NIR"
        }
      ],
      "href": "https://satellogic-earthview.s3.us-west-2.amazonaws.com/data/tif/zone=04N/region=603411_2346301/date=2022-09-15/20220915_010014_SN20_04N_603411_2346301_VISUAL.tif",
      "roles": [
        "data",
        "visual"
      ],
      "type": "image/tiff; application=geotiff; profile=cloud-optimized"
    }
  },
  "bbox": [
    -158.00360706174618,
    21.211807250957015,
    -157.9998841750412,
    21.21529819598024
  ],
  "geometry": {
    "coordinates": [
      [
        [
          -157.99990754402126,
          21.211807250957015
        ],
        [
          -157.9998841750412,
          21.21527632148101
        ],
        [
          -158.00358377917712,
          21.21529819598024
        ],
        [
          -158.00360706174618,
          21.211829121542312
        ],
        [
          -157.99990754402126,
          21.211807250957015
        ]
      ]
    ],
    "type": "Polygon"
  },
  "id": "20220915_010014_SN20_04N_603411_2346301",
  "links": [
    {
      "href": "https://satellogic-earthview.s3.us-west-2.amazonaws.com/stac/2022/2022-09/2022-09-15/catalog.json",
      "rel": "parent",
      "type": "application/json"
    },
    {
      "href": "https://satellogic-earthview.s3.us-west-2.amazonaws.com/stac/catalog.json",
      "rel": "root",
      "type": "application/json"
    },
    {
      "href": "https://satellogic-earthview.s3.us-west-2.amazonaws.com/data/json/zone=04N/region=603411_2346301/date=2022-09-15/20220915_010014_SN20_04N_603411_2346301_metadata.json",
      "rel": "self",
      "type": "application/geo+json"
    }
  ],
  "properties": {
    "datetime": "2022-09-15T01:00:14.573735+00:00",
    "grid:code": "04N-603411_2346301",
    "gsd": 1,
    "license": "CC-BY-4.0",
    "platform": "newsat20",
    "proj:epsg": "32604",
    "proj:shape": [
      384,
      384
    ],
    "proj:transform": [
      1,
      0,
      603411,
      0,
      -1,
      2346301,
      0,
      0,
      1
    ],
    "providers": [
      {
        "name": "Satellogic",
        "roles": [
          "processor",
          "producer",
          "licensor"
        ],
        "url": "https://www.satellogic.com"
      },
      {
        "description": "AWS Open Data Sponsorship Program, hosting the dataset.",
        "name": "Amazon Web Services",
        "roles": [
          "host"
        ],
        "url": "https://registry.opendata.aws/"
      }
    ],
    "satl:altitude": 456.8651464515633,
    "satl:altitude_units": "km",
    "satl:product_name": "L1",
    "view:azimuth": 97.1382808902086,
    "view:off_nadir": 19.474754820772176,
    "view:sun_azimuth": 249.34650770444287,
    "view:sun_elevation": 48.67771372102758
  },
  "stac_extensions": [
    "https://stac-extensions.github.io/projection/v1.1.0/schema.json",
    "https://stac-extensions.github.io/eo/v1.1.0/schema.json",
    "https://stac-extensions.github.io/view/v1.0.0/schema.json",
    "https://stac-extensions.github.io/grid/v1.1.0/schema.json"
  ],
  "stac_version": "1.0.0",
  "type": "Feature"
}

Qatar Mosaic Example

$ echo "SELECT Key
        FROM   s3
        WHERE  ST_X(geom) BETWEEN 50.6515 AND 51.8086
        AND    ST_Y(geom) BETWEEN 24.3595 AND 26.3612" \
        | ~/duckdb -csv -noheader satellogic.duckdb \
        > manifest.txt
$ mkdir qatar
$ cd qatar
$ cat ../manifest.txt \
    | xargs \
        -P8 \
        -I% \
        wget -c "https://satellogic-earthview.s3.us-west-2.amazonaws.com/%"
$ cd ../

$ du -hs qatar # 252 MB
$ find qatar/ \
    -type f \
    -exec cat {} + \
    > qatar.json
$ ~/duckdb
COPY (
    SELECT properties.* EXCLUDE(providers,
                                "proj:shape",
                                "proj:transform"),
           assets.preview.href          AS url_preview,
           assets.visual.href           AS url_visual,
           assets.analytic.href         AS url_analytic,
           assets.thumbnail.href        AS url_thumbnail,
           ST_GEOMFROMGEOJSON(geometry) AS geom
    FROM   'qatar.json'
) TO 'qatar.footprints.gpkg'
    WITH (FORMAT GDAL,
          DRIVER 'GPKG',
          LAYER_CREATION_OPTIONS 'WRITE_BBOX=YES');
$ echo "FROM  ST_READ('qatar.footprints.gpkg')
        LIMIT 1" \
    | ~/duckdb -json \
    | jq -S .
[
  {
    "datetime": "2022-07-14T11:06:22.192899+00:00",
    "geom": "POLYGON ((50.65626187904185 26.24828637962653, 50.656251675908884 26.25175366324302, 50.65240649343577 26.25174441056527, 50.652416810695826 26.24827712835427, 50.65626187904185 26.24828637962653))",
    "grid:code": "39N-465288_2903610",
    "gsd": 1,
    "license": "CC-BY-4.0",
    "platform": "newsat21",
    "proj:epsg": "32639",
    "satl:altitude": 461.4741909855675,
    "satl:altitude_units": "km",
    "satl:product_name": "L1",
    "url_analytic": "https://satellogic-earthview.s3.us-west-2.amazonaws.com/data/tif/zone=39N/region=465288_2903610/date=2022-07-14/20220714_110622_SN21_39N_465288_2903610_TOA.tif",
    "url_preview": "https://satellogic-earthview.s3.us-west-2.amazonaws.com/data/png/zone=39N/region=465288_2903610/date=2022-07-14/20220714_110622_SN21_39N_465288_2903610_preview.png",
    "url_thumbnail": "https://satellogic-earthview.s3.us-west-2.amazonaws.com/data/png/zone=39N/region=465288_2903610/date=2022-07-14/20220714_110622_SN21_39N_465288_2903610_thumbnail.png",
    "url_visual": "https://satellogic-earthview.s3.us-west-2.amazonaws.com/data/tif/zone=39N/region=465288_2903610/date=2022-07-14/20220714_110622_SN21_39N_465288_2903610_VISUAL.tif",
    "view:azimuth": 97.95684658243894,
    "view:off_nadir": 9.250787005630361,
    "view:sun_azimuth": 269.67692021214305,
    "view:sun_elevation": 57.10934720889825
  }
]

They images look very different from one another and don't blend seamlessly. They seem to all have been captured at the same time though. L1 processing might explain this.

.mode line

SELECT   datetime,
         platform,
         "satl:product_name",
         "view:azimuth",
         "view:off_nadir",
         "view:sun_azimuth",
         "view:sun_elevation",
         COUNT(*)
FROM     ST_READ('qatar.footprints2.gpkg')
WHERE    ST_X(ST_CENTROID(geom)) BETWEEN 50.811115 AND 50.829002
AND      ST_Y(ST_CENTROID(geom)) BETWEEN 25.522661 AND 25.542736
GROUP BY 1, 2, 3, 4, 5, 6, 7;
          datetime = 2022-10-10T07:11:56.894566+00:00
          platform = newsat16
 satl:product_name = L1
      view:azimuth = 283.5615135582265
    view:off_nadir = 23.243003077262802
  view:sun_azimuth = 149.1609429350105
view:sun_elevation = 53.39441361193675
      count_star() = 24
$ echo "SELECT url_visual
        FROM   ST_READ('qatar.footprints2.gpkg')
        WHERE  ST_X(ST_CENTROID(geom)) BETWEEN 50.811115 AND 50.829002
        AND    ST_Y(ST_CENTROID(geom)) BETWEEN 25.522661 AND 25.542736" \
        | ~/duckdb -csv -noheader \
        > url_visual.txt

$ mkdir imagery
$ cd imagery

$ cat ../url_visual.txt \
    | xargs \
        -P8 \
        -I% \
        wget -c "%"

The above downloaded 24 GeoTIFFs totalling 9.1 MB. Each image is 384x384 pixels.

$ echo "SELECT url_visual
        FROM   ST_READ('qatar.footprints2.gpkg')
        WHERE  ST_X(ST_CENTROID(geom)) BETWEEN 50.811115 AND 50.829002
        AND    ST_Y(ST_CENTROID(geom)) BETWEEN 25.522661 AND 25.542736" \
        | ~/duckdb -csv -noheader \
        > url_visual.txt

$ mkdir imagery
$ cd imagery

$ cat ../url_visual.txt \
    | xargs \
        -P8 \
        -I% \
        wget -c "%"

The above downloaded 24 GeoTIFFs totalling 9.1 MB. Each image is 384x384 pixels.

$ gdalbuildvrt mosaic.vrt *VISUAL.tif
$ gdal_translate \
    -co NUM_THREADS=ALL_CPUS \
    -of COG \
    -co COMPRESS=WEBP \
    -co PREDICTOR=2 \
    mosaic.vrt \
    mosaic.tif
@marklit
Copy link
Author

marklit commented Mar 3, 2025

qgis-bin_ZBoY0S4gxb
qgis-bin_ISlxCDKTTQ
qgis-bin_gfxsBb5stx
qgis-bin_QDvTFxYjIN
qgis-bin_7L3wLAfGAc

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment