Mapbox Static Tile Downloader

mapbox_dl.go

// Package main downloads tiles from the Mapbox Static Tiles API.
// See https://docs.mapbox.com/api/maps/static-tiles/#retrieve-raster-tiles-from-styles.
//
// Usage:
// go run mapbox_dl.go -h
package main

import (
	"flag"
	"fmt"
	"image"
	"image/draw"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	// Supported tile extensions:
	"image/jpeg"
	"image/png"
)

var (
	token     = flag.String("access_token", "", "Mapbox API access token.")
	username  = flag.String("username", "mapbox", "Mapbox style owner's username.")
	style     = flag.String("style_id", "satellite-v9", "Mapbox map style ID.")
	ext       = flag.String("ext", ".jpg", "File extension (usually .png for generated maps, .png for satellite imagery).")
	zoom      = flag.Int("zoom", 0, "Zoom level")
	x2        = flag.Bool("2x", true, "Fetch double-resolution tiles (@2x).")
	printURLs = flag.Bool("print", false, "Whether to print URLs to download.")
	download  = flag.Bool("download", false, "Whether to download tiles or just print the URLs.")
	interval  = flag.Duration("download_interval", 0, "How much time to wait between downloads (or printouts).")
	overwrite = flag.Bool("overwrite", false, "Whether to re-download existing tiles.")
	merge     = flag.Bool("merge", false, "Whether to merge all tiles into one big square image.")
	quality   = flag.Int("merge_quality", jpeg.DefaultQuality, "Quality for merging when encoding to JPEG.")
)

// Tileset contains all tiles for a zoom level.
type Tileset struct {
	username, style, ext string
	zoom                 int
	x2                   bool
}

// Tile represents a single tile.
type Tile struct {
	*Tileset
	x, y int
}

// NewTileset creates a new Tileset from flag values.
func NewTileset() *Tileset {
	return &Tileset{
		username: *username,
		style:    *style,
		ext:      *ext,
		zoom:     *zoom,
		x2:       *x2,
	}
}

// Size returns the number of of rows/cols.
func (s *Tileset) Size() int {
	return int(math.Pow(2, float64(s.zoom)))
}

// Tiles returns a channel providing every tile in the tileset.
func (s *Tileset) Tiles() <-chan Tile {
	ch := make(chan Tile)
	go func() {
		for x := 0; x < s.Size(); x++ {
			for y := 0; y < s.Size(); y++ {
				ch <- Tile{
					Tileset: s,
					x:       x,
					y:       y,
				}
			}
		}
		close(ch)
	}()
	return ch
}

// Image returns a blank Image capable to holde every tile.
func (s *Tileset) Image() draw.Image {
	size := s.Size() * 512
	if s.x2 {
		size *= 2
	}
	return image.NewRGBA(image.Rect(0, 0, size, size))
}

// Path returns the path to the merged image.
func (s *Tileset) Path() string {
	x2 := ""
	if s.x2 {
		x2 = "@2x"
	}
	return fmt.Sprintf("styles/v1/%s/%s/tiles/%d%s%s", s.username, s.style, s.zoom, x2, s.ext)
}

// URL returns the URL of the tile.
func (t *Tile) URL() string {
	p := t.Path()
	p = strings.TrimSuffix(p, filepath.Ext(p))
	return fmt.Sprintf("https://api.mapbox.com/%s?access_token=%s", p, *token)
}

// Path returns the path to the local file.
func (t *Tile) Path() string {
	x2 := ""
	if t.x2 {
		x2 = "@2x"
	}
	return fmt.Sprintf("styles/v1/%s/%s/tiles/%d/%d/%d%s%s", t.username, t.style, t.zoom, t.x, t.y, x2, t.ext)
}

// Image reads the downloaded tile as an image.
func (t *Tile) Image() (image.Image, error) {
	f, err := os.Open(t.Path())
	if err != nil {
		return nil, fmt.Errorf("error reading tile %q: %w", t.Path(), err)
	}
	defer func() {
		cerr := f.Close()
		if err == nil {
			err = cerr
		}
	}()

	var img image.Image
	if img, _, err = image.Decode(f); err != nil {
		return nil, fmt.Errorf("error decoding tile %q: %w", t.Path(), err)
	}
	// Allow the defer above to set the error.
	return img, err
}

// Rect identifies the tile's position in the merged tileset.
func (t *Tile) Rect() image.Rectangle {
	size := 512
	if t.x2 {
		size *= 2
	}
	return image.Rect(t.x*size, t.y*size, (t.x+1)*size, (t.y+1)*size)
}

// Download fetches and stores the tile locally.
// Returns true if the file was downloaded, false if failed or cached.
func (t *Tile) Download() (bool, error) {
	p := t.Path()
	dir := filepath.Dir(p)
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		// NOTE: ModePerm will have the umask removed.
		if err := os.MkdirAll(dir, os.ModePerm); err != nil {
			return false, fmt.Errorf("error creating directory: %w", err)
		}
	} else if err != nil {
		return false, fmt.Errorf("error checking directory: %w", err)
	}
	if _, err := os.Stat(p); os.IsNotExist(err) || *overwrite {
		err = t.DownloadTo(p)
		return err == nil, err
	} else if err != nil {
		return false, fmt.Errorf("error checking file: %w", err)
	}
	return false, nil
}

// DownloadTo fetches and stores the tile in 'path'.
func (t *Tile) DownloadTo(path string) error {
	res, err := http.Get(t.URL())
	if err != nil {
		return fmt.Errorf("error downloading tile: %w", err)
	}
	defer func() {
		cerr := res.Body.Close()
		if err == nil {
			err = cerr
		}
	}()

	if res.StatusCode != http.StatusOK {
		body, _ := ioutil.ReadAll(res.Body)
		return fmt.Errorf("error downloading tile: %q: %s", res.Status, body)
	}

	f, err := os.Create(path)
	if err != nil {
		return fmt.Errorf("error creating file: %w", err)
	}
	defer func() {
		cerr := f.Close()
		if err == nil {
			err = cerr
		}
	}()

	_, err = io.Copy(f, res.Body)
	return err
}

func main() {
	flag.Parse()
	if (*download || *printURLs) && *token == "" {
		println("If -download or -print is set, -access_token must also be set.")
		os.Exit(1)
	}

	s := NewTileset()
	for t := range s.Tiles() {
		if *download {
			fmt.Printf("%s… ", t.Path())
			if dl, err := t.Download(); err != nil {
				println("ERROR:")
				println(err.Error())
			} else if dl {
				fmt.Println("OK")
				time.Sleep(*interval)
			} else {
				fmt.Println("CACHED")
			}
		} else if *printURLs {
			fmt.Println(t.URL())
		}
	}
	if !*merge {
		return
	}

	img := s.Image()
	f, err := os.Create(s.Path())
	if err != nil {
		print("error creating merged file: ")
		println(err.Error())
		os.Exit(1)
	}
	defer func() {
		if err = f.Close(); err != nil {
			print("error closing merged file: ")
			println(err.Error())
			os.Exit(1)
		}
	}()
	fmt.Printf("%s… ", s.Path())

	for t := range s.Tiles() {
		i, err := t.Image()
		if err != nil {
			print("error reading tile: ")
			println(err.Error())
			os.Exit(1)
		}
		draw.Draw(img, t.Rect(), i, image.Point{}, draw.Over)
	}
	if *ext == ".png" {
		if err = png.Encode(f, img); err != nil {
			print("error saving merged file: ")
			println(err.Error())
			os.Exit(1)
		}
		fmt.Println("OK")
		return
	}
	if *ext == ".jpg" {
		if err = jpeg.Encode(f, img, &jpeg.Options{
			Quality: *quality,
		}); err != nil {
			print("error saving merged file: ")
			println(err.Error())
			os.Exit(1)
		}
		fmt.Println("OK")
		return
	}
	print("extension not supported: ")
	println(*ext)
	os.Exit(1)
}

This is quite old and not really the best idea for mass downloading. Maybe you could use it to just generate the URLs, then use scrapy or something to fetch them in parallel. Even then, (a) Mapbox probably won't let you do it, and (b) you'll use up your API tokens quite quickly if you want mass download. I'd suggest finding a ready-made archive of pre-rendered images. I haven't read the ToS though. Indeed, Google won't let you even store the map tiles, not even in a cache if I'm not mistaken, but I'm not sure about Mapbox.

Alternatively, you could render the tiles yourself using mapnik or some more modern alternative.

Sorry if this didn't help much. It also depends on how many tiles you need, if it's <100k then maybe you should just go ahead and try, and see what happens. You say "hundreds" of images so that doesn't sound like something that would even be noticed anywhere.

You're great, thank you for the advices. Best regards Il ven 12 gen 2024, 15:08 Attila Oláh ***@***.***> ha scritto:

…

***@***.**** commented on this gist. ------------------------------ This is quite old and not really the best idea for mass downloading. Maybe you could use it to just generate the URLs, then use scrapy or something to fetch them in parallel. Even then, (a) Mapbox probably won't let you do it, and (b) you'll use up your API tokens quite quickly if you want mass download. I'd suggest finding a ready-made archive of pre-rendered images. I haven't read the ToS though. Indeed, Google won't let you even store the map tiles, not even in a cache if I'm not mistaken, but I'm not sure about Mapbox. Alternatively, you could render the tiles yourself using mapnik or some more modern alternative. Sorry if this didn't help much. It also depends on how many tiles you need, if it's <100k then maybe you should just go ahead and try, and see what happens. You say "hundreds" of images so that doesn't sound like something that would even be noticed anywhere. — Reply to this email directly, view it on GitHub <https://gist.github.com/attilaolah/6a5a9c1f54463dcb9fda6188d856c2a2#gistcomment-4828156> or unsubscribe <https://github.com/notifications/unsubscribe-auth/A43RHSCGRDEWBJL63QCDDP3YOE7W7BFKMF2HI4TJMJ2XIZLTSKBKK5TBNR2WLJDUOJ2WLJDOMFWWLO3UNBZGKYLEL5YGC4TUNFRWS4DBNZ2F6YLDORUXM2LUPGBKK5TBNR2WLJDHNFZXJJDOMFWWLK3UNBZGKYLEL52HS4DFVRZXKYTKMVRXIX3UPFYGLK2HNFZXIQ3PNVWWK3TUUZ2G64DJMNZZDAVEOR4XAZNEM5UXG5FFOZQWY5LFVEYTANZRGM2TAOBSU52HE2LHM5SXFJTDOJSWC5DF> . You are receiving this email because you commented on the thread. Triage notifications on the go with GitHub Mobile for iOS <https://apps.apple.com/app/apple-store/id1477376905?ct=notification-email&mt=8&pt=524675> or Android <https://play.google.com/store/apps/details?id=com.github.android&referrer=utm_campaign%3Dnotification-email%26utm_medium%3Demail%26utm_source%3Dgithub> .