Downloader for AWS Modis data

aws_modis_downloader.go

package main

import (
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"net/http"
	"path"
	"os"
	"io"
)

const indexURL = "https://modis-pds.s3.amazonaws.com/?list-type=2&prefix=MCD43A4.006/%02d/%02d/2017007"
const baseURL = "https://modis-pds.s3.amazonaws.com/"

type Contents struct {
	Key          string
	LastModified string
	ETag         string
	Size         int
	StorageClass string
}

type ListBucketResult struct {
	Name                  string
	NextContinuationToken string
	KeyCount              int
	MaxKeys               int
	IsTruncated           bool
	Contents              []Contents
}

func ModisPathGenerator() chan string {

	out := make(chan string)

	go func() {
		for v := 2; v < 15; v++ {
			for h := 0; h < 36; h++ {

				resp, err := http.Get(fmt.Sprintf(indexURL, h, v))
				if err != nil {
					panic(err)
				}

				defer resp.Body.Close()
				conts, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					panic(err)
				}

				var lbRes ListBucketResult
				err = xml.Unmarshal(conts, &lbRes)
				if err != nil {
					panic(err)
				}

				for _, lb := range lbRes.Contents {
					switch lb.Key[len(lb.Key)-6 : len(lb.Key)] {
					case "01.TIF":
						out <- lb.Key
					case "02.TIF":
						out <- lb.Key
					}
				}

			}
		}
	}()

	return out

}

func main() {

	for p := range ModisPathGenerator() {
		fName := path.Base(p)
		fURL := baseURL + p
		fmt.Println(fURL)


		out, err := os.Create(fmt.Sprintf("/Users/pablo/Desktop/modis/%s", fName))

		resp, err := http.Get(fURL)
		if err != nil {
			fmt.Println("!!!!!!!!", err)
			continue
		}
		

		_, err = io.Copy(out, resp.Body)
		if err != nil {
			fmt.Println("!!!!!!!!", err)
		}

		out.Close()
		resp.Body.Close()
	}

}