Argo data collector

argoDataCollector.go

// miDataCollectorArgo
package main

import (
	"fmt"
	"os"
	"io/ioutil"
	"encoding/json"
	"strings"
	"strconv"
	"net/http"
	"github.com/fhs/go-netcdf/netcdf"
)

type ControlFile struct {
	Ncdump string 						`json:"ncdump"`
	Kafka struct {
		Topic string 					`json:"topic"`
		Server string 					`json:"server"`
	} 									`json:"kafka"`
	BaseURL struct {
		URL string 						`json:"url"`
		PadProfileID int 				`json:"padProfileID"`
	} 									`json:"baseURL"`
	Floats []struct {
		ID int 							`json:"id"`
		Profile int 					`json:"profile"`
	} 									`json:"floats"`
}

//
// Builds the URL for the profile NetCDF file
//	TODO Function documentaion
//	TODO Unit test for function
//
func ProfileUrl (floatID int, profileID int, baseURL string) string {
	return strings.Replace(
		strings.Replace(baseURL,"{floatID}",strconv.Itoa(floatID),-1),
		"{profileID}",strconv.Itoa(profileID),-1)
}

func main() {
//
// Fudge for  proxy servers
//	TODO Needs fixing / doing properly...
//
	os.Setenv("HTTP_PROXY", "http://0.0.0.0:80")

//
// Read the control file into memory
//		TODO Choose control file from the command line
//
	file, e := 
		ioutil.ReadFile(
			"/Documents/js/argoAcquire/argo.json")
	if e != nil {
        fmt.Printf("File error: %v\n", e)
        os.Exit(1)
    }
//
// Parse the control file to a structure
//
	var jsontype ControlFile
    json.Unmarshal(file, &jsontype)
//
// Iterate over the floats
//
	var profileNotFound int;
	for _, float := range jsontype.Floats{
		//
		// Iterate over the profiles
		//
		profileNotFound = 0
		for profileNotFound < 1 {
			//
			// Create the URL to call
			//
			fmt.Printf("%s\n",
				ProfileUrl(float.ID,float.Profile,jsontype.BaseURL.URL))
			resp, err := 
				http.Get(ProfileUrl(float.ID,float.Profile,jsontype.BaseURL.URL)) 
			if err != nil {
				panic(err)
			}
			defer resp.Body.Close()
			//
			// If we get a 404, we have gone past the last profile for this 
			// float
			//
			if strings.Compare(resp.Status,"404 Not Found") == 0 {
				profileNotFound = 1	
			} else {
				ff, _ := os.Open(ProfileUrl(float.ID,float.Profile,jsontype.BaseURL.URL))
				f, _ := cdf.Open(ff)
				r := f.Reader("TEMP", nil, nil)
				buf := r.Zero(100)      // a []T of the right T for the variable.
				n, err := r.Read(buf)   // similar to io.Read, but reads T's instead of bytes.
				if err != nil {
					panic(err)
				}
				fmt.Printf("%s\n",n)
				float.Profile = float.Profile + 1
			}
		}
	}
//
// TODO Write the new command file out
//
}

control.json

{
    "ncdump": "ncdump -i -v JULD,TEMP,PRES,PSAL,LATITUDE,LONGITUDE {file} | sed -e \"1,/data:/d\" -e \"$d\"", 
    "kafka": {
        "topic": "argo", 
        "server": "0.0.0"
    }, 
    "baseURL": {
        "url": "http://www.usgodae.org/ftp/outgoing/argo/dac/bodc/{floatID}/profiles/R{floatID}_{profileID}.nc", 
        "padProfileID": 3
    }, 
    "floats": [
        {"id": 6900444, "profile": 185}, 
        {"id": 6900658, "profile": 199}
    ]
}