IPT Sync script

sync.js

var srequest = require('sync-request');
var fs = require('fs');

// A very hacky script to issue an inventory request to each registered IPT (with reasonable timeout) and summarise the counts of the
// IPT opinion versus the GBIF.org index opinion.  This only to get an idea of the extent of mismatch.
// NOTE: many fail due to https://github.com/gbif/ipt/issues/1344

var res = srequest('GET', 'http://api.gbif.org/v1/installation?type=IPT_INSTALLATION&limit=10000');
var json = JSON.parse(res.getBody())

var results = []

var count = 0;
for (let installation of json.results) {

    if (installation.type === "IPT_INSTALLATION") {
        try {
            var inventoryUrl = installation.endpoints[0].url.replace("rss.do", "inventory/dataset")
            console.log(count++ + ": Calling inventory: " + inventoryUrl);

            var inventoryResponse = srequest('GET', inventoryUrl, {timeout: 20000});
            var inventoryJson = JSON.parse(inventoryResponse.getBody())
            if (inventoryJson.registeredResources != null) {
                for (let resource of inventoryJson.registeredResources) {
                    if (resource.type=== "OCCURRENCE") {
                        console.log(installation.key + "|" + installation.title + "|" + resource.gbifKey + "|" + resource.records);

                        var gbifCount = srequest('GET', 'http://api.gbif.org/v1/occurrence/count?datasetKey=' + resource.gbifKey);

                        let result = {
                            installationKey: installation.key,
                            installationTitle: installation.title.replace('\n', ' ').replace('\r', ' '),
                            datasetKey: resource.gbifKey,
                            datasetCountInIPT: resource.records,
                            datasetCountInGBIF: gbifCount.getBody()
                        }
                        results.push(result)

                    } else {
                        console.log("Skipping non occurrence dataset")
                    }


                }
            } else {
                console.log("No records");
            }

        } catch(e) {
            console.log("Error: ", e);
        }

    }
}

fs.writeFile(

    '/tmp/inventory.csv',

    results.map(function (r) {
        return r.installationKey + "|" +
                r.installationTitle + "|" +
                r.datasetKey+ "|" +
                r.datasetCountInIPT+ "|" +
                r.datasetCountInGBIF+ "|" +
                (r.datasetCountInIPT==r.datasetCountInGBIF)
    }).join("\n"),

    function (err) {
        if (err) {
            console.error('Failed writing file');
        }
    }
);