connect to Panoptes JS API and remove subjects

remove-subjects.js

#!/usr/bin/env node
require('es6-promise').polyfill();

// staging
//var PROJECTID = "1611";
//var WORKFLOWID = "2315";
//var SUBJECTSETID = "3700";
//var SUBJECT_IDS_TO_DELETE = ["45046"];

// production
var PROJECTID = "988";
var WORKFLOWID = "512";
var SUBJECTSETID = "4282";

var SPECIES_TO_DELETE = ["Hartebeest", "Wildebeest", "Buffalo", "Guinea Fowls", "Warthogs"];
var SUBJECT_IDS_TO_DELETE = [];
var FastCSV = require('fast-csv');
var fs = require('fs');
var glob = require('glob');
var argv = require('yargs').argv;
var request = require('request');
var auth = require('panoptes-client/lib/auth');
var api = require('panoptes-client/lib/api-client');

var subject_set_name = null;
var workflow_name = null;
var csv_name = argv.csv;

if (typeof(csv_name)=="undefined") {
  throw new Error("Please launch with \"node remove-subjects.js -u username -p password --csv <your_csv_filename>\"");
}

var input = fs.createReadStream( csv_name );
var count = 0;

function processLine(line) {
  workflow_id = (eval(line.workflow_ids)[0]).toString();
  metadata = JSON.parse(line.metadata);
  species = metadata["Crowd Determination"]
  if (PROJECTID==line.project_id && WORKFLOWID==workflow_id && SUBJECTSETID==line.subject_set_id && SPECIES_TO_DELETE.indexOf(species) >= 0) {
    SUBJECT_IDS_TO_DELETE.push(line.subject_id);
  }
  count = count + 1;
}

// this should work and tell us staging or production:
// console.log ("Connected to ",api.host);
// (can change by setting NODE_ENV=staging or NODE_ENV=production)

function signIn() {
  login = {
    login: argv.u,
    password: argv.p,
    remember_me: true
  }
  return auth.signIn(login);
}

function testGetWorkflow() {
  console.log(`Doing a test GET of workflow ${WORKFLOWID}...`);
  api.type('workflows').get(WORKFLOWID)
  .then( function (workflow) {
    workflow_name = workflow.display_name
    console.log(`  Got workflow ${workflow.id}: '${workflow_name}'`);
  })
  .catch(function (error) {
    console.log(`  Error getting workflow: ${error}`);
  });
}

function testGetSubjectSet() {
  console.log(`Doing a test GET of subject set ${SUBJECTSETID}...`);
  api.type('subject_sets').get(SUBJECTSETID)
  .then( function (subject_set) {
    subject_set_name = subject_set.display_name
    console.log(`  Got subject set ${subject_set.id}: '${subject_set_name}'`);
  })
  .catch(function (error) {
    console.log(`  Error getting subject set ${error}`);
  });
}

function testDeleteSubjects() {
  console.log(`Doing a test DELETE from subject set ${SUBJECTSETID} of subjects ${SUBJECT_IDS_TO_DELETE}...`);
  api.type('subject_sets').get(SUBJECTSETID)
  .then( function (subject_set) {
    subject_set_name = subject_set.display_name;
    console.log(`  Got subject set ${subject_set.id}: '${subject_set_name}', which currently has ${subject_set.set_member_subjects_count} subjects. Deleting...`);
    subject_set.removeLink('subjects', SUBJECT_IDS_TO_DELETE)
    .then( function(new_subject_sets) {
      new_subject_set = new_subject_sets[0]
      console.log(`  Deleted subjects successfully. Set now has ${new_subject_set.set_member_subjects_count} subjects remaining.`);
    })
    .catch( function(e) {
      console.log(e);
    });
  })
  .catch(function (error) {
    console.log(`Error getting subject set ${SUBJECTSETID} ready to delete from it`, error);
  });
}

function testGetSubjectMessageSet() {
  console.log(`Doing a test get of subject message set for subject set ${SUBJECTSETID} and workflow '${WORKFLOWID}...`);
  api.type('set_member_subjects').get({subject_set_id: SUBJECTSETID, workflow: WORKFLOWID})
  .then( function (setMemberSubjects) {
    console.log(`Found subject message set for subject set '${subject_set_name}' and workflow '${workflow_name}' with ${setMemberSubjects.length} linked subjects.`);
    console.log("Getting subjects...");
    setMemberSubjects.forEach( function (setMemberSubject) {
      sms_id = setMemberSubject.id;
      subject_id = setMemberSubject.links.subject;
      // note these are equal

      doIt = true;
      subjectsToDelete=[];

      api.type('subjects').get(subject_id)
      .then( function (subject) {
        console.log(`  ID: ${subject.id}`);
        console.log("    Media:");
        subject.locations.forEach( function (location) {
          for (var type in location) {
            if (location.hasOwnProperty(type)) {
              console.log(`      ${location[type]} (${type})`);
            }
          }
        });
        console.log("    Metadata:");
        console.log(`      ${JSON.stringify(subject.metadata)}`);
      })
      .catch(function (error) {
        console.log(`Error getting subject ${subject_id}`, error);
      });

      api.type('subject_sets').
        testDeleteSubject(setMemberSubjects,subjectsToDelete);
    });
  })
  .catch(function (error) {
    console.log(`Error getting subject message set: {$error}`);
  });
}

function deleteSubjects(subject_IDs_to_delete) {
  console.log(subject_IDs_to_delete);
  api.type('subject_sets').get(SUBJECTSETID)
  .then( function (subject_set) {
    subject_set_name = subject_set.display_name;
    console.log(`  Got subject set ${subject_set.id}: '${subject_set_name}', which currently has ${subject_set.set_member_subjects_count} subjects. Deleting ${subject_IDs_to_delete.length} subjects...`);
    subject_set.removeLink('subjects', subject_IDs_to_delete)
    .then( function(new_subject_sets) {
      new_subject_set = new_subject_sets[0];
      console.log(`  Successfully submitted request to delete subjects. Set currently has ${new_subject_set.set_member_subjects_count} subjects remaining (but check back soon and they all will have been deleted.`);
    })
    .catch( function(e) {
      console.log(e);
    });
  })
  .catch(function (error) {
    console.log(`Error getting subject set ${SUBJECTSETID} ready to delete from it`, error);
  });
}

function processedCSV() {
  console.log(`Processed CSV. Processed ${count} lines and found ${SUBJECT_IDS_TO_DELETE.length} subject IDs to be deleted.`);
  var chunkOfBigSubjectIDsList = [];
  var chunkSize = 100;
  for (i=0, j=SUBJECT_IDS_TO_DELETE.length; i<j; i+=chunkSize) {
    chunkOfBigSubjectIDsList = SUBJECT_IDS_TO_DELETE.slice(i, i + chunkSize);
    deleteSubjects(chunkOfBigSubjectIDsList);
  }
  console.log ("\nDone.\n");
}

signIn()
  .then( function (user) {
    api.update({'params.admin' : user.admin})

    Promise.all([
      api.type('workflows').get(WORKFLOWID),
      api.type('projects').get(PROJECTID),
      api.type('subject_sets').get(SUBJECTSETID)
    ])
    .then( function (values) {
      console.log(`WORKFLOW ${values[0].display_name}`);
      console.log(`PROJECT ${values[1].display_name}`);
      console.log(`SUBJECT SET ${values[2].display_name}`);
      subject_set_name = values[2].display_name;
      //testGetSubjectSet();
      //testGetWorkflow();
      //testGetSubjectMessageSet();
      //testDeleteSubjects();

      var stream = fs.createReadStream(csv_name)
      FastCSV
        .fromStream(stream, {headers : true})
        .on("data", processLine )
        .on("end", processedCSV );
    })
    .catch(function (error) {
      console.log(`Error getting workflow and project: ${error}`);
    });
  })
  .catch(function(error){
    console.log('Done', error);
  });