Batch object processing code intended for use in the Parse Dashboard Javascript API console

parse_dashboard_js_console_batch.js

///////////    Job Settings    ///////////
const itemsPerCycle = 1000; // Recommended < 1001
const maxCycles = 10;
const asyncBatchSize = 10;  // Recommended < 11 to not slow down server
const msgPerCycle = 1;      // Set low to prevent log spam with large cycle counts
const objectType = new Parse.Object("Product");

// Context values
let flagQ = new Parse.Query(new Parse.Object("Flag"));
flagQ.equalTo("name", "clothing");
let flag = await flagQ.first({useMasterKey: true});

///////////   Work Functions   ///////////
/* Return a Parse.Query() that selects the items we want to work on           */
/* Called once per cycle. THIS QUERY SHOULD EXCLUDE ALREADY PROCESSED ITEMS!! */
function workQuery(){
  let itemQuery = new Parse.Query(objectType);
  itemQuery.exists("eBayStoreCategoryName");
  itemQuery.contains("eBayStoreCategoryName", "CLOTHES SHOES & ACCESSORIES > Luggage and Travel");
  //itemQuery.doesNotMatchQuery("flags", flagQ);
  itemQuery.matchesQuery("flags", flagQ);
  return itemQuery;
}

/* Perform whatever task we want to do on each item and return a Future() that resolves when we're done */
async function work(item) {
  let flags = item.get("flags");
  flags.remove(flag);
  return item.save(null, {useMasterKey: true});
}

///////////     Main Loop      ///////////
let remainingItems = 1;
let cycles = 0;
console.log(`Starting job...`);
while (remainingItems > 0 && (cycles < maxCycles)){
  let itemQuery = workQuery();
  remainingItems = await itemQuery.count({useMasterKey: true});
  if (remainingItems < 1) break;
  console.log(`Collecting ${Math.min(remainingItems, itemsPerCycle)} of ${remainingItems} matching items...`);
  itemQuery.limit(itemsPerCycle);
  let items = await itemQuery.find({useMasterKey: true});
  let futures = [];
  let i = 0;
  for (let item of items){
    if (futures.length >= asyncBatchSize || i === (items.length-1)){
      await Promise.all(futures);
      futures = [];
      if (msgPerCycle > 1 && i % (Math.ceil(itemsPerCycle / msgPerCycle)) == 0) console.log(`Processed ${i} items`)
    }
    futures.push(work(item));
    i++;
  }
  // Await any remaining promises, just in case
  await Promise.all(futures);
  if (msgPerCycle > 0) console.log(`Processed ${i} items`)
  cycles++;
}

console.log(`############# DONE - ${cycles} cycles #############`);
return;