Copy firestore database

copyFirestoreDB.js

const firebase = require('firebase-admin');

var serviceAccountSource = require("./source.json"); // source DB key
var serviceAccountDestination = require("./destination.json"); // destiny DB key

const sourceAdmin = firebase.initializeApp({
  credential: firebase.credential.cert(serviceAccountSource)
});

const destinyAdmin = firebase.initializeApp({
  credential: firebase.credential.cert(serviceAccountDestination)
}, "destination");

/* this schema is how your DB is organized in a tree structure. You don't have to care about the Documents
  but you do need to inform the name of your collections and any subcollections, in this
  case we have two collections called users and groups, the all have their documents, but 
  the collection users has its own subcollections, friends and groups, which again have their
  own subcollection, messages.
*/
const schema = {
  users: {
    friends: {
      messages: {},
    },
    groups: {
      messages: {},
    },
  },
  groups: {},
};

var source = sourceAdmin.firestore();
var destination = destinationAdmin.firestore();

const copy = (sourceDBrep, destinationDBref, aux) => {
  return Promise.all(Object.keys(aux).map((collection) => {
    return sourceDBrep.collection(collection).get()
      .then((data) => {
        let promises = [];
        data.forEach((doc) => {
          const data = doc.data();
          promises.push(
            destinationDBref.collection(collection).doc(doc.id).set(data).then((data) => {
              return copy(sourceDBrep.collection(collection).doc(doc.id),
              destinationDBref.collection(collection).doc(doc.id),
              aux[collection])
            })
          ); 
        })
      return Promise.all(promises);
    })
  }));
};

copy(source, destination, aux).then(() => {
  console.log('copied');
});

Line 10: destinyAdmin should be destinationAdmin

Line 55: ReferenceError: aux is not defined
Needs:
let aux = { ...schema };
Right above where the copy function is called.

Bug: For very large collection transfers (over 20,000 documents), transfer fails.

(node:9785) UnhandledPromiseRejectionWarning: Unhandled promise rejection (rejection id: 1): Error: Deadline Exceeded
(node:9785) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code.

Need to incorporate batched writes . A batched write can contain up to 500 operations. So the feature needs to read up to 500 items at the origin, and batch write those to the destination, and continue to do this until there is nothing left to read at the origin. This batching process also cuts the total number of reads / writes, which is less expensive.

+1 to @boon4376

+1 to @boon4376 from me too

Amazing thanks 🙏🙌

this was working great and has recently stopped. is anyone still able to run it? Any known changes that could have caused issues?

change the source code at line 33
from
var destination = destinationAdmin.firestore();
to
var destination = destinyAdmin .firestore();

change line 20
from
const schema = {
to
const aux = {

trying for big data, halt on 5347 record
FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory

running on 8gb Ram server, 16 core processor
using node 12