firestore to json & json to firestore

readme.md

The origin of this gist: https://blog.cloudboost.io/copy-export-a-cloud-firestore-database-388cde99259b

The schema is for example only, it's must be modified according to the firestore database.

Quoting Bruno Braga

"this schema is how your DB is organized in a tree structure. You don't have to care about the Documents but you do need to inform the name of your collections and any subcollections, in this case we have two collections called users and groups, the all have their documents, but the collection users has its own subcollections, friends and groups, which again have their own subcollection, messages."

For any doubt, add it here, but let me know by email that the question has been added, I usually go unnoticed when someone comments here.

[email protected]

firestore2json.js

const admin = require('firebase-admin');
const fs = require('fs');

const serviceAccount = require('../../../../../../Private/myschool-data_transfer-key.json');

admin.initializeApp({ credential: admin.credential.cert(serviceAccount) });

const schema = require('./schema').schema;

const firestore2json = (db, schema, current) => {
  return Promise.all(
    Object.keys(schema).map(collection => {
      return db
        .collection(collection)
        .get()
        .then(data => {
          let promises = [];
          data.forEach(doc => {
            if (!current[collection]) current[collection] = { __type__: 'collection' };
            current[collection][doc.id] = doc.data();
            promises.push(
              firestore2json(
                db.collection(collection).doc(doc.id),
                schema[collection],
                current[collection][doc.id]
              )
            );
          });
          return Promise.all(promises);
        });
    })
  ).then(() => current);
};

firestore2json(admin.firestore(), { ...schema }, {}).then(res =>
  fs.writeFileSync('./src/native_web/firebase/local_db.json', JSON.stringify(res, null, 2), 'utf8')
);

json2firestore.js

const admin = require('firebase-admin');
const fs = require('fs');

const serviceAccount = require('../../../../../../Private/myschool-data_transfer-key.json');

admin.initializeApp({ credential: admin.credential.cert(serviceAccount) });

const schema = require('./schema').schema;

const json2firestore = (_JSON, db, schema) => {
  return Promise.all(
    Object.keys(schema).map(collection => {
      let promises = [];
      Object.keys(_JSON[collection]).map(_doc => {
        const doc_id = _doc;
        if (_doc === '__type__') return;
        let doc_data = Object.assign({}, _JSON[collection][_doc]);
        Object.keys(doc_data).map(_doc_data => {
          if (doc_data[_doc_data] && doc_data[_doc_data].__type__) delete doc_data[_doc_data];
        });
        promises.push(
          db
            .collection(collection)
            .doc(doc_id)
            .set(doc_data)
            .then(() => {
              return json2firestore(
                _JSON[collection][_doc],
                db.collection(collection).doc(doc_id),
                schema[collection]
              );
            })
        );
      });
      return Promise.all(promises);
    })
  );
};

json2firestore(
  JSON.parse(fs.readFileSync('./src/native_web/firebase/local_db.json', 'utf8')),
  admin.firestore(),
  { ...schema }
).then(() => console.log('done'));

schema.js

exports.schema = {
	all_users: {},
	feedback: {},
	reports: {},
	schools: {
		grades: {},
		notifications: {},
		publications: {},
		users: {},
	}
};

Workaround for dates:

const fixDates = (document) => {
  for (var key in document) {
    const val = document[key];

    if (val != null && val["_seconds"] != null && val["_nanoseconds"] != null) {
      document[key] = new admin.firestore.Timestamp(
        val["_seconds"],
        val["_nanoseconds"]
      );
    }
    if (val != null && val.constructor === Object) {
      fixDates(val);
    }
  }
  return document;
};

Then add doc_data = fixDates(doc_data);
right before setting the data

Its working well! Thanks a lot @klivin

So amazing to see that this is still helping people @sturmenta thanks for keeping it up to date.

@sturmenta can you please make a package with this ? I could improve a few things for handling very large collections (1M+ documents)

export / import command line

Project to export
Goto -> project settings -> Service account -> Generate new private key -> save it as exportedDB.json

Project to import
Goto -> project settings -> Service account -> Generate new private key -> save it as importedDB.json

run these 2 commands from the folder where u saved the files

Export:
npx -p node-firestore-import-export firestore-export -a exportedDB.json -b backup.json

Import:
npx -p node-firestore-import-export firestore-import -a importedDB.json -b backup.json

Hope it helps

Hi @sadortun , I am bruno the initial creator of this script :)

can you expand on your use case and why https://firebase.google.com/docs/firestore/manage-data/export-import does not fix your problem?

I would gladly tackle this issue and create a package if it seems like something that could benefit others :)

Workaround for dates:

const fixDates = (document) => {
  for (var key in document) {
    const val = document[key];

    if (val != null && val["_seconds"] != null && val["_nanoseconds"] != null) {
      document[key] = new admin.firestore.Timestamp(
        val["_seconds"],
        val["_nanoseconds"]
      );
    }
    if (val != null && val.constructor === Object) {
      fixDates(val);
    }
  }
  return document;
};

Then add doc_data = fixDates(doc_data); right before setting the data

This was not working for document with arrays and dates inside. Here the code fixed:

const fixDates = (document) => {
    for (var key in document) {
        const val = document[key];
        if (val != null && val["_seconds"] != null && val["_nanoseconds"] != null) {
            document[key] = new admin.firestore.Timestamp(
                   val["_seconds"],
                   val["_nanoseconds"]
            );
        }
        if (val != null && val.constructor === Object) {
            fixDates(val);
        }
        if (val != null && val.constructor === Array) {
            for(var arrayKey of val){
                fixDates(arrayKey);
            }
        }
    }
    return document;
};