firestore to json & json to firestore

readme.md

The origin of this gist: https://blog.cloudboost.io/copy-export-a-cloud-firestore-database-388cde99259b

The schema is for example only, it's must be modified according to the firestore database.

Quoting Bruno Braga

"this schema is how your DB is organized in a tree structure. You don't have to care about the Documents but you do need to inform the name of your collections and any subcollections, in this case we have two collections called users and groups, the all have their documents, but the collection users has its own subcollections, friends and groups, which again have their own subcollection, messages."

For any doubt, add it here, but let me know by email that the question has been added, I usually go unnoticed when someone comments here.

[email protected]

firestore2json.js

const admin = require('firebase-admin');
const fs = require('fs');

const serviceAccount = require('../../../../../../Private/myschool-data_transfer-key.json');

admin.initializeApp({ credential: admin.credential.cert(serviceAccount) });

const schema = require('./schema').schema;

const firestore2json = (db, schema, current) => {
  return Promise.all(
    Object.keys(schema).map(collection => {
      return db
        .collection(collection)
        .get()
        .then(data => {
          let promises = [];
          data.forEach(doc => {
            if (!current[collection]) current[collection] = { __type__: 'collection' };
            current[collection][doc.id] = doc.data();
            promises.push(
              firestore2json(
                db.collection(collection).doc(doc.id),
                schema[collection],
                current[collection][doc.id]
              )
            );
          });
          return Promise.all(promises);
        });
    })
  ).then(() => current);
};

firestore2json(admin.firestore(), { ...schema }, {}).then(res =>
  fs.writeFileSync('./src/native_web/firebase/local_db.json', JSON.stringify(res, null, 2), 'utf8')
);

json2firestore.js

const admin = require('firebase-admin');
const fs = require('fs');

const serviceAccount = require('../../../../../../Private/myschool-data_transfer-key.json');

admin.initializeApp({ credential: admin.credential.cert(serviceAccount) });

const schema = require('./schema').schema;

const json2firestore = (_JSON, db, schema) => {
  return Promise.all(
    Object.keys(schema).map(collection => {
      let promises = [];
      Object.keys(_JSON[collection]).map(_doc => {
        const doc_id = _doc;
        if (_doc === '__type__') return;
        let doc_data = Object.assign({}, _JSON[collection][_doc]);
        Object.keys(doc_data).map(_doc_data => {
          if (doc_data[_doc_data] && doc_data[_doc_data].__type__) delete doc_data[_doc_data];
        });
        promises.push(
          db
            .collection(collection)
            .doc(doc_id)
            .set(doc_data)
            .then(() => {
              return json2firestore(
                _JSON[collection][_doc],
                db.collection(collection).doc(doc_id),
                schema[collection]
              );
            })
        );
      });
      return Promise.all(promises);
    })
  );
};

json2firestore(
  JSON.parse(fs.readFileSync('./src/native_web/firebase/local_db.json', 'utf8')),
  admin.firestore(),
  { ...schema }
).then(() => console.log('done'));

schema.js

exports.schema = {
	all_users: {},
	feedback: {},
	reports: {},
	schools: {
		grades: {},
		notifications: {},
		publications: {},
		users: {},
	}
};

HI,
Sorry to ask very basic question, but I am not sure how to use this script..
I put it inside a new folder in my desktop, then I open a terminal (moving into the desktop folder), then I run npm install firebase-admin then I run node firestore2json.js but without changing anything in the script... 1/ is it the good method ? - 2/I am sure I have to change something inside the code to access to my own database, but I do not know what to change ? I have download the new private key from firebase but also do not know how to use it.. Thanks for help

hello @YLM-UJM since you answered me via email that you could solve your problem, it would be good if you could add the solution here, if it is useful for someone, thank you very much!

The export leaves timestamps as objects:
"createdAt": { "_seconds": 1588657677, "_nanoseconds": 660000000 },

The following has no effect on storing or parsing of the timestamps:
firestore.settings({ timestampsInSnapshots: true });

Am I missing something? thanks!

Workaround for dates:

const fixDates = (document) => {
  for (var key in document) {
    const val = document[key];

    if (val != null && val["_seconds"] != null && val["_nanoseconds"] != null) {
      document[key] = new admin.firestore.Timestamp(
        val["_seconds"],
        val["_nanoseconds"]
      );
    }
    if (val != null && val.constructor === Object) {
      fixDates(val);
    }
  }
  return document;
};

Then add doc_data = fixDates(doc_data);
right before setting the data

Workaround for dates:

const fixDates = (document) => {
  for (var key in document) {
    const val = document[key];

    if (val != null && val["_seconds"] != null && val["_nanoseconds"] != null) {
      document[key] = new admin.firestore.Timestamp(
        val["_seconds"],
        val["_nanoseconds"]
      );
    }
    if (val != null && val.constructor === Object) {
      fixDates(val);
    }
  }
  return document;
};

Then add doc_data = fixDates(doc_data);
right before setting the data

Its working well! Thanks a lot @klivin

So amazing to see that this is still helping people @sturmenta thanks for keeping it up to date.

@sturmenta can you please make a package with this ? I could improve a few things for handling very large collections (1M+ documents)

export / import command line

Project to export
Goto -> project settings -> Service account -> Generate new private key -> save it as exportedDB.json

Project to import
Goto -> project settings -> Service account -> Generate new private key -> save it as importedDB.json

run these 2 commands from the folder where u saved the files

Export:
npx -p node-firestore-import-export firestore-export -a exportedDB.json -b backup.json

Import:
npx -p node-firestore-import-export firestore-import -a importedDB.json -b backup.json

Hope it helps

Hi @sadortun , I am bruno the initial creator of this script :)

can you expand on your use case and why https://firebase.google.com/docs/firestore/manage-data/export-import does not fix your problem?

I would gladly tackle this issue and create a package if it seems like something that could benefit others :)

Workaround for dates:

const fixDates = (document) => {
  for (var key in document) {
    const val = document[key];

    if (val != null && val["_seconds"] != null && val["_nanoseconds"] != null) {
      document[key] = new admin.firestore.Timestamp(
        val["_seconds"],
        val["_nanoseconds"]
      );
    }
    if (val != null && val.constructor === Object) {
      fixDates(val);
    }
  }
  return document;
};

Then add doc_data = fixDates(doc_data); right before setting the data

This was not working for document with arrays and dates inside. Here the code fixed:

const fixDates = (document) => {
    for (var key in document) {
        const val = document[key];
        if (val != null && val["_seconds"] != null && val["_nanoseconds"] != null) {
            document[key] = new admin.firestore.Timestamp(
                   val["_seconds"],
                   val["_nanoseconds"]
            );
        }
        if (val != null && val.constructor === Object) {
            fixDates(val);
        }
        if (val != null && val.constructor === Array) {
            for(var arrayKey of val){
                fixDates(arrayKey);
            }
        }
    }
    return document;
};