Firebase Hosting Fetch All Files

README.md

Fetch All Files from Firebase Hosting

This script fetches all of the files from the currently deployed version of a Firebase Hosting site. You must be signed in via the Firebase CLI and have "Site Viewer" permission on the site in question to be able to properly run the script.

Running via NPX

npx https://gist.github.com/mbleigh/9c8680cf319ace2f506f57380da66e7d <site_name>

Where <site_name> is your Firebase Hosting site (e.g. my-site from my-site.firebaseapp.com). The files will be downloaded into a folder <site_name>_<version_id> where the version ID is the current version of your site.

fetchFiles.js

#!/usr/bin/env node

const requireAuth = require('firebase-tools/lib/requireAuth');
const api = require('firebase-tools/lib/api');
const fs = require('fs-extra');
const request = require('request');

if (!process.argv[2]) {
    console.error(`
ERROR: Must supply a site name. Usage:

  node fetchFiles.js <site_name>`);
  process.exit(1);
}

const site = process.argv[2];

async function getLatestVersionName() {
  const result = await api.request('GET', `/v1beta1/sites/${site}/releases?pageSize=1`, {
    auth: true,
    origin: api.hostingApiOrigin,
  });
  const release = (result.body.releases || [])[0];
  if (release) {
    return release.version.name;
  }
  return null;
}

const LIST_PAGE_SIZE = 1000;
async function listFiles(versionName, existing = [], pageToken = null) {
  const result = await api.request('GET', `/v1beta1/${versionName}/files?pageSize=${LIST_PAGE_SIZE}${pageToken ? `&pageToken=${pageToken}` : ''}`, {auth: true, origin: api.hostingApiOrigin});
  result.body.files.forEach(file => existing.push(file.path));
  if (result.body.nextPageToken) {
    return await listFiles(versionName, existing, result.body.nextPageToken);
  }
  return existing;
}

const MAX_FETCHES = 100;

(async function() {
  try {
    await requireAuth({}, ['https://www.googleapis.com/auth/cloud-platform']);
    const v = await getLatestVersionName();
    const vid = v.split('/')[v.split('/').length - 1];
    const toFetch = await listFiles(v);
    const dirName = `${site}_${vid}`;
    let fetchesOutstanding = 0;
    let fetchCount = 0;
    function fetch() {
      if (fetchesOutstanding >= MAX_FETCHES) {
        return;
      } else if (toFetch.length === 0) {
        console.log();
        console.log("Complete. Fetched", fetchCount, "files.");
        return;
      }

      const f = toFetch.shift();
      console.log('Fetching', f);
      fetchesOutstanding++;
      fetchCount++;
      fs.ensureFileSync(dirName + f);
      const q = request(`https://${site}.firebaseapp.com${f}`)
      const ws = fs.createWriteStream(dirName + f);
      q.pipe(ws);
      ws.on('finish', () => {
        console.log('Fetched ', f);
        fetchesOutstanding--;
        fetch();
      });
    }

    fetch();
  } catch (e) {
    console.error("ERROR:", e.stack);
    process.exit(1);
  }
})();

package.json

{
  "name": "firebase-hosting-fetch-version",
  "version": "0.1.0",
  "bin": "./fetchFiles.js",
  "dependencies": {
    "firebase-tools": "^7.4.0",
    "fs-extra": "^8.1.0",
    "request": "^2.88.0"
  }
}

(node:7784) Warning: Accessing non-existent property 'padLevels' of module exports inside circular dependency (Use node --trace-warnings ... to show where the warning was created) ERROR: FirebaseError: HTTP Error: 404, Not Found at module.exports (C:\Users\Administrator\AppData\Roaming\npm-cache\_npx\7784\node_modules\firebase-hosting-fetch-version\node_modules\firebase-tools\lib\responseToError.js:38:12) at Request._callback (C:\Users\Administrator\AppData\Roaming\npm-cache\_npx\7784\node_modules\firebase-hosting-fetch-version\node_modules\firebase-tools\lib\api.js:40:35) at Request.self.callback (C:\Users\Administrator\AppData\Roaming\npm-cache\_npx\7784\node_modules\firebase-hosting-fetch-version\node_modules\request\request.js:185:22) at Request.emit (events.js:315:20) at Request.<anonymous> (C:\Users\Administrator\AppData\Roaming\npm-cache\_npx\7784\node_modules\firebase-hosting-fetch-version\node_modules\request\request.js:1154:10) at Request.emit (events.js:315:20) at IncomingMessage.<anonymous> (C:\Users\Administrator\AppData\Roaming\npm-cache\_npx\7784\node_modules\firebase-hosting-fetch-version\node_modules\request\request.js:1076:12) at Object.onceWrapper (events.js:421:28) at IncomingMessage.emit (events.js:327:22) at endReadableNT (_stream_readable.js:1327:12)

I am Getting this error.
I am the site owner .

This is work

I tried like this:
(1) my site name is HelloWorld so...
(2) tried below - npx https://gist.github.com/mbleigh/9c8680cf319ace2f506f57380da66e7d HelloWorld

This is really working! Thank you so much you saved my project and time!!!

This is work

I tried like this:
(1) my site name is HelloWorld so...
(2) tried below - npx https://gist.github.com/mbleigh/9c8680cf319ace2f506f57380da66e7d HelloWorld

This is really working! Thank you so much you saved my project and time!!!

It is working for me too. just use your app's name without firebaseapp.com

In your advice, I have this question: " "Site Viewer" permission on the site" How to do this ?.
I did all the things you have shown except "site viewer" cause idk how to do this.

After those things, I got this error:
(node:8648) Warning: Accessing non-existent property 'padLevels' of module exports inside circular dependency
(Use node --trace-warnings ... to show where the warning was created)
ERROR: FirebaseError: HTTP Error: 404, Not Found

Please help me, I really need the source code to review and learn for an upcoming interview.
God blesses you.
Thankssssssss

Use npx https://gist.github.com/mbleigh/9c8680cf319ace2f506f57380da66e7d app name

Note, use app name without firebaseapp.com

I have error calling that "Unexpected token function"

In order to make this code work on my machine, I've changed line 44 from:

await requireAuth({}, ['https://www.googleapis.com/auth/cloud-platform']);

to

await requireAuth.requireAuth({}, ['https://www.googleapis.com/auth/cloud-platform']);

My node version is 16.13.0.
firebase-tools version 9.21.0

OS version:

Ubuntu 20.04.3 LTS
4.4.0-19041-Microsoft #1237-Microsoft Sat Sep 11 14:32:00 PST 2021 x86_64 x86_64 x86_64 GNU/Linux

In continuation to my previous comment https://gist.github.com/mbleigh/9c8680cf319ace2f506f57380da66e7d#gistcomment-3949726
Change has been needed, because I've used node instead of npx, as recommended in README.md.
Now I run again using npx and everything worked well.

Thanks @KurisuWong worked like a charm.

after the files are download - how do you actually view the code? in the files download, I don't see any familiar code from the react-application that was my latest deployment to firebase.

Great tool! Thank you!

I got the "Warning: Accessing non-existent property 'padLevels' of module exports inside circular dependency" error too.

Changed the site to exclude "firebaseapp.com" and it worked like a charm. Thanks.

thank you! is there something similar for fetch functions?

Using the above method I am only able to fetch the compiled version of dart files, not the actual source code. If anyone knows a workaround please help.

Firebase Hosting only has the files that were uploaded from your public directory. It is not a version control system and does not retain original source files.

…

On Sun, Apr 24, 2022, 5:36 AM raviganwal ***@***.***> wrote: ***@***.**** commented on this gist. ------------------------------ Using the above method I am only able to fetch the compiled version of dart files, not the actual project. If anyone knows a workaround please help. — Reply to this email directly, view it on GitHub <https://gist.github.com/9c8680cf319ace2f506f57380da66e7d#gistcomment-4143251>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AAAAH7SEKRABSMGWFS3E3DDVGU55BANCNFSM4JJ3CEDA> . You are receiving this because you authored the thread.Message ID: ***@***.***>

Should your files contain characters unacceptable in URLs, use this path to escape them first

diff --git a/fetchFiles.js b/fetchFiles.js
index 78e59ab..3409fc2 100644
--- a/fetchFiles.js
+++ b/fetchFiles.js
@@ -37,6 +37,11 @@ async function listFiles(versionName, existing = [], pageToken = null) {
   return existing;
 }

+function escapePathForURL(path) {
+  const components = path.split('/');
+  return components.map(encodeURIComponent).join('/');
+}
+
 const MAX_FETCHES = 100;

 (async function() {
@@ -62,7 +67,7 @@ const MAX_FETCHES = 100;
       fetchesOutstanding++;
       fetchCount++;
       fs.ensureFileSync(dirName + f);
-      const q = request(`https://${site}.firebaseapp.com${f}`)
+      const q = request(`https://${site}.firebaseapp.com${escapePathForURL(f)}`)
       const ws = fs.createWriteStream(dirName + f);
       q.pipe(ws);
       ws.on('finish', () => {

@mbleigh Feel free to apply it to your Gist.

worked like charm

THANK YOU!

I had same 404 issue,
as @jinmanpak mentioned we should use site name (not website URL).
Thanks @jinmanpak

@mbleigh Worked for me thank you a alot.

I posted the above url fix myself for convinience
https://gist.github.com/gaberilde/4c2d55d8c6de0772ca49c8b473dff442

We are so confused. We have Node.js and NPM downloaded. We are logged in to our Firebase through CLI. We requested Firebase to list projects we have access to. It gave us the Project ID. We have pasted the gist.github.com/mbleigh link above and at the end we have listed our Project ID (which does not include .firebase.app at the end). We have tried with a space between the link and the project name, without a space, with <> and without <>. We continue getting this response.
npm ERR! code ENOENT
npm ERR! syscall spawn git
npm ERR! path git
npm ERR! errno -4058
npm ERR! enoent An unknown git error occurred
npm ERR! enoent This is related to npm not being able to find a file.
npm ERR! enoent

Please help anyone?! Thank you!

@mbleigh Worked for me thank you a alot. 👍

Was getting
FirebaseError: HTTP Error: 401, Request had invalid authentication credentials. Expected OAuth 2 access token, login cookie or other valid authentication credential.

Had to do:
firebase login --reauth

After which it worked fine.

for anyone that runs into a 404, i had to use the project id instead of the name and it worked

I had the 404 issue. I used the project ID instead of the <site_name>, and it resolved the issue.

<project_id>.firebaseapp.com

npx https://gist.github.com/mbleigh/9c8680cf319ace2f506f57380da66e7d <project_id>

I had the 404 issue. I used the project ID instead of the <site_name>, and it resolved the issue.

<project_id>.firebaseapp.com

npx https://gist.github.com/mbleigh/9c8680cf319ace2f506f57380da66e7d <project_id>

Yes, this is the only solution that worked for me. However, is the complete website being pulled down? How can one confirm?

Is there a similar way to retrieve firebase functions ?

Is there a similar way to retrieve firebase functions ?

It looks like for Firebase functions we can get the source code from Google cloud.

• 1. Go to Run cloud : https://console.cloud.google.com/run
• 2. Click on the function you want the source code
     
• 3. Click on the "Source" tab
• 4. Click on "Download Zip"