Nodejs, Golang pull multiple file from the server, then archives all files to ZIP and all running processes in memory without writing them to disc

index.js

const rp = require('request-promise');
const Stream = require('stream');
const archiver = require('archiver');
const fs = require('fs');

const load = async () => {
    let streams = [];
    for (let i = 0; i < 3; i++) {
        let res = await rp.get(
            {
                url: 'https://mystorage.com/logo.png',
                encoding: null
            }
        );
        
        // write png buffer to in memory readable stream
        const readableStream = new Stream.Readable({ read: () => {} });
        readableStream.push(res);

        // send null indicate stream are end (EOF)
        readableStream.push(null);

        streams.push(readableStream);
    }

    return streams;
};

const zipAll = (streams) => {
    const archive = archiver('zip', {
        zlib: { level: 9 } // Sets the compression level.
    });

    archive.on('warning', (err) => {
        if (err.code === 'ENOENT') {
            // log warning
            console.log(err);
        } else {
            console.log(err);
        }
    });

    // good practice to catch this error explicitly
    archive.on('error', (err) => {
        console.log(err);
    });
    
    let i = 0;
    for (const s of streams) {
        // append each png buffer from streams to in memory readable stream
        archive.append(s, { name: `file-${i}.png` });
        i++;
    }

    archive.finalize();

    return archive;
};

load().then(streams => {
    const arc = zipAll(streams);
    console.log(arc.writable);

    // arc is a writable and readable stream
    // you can write to and read from
    arc.pipe(fs.createWriteStream('out.zip', { flags: 'w' }));
});

index_encrypted.js

const rp = require('request-promise');
const Stream = require('stream');
const archiver = require('archiver');
const fs = require('fs');

archiver.registerFormat('zip-encrypted', require("archiver-zip-encrypted"));

const load = async () => {
    let streams = [];
    for (let i = 0; i < 3; i++) {
        let res = await rp.get(
            {
                url: 'https://mystorage/logo.png',
                encoding: null
            }
        );
        
        // write png buffer to in memory readable stream
        const readableStream = new Stream.Readable({ read: () => {} });
        readableStream.push(res);

        // send null indicate stream are end (EOF)
        readableStream.push(null);

        streams.push(readableStream);
    }

    return streams;
};

const zipAll = (streams) => {
    const archive = archiver('zip-encrypted', {
        zlib: { level: 9 }, // Sets the compression level.
        encryptionMethod: 'aes256', password: '123'
    });

    archive.on('warning', (err) => {
        if (err.code === 'ENOENT') {
            // log warning
            console.log(err);
        } else {
            console.log(err);
        }
    });

    // good practice to catch this error explicitly
    archive.on('error', (err) => {
        console.log(err);
    });
    
    let i = 0;
    for (const s of streams) {
        // append each png buffer from streams to in memory readable stream
        archive.append(s, { name: `file-${i}.png` });
        i++;
    }

    archive.finalize();

    return archive;
};

load().then(streams => {
    const arc = zipAll(streams);
    console.log(arc.writable);

    // arc is a writable and readable stream
    // you can write to and read from
    arc.pipe(fs.createWriteStream('out.zip', { flags: 'w' }));
});

install.md

$ npm install request-promise archiver archiver-zip-encrypted request --save

main.go

// similar implementation with Golang
package main

import (
	"archive/zip"
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"time"
)

// ZipSources type
type ZipSources map[string]io.Reader

func main() {
	// create zip output
	fileName := "out.zip"
	fileZipOut, err := os.Create(fileName)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	defer fileZipOut.Close()

	// download
	url := "https://myweb/logo.png"

	sources := make(ZipSources)
	for i := 0; i < 3; i++ {
		response, err := httpGet(url)
		if err != nil {
			fmt.Println("cannot perform a request")
			os.Exit(1)
		}

		defer response.Body.Close()

		name := fmt.Sprintf("file-%d.png", i)
		sources[name] = response.Body
	}

	if err := Zip(fileZipOut, sources); err != nil {
		fmt.Println("cannot write zip file")
		os.Exit(1)
	}

}

func Zip(out io.Writer, sources ZipSources) error {
	zipWriter := zip.NewWriter(out)

	closeZip := func() error {
		if err := zipWriter.Close(); err != nil {
			return err
		}

		return nil
	}

	for name, source := range sources {
		w, err := zipWriter.Create(name)
		if err != nil {
			closeZip()
			return err
		}

		if _, err := io.Copy(w, source); err != nil {
			closeZip()
			return err
		}

	}

	return closeZip()
}

func httpGet(url string) (*http.Response, error) {
	transport := &http.Transport{
		Dial: (&net.Dialer{
			Timeout: 5 * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 5 * time.Second,
		IdleConnTimeout:     10 * time.Second,
	}

	httpClient := &http.Client{
		//Timeout:   time.Second * 10,
		Transport: transport,
	}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	response, err := httpClient.Do(req)
	if err != nil {
		return nil, err
	}

	return response, nil
}

main_encrypted.go

package main

import (
	"github.com/yeka/zip"
	"fmt"
	"io"
	"net"
	"net/http"
	"os"
	"time"
)

// ZipSources type
type ZipSources map[string]io.Reader

func main() {
	// create zip output
	fileName := "out.zip"
	fileZipOut, err := os.Create(fileName)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	defer fileZipOut.Close()

	// download
	url := "https://mystorage.com/logo.png"

	sources := make(ZipSources)
	for i := 0; i < 3; i++ {
		response, err := httpGet(url)
		if err != nil {
			fmt.Println("cannot perform a request")
			os.Exit(1)
		}

		defer response.Body.Close()

		name := fmt.Sprintf("file-%d.png", i)
		sources[name] = response.Body
	}

	if err := Zip(fileZipOut, sources, "123"); err != nil {
		fmt.Println("cannot write zip file")
		os.Exit(1)
	}

}

func Zip(out io.Writer, sources ZipSources, password string) error {
	zipWriter := zip.NewWriter(out)

	closeZip := func() error {
		if err := zipWriter.Close(); err != nil {
			return err
		}

		return nil
	}

	for name, source := range sources {
		w, err := zipWriter.Encrypt(name, password, zip.AES256Encryption)
		if err != nil {
			closeZip()
			return err
		}

		if _, err := io.Copy(w, source); err != nil {
			closeZip()
			return err
		}

	}

	return closeZip()
}

func httpGet(url string) (*http.Response, error) {
	transport := &http.Transport{
		Dial: (&net.Dialer{
			Timeout: 5 * time.Second,
		}).Dial,
		TLSHandshakeTimeout: 5 * time.Second,
		IdleConnTimeout:     10 * time.Second,
	}

	httpClient := &http.Client{
		//Timeout:   time.Second * 10,
		Transport: transport,
	}
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	response, err := httpClient.Do(req)
	if err != nil {
		return nil, err
	}

	return response, nil
}