cloudflare

r2-sdk-v3.ts

import { GetObjectCommand, GetObjectCommandOutput, S3Client, ListObjectsV2Command , S3 } from "@aws-sdk/client-s3";
import "dotenv/config";
import fs from "node:fs";
import path from "path";
import {Writable} from "stream"
const { AccessKeyId, SecretAccessKey, S3ClientLink } = process.env;

if (!AccessKeyId || !SecretAccessKey || !S3ClientLink) throw new Error("Missing environment variables");

class S3Handler {
	
	constructor(private readonly client = new S3Client({ region: "auto", credentials: { accessKeyId: AccessKeyId ?? "error", secretAccessKey: SecretAccessKey ?? "error" }, endpoint: S3ClientLink ?? "error" })) {
		console.log("S3Handler created -> S3 initialised");
	
	}

	public async ListFilesFromS3(prefix: string) {
		console.log("List Files from S3 function : ", prefix);
		try {
			
			const listObjectsResponse = await this.client.send(new ListObjectsV2Command({
				Bucket: "vercel",
				Prefix: prefix,
			}));

			// const listObjectsResponseAggregated = await this.client.listObjects({ Bucket: "sellerhub-bucket", Prefix: prefix });
			

			console.log("listObjectsResponse from S3 -> ", listObjectsResponse);
			return listObjectsResponse;
		} catch (error) {
			console.error("An error occurred:", error);
		}
	}

	public async DownloadFileFromS3(key: string | undefined) {
		if (!key) return Promise.resolve("");
		console.log("Downloading file from S3 -> ", key);
		const finalOutputPath = path.join(__dirname, key);
		const OutputFile = fs.createWriteStream(finalOutputPath);

		const dirname = path.dirname(finalOutputPath);

		if (!fs.existsSync(dirname)) {
			fs.mkdirSync(dirname, { recursive: true });
		}
		try {
			const getObjectCommand = new GetObjectCommand({
				Bucket: "sellerhub-bucket",
				Key: key,
			});
			const { Body } = (await this.client.send(getObjectCommand)) as GetObjectCommandOutput;
			// const {Body} = await this.client.getObject({ Bucket: "sellerhub-bucket", Key: key }) as GetObjectCommandOutput;
			if (!Body) {
				console.log("No file found in S3 -> ", key);
				return Promise.resolve("");
			}
			const nodeStream = await Body.transformToWebStream();
			console.log(await Body.transformToString() )
			nodeStream.pipeTo(Writable.toWeb(OutputFile))

		} catch (error) {
			console.error("An error occurred in DownloadFileFromS3:", error);
		}
	}


	public async RunLoopForDownloadingFiles(prefix: string) {
		try { 
			const listObjectsResponse = await this.ListFilesFromS3(prefix);
			if (!listObjectsResponse) return Promise.resolve("");
			const allPromises = listObjectsResponse.Contents?.map(async ({ Key }) => {
				await this.DownloadFileFromS3(Key);
			}) || [];

			await Promise.all(allPromises.filter(p => p !== undefined));
		}
		catch (error) {
			console.error("An error occurred in RunLoopForDownloadingFiles:", error);
		}
	}
}

export default new S3Handler();



// EXAMPLE USAGE OF THIS S3 handler class
//import { createClient, commandOptions } from 'redis';
// import S3Handler from './aws';

// const subscriber = createClient();

// subscriber.connect().then(() => {
//     console.log('connected');
// });

// async function main() { 
//     while (true) {
//         const res = await subscriber.brPop(commandOptions({ isolated: true }), 'build-queue', 0);
//         //?  just some element taken from the queue -> res.element is of type string | undefined
//         console.log(res);
//         if (!res) continue;
//         const context = await S3Handler.ListFilesFromS3("/output/" + res.element);
//         if (!context?.Contents) console.log("No files found");
//         // ^ undefined .... idk why maybe im dumb :) 
//         await S3Handler.RunLoopForDownloadingFiles("/output/" + res.element);
//     }
// };
// main();

Contents does not exist on this