This script processes a single file containing image URLs, removes duplicates, and downloads the images to a specified output folder. Simply replace inputFilePath and outputFolderPath with your desired file path and folder path. Ensure you have the axios package installed (npm install axios) to handle HTTP requests for image downloads.

downloadImagesFromURLs.js

// This script processes a single file containing image URLs and downloads the images to a specified folder.
// To use, replace the inputFilePath and outputFolderPath with your desired paths.
// Make sure to install the axios package using npm install axios.
// The script reads the file, extracts image URLs, removes duplicates, and downloads the images.

const fs = require("fs")
const path = require("path")
const axios = require("axios")

// Function to extract image URLs from a file
const extractImageUrlsFromFile = (filePath) => {
	return new Promise((resolve, reject) => {
		fs.readFile(filePath, "utf8", (err, data) => {
			if (err) {
				reject(err)
				return
			}
			// Regular expression to match URLs ending with .png, .jpg, .jpeg, .gif, or .webp
			const urls = data.match(
				/\bhttps?:\/\/\S+\.(?:png|jpg|jpeg|gif|webp)\b/g
			)
			resolve(urls || [])
		})
	})
}

// Function to download an image
const downloadImage = async (imageUrl, folderPath, imageIndex) => {
	try {
		const response = await axios.get(imageUrl, {
			responseType: "arraybuffer",
		})
		const extension = path.extname(imageUrl).split("?")[0] // Ensure the extension is correct
		const imageName = `${imageIndex}${extension}`
		const imagePath = path.join(folderPath, imageName)
		fs.writeFileSync(imagePath, response.data)
		console.log(`Downloaded and saved ${imageUrl} as ${imageName}`)
	} catch (error) {
		console.error(`Error downloading ${imageUrl}: ${error.message}`)
	}
}

// Main function to process a single file
const processFile = async (inputFilePath, outputFolderPath) => {
	try {
		const imageUrls = await extractImageUrlsFromFile(inputFilePath)
		const uniqueUrls = [...new Set(imageUrls)]
		const folderName = path.basename(
			inputFilePath,
			path.extname(inputFilePath)
		)
		const folderPathForImages = path.join(outputFolderPath, folderName)

		if (!fs.existsSync(folderPathForImages)) {
			fs.mkdirSync(folderPathForImages, { recursive: true })
		}

		for (let i = 0; i < uniqueUrls.length; i++) {
			await downloadImage(uniqueUrls[i], folderPathForImages, i + 1)
		}

		console.log(`Processing of file ${inputFilePath} completed.`)
	} catch (error) {
		console.error("Error processing file:", error)
	}
}

// Replace inputFilePath and outputFolderPath with your desired paths
const inputFilePath = "path/to/your/input/file.txt"
const outputFolderPath = "path/to/your/output/folder"

processFile(inputFilePath, outputFolderPath)