MongoDB Database Copy Script (alternative to mongodump/mongorestore)

mongo-copy.js

// MongoDB Refactored Copy Script with Modular Functions and Index Copying
// This script copies all collections and their indexes from a source database to a target database without using mongodump.

// === Configuration ===

// Source MongoDB connection details
const sourceConfig = {
    host: "source_host",           // e.g., "localhost"
    port: 27017,                   // e.g., 27017
    username: "sourceUser",        // Source DB username
    password: "sourcePassword",    // Source DB password
    authDb: "admin",               // Authentication database for source (often "admin" or the db itself)
    dbName: "sourceDatabase"       // Source database name to copy
};

// Target MongoDB connection details
const targetConfig = {
    host: "target_host",           // e.g., "localhost"
    port: 27017,                   // e.g., 27017
    username: "targetUser",        // Target DB username (if authentication is required)
    password: "targetPassword",    // Target DB password
    authDb: "admin",               // Authentication database for target (if authentication is required)
    dbName: "targetDatabase"       // Target database name where data will be copied
};

// Batch size for bulk insert operations
const BATCH_SIZE = 1000;

// === Helper Functions ===

/**
 * Constructs a MongoDB URI with proper encoding for username and password.
 * @param {Object} config - Configuration object containing connection details.
 * @returns {String} - Constructed MongoDB URI.
 */
function buildMongoURI(config) {
    // Encode username and password to handle special characters
    const encodedUsername = encodeURIComponent(config.username);
    const encodedPassword = encodeURIComponent(config.password);

    // Construct the URI with authentication
    return `mongodb://${encodedUsername}:${encodedPassword}@${config.host}:${config.port}/${config.dbName}?authSource=${config.authDb}`;
}

/**
 * Connects to a MongoDB instance using the provided URI.
 * @param {String} uri - MongoDB connection URI.
 * @param {String} dbName - Database name to connect to.
 * @returns {Object} - Connected MongoDB database object.
 */
function connectToDatabase(uri, dbName) {
    try {
        const conn = new Mongo(uri);
        const db = conn.getDB(dbName);
        // Verify connection
        db.runCommand({ ping: 1 });
        log(`Successfully connected to database: ${dbName}`);
        return { conn, db };
    } catch (e) {
        log(`Failed to connect to database '${dbName}': ${e}`);
        quit(1);
    }
}

/**
 * Retrieves all collection names from a given database, excluding system collections.
 * @param {Object} db - MongoDB database object.
 * @returns {Array} - List of collection names.
 */
function getCollections(db) {
    return db.getCollectionNames().filter(function(collName) {
        return !collName.startsWith("system.");
    });
}

/**
 * Copies all indexes from a source collection to a target collection.
 * @param {Object} sourceCollection - Source MongoDB collection object.
 * @param {Object} targetCollection - Target MongoDB collection object.
 */
function copyIndexes(sourceCollection, targetCollection) {
    log(`  Copying indexes for collection: ${sourceCollection.getName()}`);
    const indexes = sourceCollection.getIndexes();

    indexes.forEach(function(index) {
        // Skip the default _id index as MongoDB automatically creates it
        if (index.name === "_id_") {
            log(`    Skipping default _id index.`);
            return;
        }

        // Prepare index options by excluding fields not needed for creation
        const indexOptions = Object.assign({}, index);
        delete indexOptions.v;
        delete indexOptions.ns;
        delete indexOptions.key;
        delete indexOptions.name;

        try {
            targetCollection.createIndex(index.key, indexOptions);
            log(`    Created index: ${index.name}`);
        } catch (e) {
            log(`    Error creating index '${index.name}': ${e}`);
        }
    });
}

/**
 * Copies all documents from a source collection to a target collection in batches.
 * @param {Object} sourceCollection - Source MongoDB collection object.
 * @param {Object} targetCollection - Target MongoDB collection object.
 */
function copyCollectionData(sourceCollection, targetCollection) {
    log(`  Starting data copy for collection: ${sourceCollection.getName()}`);
    const cursor = sourceCollection.find();
    let batch = [];
    let totalInserted = 0;

    while (cursor.hasNext()) {
        const doc = cursor.next();
        batch.push(doc);

        if (batch.length === BATCH_SIZE) {
            try {
                targetCollection.insertMany(batch);
                totalInserted += batch.length;
                log(`    Inserted ${totalInserted} documents so far...`);
                batch = [];
            } catch (e) {
                log(`    Error inserting batch: ${e}`);
            }
        }
    }

    // Insert any remaining documents in the batch
    if (batch.length > 0) {
        try {
            targetCollection.insertMany(batch);
            totalInserted += batch.length;
            log(`    Inserted a total of ${totalInserted} documents into '${sourceCollection.getName()}'.`);
        } catch (e) {
            log(`    Error inserting final batch: ${e}`);
        }
    }

    log(`  Completed data copy for collection: ${sourceCollection.getName()}`);
}

/**
 * Logs messages with a timestamp for better traceability.
 * @param {String} message - Message to log.
 */
function log(message) {
    const timestamp = new Date().toISOString();
    print(`[${timestamp}] ${message}`);
}

// === Main Execution Flow ===

(function() {
    // Build connection URIs
    const sourceUri = buildMongoURI(sourceConfig);
    const targetUri = buildMongoURI(targetConfig);

    // Connect to source and target databases
    log("Connecting to source MongoDB...");
    const { conn: sourceConn, db: sourceDB } = connectToDatabase(sourceUri, sourceConfig.dbName);

    log("Connecting to target MongoDB...");
    const { conn: targetConn, db: targetDB } = connectToDatabase(targetUri, targetConfig.dbName);

    // Retrieve collections from source database
    const collections = getCollections(sourceDB);
    log(`\nFound ${collections.length} collections to copy from '${sourceConfig.dbName}' to '${targetConfig.dbName}'.`);

    // Iterate through each collection and perform copy operations
    collections.forEach(function(collName) {
        log(`\nProcessing collection: ${collName}`);

        const sourceCollection = sourceDB.getCollection(collName);
        const targetCollection = targetDB.getCollection(collName);

        // Optional: Drop the target collection if it exists to start fresh
        if (targetDB.getCollectionNames().includes(collName)) {
            log(`  Dropping existing collection '${collName}' in target database.`);
            targetCollection.drop();
        }

        // Create the collection in the target database
        try {
            targetDB.createCollection(collName);
            log(`  Created collection '${collName}' in target database.`);
        } catch (e) {
            log(`  Error creating collection '${collName}': ${e}`);
        }

        // Copy indexes
        copyIndexes(sourceCollection, targetCollection);

        // Copy documents
        copyCollectionData(sourceCollection, targetCollection);

        log(`Finished processing collection: ${collName}`);
    });

    log(`\nDatabase copy from '${sourceConfig.dbName}' to '${targetConfig.dbName}' completed successfully.`);

    // Close the connections
    sourceConn.close();
    targetConn.close();
})();