cloudinary migration

cloudinaryMigration.ts

import { prisma } from "@princess/db";
import pool from "@ricokahler/pool";
import { type ResourceApiResponse, v2 } from "cloudinary";

// this is the old cloudinary
const cloudName = "dg4lbnuii";
const apiSecret = "e2L-_EHhcno3FLYZcHYbokaY-EU";
const apiKey = "673961598382791";

const newCloudNamePreview = "wemakeapp-preview";
const newApiKeyPreview = "837712146135957";
const newUploadPresetPreview = "princess";
const newApiSecretPreview = "wqA92OuTVsYbNcIBLtSPUn_0TiU";

const newCloudNameProduction = "wemakeapp-production";
const newApiKeyProduction = "773825855679162";
const newUploadPresetProduction = "princess";
const newApiSecretProduction = "t70r3ya5K8iFFxvqyyiqJ_VNt0E";

const cloudinary = v2;

const setOldCloudinary = () => {
  cloudinary.config({
    cloud_name: cloudName,
    api_key: apiKey,
    api_secret: apiSecret,
    secure: true,
  });
};

const setNewCloudinaryPreview = () => {
  cloudinary.config({
    cloud_name: newCloudNamePreview,
    api_key: newApiKeyPreview,
    api_secret: newApiSecretPreview,
    secure: true,
  });
};

const setNewCloudinaryProduction = () => {
  cloudinary.config({
    cloud_name: newCloudNameProduction,
    api_key: newApiKeyProduction,
    api_secret: newApiSecretProduction,
    secure: true,
  });
};

const removeDanglingImages = async () => {
  const images = await prisma.cloudinaryImage.findMany({
    select: {
      id: true,
      publicId: true,
      updatedAt: true,
    },
    orderBy: {
      updatedAt: "asc",
    },
  });
  const count = await prisma.cloudinaryImage.count();
  //   38 = 1 + 15 + 22
  console.log({ count });

  // set the old cloudinary
  cloudinary.config({
    cloud_name: cloudName,
    api_key: apiKey,
    api_secret: apiSecret,
    secure: true,
  });

  const oldImageAppProduction = cloudinary.api.resources_by_tag(
    "appEnv:production",
    {
      max_results: 2000,
    }
  );

  const oldImageVercelProduction = cloudinary.api.resources_by_tag(
    "vercelEnv:production",
    {
      max_results: 2000,
    }
  );
  const [result1, result2] = await Promise.all([
    oldImageAppProduction,
    oldImageVercelProduction,
  ]);
  const result = [...result1.resources, ...result2.resources];
  console.log("==== the difference ====");
  console.log(
    {
      cloudinaryImageCount: result.length,
      databaseCount: count,
    },
    "cloudinary image should be more"
  );

  const in_database = result
    .filter((r) => images.some((i) => i.publicId === r.public_id))
    .map((r) => r.public_id);
  console.log("in the database: ", in_database.length);
  const not_in_database = result
    .filter((r) => !images.some((i) => i.publicId === r.public_id))
    .map((r) => ({
      publicId: r.public_id,
      createdAt: r.created_at,
    }));
  console.log(
    `not in the database (${not_in_database.length}): `,
    not_in_database.map((r) => r.publicId)
  );

  // remove everything not in the db
  const delete_result = await cloudinary.api.delete_resources(
    not_in_database.map((r) => r.publicId)
  );
  console.log(delete_result);
};

function* chunks<T>(arr: T[], n: number): Generator<T[], void> {
  for (let i = 0; i < arr.length; i += n) {
    yield arr.slice(i, i + n);
  }
}
const migrateImages = async () => {
  try {
    // this is the old cloudinary
    setOldCloudinary();

    const images = await prisma.cloudinaryImage.findMany({
      select: {
        id: true,
        publicId: true,
        updatedAt: true,
      },
      orderBy: {
        updatedAt: "asc",
      },
    });
    console.log(
      images.length,
      images.map((i) => i.publicId)
    );

    const imageChunks = [
      ...chunks(
        images.map((i) => i.publicId),
        100
      ),
    ];
    const promises = imageChunks.map((ic) =>
      cloudinary.api.resources_by_ids(ic, {
        max_results: 500,
        tags: true,
      })
    );
    const results = await Promise.all(promises);
    // @ts-ignore
    const resources: ResourceApiResponse["resources"] = results.reduce(
      // @ts-ignore
      (acc, r) => [...acc, ...r.resources],
      [] as unknown as ResourceApiResponse["resources"]
    );

    // get all the publicId
    const wantedInfo = resources.map((r) => ({
      publicId: r.public_id,
      tags: r.tags,
      url: r.secure_url,
    }));
    console.log(wantedInfo.length);
    // set the new cloudinary
    setNewCloudinaryProduction();
    const uploadPromises = wantedInfo.map((r) => {
      return cloudinary.uploader.unsigned_upload(
        r.url,
        newUploadPresetPreview,
        {
          tags: r.tags,
          public_id: r.publicId,
        }
      );
    });
    const uploadResults = await Promise.all(uploadPromises);
    console.log(uploadResults);
  } catch (e) {
    console.log((e as Error).message);
  }
};

// migrate the image to different folder through rename
const migrateFolder = async () => {
  try {
    const results = await prisma.cloudinaryImage.findMany({
      select: {
        id: true,
        userAvatar: {
          select: {
            id: true,
          },
        },
        userAvatarHalf: {
          select: {
            id: true,
          },
        },
        userAvatarFull: {
          select: {
            id: true,
          },
        },
        publicId: true,
      },
    });
    const images = results.map((i) => ({
      id: Number(i.id),
      publicId: i.publicId,
      userId: Number(
        i.userAvatar[0]?.id ||
          i.userAvatarHalf[0]?.id ||
          i.userAvatarFull[0]?.id
      ),
    }));
    console.log(images, images.length);
    // set the new cloudinary
    setNewCloudinaryProduction();
    const filteredImages = images
      // we only migrate the images which are before the cloudinary migration PR
      .filter((i) => !i.publicId.includes("princess/"));
    // change the image publicid
    for (let i = 0; i < filteredImages.length; i++) {
      try {
        const image = filteredImages[i]!;
        const result = await cloudinary.uploader.rename(
          `princess/${image.publicId}`,
          `princess/Avatar/${image.userId}/${image.publicId}`,
          {
            overwrite: true,
          }
        );
        console.log(
          `✅ renaming ${i + 1} of ${filteredImages.length}: princess/${
            image.publicId
          } -> princess/Avatar/${image.userId}/${image.publicId}`
        );
      } catch (e) {
        console.error(
          `❌ renaming ${i + 1} of ${filteredImages.length}: `,
          (e as Error).message
        );
      }
    }
  } catch (e) {
    console.log((e as Error).message);
  }
};

const renamePublicId = async () => {
  const results = await prisma.cloudinaryImage.findMany({
    select: {
      id: true,
      userAvatar: {
        select: {
          id: true,
        },
      },
      userAvatarHalf: {
        select: {
          id: true,
        },
      },
      userAvatarFull: {
        select: {
          id: true,
        },
      },
      publicId: true,
    },
    where: {
      publicId: {
        not: {
          startsWith: "princess/",
        },
      },
    },
  });
  const images = results.map((i) => ({
    id: Number(i.id),
    publicId: i.publicId,
    userId: Number(
      i.userAvatar[0]?.id || i.userAvatarHalf[0]?.id || i.userAvatarFull[0]?.id
    ),
  }));
  console.log({
    images,
    count: images.length,
  });
  // const urls = images.map((image) => {
  //   const newPublicId = `princess/Avatar/${image.userId}/${image.publicId}`;
  //   const newUrl = `https://res.cloudinary.com/${newCloudNameProduction}/image/upload/${newPublicId}.jpg`;
  //   return newPublicId;
  // });

  const result = await pool({
    collection: images,
    maxConcurrency: 10, // only 10 connections at a time
    task: async (image) => {
      const newPublicId = `princess/Avatar/${image.userId}/${image.publicId}`;
      const newUrl = `https://res.cloudinary.com/${newCloudNameProduction}/image/upload/${newPublicId}.jpg`;
      return prisma.cloudinaryImage.update({
        where: {
          publicId: image.publicId,
        },
        data: {
          publicId: newPublicId,
          url: newUrl,
        },
      });
    },
  });

  console.log(result.map((r) => r.publicId));
};

const fixOldCloudinaryDanglingIssue = async () => {
  const results = await prisma.cloudinaryImage.findMany({
    where: {
      publicId: {
        not: {
          startsWith: "princess/",
        },
      },
    },
    select: {
      id: true,
      publicId: true,
    },
  });

  setOldCloudinary();
  for (const result of results) {
    // get the new public id from tags
    const { resources: images } = await cloudinary.api.resources_by_ids(
      result.publicId,
      {
        tags: true,
      }
    );
    const newPublicId = images[0].tags
      .find((tag) => tag.includes("newPublicId:"))
      ?.replace("newPublicId:", "princess");
    if (newPublicId) {
      // log the new public ids
      const newUrl = `https://res.cloudinary.com/${newCloudNameProduction}/image/upload/${newPublicId}`;
      console.log({
        newPublicId,
        newUrl,
      });
      //   update the db with new public id
      await prisma.cloudinaryImage.update({
        where: {
          id: result.id,
        },
        data: {
          publicId: newPublicId,
          url: newUrl,
        },
      });
    }
  }
};

// renamePublicId();
// migrateFolder();
renamePublicId();