Migrate to Self Hosted Redash

migrate.js

/* eslint-disable no-param-reassign */

const fetch = require("node-fetch");
const _ = require("lodash");
const fs = require("fs");
const { format } = require("util");

const SOURCE_REDASH = "https://app.redash.io/domain";
const SOURCE_REDASH_API_KEY = "XXXXXXXXXXXXXXXXXXXXXXXXXXX";

const TARGET_REDASH = "https://redash.domain.io";
const TARGET_REDASH_API_KEY = "XXXXXXXXXXXXXXXXXXXXXXXXXXX";

function log(level, fmt, ...values) {
  return console[level](format(fmt, ...values));
}

async function req(method, url, key, body) {
  const res = await fetch(url, {
    method,
    headers: { Authorization: key },
    body: body ? JSON.stringify(body) : undefined,
  });
  if (res.status !== 200) {
    throw new Error(await res.text());
  }
  return res.json();
}

let cache = {
  source: {
    users: {},
    queries: {},
    dashboards: {},
    usersQueries: {},
    usersDashboards: {},
    dataSources: {},
    visualizations: {},
  },
  target: {
    users: {},
    queries: {},
    dashboards: {},
    usersQueries: {},
    usersDashboards: {},
    dataSources: {},
    visualizations: {},
  },
  migration: {
    users: {},
    queries: {},
    dashboards: {},
    dataSources: {},
    visualizations: {},
  },
};

try {
  cache = JSON.parse(fs.readFileSync("cache.json", { encoding: "utf-8" }));
} catch (err) {
  log("warn", err.message);
}

function onShutdown() {
  log("debug", "shutdown...");
  fs.writeFileSync("cache.json", JSON.stringify(cache));
}

async function pullData({ baseUrl, apiKey, store }) {
  log("debug", `pulling data sources from ${baseUrl}`);

  const dataSources = await req("GET", `${baseUrl}/api/data_sources`, apiKey);

  for (const { id } of dataSources) {
    if (id in store.dataSources) {
      log("debug", "skip pulling data source %s from %s", id, baseUrl);
      continue;
    }
    log("debug", "pulling data source %s from %s", id, baseUrl);
    const dataSource = await req("GET", `${baseUrl}/api/data_sources/${id}`, apiKey);
    store.dataSources[id] = dataSource;
  }

  log("debug", `pulling active users from ${baseUrl}`);

  const { results: activeUsers } = await req(
    "GET",
    `${baseUrl}/api/users?page_size=250&order=created_at`,
    apiKey
  );

  for (const user of activeUsers) {
    if (user.id in store.users) {
      log("debug", "skiping user %s", user.id);
      continue;
    }

    log("debug", "pulling user's data %s from %s", user.id, baseUrl);
    store.users[user.id] = await req("GET", `${baseUrl}/api/users/${user.id}`, apiKey);
  }

  log("debug", "pulling disabled users from %s", baseUrl);

  const { results: disabledUsers } = await req(
    "GET",
    `${baseUrl}/api/users?page_size=250&order=created_at&disabled=true`,
    apiKey
  );

  for (const user of disabledUsers) {
    if (user.id in store.users) {
      log("debug", "skiping user %s", user.id);
      continue;
    }

    log("debug", "pulling user's data %s from %s", user.id, baseUrl);
    store.users[user.id] = await req("GET", `${baseUrl}/api/users/${user.id}`, apiKey);
  }

  for (const user of Object.values(store.users)) {
    if (user.id in store.usersQueries) {
      log("debug", "skip user's queries %s from %s", user.id, baseUrl);
      continue;
    }
    if (user.is_disabled) {
      log("debug", "skip user's queries %s from %s", user.id, baseUrl);
      continue;
    }

    log("debug", "pulling user's queries %s from %s", user.id, baseUrl);
    const { results: queries } = await req(
      "GET",
      `${baseUrl}/api/queries?page_size=250&order=created_at`,
      user.api_key
    );

    for (const query of queries) {
      if (query.id in store.queries) {
        log("debug", "skip query %s from %s", query.id, baseUrl);
        continue;
      }

      log("debug", "pulling query data %s from %s", query.id, baseUrl);
      store.queries[query.id] = await req(
        "GET",
        `${baseUrl}/api/queries/${query.id}`,
        user.api_key
      );
    }

    store.usersQueries[user.id] = queries;
  }

  for (const user of Object.values(store.users)) {
    if (user.id in store.usersDashboards) {
      log("debug", "skip pulling user's dashboards %s from %s", user.id, baseUrl);
      continue;
    }

    if (user.is_disabled) {
      log("debug", "skip pulling user's dashboards %s from %s", user.id, baseUrl);
      continue;
    }

    log("debug", "pulling user's dashboards %s from %s", user.id, baseUrl);
    const { results: dashboards } = await req(
      "GET",
      `${baseUrl}/api/dashboards?page_size=250&order=created_at`,
      user.api_key
    );

    for (const dashboard of dashboards) {
      if (dashboard.id in store.dashboards) {
        log("debug", "skip pulling dashboard's data %s from %s", dashboard.id, baseUrl);
        continue;
      }

      log("debug", "pulling dashboard's data %s from %s", dashboard.id, baseUrl);
      store.dashboards[dashboard.id] = await req(
        "GET",
        `${baseUrl}/api/dashboards/${dashboard.slug}`,
        user.api_key
      );
    }

    store.usersDashboards[user.id] = dashboards;
  }

  return store;
}

(async () => {
  try {
    await pullData({
      baseUrl: SOURCE_REDASH,
      apiKey: SOURCE_REDASH_API_KEY,
      store: cache.source,
    });

    await pullData({
      baseUrl: TARGET_REDASH,
      apiKey: TARGET_REDASH_API_KEY,
      store: cache.target,
    });

    // Migrate data from source to target

    for (const sUser of Object.values(cache.source.users)) {
      if (sUser.id in cache.migration.users) {
        log("debug", "skip migrating user %s from %s", sUser.id, TARGET_REDASH);
        continue;
      }

      log("debug", "migrating user %s to %s", sUser.id, TARGET_REDASH);

      const tUser = await req("POST", `${TARGET_REDASH}/api/users`, TARGET_REDASH_API_KEY, {
        ...sUser,
        id: undefined,
        api_key: sUser.api_key,
        groups: [],
      });

      cache.target.users[tUser.id] = await req(
        "GET",
        `${TARGET_REDASH}/api/users/${tUser.id}`,
        TARGET_REDASH_API_KEY
      );

      cache.migration.users[sUser.id] = tUser.id;
    }

    for (const sDataSource of Object.values(cache.source.dataSources)) {
      if (sDataSource.id in cache.migration.dataSources) {
        log("debug", "skip migrating data source %s from %s", sDataSource.id, SOURCE_REDASH);
        continue;
      }

      log("debug", "migrating data source %s from %s", sDataSource.id, SOURCE_REDASH);

      const tDataSource = await req(
        "POST",
        `${TARGET_REDASH}/api/data_sources`,
        TARGET_REDASH_API_KEY,
        {
          ...sDataSource,
          id: undefined,
          groups: [],
        }
      );

      cache.target.dataSources[tDataSource.id] = tDataSource;
      cache.migration.dataSources[sDataSource.id] = tDataSource.id;
    }

    for (const sQuery of Object.values(cache.source.queries)) {
      try {
        if (sQuery.id in cache.migration.queries) {
          continue;
        }

        const tUserId = cache.migration.users[sQuery.user.id];

        if (!tUserId) {
          throw new Error(`Couldn't find ${sQuery.user.id} in cache.migration.users`);
        }

        const tUser = cache.target.users[tUserId];

        if (!tUser) {
          throw new Error(`Couldn't find ${tUserId} in cache.target.users`);
        }

        const tDataSourceId = cache.migration.dataSources[sQuery.data_source_id];

        if (!tDataSourceId) {
          throw new Error(`Couldn't find ${sQuery.data_source_id} in cache.migration.dataSources`);
        }

        log("debug", "migrating query %s from %s", sQuery.id, SOURCE_REDASH);

        const tQuery = await req("POST", `${TARGET_REDASH}/api/queries`, tUser.api_key, {
          ...sQuery,
          data_source_id: tDataSourceId,
          last_modified_by: tUser,
          user: tUser,
          id: undefined,
          is_favorite: undefined,
          can_edit: undefined,
          api_key: undefined,
          latest_query_data_id: undefined,
          is_safe: undefined,
          visualizations: undefined,
        });

        const [defaultVisualization] = tQuery.visualizations;

        await req(
          "DELETE",
          `${TARGET_REDASH}/api/visualizations/${defaultVisualization.id}`,
          tUser.api_key
        );

        for (const sVisualization of sQuery.visualizations) {
          if (sVisualization.id in cache.migration.visualizations) {
            continue;
          }

          const tVisualization = await req(
            "POST",
            `${TARGET_REDASH}/api/visualizations`,
            tUser.api_key,
            {
              ...sVisualization,
              id: undefined,
              query_id: tQuery.id,
            }
          );

          cache.target.visualizations[tVisualization.id] = tVisualization;
          cache.migration.visualizations[sVisualization.id] = tVisualization.id;
        }

        if (sQuery.is_draft === false) {
          await req("POST", `${TARGET_REDASH}/api/queries/${tQuery.id}`, tUser.api_key, {
            id: tQuery.id,
            is_draft: false,
            version: 1,
          });
        }

        cache.target.queries[tQuery.id] = tQuery;
        cache.migration.queries[sQuery.id] = tQuery.id;
      } catch (err) {
        log("error", err.message);
      }
    }

    for (const sDashboard of Object.values(cache.source.dashboards)) {
      if (sDashboard.id in cache.migration.dashboards) {
        log("debug", "skip migrating dashboard %s from %s", sDashboard.id, TARGET_REDASH);
        continue;
      }

      const tUserId = cache.migration.users[sDashboard.user.id];

      if (!tUserId) {
        throw new Error(`Couldn't find ${sDashboard.user.id} in cache.migration.users`);
      }

      const tUser = cache.target.users[tUserId];

      if (!tUser) {
        throw new Error(`Couldn't find ${tUserId} in cache.target.users`);
      }

      log("debug", "migrating dashboard %s to %s", sDashboard.id, TARGET_REDASH);

      const tDashboard = await req("POST", `${TARGET_REDASH}/api/dashboards`, tUser.api_key, {
        ...sDashboard,
        id: undefined,
        user: tUser,
        user_id: tUser.id,
        widgets: [],
        is_favorite: undefined,
        is_safe: undefined,
        can_edit: undefined,
      });

      for (const sWidget of sDashboard.widgets) {
        const tVisualizationId = cache.migration.visualizations[sWidget.visualization.id];

        if (!tVisualizationId) {
          throw new Error(
            `Failed to find ${sWidget.visualization.id} in cache.migration.visualizations`
          );
        }

        await req("POST", `${TARGET_REDASH}/api/widgets`, tUser.api_key, {
          ...sWidget,
          id: undefined,
          dashboard_id: tDashboard.id,
          visualization_id: tVisualizationId,
        });
      }

      if (sDashboard.is_draft === false) {
        await req("POST", `${TARGET_REDASH}/api/dashboards/${tDashboard.id}`, tUser.api_key, {
          is_draft: false,
          name: tDashboard.name,
          slug: tDashboard.id,
        });
      }

      cache.target.dashboards[tDashboard.id] = tDashboard;
      cache.migration.dashboards[sDashboard.id] = tDashboard.id;
    }

    for (const sUser of Object.values(cache.source.users)) {
      const tUserId = cache.migration.users[sUser.id];
      if (!tUserId) {
        throw new Error(`Failed to find ${sUser.id} in cache.migration.users`);
      }
      const tUser = cache.target.users[tUserId];
      if (!tUser) {
        throw new Error(`Failed to find ${tUserId} in cache.target.users`);
      }
      if (sUser.is_disabled) {
        await req("POST", `${TARGET_REDASH}/api/users/${tUser.id}/disable`, TARGET_REDASH_API_KEY);
      }
    }
  } finally {
    onShutdown();
  }
})();