Skip to content

Instantly share code, notes, and snippets.

@Mecanik
Last active November 5, 2024 06:59
Show Gist options
  • Save Mecanik/e95961d091065a71a4589440beefc73e to your computer and use it in GitHub Desktop.
Save Mecanik/e95961d091065a71a4589440beefc73e to your computer and use it in GitHub Desktop.
Send or Delete bulk data to Cloudflare Workers KV (TypeScript)
export default class Utils {
/**
* Delete items in bulk from Cloudflare KV Storage
* This assumes you have defined your ENV_ variables using wrangler secrets
* @param {namespace} The namespace binding you defined for your script
*/
static async deleteInBulk(namespace): Promise<Boolean> {
const delete_endpoint =
"https://api.cloudflare.com/client/v4/accounts/" +
EN_ACC_ID +
"/storage/kv/namespaces/" +
EN_KV_ID +
"/bulk";
// list all keys from a given NAMESPACE by prefix
let entries = await namespace.list();
// get cursor, if returned
let cursor = entries.cursor;
// get entries from first call above
let to_delete: any[] = [...entries.keys];
// as long as `entries.list_complete` is false, loop over the next block
while (!entries.list_complete) {
let next_value = await namespace.list({ cursor: cursor });
// update `entries.list_complete` to break the loop as soon as we've got all keys
entries.list_complete = next_value.list_complete;
// update cursor for the next call
cursor = next_value.cursor;
// push to array keys to delete
to_delete.push(...next_value.keys);
}
// List returns us "name" -> key, so we must make a new array only with keys.........
let delete_array: any[] = [];
for (const [key, value] of Object.entries(to_delete)) {
delete_array.push(value.name);
}
if (Object.keys(delete_array).length > 10000) {
// Split delete_array into smaller chunks
let parted_array = Utils.partition(
delete_array,
Math.round(Object.keys(delete_array).length / 1000)
);
// Iterate smaller chunks and send them in bulk
for (const [key, value] of Object.entries(parted_array)) {
if (Object.keys(value).length != 0) {
let delete_response = await fetch(delete_endpoint, {
body: JSON.stringify(value),
method: "DELETE",
headers: {
"content-type": "application/json;charset=UTF-8",
Authorization: "Bearer " + EN_API_KEY,
},
});
let delete_result = await delete_response.json();
if (!delete_result["success"]) return false;
}
}
return true;
} else {
let delete_response = await fetch(delete_endpoint, {
body: JSON.stringify(delete_array),
method: "DELETE",
headers: {
"content-type": "application/json;charset=UTF-8",
Authorization: "Bearer " + EN_API_KEY,
},
});
let delete_result = await delete_response.json();
return delete_result["success"];
}
return false;
}
/**
* Store items in bulk into Cloudflare KV Storage
* This assumes you have defined your ENV_ variables using wrangler secrets
* @note You must JSON encode each objects value ([{key: "foo", value: JSON.stringify("bar")}])
* @param {array} A formatted array of objects ([{key: "foo", value: "bar"}])
*/
static async sendInBulk(array): Promise<Boolean> {
const send_endpoint =
"https://api.cloudflare.com/client/v4/accounts/" +
EN_ACC_ID +
"/storage/kv/namespaces/" +
EN_KV_ID +
"/bulk";
const headers = { "Content-type": "application/json;charset=UTF-8" };
// Do we have more than 10,000 keys?
if (Object.keys(array).length > 10000) {
// Split array into smaller chunks
let parted_array = Utils.partition(
array,
Math.round(Object.keys(array).length / 1000)
);
// Iterate smaller chunks and send them in bulk
for (const [key, value] of Object.entries(parted_array)) {
if (value.length != 0) {
let send_response = await fetch(send_endpoint, {
body: JSON.stringify(value),
method: "PUT",
headers: {
"content-type": "application/json;charset=UTF-8",
Authorization: "Bearer " + EN_API_KEY,
},
});
let send_result = await send_response.json();
if (!send_result["success"]) return false;
}
}
return true;
} else {
// Ok, not more than 10,000; just send them.
let send_response = await fetch(send_endpoint, {
body: JSON.stringify(array),
method: "PUT",
headers: {
"content-type": "application/json;charset=UTF-8",
Authorization: "Bearer " + EN_API_KEY,
},
});
let send_result = await send_response.json();
return send_result["success"];
}
return false;
}
static partition(list = [], n = 1) {
const isPositiveInteger = Number.isSafeInteger(n) && n > 0;
if (!isPositiveInteger) {
throw new RangeError("n must be a positive integer");
}
const q = Math.floor(list.length / n);
const r = list.length % n;
let i; // denotes the offset of the start of the slice
let j; // denotes the zero-relative partition number
let len; // denotes the computed length of the slice
let partitions: any[] = [];
for (i = 0, j = 0, len = 0; i < list.length; i += len, ++j) {
len = j < r ? q + 1 : q;
let partition: any[] = list.slice(i, i + len);
partitions.push(partition);
}
return partitions;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment