Last active
November 5, 2024 06:59
-
-
Save Mecanik/e95961d091065a71a4589440beefc73e to your computer and use it in GitHub Desktop.
Send or Delete bulk data to Cloudflare Workers KV (TypeScript)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
export default class Utils { | |
/** | |
* Delete items in bulk from Cloudflare KV Storage | |
* This assumes you have defined your ENV_ variables using wrangler secrets | |
* @param {namespace} The namespace binding you defined for your script | |
*/ | |
static async deleteInBulk(namespace): Promise<Boolean> { | |
const delete_endpoint = | |
"https://api.cloudflare.com/client/v4/accounts/" + | |
EN_ACC_ID + | |
"/storage/kv/namespaces/" + | |
EN_KV_ID + | |
"/bulk"; | |
// list all keys from a given NAMESPACE by prefix | |
let entries = await namespace.list(); | |
// get cursor, if returned | |
let cursor = entries.cursor; | |
// get entries from first call above | |
let to_delete: any[] = [...entries.keys]; | |
// as long as `entries.list_complete` is false, loop over the next block | |
while (!entries.list_complete) { | |
let next_value = await namespace.list({ cursor: cursor }); | |
// update `entries.list_complete` to break the loop as soon as we've got all keys | |
entries.list_complete = next_value.list_complete; | |
// update cursor for the next call | |
cursor = next_value.cursor; | |
// push to array keys to delete | |
to_delete.push(...next_value.keys); | |
} | |
// List returns us "name" -> key, so we must make a new array only with keys......... | |
let delete_array: any[] = []; | |
for (const [key, value] of Object.entries(to_delete)) { | |
delete_array.push(value.name); | |
} | |
if (Object.keys(delete_array).length > 10000) { | |
// Split delete_array into smaller chunks | |
let parted_array = Utils.partition( | |
delete_array, | |
Math.round(Object.keys(delete_array).length / 1000) | |
); | |
// Iterate smaller chunks and send them in bulk | |
for (const [key, value] of Object.entries(parted_array)) { | |
if (Object.keys(value).length != 0) { | |
let delete_response = await fetch(delete_endpoint, { | |
body: JSON.stringify(value), | |
method: "DELETE", | |
headers: { | |
"content-type": "application/json;charset=UTF-8", | |
Authorization: "Bearer " + EN_API_KEY, | |
}, | |
}); | |
let delete_result = await delete_response.json(); | |
if (!delete_result["success"]) return false; | |
} | |
} | |
return true; | |
} else { | |
let delete_response = await fetch(delete_endpoint, { | |
body: JSON.stringify(delete_array), | |
method: "DELETE", | |
headers: { | |
"content-type": "application/json;charset=UTF-8", | |
Authorization: "Bearer " + EN_API_KEY, | |
}, | |
}); | |
let delete_result = await delete_response.json(); | |
return delete_result["success"]; | |
} | |
return false; | |
} | |
/** | |
* Store items in bulk into Cloudflare KV Storage | |
* This assumes you have defined your ENV_ variables using wrangler secrets | |
* @note You must JSON encode each objects value ([{key: "foo", value: JSON.stringify("bar")}]) | |
* @param {array} A formatted array of objects ([{key: "foo", value: "bar"}]) | |
*/ | |
static async sendInBulk(array): Promise<Boolean> { | |
const send_endpoint = | |
"https://api.cloudflare.com/client/v4/accounts/" + | |
EN_ACC_ID + | |
"/storage/kv/namespaces/" + | |
EN_KV_ID + | |
"/bulk"; | |
const headers = { "Content-type": "application/json;charset=UTF-8" }; | |
// Do we have more than 10,000 keys? | |
if (Object.keys(array).length > 10000) { | |
// Split array into smaller chunks | |
let parted_array = Utils.partition( | |
array, | |
Math.round(Object.keys(array).length / 1000) | |
); | |
// Iterate smaller chunks and send them in bulk | |
for (const [key, value] of Object.entries(parted_array)) { | |
if (value.length != 0) { | |
let send_response = await fetch(send_endpoint, { | |
body: JSON.stringify(value), | |
method: "PUT", | |
headers: { | |
"content-type": "application/json;charset=UTF-8", | |
Authorization: "Bearer " + EN_API_KEY, | |
}, | |
}); | |
let send_result = await send_response.json(); | |
if (!send_result["success"]) return false; | |
} | |
} | |
return true; | |
} else { | |
// Ok, not more than 10,000; just send them. | |
let send_response = await fetch(send_endpoint, { | |
body: JSON.stringify(array), | |
method: "PUT", | |
headers: { | |
"content-type": "application/json;charset=UTF-8", | |
Authorization: "Bearer " + EN_API_KEY, | |
}, | |
}); | |
let send_result = await send_response.json(); | |
return send_result["success"]; | |
} | |
return false; | |
} | |
static partition(list = [], n = 1) { | |
const isPositiveInteger = Number.isSafeInteger(n) && n > 0; | |
if (!isPositiveInteger) { | |
throw new RangeError("n must be a positive integer"); | |
} | |
const q = Math.floor(list.length / n); | |
const r = list.length % n; | |
let i; // denotes the offset of the start of the slice | |
let j; // denotes the zero-relative partition number | |
let len; // denotes the computed length of the slice | |
let partitions: any[] = []; | |
for (i = 0, j = 0, len = 0; i < list.length; i += len, ++j) { | |
len = j < r ? q + 1 : q; | |
let partition: any[] = list.slice(i, i + len); | |
partitions.push(partition); | |
} | |
return partitions; | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment