Skip to content

Instantly share code, notes, and snippets.

@marcelofern
Created June 3, 2023 21:54
Show Gist options
  • Save marcelofern/82f6fc8379436c3258845e17bcfaf3bd to your computer and use it in GitHub Desktop.
Save marcelofern/82f6fc8379436c3258845e17bcfaf3bd to your computer and use it in GitHub Desktop.
Nova ro market scrapper
const BASE_URL = 'https://www.novaragnarok.com/?module=vending&action=item&id=';
let table = $("table#itemtable").DataTable();
let data = table.rows().data();
let ITEMS = {};
for (let i = 0; i < data.length; i++) {
let item_id = data[i][1];
let quantity = data[i][4];
ITEMS[item_id] = quantity;
}
const ITEMS_LENGTH = Object.keys(ITEMS).length;
const result = {};
const fetchData = (url) => {
return new Promise((resolve, reject) => {
const request = new XMLHttpRequest();
request.open('GET', url, true); // Make the request asynchronous
request.onload = function () {
if (request.status === 200) {
const html = request.responseText;
const parser = new DOMParser();
const doc = parser.parseFromString(html, 'text/html');
const table = doc.getElementById('nova-table-stats');
if (!table) {
console.log('No market table found in the page.');
resolve();
return;
}
const tbody = table.querySelector('tbody');
if (!tbody) {
console.log('The table doesn\'t have a body!');
resolve();
return;
}
const rows = tbody.querySelectorAll('tr');
if (rows.length !== 2) {
console.log('Too many rows in the table. Not sure what to do!');
resolve();
return;
}
const secondRow = rows[1];
const columns = secondRow.querySelectorAll('td');
if (columns.length !== 6) {
console.log('Row structure is different than the expected!');
resolve();
return;
}
const rawAverageMonthlyPrice = columns[4].textContent;
const strippedString = rawAverageMonthlyPrice.replace(/,/g, "").replace("z", "");
const averageMonthlyPrice = parseInt(strippedString, 10);
const pageTitleElement = doc.querySelector('title');
const pageTitle = pageTitleElement ? pageTitleElement.textContent : '';
const itemName = pageTitle.split(' - ')[0];
result[itemName] = averageMonthlyPrice;
resolve();
} else {
console.log('Request failed:', request.status);
resolve();
}
};
request.onerror = function () {
console.log('Request failed:', request.status);
resolve();
};
request.send();
});
};
const fetchItems = async () => {
for (const [index, [item_id, quantity]] of Object.entries(ITEMS).entries()) {
const url = `${BASE_URL}${item_id}`;
await fetchData(url, index, item_id, quantity);
console.log(`Fetched item ${index+1} out of ${ITEMS_LENGTH}...`);
}
};
fetchItems().then(( => {
// Transform the object into an array and sort based on the highest value.
const entries = Object.entries(result);
entries.sort((a, b) => b[1] - a[1]);
const sortedResult = entries.map(([key, value]) => ({
key,
value: value.toLocaleString() + "z"
}));
const output = sortedResult.map(obj => `${obj.key}: ${obj.value}`).join('\n');
console.log(output);
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment