Created
June 3, 2023 21:54
-
-
Save marcelofern/82f6fc8379436c3258845e17bcfaf3bd to your computer and use it in GitHub Desktop.
Nova ro market scrapper
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const BASE_URL = 'https://www.novaragnarok.com/?module=vending&action=item&id='; | |
let table = $("table#itemtable").DataTable(); | |
let data = table.rows().data(); | |
let ITEMS = {}; | |
for (let i = 0; i < data.length; i++) { | |
let item_id = data[i][1]; | |
let quantity = data[i][4]; | |
ITEMS[item_id] = quantity; | |
} | |
const ITEMS_LENGTH = Object.keys(ITEMS).length; | |
const result = {}; | |
const fetchData = (url) => { | |
return new Promise((resolve, reject) => { | |
const request = new XMLHttpRequest(); | |
request.open('GET', url, true); // Make the request asynchronous | |
request.onload = function () { | |
if (request.status === 200) { | |
const html = request.responseText; | |
const parser = new DOMParser(); | |
const doc = parser.parseFromString(html, 'text/html'); | |
const table = doc.getElementById('nova-table-stats'); | |
if (!table) { | |
console.log('No market table found in the page.'); | |
resolve(); | |
return; | |
} | |
const tbody = table.querySelector('tbody'); | |
if (!tbody) { | |
console.log('The table doesn\'t have a body!'); | |
resolve(); | |
return; | |
} | |
const rows = tbody.querySelectorAll('tr'); | |
if (rows.length !== 2) { | |
console.log('Too many rows in the table. Not sure what to do!'); | |
resolve(); | |
return; | |
} | |
const secondRow = rows[1]; | |
const columns = secondRow.querySelectorAll('td'); | |
if (columns.length !== 6) { | |
console.log('Row structure is different than the expected!'); | |
resolve(); | |
return; | |
} | |
const rawAverageMonthlyPrice = columns[4].textContent; | |
const strippedString = rawAverageMonthlyPrice.replace(/,/g, "").replace("z", ""); | |
const averageMonthlyPrice = parseInt(strippedString, 10); | |
const pageTitleElement = doc.querySelector('title'); | |
const pageTitle = pageTitleElement ? pageTitleElement.textContent : ''; | |
const itemName = pageTitle.split(' - ')[0]; | |
result[itemName] = averageMonthlyPrice; | |
resolve(); | |
} else { | |
console.log('Request failed:', request.status); | |
resolve(); | |
} | |
}; | |
request.onerror = function () { | |
console.log('Request failed:', request.status); | |
resolve(); | |
}; | |
request.send(); | |
}); | |
}; | |
const fetchItems = async () => { | |
for (const [index, [item_id, quantity]] of Object.entries(ITEMS).entries()) { | |
const url = `${BASE_URL}${item_id}`; | |
await fetchData(url, index, item_id, quantity); | |
console.log(`Fetched item ${index+1} out of ${ITEMS_LENGTH}...`); | |
} | |
}; | |
fetchItems().then(( => { | |
// Transform the object into an array and sort based on the highest value. | |
const entries = Object.entries(result); | |
entries.sort((a, b) => b[1] - a[1]); | |
const sortedResult = entries.map(([key, value]) => ({ | |
key, | |
value: value.toLocaleString() + "z" | |
})); | |
const output = sortedResult.map(obj => `${obj.key}: ${obj.value}`).join('\n'); | |
console.log(output); | |
}); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment