Created
May 16, 2024 21:03
-
-
Save ClementGre/5669bb6def66edd1434c8beadc38c4fb to your computer and use it in GitHub Desktop.
INKK Hébergement floors maps scrapping
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const fs = require('fs'); | |
const path = require('path'); | |
const axios = require("axios"); | |
const https = require("https"); | |
const httpsAgent = new https.Agent({rejectUnauthorized: false}); | |
const capaignId = 'clvwo2kc40001ncj8jgsroyoz' | |
const get_buildings_query = `query getAvailableEntries($buildingId: ID, $floorId: ID, $roomId: ID, $campaignId: ID!, $bedsCount: Int!) { | |
getAvailableEntries( | |
buildingId: $buildingId | |
floorId: $floorId | |
roomId: $roomId | |
campaignId: $campaignId | |
bedsCount: $bedsCount | |
) { | |
map { | |
url | |
positions | |
__typename | |
} | |
entries { | |
id | |
name | |
reference | |
position | |
available | |
__typename | |
} | |
__typename | |
} | |
} | |
` | |
const get_floors_query = `query getAvailableEntries($buildingId: ID, $floorId: ID, $roomId: ID, $campaignId: ID!, $bedsCount: Int!) { | |
getAvailableEntries( | |
buildingId: $buildingId | |
floorId: $floorId | |
roomId: $roomId | |
campaignId: $campaignId | |
bedsCount: $bedsCount | |
) { | |
map { | |
url | |
positions | |
__typename | |
} | |
entries { | |
id | |
name | |
reference | |
position | |
available | |
__typename | |
} | |
__typename | |
} | |
} | |
` | |
const get_rooms_query = `query getAvailableEntries($buildingId: ID, $floorId: ID, $roomId: ID, $campaignId: ID!, $bedsCount: Int!) { | |
getAvailableEntries( | |
buildingId: $buildingId | |
floorId: $floorId | |
roomId: $roomId | |
campaignId: $campaignId | |
bedsCount: $bedsCount | |
) { | |
map { | |
url | |
positions | |
__typename | |
} | |
entries { | |
id | |
name | |
reference | |
position | |
available | |
__typename | |
} | |
__typename | |
} | |
}` | |
async function scrap_all(){ | |
let building_res = await graphQL_query(get_buildings_query, {campaignId: capaignId, bedsCount: 1}) | |
let campus_map_url = building_res['getAvailableEntries'].map.url | |
await download_url_to_file(campus_map_url, 'out/campus_map.png') | |
for(const building of building_res['getAvailableEntries'].entries){ | |
let building_id = building.id | |
let building_name = building.name | |
let floors_res = await graphQL_query(get_floors_query, {campaignId: capaignId, buildingId: building_id, bedsCount: 1}) | |
for(const floor of floors_res['getAvailableEntries'].entries){ | |
let floor_id = floor.id | |
let floor_name = floor.name | |
let floor_ref = floor.reference | |
let rooms_res = await graphQL_query(get_rooms_query, {campaignId: capaignId, buildingId: building_id, floorId: floor_id, bedsCount: 1}) | |
let floor_map_url = rooms_res['getAvailableEntries'].map.url | |
await download_url_to_file(floor_map_url, `out/${building_name}/${floor_ref}.png`) | |
} | |
} | |
return "done." | |
} | |
async function graphQL_query(query, values){ | |
return axios | |
.post("https://ks.inkk-hebergement.insa-lyon.fr/api/graphql", { | |
operationName: 'getAvailableEntries', | |
query: query, | |
variables: values | |
}, { | |
httpsAgent: httpsAgent, | |
headers: { | |
'Content-Type': 'application/json', | |
}, | |
}) | |
.then((res) => { | |
console.log(res.data.data) | |
return res.data.data | |
}) | |
.catch((err) => { | |
console.log(err) | |
return null | |
}) | |
} | |
async function download_url_to_file(url, file_path){ | |
ensureDirectoryExistence(file_path) | |
try { | |
const response = await axios.get(url, { | |
httpsAgent, | |
responseType: 'stream' | |
}); | |
// Create a writable stream | |
const writer = fs.createWriteStream(file_path); | |
// Pipe the image data to the writable stream | |
response.data.pipe(writer); | |
// Wait for the stream to finish writing the file | |
await new Promise((resolve, reject) => { | |
writer.on('finish', resolve); | |
writer.on('error', reject); | |
}); | |
console.log('Image downloaded successfully.'); | |
} catch (error) { | |
console.error('Error downloading image:', error.message); | |
} | |
} | |
const ensureDirectoryExistence = (filePath) => { | |
const dirname = path.dirname(filePath); | |
if (fs.existsSync(dirname)) { | |
return true; | |
} | |
ensureDirectoryExistence(dirname); | |
fs.mkdirSync(dirname); | |
}; | |
scrap_all().then(r => console.log(r)) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment