Skip to content

Instantly share code, notes, and snippets.

@WisaniShilumani
Last active October 5, 2019 18:18
Show Gist options
  • Save WisaniShilumani/03192298e9b5a494304d371cc1b76ade to your computer and use it in GitHub Desktop.
Save WisaniShilumani/03192298e9b5a494304d371cc1b76ade to your computer and use it in GitHub Desktop.
Scrapes a single property 24 page and sends it to an endpoint
const scrape = () => {
const showEmail = document.getElementById('P24_ToggleAgentEmailLink')
const showNumber = document.getElementById('P24_ToggleAgentNumbersLink')
const readMore = document.querySelector('a[title="Read More"]');
let nextClicks = 0
let nextClickInterval = null
const showEmailAndThenShowNumber = () => {
if (showEmail) showEmail.click()
setTimeout(showNumberAndThenReadMore, 3000)
}
const showNumberAndThenReadMore = () => {
if (showNumber) showNumber.click()
setTimeout(readMoreAndThenBeginClicks, 3000)
}
const readMoreAndThenBeginClicks = () => {
if (readMore) readMore.click()
nextClickInterval = setInterval(clickImageNext, 200)
}
const clickImageNext = () => {
document.querySelector('.p24_next').click()
nextClicks++;
if (nextClicks > 10) {
clearInterval(nextClickInterval)
setTimeout(getAllDetails, 1000)
}
}
const defaultElement = { innerText: ''}
const getAllDetails = () => {
const agentNumbers = []
const images = []
const title = document.querySelector('h1').innerText;
const price = document.querySelector('.panel-body .p24_price').innerText.replace(/(R|\s)/g, '').trim();
const agentName = (document.querySelector('.js-P24_AgentName') || defaultElement).innerText;
const agentEmail = (document.querySelector('.js-P24_Email') || defaultElement).innerText;
(document.querySelectorAll('.p24_sidebarAgentContactNumber') || []).forEach(number => agentNumbers.push(number.innerText.trim()))
const bedrooms = document.querySelector('li[title="Bedrooms"]').innerText.trim();
const bathrooms = document.querySelector('li[title="Bathrooms"]').innerText.trim();
const floorSize = document.querySelector('li[title="Floor Size"]').innerText.replace(/(m|²)/g, '').trim();
const description = document.querySelector('.js_readMoreText').innerText;
document.querySelectorAll('#main-gallery img.lazyImageLoading').forEach(image => {
if (image.src.indexOf('.gif') === -1) {
images.push({
src: image.src,
alt: image.alt,
title: image.title
})
}
})
const data = {
title,
price,
agentName,
agentEmail,
agentNumbers,
bedrooms,
bathrooms,
floorSize,
description,
images
}
console.log(JSON.stringify(data, null, 2))
const xhr = new XMLHttpRequest();
const url='https://jsonplaceholder.typicode.com/posts'; // TODO: replace with firebase endpoint
xhr.open('POST', url, true);
xhr.setRequestHeader('Content-Type', 'application/json');
xhr.onreadystatechange = function() {
if (this.readyState === XMLHttpRequest.DONE && this.status === 200) {
console.log('INFO SENT')
}
}
xhr.send(JSON.stringify(data));
}
showEmailAndThenShowNumber()
}
scrape()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment