Last active
October 5, 2019 18:18
-
-
Save WisaniShilumani/03192298e9b5a494304d371cc1b76ade to your computer and use it in GitHub Desktop.
Scrapes a single property 24 page and sends it to an endpoint
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const scrape = () => { | |
const showEmail = document.getElementById('P24_ToggleAgentEmailLink') | |
const showNumber = document.getElementById('P24_ToggleAgentNumbersLink') | |
const readMore = document.querySelector('a[title="Read More"]'); | |
let nextClicks = 0 | |
let nextClickInterval = null | |
const showEmailAndThenShowNumber = () => { | |
if (showEmail) showEmail.click() | |
setTimeout(showNumberAndThenReadMore, 3000) | |
} | |
const showNumberAndThenReadMore = () => { | |
if (showNumber) showNumber.click() | |
setTimeout(readMoreAndThenBeginClicks, 3000) | |
} | |
const readMoreAndThenBeginClicks = () => { | |
if (readMore) readMore.click() | |
nextClickInterval = setInterval(clickImageNext, 200) | |
} | |
const clickImageNext = () => { | |
document.querySelector('.p24_next').click() | |
nextClicks++; | |
if (nextClicks > 10) { | |
clearInterval(nextClickInterval) | |
setTimeout(getAllDetails, 1000) | |
} | |
} | |
const defaultElement = { innerText: ''} | |
const getAllDetails = () => { | |
const agentNumbers = [] | |
const images = [] | |
const title = document.querySelector('h1').innerText; | |
const price = document.querySelector('.panel-body .p24_price').innerText.replace(/(R|\s)/g, '').trim(); | |
const agentName = (document.querySelector('.js-P24_AgentName') || defaultElement).innerText; | |
const agentEmail = (document.querySelector('.js-P24_Email') || defaultElement).innerText; | |
(document.querySelectorAll('.p24_sidebarAgentContactNumber') || []).forEach(number => agentNumbers.push(number.innerText.trim())) | |
const bedrooms = document.querySelector('li[title="Bedrooms"]').innerText.trim(); | |
const bathrooms = document.querySelector('li[title="Bathrooms"]').innerText.trim(); | |
const floorSize = document.querySelector('li[title="Floor Size"]').innerText.replace(/(m|²)/g, '').trim(); | |
const description = document.querySelector('.js_readMoreText').innerText; | |
document.querySelectorAll('#main-gallery img.lazyImageLoading').forEach(image => { | |
if (image.src.indexOf('.gif') === -1) { | |
images.push({ | |
src: image.src, | |
alt: image.alt, | |
title: image.title | |
}) | |
} | |
}) | |
const data = { | |
title, | |
price, | |
agentName, | |
agentEmail, | |
agentNumbers, | |
bedrooms, | |
bathrooms, | |
floorSize, | |
description, | |
images | |
} | |
console.log(JSON.stringify(data, null, 2)) | |
const xhr = new XMLHttpRequest(); | |
const url='https://jsonplaceholder.typicode.com/posts'; // TODO: replace with firebase endpoint | |
xhr.open('POST', url, true); | |
xhr.setRequestHeader('Content-Type', 'application/json'); | |
xhr.onreadystatechange = function() { | |
if (this.readyState === XMLHttpRequest.DONE && this.status === 200) { | |
console.log('INFO SENT') | |
} | |
} | |
xhr.send(JSON.stringify(data)); | |
} | |
showEmailAndThenShowNumber() | |
} | |
scrape() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment