hendrixroa

public async getPage() {
    if (!this.browser) {
      await this.initBrowser();
    }
    const page = await this.browser.newPage();

    // Avoiding Bot detection
    const userAgent =
      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36';
    await page.setUserAgent(userAgent);

hendrixroa

public closeBrowser() {
    if (this.browser) {
      this.browser.close();
    }
  }

hendrixroa

export class Scraper {
  protected name: string;
  protected baseURL: string;
  protected browser = new HeadlessBrowser();

  constructor() {
    this.name = 'Scraper';
    this.baseURL = 'https://github.com/trending';
  }
}  

hendrixroa

const page = await this.browser.getPage();

await page.goto(this.baseURL, {
  timeout: 300000,
  waitUntil: 'networkidle0',
});

hendrixroa

const data = await page.evaluate(() => {
  // Your scraping code here 
});

hendrixroa

const allReposArticles = document.querySelectorAll(
  '.Box-row h1.lh-condensed a',
);
const allReposArray = Array.from(allReposArticles);
const allNamesRepos = allReposArray.map((item: HTMLElement | any) => {
  return { name: item.innerText };
});

hendrixroa

const regexMatchDigits = /\d+/g;
const allStarArticles = document.querySelectorAll(
  '.Box-row .d-inline-block.float-sm-right',
);
const allStarReposArray = Array.from(allStarArticles);
const allStarsRepos = allStarReposArray.map((item: HTMLElement | any) => {

  const starDigits = item.innerText.match(regexMatchDigits);
  return { stars: Number(starDigits[0]) };

hendrixroa

const dataMerged = allNamesRepos.map((repo: any, index: number) => {
  const obj = {
    name: repo.name,
    starsToday: allStarsRepos[index].stars,
  };
  return obj;
});

hendrixroa

import { Scraper } from './scraper';

async function bootstrap() {
  const scraper = new Scraper();

  try {
    const data = await scraper.scrape();
    console.log(data);
  } catch (e) {
    console.error('error', e);

hendrixroa

terraform {
  backend "s3" {
    bucket = "your bucket to store state"
    key    = "terraform.state"
    region = "us-east-1"
  }

  required_providers {
    aws = {
      source  = "hashicorp/aws"