tridungle

from airflow import DAG
from airflow.operators.docker_operator import DockerOperator

dag = DAG(dag_id="example DAG", default_args=default_args)


run_task_with_docker = DockerOperator(
    task_id='run_task_with_docker',

		# Assuming this image is already pulled

tridungle

curl -X POST   http://localhost:8081/api/experimental/dags/hello_world_a/dag_runs   -H 'Cache-Control: no-cache'   -H 'Content-Type: application/json'   -d '{"conf":"{\"task_payload\":\"payload1\"}"}'
curl -X POST   http://localhost:8081/api/experimental/dags/hello_world_a/dag_runs   -H 'Cache-Control: no-cache'   -H 'Content-Type: application/json'   -d '{"conf":"{\"task_payload\":\"payload2\"}"}'
curl -X POST   http://localhost:8081/api/experimental/dags/hello_world_a/dag_runs   -H 'Cache-Control: no-cache'   -H 'Content-Type: application/json'   -d '{"conf":"{\"task_payload\":\"payload3\"}"}'

tridungle

import json
from pprint import pprint
import requests
from bs4 import BeautifulSoup as BS
import facebook

class FbBaseCrawler(object):
    
    default_headers = {
        'Accept'                    :'*/*',

tridungle

from bs4 import BeautifulSoup, SoupStrainer
from scrapy.selector import HtmlXPathSelector
from urllib import request
import requests
from scrapy.linkextractors import LinkExtractor
import requests.utils
import scrapy.link
import scrapy
import scrapy.spiders
from scrapy.spiders.crawl import CrawlSpider, Rule

tridungle

AWSTemplateFormatVersion: "2010-09-09"
Transform: "AWS::Serverless-2016-10-31"

Description: >
    CloudFormation in Action: An example with seven AWS services (SNS, SQS, Lambda, Kinesis, S3, Glue and Athena)
    Use case: How to set-up an environment in AWS to perform data analysis on the daily numbers of Covid-19?
    Author: Muttalip Kucuk
    Date: 01-11-2020

Globals:

tridungle

const mongoose = require("mongoose");
const Schema = mongoose.Schema;
const hotelSchema = new Schema({
  hotel_name: {type: String},
  location: {type: String},
  city: {type: String},
  state: {type: String},
  present_price: {type: String},
  features: {type: [String]},
  hotelsNg_link: {type: String},

tridungle

const crawler = require("web-crawljs");

/**//
 *  @description removes duplicate hotels for the array
 * @param arr
 * @returns {Array.<T>|*}
 */
function makeUnique(arr) {
  "use strict";
  const key = {};

tridungle

function getPageType() {
	var path = window.location.pathname;
	if (path.split('-')[0] == "/Hotel_Review") { return "search"; }
	if (path.split('-')[0] == "/Hotels") { return "details"; }
	return "unrecognized";
}

function get_location_name() {
	var PT = getPageType();
	if (PT === "details") {

tridungle

<?php
/**
{
    "require": {
        "symfony/dom-crawler": "2.*",
        "symfony/css-selector": "2.*"
    }
}
 */

tridungle

package crawler

import (
	"io/ioutil"
	"log"
	//"net"
	"net/http"
	//"net/url"
	"crypto/tls"
	"encoding/json"