Skip to content

Instantly share code, notes, and snippets.

@albertsun

albertsun/rss-full-text-scrape-to-stevedore.json

Last active April 19, 2016 15:22
Show Gist options
  • Save albertsun/7e5cffc84a450c7d587f05f9f5b6938e to your computer and use it in GitHub Desktop.
Save albertsun/7e5cffc84a450c7d587f05f9f5b6938e to your computer and use it in GitHub Desktop.
Download ZIP
Huginn scenarios
Raw
rss-full-text-scrape-to-stevedore.json
{
"schema_version": 1,
"name": "RSS Full Text Scrape to Stevedore",
"description": "Follows an RSS feed and visits every feed item to save the full text into Stevedore",
"source_url": "https://mighty-retreat-48764.herokuapp.com/scenarios/7/export",
"guid": "1dcba6751110ed460f0a83273953535e",
"tag_fg_color": "#ffffff",
"tag_bg_color": "#5bc0de",
"exported_at": "2016-04-18T22:41:41Z",
"agents": [
{
"type": "Agents::PostAgent",
"name": "EEOC Post to Stevedore",
"disabled": false,
"guid": "a3817606515748c31c3bfa32465ff8e2",
"options": {
"post_url": "http://example.com/stevedore",
"expected_receive_period_in_days": "1",
"content_type": "json",
"method": "post",
"payload": {
"id": "{{ id }}",
"title": "{{ title }}",
"source_url": "{{ url }}",
"file": {
"title": "{{ title }}",
"file": "{{ body_text }}"
},
"analyzed": {
"body": "{{ body_text }}",
"metadata": {
"Content-Type": "text/plain"
}
}
},
"headers": {
},
"emit_events": "false",
"no_merge": "true"
},
"schedule": "never",
"keep_events_for": 0,
"propagate_immediately": true
},
{
"type": "Agents::RssAgent",
"name": "EEOC RSS",
"disabled": false,
"guid": "bcff007b3576fb801ffbb1288e95e920",
"options": {
"expected_update_period_in_days": "5",
"clean": "false",
"url": "https://www1.eeoc.gov/eeoc/newsroom/index.cfm?xml=eeoc_press,RSS2.0full"
},
"schedule": "every_5h",
"keep_events_for": 2592000
},
{
"type": "Agents::WebsiteAgent",
"name": "EEOC News Page Scrape",
"disabled": false,
"guid": "f68d5d70c4d0265eef284486a299dbd4",
"options": {
"expected_update_period_in_days": "2",
"url_from_event": "{{url}}",
"type": "html",
"mode": "merge",
"extract": {
"title": {
"css": "div h1",
"value": "normalize-space(.)"
},
"body_text": {
"css": "#CS_Element_Content",
"value": "normalize-space(.)"
}
}
},
"schedule": "never",
"keep_events_for": 0,
"propagate_immediately": true
}
],
"links": [
{
"source": 1,
"receiver": 2
},
{
"source": 2,
"receiver": 0
}
],
"control_links": [
]
}
Raw
website-email-notifier.json
{
"schema_version": 1,
"name": "Website Notifier",
"description": "Scrape a website and notify via email when it changes",
"source_url": "https://mighty-retreat-48764.herokuapp.com/scenarios/1/export",
"guid": "ce4d15de1dc26cd4dbc17c88cfc03c1c",
"tag_fg_color": "#ffffff",
"tag_bg_color": "#5bc0de",
"exported_at": "2016-04-18T16:56:11Z",
"agents": [
{
"type": "Agents::EmailAgent",
"name": "Donald Trump Email Notifier",
"disabled": false,
"guid": "9c61eccfd02f22c23d8fe461bbc46805",
"options": {
"subject": "Donald Trump's Website Changed",
"headline": "Your notification:",
"expected_receive_period_in_days": "2"
},
"propagate_immediately": true
},
{
"type": "Agents::WebsiteAgent",
"name": "Donald Trump Wall",
"disabled": false,
"guid": "f820bf7d25c0fc527280a0c05d905b1f",
"options": {
"expected_update_period_in_days": "2",
"url": "https://www.donaldjtrump.com/positions/pay-for-the-wall",
"type": "html",
"mode": "on_change",
"extract": {
"text": {
"css": "article.press_item.entry",
"value": "normalize-space(.)"
}
}
},
"schedule": "every_30m",
"keep_events_for": 0,
"propagate_immediately": false
}
],
"links": [
{
"source": 1,
"receiver": 0
}
],
"control_links": [
]
}
Raw
website-slack-notifier.json
{
"schema_version": 1,
"name": "Website Slack Notifier",
"description": "Scrape a website into slack",
"source_url": false,
"guid": "2176cbdc72468e0044c7895dff5cb6e4",
"tag_fg_color": "#ffffff",
"tag_bg_color": "#5bc0de",
"exported_at": "2016-04-18T21:05:01Z",
"agents": [
{
"type": "Agents::SlackAgent",
"name": "codeconvening-test slack",
"disabled": false,
"guid": "b90e88f7babfdef67940437f852e90f8",
"options": {
"webhook_url": "{% credential slack_webhook %}",
"channel": "#codeconvening-test",
"username": "Huginn",
"message": "{{text}}",
"icon": ""
},
"propagate_immediately": false
},
{
"type": "Agents::WebsiteAgent",
"name": "Donald Trump Wall",
"disabled": false,
"guid": "f820bf7d25c0fc527280a0c05d905b1f",
"options": {
"expected_update_period_in_days": "2",
"url": "https://www.donaldjtrump.com/positions/pay-for-the-wall",
"type": "html",
"mode": "on_change",
"extract": {
"text": {
"css": "article.press_item.entry",
"value": "normalize-space(.)"
}
}
},
"schedule": "every_30m",
"keep_events_for": 0,
"propagate_immediately": false
}
],
"links": [
{
"source": 1,
"receiver": 0
}
],
"control_links": [
]
}
Raw
website-to-stevedore.json
{
"schema_version": 1,
"name": "Website to Stevedore",
"description": "Scrape a website and insert it's changes into Stevedore",
"source_url": false,
"guid": "4c970fc0789eb9c9117a33b4a60bf776",
"tag_fg_color": "#ffffff",
"tag_bg_color": "#5bc0de",
"exported_at": "2016-04-18T21:42:51Z",
"agents": [
{
"type": "Agents::PostAgent",
"name": "Donald Trump Stevedore Insert Agent",
"disabled": false,
"guid": "83ebd85c425a91a222711563e83f6425",
"options": {
"post_url": "http://example.com/stevedore",
"expected_receive_period_in_days": "1",
"content_type": "json",
"method": "post",
"payload": {
"id": "https://www.donaldjtrump.com/positions/pay-for-the-wall",
"title": "Donald Trump Pay for the Wall",
"source_url": "https://www.donaldjtrump.com/positions/pay-for-the-wall",
"file": {
"title": "Donald Trump Pay for the Wall",
"file": "{{ text }}"
},
"analyzed": {
"body": "{{ text }}",
"metadata": {
"Content-Type": "text/plain"
}
}
},
"headers": {
},
"emit_events": "false",
"no_merge": "true"
},
"schedule": "never",
"keep_events_for": 0,
"propagate_immediately": true
},
{
"type": "Agents::WebsiteAgent",
"name": "Donald Trump Wall",
"disabled": false,
"guid": "f820bf7d25c0fc527280a0c05d905b1f",
"options": {
"expected_update_period_in_days": "2",
"url": "https://www.donaldjtrump.com/positions/pay-for-the-wall",
"type": "html",
"mode": "on_change",
"extract": {
"text": {
"css": "article.press_item.entry",
"value": "normalize-space(.)"
}
}
},
"schedule": "every_30m",
"keep_events_for": 0,
"propagate_immediately": false
}
],
"links": [
{
"source": 1,
"receiver": 0
}
],
"control_links": [
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment