Skip to content

Instantly share code, notes, and snippets.

@costr

costr/Node-Red-WebScrapper_to_Home_Assistant_Example.js

Created June 29, 2019 15:10
Show Gist options
  • Save costr/db0959cc5746f1c35b366bd301a4254b to your computer and use it in GitHub Desktop.
Save costr/db0959cc5746f1c35b366bd301a4254b to your computer and use it in GitHub Desktop.
Download ZIP
As someone who fishes near Okoboji, IA I wanted fishing reports but I found that the IA DNR site had annoying modals and required several clicks to get what I wanted. Instead I wanted to compile the fishing information and pump the information into Home Assistant for faster digestion.
Raw
Node-Red-WebScrapper_to_Home_Assistant_Example.js
[
{
"id": "338f10b4.b5013",
"type": "tab",
"label": "Fishing",
"disabled": false,
"info": ""
},
{
"id": "c551c71b.11cd28",
"type": "html",
"z": "338f10b4.b5013",
"name": "Get Fishing Forcast",
"property": "payload",
"outproperty": "payload",
"tag": "td.fishingreportdetail",
"ret": "text",
"as": "single",
"x": 790,
"y": 380,
"wires": [
[
"6872d86a.611658"
]
]
},
{
"id": "a6c8eca1.6aaff",
"type": "inject",
"z": "338f10b4.b5013",
"name": "",
"topic": "",
"payload": "",
"payloadType": "date",
"repeat": "21600",
"crontab": "",
"once": false,
"onceDelay": 0.1,
"x": 110,
"y": 380,
"wires": [
[
"639c3245.16131c",
"61410b97.980604",
"e81a678a.4dd6f8",
"a1bf5df6.98a41",
"64bceac4.713924",
"e64b88de.1e1c18",
"8dd5a2e8.36e8b",
"b9d0ec96.ee2"
]
]
},
{
"id": "b9d0ec96.ee2",
"type": "http request",
"z": "338f10b4.b5013",
"name": "Get East Okoboji Info",
"method": "GET",
"ret": "txt",
"paytoqs": false,
"url": "https://www.iowadnr.gov/Fishing/Where-to-Fish/Lakes-Ponds-Reservoirs/LakeDetails/lakeCode/EOK30",
"tls": "",
"proxy": "",
"authType": "basic",
"x": 340,
"y": 180,
"wires": [
[
"f5e08995.b76cf8"
]
]
},
{
"id": "8dd5a2e8.36e8b",
"type": "http request",
"z": "338f10b4.b5013",
"name": "Get West Okoboji Info",
"method": "GET",
"ret": "txt",
"paytoqs": false,
"url": "https://www.iowadnr.gov/Fishing/Where-to-Fish/Lakes-Ponds-Reservoirs/LakeDetails/lakeCode/WOK30",
"tls": "",
"proxy": "",
"authType": "basic",
"x": 343,
"y": 240,
"wires": [
[
"7e26598a.4e07d8"
]
]
},
{
"id": "e64b88de.1e1c18",
"type": "http request",
"z": "338f10b4.b5013",
"name": "Get Upper Gar Info",
"method": "GET",
"ret": "txt",
"paytoqs": false,
"url": "https://www.iowadnr.gov/Fishing/Where-to-Fish/Lakes-Ponds-Reservoirs/LakeDetails/lakeCode/UGR30",
"tls": "",
"proxy": "",
"authType": "basic",
"x": 330,
"y": 300,
"wires": [
[
"41106959.674eb8"
]
]
},
{
"id": "64bceac4.713924",
"type": "http request",
"z": "338f10b4.b5013",
"name": "Get Minnewashta lake Info",
"method": "GET",
"ret": "txt",
"paytoqs": false,
"url": "https://www.iowadnr.gov/Fishing/Where-to-Fish/Lakes-Ponds-Reservoirs/LakeDetails/lakeCode/MIN30",
"tls": "",
"proxy": "",
"authType": "basic",
"x": 360,
"y": 360,
"wires": [
[
"fbd07abd.3cd138"
]
]
},
{
"id": "a1bf5df6.98a41",
"type": "http request",
"z": "338f10b4.b5013",
"name": "Get Lower Gar Lake Info",
"method": "GET",
"ret": "txt",
"paytoqs": false,
"url": "https://www.iowadnr.gov/Fishing/Where-to-Fish/Lakes-Ponds-Reservoirs/LakeDetails/lakeCode/LGA30",
"tls": "",
"proxy": "",
"authType": "basic",
"x": 350,
"y": 420,
"wires": [
[
"1ed7e6e2.55c3d9"
]
]
},
{
"id": "1ed7e6e2.55c3d9",
"type": "change",
"z": "338f10b4.b5013",
"name": "set topic",
"rules": [
{
"t": "set",
"p": "topic",
"pt": "msg",
"to": "Lower Gar",
"tot": "str"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 560,
"y": 420,
"wires": [
[
"c551c71b.11cd28"
]
]
},
{
"id": "fbd07abd.3cd138",
"type": "change",
"z": "338f10b4.b5013",
"name": "set topic",
"rules": [
{
"t": "set",
"p": "topic",
"pt": "msg",
"to": "Minnewashta",
"tot": "str"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 560,
"y": 360,
"wires": [
[
"c551c71b.11cd28"
]
]
},
{
"id": "41106959.674eb8",
"type": "change",
"z": "338f10b4.b5013",
"name": "set topic",
"rules": [
{
"t": "set",
"p": "topic",
"pt": "msg",
"to": "Upper Gar",
"tot": "str"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 560,
"y": 300,
"wires": [
[
"c551c71b.11cd28"
]
]
},
{
"id": "7e26598a.4e07d8",
"type": "change",
"z": "338f10b4.b5013",
"name": "set topic",
"rules": [
{
"t": "set",
"p": "topic",
"pt": "msg",
"to": "West Okoboji",
"tot": "str"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 560,
"y": 240,
"wires": [
[
"c551c71b.11cd28"
]
]
},
{
"id": "f5e08995.b76cf8",
"type": "change",
"z": "338f10b4.b5013",
"name": "set topic",
"rules": [
{
"t": "set",
"p": "topic",
"pt": "msg",
"to": "East Okoboji",
"tot": "str"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 560,
"y": 180,
"wires": [
[
"c551c71b.11cd28"
]
]
},
{
"id": "e81a678a.4dd6f8",
"type": "http request",
"z": "338f10b4.b5013",
"name": "Get Center Lake Info",
"method": "GET",
"ret": "txt",
"paytoqs": false,
"url": "https://www.iowadnr.gov/idnr/Fishing/Where-to-Fish/Lakes-Ponds-Reservoirs/LakeDetails?lakeCode=CEN30",
"tls": "",
"proxy": "",
"authType": "basic",
"x": 340,
"y": 480,
"wires": [
[
"c0b004af.aff6a8"
]
]
},
{
"id": "c0b004af.aff6a8",
"type": "change",
"z": "338f10b4.b5013",
"name": "set topic",
"rules": [
{
"t": "set",
"p": "topic",
"pt": "msg",
"to": "Center",
"tot": "str"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 560,
"y": 480,
"wires": [
[
"c551c71b.11cd28"
]
]
},
{
"id": "61410b97.980604",
"type": "http request",
"z": "338f10b4.b5013",
"name": "Get Spirit Lake Info",
"method": "GET",
"ret": "txt",
"paytoqs": false,
"url": "https://www.iowadnr.gov/idnr/Fishing/Where-to-Fish/Lakes-Ponds-Reservoirs/LakeDetails?lakeCode=SPL30",
"tls": "",
"proxy": "",
"authType": "basic",
"x": 330,
"y": 540,
"wires": [
[
"bd016181.27f6c"
]
]
},
{
"id": "bd016181.27f6c",
"type": "change",
"z": "338f10b4.b5013",
"name": "set topic",
"rules": [
{
"t": "set",
"p": "topic",
"pt": "msg",
"to": "Spirit",
"tot": "str"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 560,
"y": 540,
"wires": [
[
"c551c71b.11cd28"
]
]
},
{
"id": "639c3245.16131c",
"type": "http request",
"z": "338f10b4.b5013",
"name": "Get Silver Lake Info",
"method": "GET",
"ret": "txt",
"paytoqs": false,
"url": "https://www.iowadnr.gov/Fishing/Where-to-Fish/Lakes-Ponds-Reservoirs/LakeDetails?lakeCode=SIL30",
"tls": "",
"proxy": "",
"authType": "basic",
"x": 330,
"y": 600,
"wires": [
[
"4956bfe6.09f7c"
]
]
},
{
"id": "4956bfe6.09f7c",
"type": "change",
"z": "338f10b4.b5013",
"name": "set topic",
"rules": [
{
"t": "set",
"p": "topic",
"pt": "msg",
"to": "Silver Lake",
"tot": "str"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 560,
"y": 600,
"wires": [
[
"c551c71b.11cd28"
]
]
},
{
"id": "6872d86a.611658",
"type": "function",
"z": "338f10b4.b5013",
"name": "Prettify",
"func": "var lake = msg.topic;\nvar lakeUrl = msg.responseUrl;\n\nvar result = `\n# [${lake}](${lakeUrl}) #\n\n<details>\n`;\n\nvar payload = msg.payload;\n\nif(!payload || payload.length < 1) return [null, msg];\n\nfor(var forecast in payload) {\n var dashPosition = payload[forecast].indexOf('-');\n var colonPosition = payload[forecast].indexOf(':');\n \n var header = payload[forecast].substring(0, dashPosition).trim();\n var level = payload[forecast].substring(dashPosition+1, colonPosition).trim().toUpperCase();\n var body = payload[forecast].substring(colonPosition+1, payload[forecast].length).trim();\n if(header && level) {\n result += `\n## ${header} ##\n\n### Forecast Level: ${level} ###\n\n${body}\n\n`;\n }\n \n if(header && !level) {\n result += `\n## ${header} ##\n\n${body}\n\n `;\n }\n \n if(!header) {\n result += `\n${body}\n`;\n }\n}\n\nresult += `\n</details>\n`;\n\nvar newResult = {\n \"Report\" : result,\n \"Lake\": lake\n};\n\nmsg.payload = newResult;\n\nreturn [msg,null];",
"outputs": 2,
"noerr": 0,
"x": 980,
"y": 380,
"wires": [
[
"1f3396d3.d0eca9"
],
[]
]
},
{
"id": "1f3396d3.d0eca9",
"type": "join",
"z": "338f10b4.b5013",
"name": "Join Results",
"mode": "custom",
"build": "array",
"property": "payload",
"propertyType": "msg",
"key": "topic",
"joiner": "\\n",
"joinerType": "str",
"accumulate": false,
"timeout": "",
"count": "8",
"reduceRight": false,
"reduceExp": "",
"reduceInit": "",
"reduceInitType": "",
"reduceFixup": "",
"x": 770,
"y": 320,
"wires": [
[
"8ed28f5e.c7316"
]
]
},
{
"id": "e53f240e.ade2f8",
"type": "ha-api",
"z": "338f10b4.b5013",
"name": "Home",
"server": "144eb367.c8419d",
"protocol": "http",
"method": "post",
"path": "/api/states/sensor.fishing_report",
"data": "",
"location": "payload",
"locationType": "msg",
"responseType": "json",
"x": 1150,
"y": 260,
"wires": [
[]
]
},
{
"id": "8ed28f5e.c7316",
"type": "function",
"z": "338f10b4.b5013",
"name": "Create HASS payload",
"func": "var payload = msg.payload;\nvar reportAttributes = {};\n\nfor(var lake in payload) {\n reportAttributes[payload[lake].Lake] = payload[lake].Report;\n}\n\nvar sensorData = {\n \"attributes\" : reportAttributes,\n \"state\" : new Date()\n};\n\nvar newPayload = {\n \"data\": sensorData\n};\n\nmsg.payload = newPayload;\n\nreturn msg;",
"outputs": 1,
"noerr": 0,
"x": 960,
"y": 320,
"wires": [
[
"e53f240e.ade2f8",
"8db3dd70.8c4a8"
]
]
},
{
"id": "8db3dd70.8c4a8",
"type": "ha-api",
"z": "338f10b4.b5013",
"name": "Fishing",
"server": "bce12445.22b7c8",
"protocol": "http",
"method": "post",
"path": "/api/states/sensor.fishing_report",
"data": "",
"location": "payload",
"locationType": "msg",
"responseType": "json",
"x": 1150,
"y": 380,
"wires": [
[]
]
},
{
"id": "144eb367.c8419d",
"type": "server",
"z": "",
"name": "Home Assistant",
"legacy": false,
"hassio": false,
"rejectUnauthorizedCerts": false,
"ha_boolean": "y|yes|true|on|home|open",
"connectionDelay": true
}
]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment