-
-
Save alistairhenderson/40236705d062cfe6e005e1df22392594 to your computer and use it in GitHub Desktop.
This will create a Node-Red flow. The information for this can be seen at https://open-source.blog/2023/07/21/getting-node-red-ready-for-aws-s3-access/
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
[ | |
{ | |
"id": "2dd7c4b1fdb409ab", | |
"type": "tab", | |
"label": "Sync Files to S3", | |
"disabled": false, | |
"info": "# USE\r\nThis syncs all of the Live_data files every 30 minutes to the S3 bucket", | |
"env": [] | |
}, | |
{ | |
"id": "60027a0a1146f7aa", | |
"type": "inject", | |
"z": "2dd7c4b1fdb409ab", | |
"name": "", | |
"props": [ | |
{ | |
"p": "payload" | |
}, | |
{ | |
"p": "topic", | |
"vt": "str" | |
} | |
], | |
"repeat": "", | |
"crontab": "0 0-22 * * *", | |
"once": false, | |
"onceDelay": 0.1, | |
"topic": "", | |
"payload": "", | |
"payloadType": "date", | |
"x": 210, | |
"y": 220, | |
"wires": [ | |
[ | |
"73abdd77c90a9047" | |
] | |
] | |
}, | |
{ | |
"id": "c7e98b9c8f3e6859", | |
"type": "file in", | |
"z": "2dd7c4b1fdb409ab", | |
"name": "Read the files", | |
"filename": "filepath", | |
"filenameType": "msg", | |
"format": "utf8", | |
"chunk": false, | |
"sendError": false, | |
"encoding": "base64", | |
"allProps": false, | |
"x": 540, | |
"y": 560, | |
"wires": [ | |
[ | |
"22c8298da7b2d9a4" | |
] | |
] | |
}, | |
{ | |
"id": "747312053e2cc3d3", | |
"type": "readdir", | |
"z": "2dd7c4b1fdb409ab", | |
"name": "Loop Through copy_data directory", | |
"dir": "/opt/systems/data/live_data/", | |
"as": "multi", | |
"recursive": true, | |
"outproperty": "filepath", | |
"x": 280, | |
"y": 560, | |
"wires": [ | |
[ | |
"c7e98b9c8f3e6859" | |
] | |
] | |
}, | |
{ | |
"id": "5048f651534218ed", | |
"type": "debug", | |
"z": "2dd7c4b1fdb409ab", | |
"name": "File Listing", | |
"active": false, | |
"tosidebar": true, | |
"console": false, | |
"tostatus": false, | |
"complete": "true", | |
"targetType": "full", | |
"statusVal": "", | |
"statusType": "auto", | |
"x": 1490, | |
"y": 420, | |
"wires": [] | |
}, | |
{ | |
"id": "22c8298da7b2d9a4", | |
"type": "function", | |
"z": "2dd7c4b1fdb409ab", | |
"name": "Create all of the paramaters", | |
"func": "// Get the full path from the payload\nconst fullPath = msg.filepath;\n\n// Extract the filename from the full path\nconst filename = fullPath.substring(fullPath.lastIndexOf('/') + 1);\n\n// Remove the filename from the full path to get the folder path\nconst folderPath = fullPath.substring(0, fullPath.lastIndexOf('/'));\n\n// Extract the foldername from the folder path\nconst foldername = folderPath.substring(folderPath.lastIndexOf('/') + 1);\n\n// Extract the filetype from the filename\nconst filetype = filename.substring(filename.lastIndexOf('.') + 1);\n\n\nconst lineno = `line_no_${msg.line_no.padStart(2, '0')}`;\n\n// Concatenate msg.line_no with foldername\n//const resultString = `${lineno}/${foldername}`;\nconst fullFoldername = `${lineno}/${foldername}`;\n\n\n// Create an object with the extracted values\nconst result = {\n foldername: fullFoldername,\n filename: filename.replace(`.${filetype}`, ''),\n filetype: filetype,\n};\n\n// Output the result in msg.payload\nmsg.fileparamaters = result;\n\n// Return the modified message\nreturn msg;", | |
"outputs": 1, | |
"noerr": 0, | |
"initialize": "", | |
"finalize": "", | |
"libs": [], | |
"x": 860, | |
"y": 560, | |
"wires": [ | |
[ | |
"660ee98a0bc2acae" | |
] | |
] | |
}, | |
{ | |
"id": "660ee98a0bc2acae", | |
"type": "change", | |
"z": "2dd7c4b1fdb409ab", | |
"name": "Create Elements for the S3 Write", | |
"rules": [ | |
{ | |
"t": "move", | |
"p": "payload", | |
"pt": "msg", | |
"to": "base64", | |
"tot": "msg" | |
}, | |
{ | |
"t": "set", | |
"p": "folderName", | |
"pt": "msg", | |
"to": "fileparamaters.foldername", | |
"tot": "msg" | |
}, | |
{ | |
"t": "set", | |
"p": "fileType", | |
"pt": "msg", | |
"to": "fileparamaters.filetype", | |
"tot": "msg" | |
}, | |
{ | |
"t": "move", | |
"p": "filename", | |
"pt": "msg", | |
"to": "file_path", | |
"tot": "msg" | |
}, | |
{ | |
"t": "set", | |
"p": "userId", | |
"pt": "msg", | |
"to": "fileparamaters.filename", | |
"tot": "msg" | |
} | |
], | |
"action": "", | |
"property": "", | |
"from": "", | |
"to": "", | |
"reg": false, | |
"x": 1220, | |
"y": 560, | |
"wires": [ | |
[ | |
"5048f651534218ed", | |
"63e4aa2e98c912ec", | |
"81a988e7dd6f42f9" | |
] | |
] | |
}, | |
{ | |
"id": "73abdd77c90a9047", | |
"type": "function", | |
"z": "2dd7c4b1fdb409ab", | |
"name": "Environmental Variables", | |
"func": "msg.line_no = '01'\nreturn msg;", | |
"outputs": 1, | |
"noerr": 0, | |
"initialize": "", | |
"finalize": "", | |
"libs": [], | |
"x": 590, | |
"y": 220, | |
"wires": [ | |
[ | |
"747312053e2cc3d3" | |
] | |
] | |
}, | |
{ | |
"id": "63e4aa2e98c912ec", | |
"type": "s3-upload", | |
"z": "2dd7c4b1fdb409ab", | |
"name": "write-to-s3", | |
"target": "payload", | |
"accessKeyId": "IAM_ID", | |
"accessKeyIdType": "str", | |
"secretAccessKey": "IAM-ACCESS-KEY", | |
"secretAccessKeyType": "str", | |
"awsRegion": "eu-west-2", | |
"awsRegionType": "str", | |
"s3Bucket": "S3-BUCKET-NAME", | |
"s3BucketType": "str", | |
"x": 1510, | |
"y": 560, | |
"wires": [ | |
[], | |
[] | |
] | |
}, | |
{ | |
"id": "81a988e7dd6f42f9", | |
"type": "flogger", | |
"z": "2dd7c4b1fdb409ab", | |
"name": "Log", | |
"logfile": "linesync.log", | |
"inputchoice": "object", | |
"inputobject": "file_path", | |
"inputobjectType": "msg", | |
"inputmoustache": "Recieved payload {{payload}} and topic {{topic}}", | |
"loglevel": "INFO", | |
"logconfig": "1fda2671412d7083", | |
"sendpane": "", | |
"x": 1490, | |
"y": 740, | |
"wires": [ | |
[] | |
] | |
}, | |
{ | |
"id": "1fda2671412d7083", | |
"type": "config-log", | |
"logname": "FileLogging", | |
"logdir": "/u10/logging", | |
"stamp": "utc", | |
"logstyle": "plain", | |
"logrotate": true, | |
"logcompress": true, | |
"logrotatecount": "1", | |
"logsize": "1000" | |
} | |
] |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment