Last active
November 18, 2021 11:49
-
-
Save yai333/7cb2f778d6ff6365f48e901e7e255e6b to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "name": "[concat(parameters('factoryName'), '/DemoPipeline')]", | |
| "type": "Microsoft.DataFactory/factories/pipelines", | |
| "apiVersion": "2018-06-01", | |
| "properties": { | |
| "activities": [ | |
| { | |
| "name": "ExtractCalendarView", | |
| "description": "Get the occurrences, exceptions of events in a calendar view defined by a time range", | |
| "type": "Copy", | |
| "dependsOn": [ | |
| { | |
| "activity": "SetGlobalVariable", | |
| "dependencyConditions": [ | |
| "Succeeded" | |
| ] | |
| } | |
| ], | |
| "policy": { | |
| "timeout": "7.00:00:00", | |
| "retry": 0, | |
| "retryIntervalInSeconds": 30, | |
| "secureOutput": false, | |
| "secureInput": false | |
| }, | |
| "userProperties": [], | |
| "typeProperties": { | |
| "source": { | |
| "type": "Office365Source", | |
| "startTime": { | |
| "value": "@utcnow()", | |
| "type": "Expression" | |
| }, | |
| "endTime": { | |
| "value": "@adddays(utcnow(), 3)", | |
| "type": "Expression" | |
| }, | |
| "userScopeFilterUri": "", | |
| "outputColumns": [ | |
| { | |
| "name": "id" | |
| }, | |
| { | |
| "name": "createdDateTime" | |
| }, | |
| { | |
| "name": "lastModifiedDateTime" | |
| }, | |
| { | |
| "name": "originalStartTimeZone" | |
| }, | |
| { | |
| "name": "originalEndTimeZone" | |
| }, | |
| { | |
| "name": "iCalUId" | |
| }, | |
| { | |
| "name": "start" | |
| }, | |
| { | |
| "name": "end" | |
| }, | |
| { | |
| "name": "location" | |
| }, | |
| { | |
| "name": "isAllDay" | |
| }, | |
| { | |
| "name": "isCancelled" | |
| }, | |
| { | |
| "name": "recurrence" | |
| }, | |
| { | |
| "name": "transactionId" | |
| }, | |
| { | |
| "name": "attendees" | |
| }, | |
| { | |
| "name": "organizer" | |
| }, | |
| { | |
| "name": "originalStart" | |
| } | |
| ] | |
| }, | |
| "sink": { | |
| "type": "BinarySink", | |
| "storeSettings": { | |
| "type": "AzureBlobStorageWriteSettings" | |
| } | |
| }, | |
| "enableStaging": false | |
| }, | |
| "inputs": [ | |
| { | |
| "referenceName": "calendarDataset", | |
| "type": "DatasetReference", | |
| "parameters": {} | |
| } | |
| ], | |
| "outputs": [ | |
| { | |
| "referenceName": "outputDataset", | |
| "type": "DatasetReference", | |
| "parameters": { | |
| "outputPath": { | |
| "value": "@variables('outputPath')", | |
| "type": "Expression" | |
| } | |
| } | |
| } | |
| ] | |
| }, | |
| { | |
| "name": "LoadToS3", | |
| "description": "Load Calendar view change feed to AWS S3", | |
| "type": "DatabricksNotebook", | |
| "dependsOn": [ | |
| { | |
| "activity": "ExtractCalendarView", | |
| "dependencyConditions": [ | |
| "Succeeded" | |
| ] | |
| } | |
| ], | |
| "policy": { | |
| "timeout": "7.00:00:00", | |
| "retry": 0, | |
| "retryIntervalInSeconds": 30, | |
| "secureOutput": false, | |
| "secureInput": false | |
| }, | |
| "userProperties": [], | |
| "typeProperties": { | |
| "notebookPath": "/Users/[email protected]/Office365ETL", | |
| "baseParameters": { | |
| "inputPath": { | |
| "value": "@variables('outputPath')", | |
| "type": "Expression" | |
| } | |
| } | |
| }, | |
| "linkedServiceName": { | |
| "referenceName": "databricksLinkedService", | |
| "type": "LinkedServiceReference" | |
| } | |
| }, | |
| { | |
| "name": "SetGlobalVariable", | |
| "type": "SetVariable", | |
| "dependsOn": [], | |
| "userProperties": [], | |
| "typeProperties": { | |
| "variableName": "outputPath", | |
| "value": { | |
| "value": "@formatDateTime(utcNow(), 'yyyy/MM/dd/HH-mm')", | |
| "type": "Expression" | |
| } | |
| } | |
| }, | |
| { | |
| "name": "ETLFailActivity", | |
| "type": "Fail", | |
| "dependsOn": [ | |
| { | |
| "activity": "ExtractCalendarView", | |
| "dependencyConditions": [ | |
| "Failed" | |
| ] | |
| }, | |
| { | |
| "activity": "LoadToS3", | |
| "dependencyConditions": [ | |
| "Failed" | |
| ] | |
| } | |
| ], | |
| "userProperties": [], | |
| "typeProperties": { | |
| "message": { | |
| "value": "Pipeline @pipeline().RunId has failed.", | |
| "type": "Expression" | |
| }, | |
| "errorCode": "500" | |
| } | |
| } | |
| ], | |
| "policy": { | |
| "elapsedTimeMetric": {}, | |
| "cancelAfter": {} | |
| }, | |
| "variables": { | |
| "outputPath": { | |
| "type": "String" | |
| } | |
| }, | |
| "annotations": [], | |
| }, | |
| "dependsOn": [ | |
| "[concat(variables('factoryId'), '/datasets/calendarDataset')]", | |
| "[concat(variables('factoryId'), '/datasets/outputDataset')]", | |
| "[concat(variables('factoryId'), '/linkedServices/databricksLinkedService')]" | |
| ] | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment