Skip to content

Instantly share code, notes, and snippets.

@michael-golfi

michael-golfi/create-cosmos-collection.sh

Created November 27, 2017 21:33
Show Gist options
  • Save michael-golfi/cd7ea17af1aaec19f5f47f671fcc1f4e to your computer and use it in GitHub Desktop.
Save michael-golfi/cd7ea17af1aaec19f5f47f671fcc1f4e to your computer and use it in GitHub Desktop.
Download ZIP
Create Azure Deployment
Raw
create-cosmos-collection.sh
#!/bin/bash
# Ref
# https://docs.microsoft.com/en-us/cli/azure/cosmosdb?view=azure-cli-latest#create
# Set variables for the new account, database, and collection
resourceGroupName='myResourceGroup'
location='southcentralus'
name='docdb-test'
databaseName='docdb-test-database'
collectionName='docdb-test-collection'
# Create a resource group
az group create \
--name $resourceGroupName \
--location $location
# Create a DocumentDB API Cosmos DB account
az cosmosdb create \
--name $name \
--kind GlobalDocumentDB \
--resource-group $resourceGroupName \
--max-interval 10 \
--max-staleness-prefix 200
# Create a database
az cosmosdb database create \
--name $name \
--db-name $databaseName \
--resource-group $resourceGroupName
# Create a collection
az cosmosdb collection create \
--collection-name $collectionName \
--name $name \
--db-name $databaseName \
--resource-group $resourceGroupName
Raw
deployment.json
{
"$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json",
"contentVersion": "1.0.0.0",
"parameters": {
"DataFactorySettings": {
"value": {
"Name": "First DataFactory",
"Location": "East US"
}
},
"dataFactoryName": {
"type": "string",
"defaultValue": "[uniqueString(resourceGroup().id, deployment().name)]",
"metadata": {
"description": "Name of the data factory. Must be globally unique."
}
},
"dataFactoryLocation": {
"type": "string",
"allowedValues": [
"East US",
"East US 2"
],
"defaultValue": "East US",
"metadata": {
"description": "Location of the data factory. Currently, only East US and East US 2 are supported. "
}
},
"storageAccountName": {
"type": "string",
"metadata": {
"description": "Name of the Azure storage account that contains the input/output data."
}
},
"storageAccountKey": {
"type": "securestring",
"metadata": {
"description": "Key for the Azure storage account."
}
},
"blobContainer": {
"type": "string",
"metadata": {
"description": "Name of the blob container in the Azure Storage account."
}
},
"inputBlobFolder": {
"type": "string",
"metadata": {
"description": "The folder in the blob container that has the input file."
}
},
"inputBlobName": {
"type": "string",
"metadata": {
"description": "Name of the input file/blob."
}
},
"outputBlobFolder": {
"type": "string",
"metadata": {
"description": "The folder in the blob container that will hold the transformed data."
}
},
"outputBlobName": {
"type": "string",
"metadata": {
"description": "Name of the output file/blob."
}
},
"iotHubName": {
"type": "string",
"minLength": 3,
"metadata": {
"description": "Name of the IoT Hub instance to provision."
}
},
"skuName": {
"type": "string",
"allowedValues": [
"F1",
"S1",
"S2"
],
"defaultValue": "S1",
"metadata": {
"description": "One of the supported SKUs: F1, S1 or S2"
}
},
"capacityUnits": {
"type": "int",
"minValue": 1,
"defaultValue": 1,
"metadata": {
"description": "Number of desired IoT Hub units. Restricted to 1 unit for F1. Can be set up to maximum number allowed for subscription."
}
},
"d2cMessageRetentionInDaysPeriod": {
"type": "int",
"minValue": 1,
"maxValue": 7,
"defaultValue": 1,
"metadata": {
"description": "Retention time in days for device-to-cloud messages."
}
},
"d2cPartitionCount": {
"type": "int",
"minValue": 2,
"defaultValue": 2,
"metadata": {
"description": "Number of desired partitions for device-to-cloud event ingestion."
}
},
"streamAnalyticsJobName": {
"type": "string",
"minLength": 3,
"maxLength": 63,
"metadata": {
"description": "Stream Analytics Job Name, can contain alphanumeric characters and hypen and must be 3-63 characters long"
}
},
"streamAnalyticsJobSku": {
"type": "string",
"defaultValue": "Standard",
"allowedValues": [
"Standard",
"Free",
"P10Premium",
"P20Premium"
],
"metadata": {
"description": "Stream Analytis SKU"
}
},
"numberOfStreamingUnits": {
"type": "int",
"minValue": 1,
"maxValue": 48,
"defaultValue": 1,
"allowedValues": [
1,
3,
6,
12,
18,
24,
30,
36,
42,
48
],
"metadata": {
"description": "Number of Stream Analytis Jobs Streaming Units"
}
},
"referenceBlobAccountName": {
"type": "string",
"metadata": {
"description": "Azure Storage account name."
}
},
"referenceBlobAccountKey": {
"type": "string",
"metadata": {
"description": "Azure Storage account key."
}
},
"referenceBlobContainer": {
"type": "string",
"metadata": {
"description": "The name of a container within the associated Storage account. This container contains either the blob(s) to be read from or written to. Required on PUT (CreateOrReplace) requests."
}
},
"referenceBlobPathPattern": {
"type": "string",
"metadata": {
"description": "The blob path pattern. Not a regular expression. It represents a pattern against which blob names will be matched to determine whether or not they should be included as input or output to the job. See https://docs.microsoft.com/en-us/rest/api/streamanalytics/stream-analytics-input or https://docs.microsoft.com/en-us/rest/api/streamanalytics/stream-analytics-output for a more detailed explanation and example."
}
},
"cosmosAccountName": {
"type": "string",
"metadata": {
"description": "The name of the Cosmos DB Account"
}
},
"cosmosDatabaseName": {
"type": "string",
"metadata": {
"description": "The name of the Cosmos Database Instance"
}
},
"cosmosCollectionName": {
"type": "string",
"defaultValue": "consumption",
"metadata": {
"description": "The name of the Cosmos Database Collection"
}
},
"cosmosDocumentId": {
"type": "string",
"defaultValue": "id",
"metadata": {
"description": "The field name for the Cosmos document primary ID"
}
},
"cosmosPartitionKey": {
"type": "string",
"defaultValue": "/storeId",
"metadata": {
"description": "The key to be used as a partitioning key for the dataset. Should split data into more or less equal sizes. Should start with a '/' and not trail with a slash."
}
},
"powerbiUserDisplayName": {
"type": "string",
"minLength": 3,
"metadata": {
"description": "Display name for connecting Stream Analytics to PowerBI"
}
},
"powerbiUserLoginName": {
"type": "string",
"minLength": 3,
"defaultValue": "<user>@domain.com",
"metadata": {
"description": "Your username for connecting Stream Analytics to PowerBI"
}
},
"powerbiWorkspaceName": {
"type": "string",
"minLength": 3,
"maxLength": 63,
"metadata": {
"description": "A unique name for the PowerBI workspace collection. It should match with the following regular expression: ^(?:[a-zA-Z0-9]+-?)+$ or it will raise an error. "
}
},
"sku": {
"type": "string",
"allowedValues": [
"S1",
"s1"
],
"metadata": {
"description": "provide the sku for powerbi workspace collection."
}
}
},
"variables": {
"azureStorageLinkedServiceName": "ArmtemplateStorageLinkedService",
"inputDatasetName": "ArmtemplateTestDatasetIn",
"outputDatasetName": "ArmtemplateTestDatasetOut",
"pipelineName": "ArmtemplateSampleCopyPipeline",
"cosmosApiVersion": "[providers('Microsoft.DocumentDB','databaseAccounts').apiVersions[0]]",
"hubApiVersion": "[providers('Microsoft.Devices','IotHubs').apiVersions[0]]",
"powerBIVersion": "[providers('Microsoft.PowerBI','workspaceCollections').apiVersions[0]]"
},
"resources": [
{
"name": "[parameters('dataFactoryName')]",
"apiVersion": "2017-09-01-preview",
"type": "Microsoft.DataFactory/factories",
"location": "[parameters('dataFactoryLocation')]",
"properties": {
"loggingStorageAccountName": "[parameters('storageAccountName')]",
"loggingStorageAccountKey": "[parameters('storageAccountKey')]"
},
"resources": [
{
"type": "linkedservices",
"name": "[variables('azureStorageLinkedServiceName')]",
"dependsOn": [
"[parameters('dataFactoryName')]"
],
"apiVersion": "2017-09-01-preview",
"properties": {
"type": "AzureStorage",
"description": "Azure Storage linked service",
"typeProperties": {
"connectionString": {
"value": "[concat('DefaultEndpointsProtocol=https;AccountName=',parameters('storageAccountName'),';AccountKey=',parameters('storageAccountKey'))]",
"type": "SecureString"
}
}
}
},
{
"type": "datasets",
"name": "[variables('inputDatasetName')]",
"dependsOn": [
"[parameters('dataFactoryName')]",
"[variables('azureStorageLinkedServiceName')]"
],
"apiVersion": "2017-09-01-preview",
"properties": {
"type": "AzureBlob",
"typeProperties": {
"format": {
"type": "TextFormat",
"columnDelimiter": ",",
"nullValue": "\\N",
"treatEmptyAsNull": false,
"firstRowAsHeader": false
},
"folderPath": "[concat(parameters('blobContainer'), '/', parameters('inputBlobFolder'), '/')]",
"fileName": "[parameters('inputBlobName')]"
},
"linkedServiceName": {
"referenceName": "[variables('azureStorageLinkedServiceName')]",
"type": "LinkedServiceReference"
}
}
},
{
"type": "datasets",
"name": "[variables('outputDatasetName')]",
"dependsOn": [
"[parameters('dataFactoryName')]",
"[variables('azureStorageLinkedServiceName')]"
],
"apiVersion": "2017-09-01-preview",
"properties": {
"type": "AzureBlob",
"typeProperties": {
"format": {
"type": "TextFormat",
"columnDelimiter": ",",
"nullValue": "\\N",
"treatEmptyAsNull": false,
"firstRowAsHeader": false
},
"folderPath": "[concat(parameters('blobContainer'), '/', parameters('outputBlobFolder'), '/')]",
"fileName": "[parameters('outputBlobName')]"
},
"linkedServiceName": {
"referenceName": "[variables('azureStorageLinkedServiceName')]",
"type": "LinkedServiceReference"
}
}
},
{
"type": "pipelines",
"name": "[variables('pipelineName')]",
"dependsOn": [
"[parameters('dataFactoryName')]",
"[variables('azureStorageLinkedServiceName')]",
"[variables('inputDatasetName')]",
"[variables('outputDatasetName')]"
],
"apiVersion": "2017-09-01-preview",
"properties": {
"activities": [
{
"type": "Copy",
"typeProperties": {
"source": {
"type": "BlobSource"
},
"sink": {
"type": "BlobSink"
}
},
"name": "MyCopyActivity",
"inputs": [
{
"referenceName": "[variables('inputDatasetName')]",
"type": "DatasetReference"
}
],
"outputs": [
{
"referenceName": "[variables('outputDatasetName')]",
"type": "DatasetReference"
}
]
}
]
}
}
]
},
{
"type": "Microsoft.Devices/IotHubs",
"sku": {
"name": "[parameters('skuName')]",
"capacity": "[parameters('capacityUnits')]"
},
"name": "[parameters('iotHubName')]",
"apiVersion": "[variables('hubApiVersion')]",
"location": "[resourceGroup().location]",
"properties": {
"eventHubEndpoints": {
"events": {
"retentionTimeInDays": "[parameters('d2cMessageRetentionInDaysPeriod')]",
"partitionCount": "[parameters('d2cPartitionCount')]"
}
}
}
},
{
"type": "Microsoft.StreamAnalytics/StreamingJobs",
"apiVersion": "2016-03-01",
"name": "[parameters('streamAnalyticsJobName')]",
"location": "[resourceGroup().location]",
"properties": {
"sku": {
"name": "[parameters('streamAnalyticsJobSku')]"
},
"outputErrorPolicy": "stop",
"eventsOutOfOrderPolicy": "adjust",
"eventsOutOfOrderMaxDelayInSeconds": 0,
"eventsLateArrivalMaxDelayInSeconds": 5,
"dataLocale": "en-US",
"inputs": [
{
"name": "IoTHub",
"properties": {
"dataSource": {
"properties": {
"iotHubNamespace": "[parameters('iotHubName')]",
"sharedAccessPolicyKey": "[listkeys(resourceId('Microsoft.Devices/IotHubs/IotHubKeys',parameters('iotHubName'), 'iothubowner'), variables('hubApiVersion')).primaryKey]",
"sharedAccessPolicyName": "iothubowner",
"endpoint": "messages/events",
"consumerGroupName": "$Default"
},
"type": "Microsoft.Devices/IotHubs"
},
"serialization": {
"properties": {
"encoding": "UTF8"
},
"type": "Json"
},
"compression": {
"type": "None"
},
"type": "Stream"
}
},
{
"type": "Microsoft.Storage/Blob",
"properties": {
"storageAccounts": [
{
"accountName": "[parameters('referenceBlobAccountName')]",
"accountKey": "[parameters('referenceBlobAccountKey')]"
}
],
"container": "[parameters('referenceBlobContainer')]",
"pathPattern": "[parameters('referenceBlobPathPattern')]",
"dateFormat": "YYYY/MM/DD",
"timeFormat": "HH"
}
}
],
"transformation": {
"name": "Transformation",
"properties": {
"streamingUnits": "[parameters('numberOfStreamingUnits')]",
"query": "SELECT\r\n *\r\nINTO\r\n [CosmosDB]\r\nFROM\r\n [IoTHub]"
}
},
"outputs": [
{
"name": "CosmosDB",
"properties": {
"dataSource": {
"properties": {
"accountId": "[parameters('cosmosAccountName')]",
"accountKey": "[listKeys(resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('cosmosAccountName')), variables('cosmosApiVersion')).primaryMasterKey]",
"collectionNamePattern": "[parameters('cosmosCollectionName')]",
"database": "[parameters('cosmosDatabaseName')]",
"documentId": "[parameters('cosmosDocumentId')]",
"partitionKey": "[parameters('cosmosPartitionKey')]"
},
"type": "Microsoft.Storage/DocumentDB"
},
"diagnostics": null,
"etag": null,
"serialization": null
}
},
{
"name": "PowerBI",
"properties": {
"dataSource": {
"type": "PowerBI",
"outputPowerBISource": {
"dataSet": "[parameters('cosmosCollectionName')]",
"table": "[parameters('cosmosCollectionName')]",
"groupId": "",
"groupName": "My Workspace",
"refreshToken": "dummytoken",
"tokenUserDisplayName": "[parameters('powerbiUserDisplayName')]",
"tokenUserPrincipalName": "[parameters('powerbiUserLoginName')]"
},
"properties": {
"dataSet": "[parameters('cosmosCollectionName')]",
"table": "[parameters('cosmosCollectionName')]",
"groupId": "",
"groupName": "My Workspace",
"refreshToken": "dummytoken",
"tokenUserDisplayName": "[parameters('powerbiUserDisplayName')]",
"tokenUserPrincipalName": "[parameters('powerbiUserLoginName')]"
}
},
"serialization": null,
"diagnostics": null
}
}
]
},
"dependsOn": [
"[concat('Microsoft.Devices/IotHubs/', parameters('iotHubName'))]",
"[concat('Microsoft.DocumentDB/databaseAccounts/', parameters('cosmosAccountName'))]",
"[concat('Microsoft.DataFactory/factories', parameters('dataFactoryName'))]"
]
},
{
"type": "Microsoft.DocumentDB/databaseAccounts",
"kind": "GlobalDocumentDB",
"name": "[parameters('cosmosAccountName')]",
"apiVersion": "[variables('cosmosApiVersion')]",
"location": "[resourceGroup().location]",
"tags": {
"defaultExperience": "Table"
},
"scale": null,
"properties": {
"databaseAccountOfferType": "Standard",
"consistencyPolicy": {
"defaultConsistencyLevel": "Session",
"maxIntervalInSeconds": 5,
"maxStalenessPrefix": 100
},
"name": "[parameters('cosmosDatabaseName')]"
}
}
],
"outputs": {
"SharedAccessPolicyKey": {
"value": "[listKeys(resourceId('Microsoft.Devices/IotHubs/IotHubKeys', parameters('iotHubName'), 'iothubowner'), variables('hubApiVersion')).primaryKey]",
"type": "object"
},
"cosmosKeys": {
"value": "[listKeys(resourceId('Microsoft.DocumentDB/databaseAccounts', parameters('cosmosAccountName')), variables('cosmosApiVersion'))]",
"type": "object"
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment