Created
February 9, 2026 22:13
-
-
Save steve-fraser/285eabbec0c8a08ab3e483e62d798827 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| RETRY_SEC=${RETRY_SEC:-1} | |
| RETRY_TIMES=${RETRY_TIMES:-30} | |
| retry() { | |
| local -r cmd="$@" | |
| local -i retries=1 | |
| until $cmd; do | |
| sleep $RETRY_SEC | |
| [[ retries -eq $RETRY_TIMES ]] && echo "Something went wrong, please try again. If issue persists please notify CAST AI team" && return 1 | |
| echo "Still executing..." | |
| let retries=retries+1 | |
| done | |
| } | |
| is_true() { | |
| [[ "$(echo "$1" | tr '[:upper:]' '[:lower:]')" == "true" ]] | |
| } | |
| fatal() { | |
| echo -e "\033[31m\033[1m$1\033[0m" | |
| exit 1 | |
| } | |
| CASTAI_API_URL="${CASTAI_API_URL:-https://api.cast.ai}" | |
| CASTAI_GRPC_URL="${CASTAI_GRPC_URL:-grpc.cast.ai:443}" | |
| CASTAI_API_GRPC_URL="${CASTAI_API_GRPC_URL:-api-grpc.cast.ai:443}" | |
| CASTAI_KVISOR_GRPC_URL="${CASTAI_KVISOR_GRPC_URL:-kvisor.prod-master.cast.ai:443}" | |
| kubectl get namespace castai-agent >/dev/null 2>&1 | |
| if [ $? -eq 1 ]; then | |
| fatal "Cast AI namespace not found. Please run phase1 of the onboarding script first." | |
| fi | |
| if [ -z $NODE_GROUP ]; then | |
| fatal "NODE_GROUP environment variable was not provided" | |
| fi | |
| if [ -z $SUBSCRIPTION_ID ]; then | |
| fatal "SUBSCRIPTION_ID environment variable was not provided" | |
| fi | |
| if [ -z $CASTAI_API_TOKEN ] || [ -z $CASTAI_API_URL ] || [ -z $CASTAI_CLUSTER_ID ]; then | |
| fatal "CASTAI_API_TOKEN, CASTAI_API_URL or CASTAI_CLUSTER_ID variables were not provided" | |
| fi | |
| if ! [ -x "$(command -v az)" ]; then | |
| fatal "Error: azure cli is not installed" | |
| fi | |
| if ! [ -x "$(command -v jq)" ]; then | |
| fatal "Error: jq is not installed" | |
| fi | |
| if ! [ -x "$(command -v helm)" ]; then | |
| fatal "Error: helm is not installed. (helm is required to install castai-cluster-controller)" | |
| fi | |
| if [ -z $REGION ]; then | |
| fatal "REGION environment variable was not provided" | |
| fi | |
| function wait_for_component() { | |
| local resource_name=$1 | |
| local namespace=${2:-castai-agent} | |
| local timeout=${3:-300} # Default 5 minutes | |
| local interval=${4:-5} # Default check every 5 seconds | |
| echo "Waiting for ${resource_name} to be ready (timeout: ${timeout}s)..." | |
| local elapsed=0 | |
| local current_version="" | |
| while [ $elapsed -lt $timeout ]; do | |
| current_version=$(kubectl get component "$resource_name" -n "$namespace" -o jsonpath='{.status.currentVersion}' 2>/dev/null || true) | |
| target_version=$(kubectl get component "$resource_name" -n "$namespace" -o jsonpath='{.spec.version}' 2>/dev/null || true) | |
| if [ -n "$current_version" ] && [ "$current_version" = "$target_version" ]; then | |
| echo "${resource_name} ready with version $current_version" | |
| return 0 | |
| fi | |
| echo " Waiting... (${elapsed}s elapsed)" | |
| sleep $interval | |
| elapsed=$((elapsed + interval)) | |
| done | |
| # Timeout reached | |
| echo "Something went wrong while installing ${resource_name}, please try again." | |
| echo "Run the command 'kubectl logs -n castai-agent deployment/castware-operator --tail=100 | grep ${resource_name}' for more information. If issue persists please notify CAST AI team" | |
| exit 1 | |
| } | |
| function enable_base_components() { | |
| if is_true "$INSTALL_OPERATOR"; then | |
| if ! helm search repo castai-helm | grep -q "castware-operator"; then | |
| echo "Error: Chart 'castware-operator' not found in repository 'castai-helm'" | |
| exit 1 | |
| fi | |
| echo "Installing castware-operator." | |
| helm upgrade -i castware-operator castai-helm/castware-operator -n castai-agent --atomic \ | |
| --set apiKeySecret.apiKey="$CASTAI_API_TOKEN" \ | |
| --set defaultCluster.terraform=false \ | |
| --set defaultCluster.provider="aks" \ | |
| --set defaultCluster.api.apiUrl="$CASTAI_API_URL" \ | |
| --set defaultCluster.migrationMode=autoUpgrade \ | |
| --set defaultComponents.enabled=false \ | |
| --set extendedPermissions=true | |
| # Wait for agent migration and upgrade | |
| wait_for_component "castai-agent" | |
| echo "Finished installing castware-operator." | |
| elif is_true "$OPERATOR_MANAGED"; then | |
| echo "Enabling extended permissions for castware-operator." | |
| helm upgrade castware-operator castai-helm/castware-operator -n castai-agent --atomic \ | |
| --set apiKeySecret.apiKey="$CASTAI_API_TOKEN" \ | |
| --set extendedPermissions="true" \ | |
| --reset-then-reuse-values | |
| echo "Extended permissions for castware-operator enabled." | |
| fi | |
| echo "Installing castai-cluster-controller." | |
| if is_true "$OPERATOR_MANAGED" || is_true "$INSTALL_OPERATOR"; then | |
| echo "With operator." | |
| if kubectl apply -f - <<EOF; then | |
| apiVersion: castware.cast.ai/v1alpha1 | |
| kind: Component | |
| metadata: | |
| name: cluster-controller | |
| namespace: castai-agent | |
| spec: | |
| cluster: castai | |
| component: cluster-controller | |
| enabled: true | |
| values: | |
| autoscaling: | |
| enabled: $(is_true "${INSTALL_AUTOSCALER}" && echo "true" || echo "false") | |
| aks: | |
| enabled: true | |
| EOF | |
| echo "Successfully installed cluster-controller custom resource." | |
| else | |
| echo "Failed to install cluster-controller custom resource." | |
| exit 1 | |
| fi | |
| wait_for_component "cluster-controller" | |
| else | |
| echo -e "\033[92mWithout operator.\033[0m" | |
| helm upgrade -i cluster-controller castai-helm/castai-cluster-controller -n castai-agent \ | |
| --set castai.apiKey="$CASTAI_API_TOKEN" \ | |
| --set castai.apiURL="$CASTAI_API_URL" \ | |
| --set castai.clusterID="$CASTAI_CLUSTER_ID" \ | |
| --set aks.enabled=true \ | |
| --set autoscaling.enabled="$INSTALL_AUTOSCALER" | |
| fi | |
| echo "Finished installing castai-cluster-controller." | |
| echo "Installing castai-spot-handler." | |
| if is_true "$OPERATOR_MANAGED" || is_true "$INSTALL_OPERATOR"; then | |
| echo "With operator." | |
| if kubectl apply -f - <<EOF; then | |
| apiVersion: castware.cast.ai/v1alpha1 | |
| kind: Component | |
| metadata: | |
| name: spot-handler | |
| namespace: castai-agent | |
| spec: | |
| cluster: castai | |
| component: spot-handler | |
| enabled: true | |
| values: | |
| autoscaling: | |
| enabled: $(is_true "${INSTALL_AUTOSCALER}" && echo "true" || echo "false") | |
| EOF | |
| echo "Successfully installed spot-handler custom resource." | |
| else | |
| echo "Failed to install spot-handler custom resource." | |
| exit 1 | |
| fi | |
| wait_for_component "spot-handler" | |
| else | |
| echo -e "\033[92mWithout operator.\033[0m" | |
| # Script is opinionated and always sets clusterID on all components. As the references are mutually exclusive, we reset the values for CM/Secret refs. | |
| helm upgrade -i castai-spot-handler castai-helm/castai-spot-handler -n castai-agent \ | |
| --set castai.apiKey=$CASTAI_API_TOKEN \ | |
| --set castai.apiURL=$CASTAI_API_URL \ | |
| --set castai.clusterID=$CASTAI_CLUSTER_ID \ | |
| --set castai.provider=azure \ | |
| --set castai.clusterIdConfigMapKeyRef.name=null \ | |
| --set castai.clusterIdSecretKeyRef.name=null \ | |
| --set phase2Permissions="$INSTALL_AUTOSCALER" | |
| fi | |
| echo "Finished installing castai-spot-handler" | |
| } | |
| function enable_autoscaler_agent() { | |
| echo "Installing autoscaler" | |
| echo "Installing autoscaler cloud components" | |
| SUBSCRIPTION=$(az account list --query "[?id=='${SUBSCRIPTION_ID}'].{name:name,tenantId:tenantId}[0]") | |
| if [[ $SUBSCRIPTION == "" ]]; then | |
| fatal "Error: subscription not found" | |
| fi | |
| echo "Setting active subscription: $(echo $SUBSCRIPTION | jq -r '.name')" | |
| az account set -s $SUBSCRIPTION_ID | |
| function get_cluster() { | |
| CLUSTER=$(az aks list --query "[?nodeResourceGroup=='${NODE_GROUP}'].{name:name,resourceGroup:resourceGroup,subnet:agentPoolProfiles[0].vnetSubnetId}[0]" --output json) | |
| if [[ $CLUSTER == "" ]]; then | |
| return 1 | |
| fi | |
| } | |
| echo "Fetching cluster information" | |
| if ! retry get_cluster; then | |
| echo "Error: failed to find cluster by nodeResourceGroup $NODE_GROUP" | |
| exit 1 | |
| fi | |
| CLUSTER_NAME=$(echo $CLUSTER | jq -r '.name') | |
| CLUSTER_GROUP=$(echo $CLUSTER | jq -r '.resourceGroup') | |
| if [[ -n "$CASTAI_IMPERSONATE" && -n "$USE_MANAGED_IDENTITY" ]]; then | |
| RESOURCE_GROUP="${RESOURCE_GROUP:-${CLUSTER_NAME}-cast-rg}" | |
| IDENTITY_NAME="${IDENTITY_NAME:-${CLUSTER_NAME}-castai-identity}" | |
| FEDERATED_CRED_NAME="${FEDERATED_CRED_NAME:-${CLUSTER_NAME}-castai-credentials}" | |
| fi | |
| VNET_GROUP=$(echo $CLUSTER | jq -r '.subnet | select(. != null)' | sed -n "s|/subscriptions/$SUBSCRIPTION_ID/resourceGroups/\([[:alnum:](),_-]*\).*|\1|p") | |
| ROLE_NAME="CastAKSRole-${CASTAI_CLUSTER_ID:0:8}" | |
| SCOPES=( | |
| $([ -n "$VNET_GROUP" ] && [ $CLUSTER_GROUP != $VNET_GROUP ] && echo "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$VNET_GROUP") | |
| "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$CLUSTER_GROUP" | |
| "/subscriptions/$SUBSCRIPTION_ID/resourceGroups/$NODE_GROUP" | |
| ) | |
| ROLE_DEF='{ | |
| "Name": "'"$ROLE_NAME"'", | |
| "Description": "CAST.AI role used to manage '"$CLUSTER_NAME"' AKS cluster", | |
| "IsCustom": true, | |
| "Actions": [ | |
| "Microsoft.ContainerService/*/read" | |
| "Microsoft.Compute/*/read", | |
| "Microsoft.Compute/virtualMachines/*", | |
| "Microsoft.Compute/virtualMachineScaleSets/*", | |
| "Microsoft.Compute/disks/write", | |
| "Microsoft.Compute/disks/delete", | |
| "Microsoft.Compute/disks/beginGetAccess/action", | |
| "Microsoft.Compute/galleries/write", | |
| "Microsoft.Compute/galleries/delete", | |
| "Microsoft.Compute/galleries/images/write", | |
| "Microsoft.Compute/galleries/images/delete", | |
| "Microsoft.Compute/galleries/images/versions/write", | |
| "Microsoft.Compute/galleries/images/versions/delete", | |
| "Microsoft.Compute/snapshots/write", | |
| "Microsoft.Compute/snapshots/delete", | |
| "Microsoft.Network/*/read", | |
| "Microsoft.Network/networkInterfaces/write", | |
| "Microsoft.Network/networkInterfaces/delete", | |
| "Microsoft.Network/networkInterfaces/join/action", | |
| "Microsoft.Network/networkSecurityGroups/join/action", | |
| "Microsoft.Network/virtualNetworks/subnets/join/action", | |
| "Microsoft.Network/applicationGateways/backendhealth/action", | |
| "Microsoft.Network/applicationGateways/backendAddressPools/join/action", | |
| "Microsoft.Network/applicationSecurityGroups/joinIpConfiguration/action", | |
| "Microsoft.Network/loadBalancers/backendAddressPools/write", | |
| "Microsoft.Network/loadBalancers/backendAddressPools/join/action", | |
| "Microsoft.ContainerService/*/read", | |
| "Microsoft.ContainerService/managedClusters/runCommand/action", | |
| "Microsoft.ContainerService/managedClusters/agentPools/*", | |
| "Microsoft.ContainerService/managedClusters/write", | |
| "Microsoft.Resources/*/read", | |
| "Microsoft.Resources/tags/write", | |
| "Microsoft.Authorization/locks/read", | |
| "Microsoft.Authorization/roleAssignments/read", | |
| "Microsoft.Authorization/roleDefinitions/read", | |
| "Microsoft.ManagedIdentity/userAssignedIdentities/assign/action" | |
| ], | |
| "AssignableScopes": [ | |
| '$( | |
| tmp=$(printf '"%s",' "${SCOPES[@]}") | |
| echo ${tmp%,} | |
| )' | |
| ] | |
| }' | |
| ROLE=$(az role definition list -n $ROLE_NAME --query "[0]") | |
| if [[ $ROLE == "" ]]; then | |
| echo "Creating custom role: '$ROLE_NAME'" | |
| az role definition create -o none --role-definition ''"$ROLE_DEF"'' | |
| else | |
| echo "Role already exists. Updating..." | |
| az role definition update -o none --only-show-errors --role-definition ''"$ROLE_DEF"'' | |
| fi | |
| GRAPH_URL="https://graph.microsoft.com" | |
| if [[ $REGION == usgov* ]]; then | |
| GRAPH_URL="https://graph.microsoft.us" | |
| fi | |
| if [[ -z "$USE_MANAGED_IDENTITY" ]]; then | |
| if [ -z "$APP_NAME" ]; then | |
| APP_NAME="CAST.AI ${CLUSTER_NAME}-${CASTAI_CLUSTER_ID:0:8}" | |
| fi | |
| APP=$(az rest -m GET -u "${GRAPH_URL}/v1.0/applications\?\$filter=startswith(displayName, '${APP_NAME}')" --headers ConsistencyLevel=eventual | jq -r '.value[0]') | |
| APP_ID="" | |
| APP_OBJECT_ID="" | |
| if [[ $APP == "null" ]]; then | |
| echo "Creating app registration: '${APP_NAME}'" | |
| APP=$(az rest -m POST -u "${GRAPH_URL}/v1.0/applications" --headers Content-Type=application/json -b "{'displayName':'${APP_NAME}'}") | |
| else | |
| echo "Using existing app registration: '${APP_NAME}'" | |
| fi | |
| APP_ID=$(echo $APP | jq -r '.appId') | |
| APP_OBJECT_ID=$(echo $APP | jq -r '.id') | |
| fi | |
| if [[ -n $CASTAI_IMPERSONATE ]]; then | |
| echo "Using impersonation flow" | |
| echo "Sending request to create service account for impersonation and impersonate SA $SERVICE_ACCOUNT_EMAIL..." | |
| RESPONSE=$(curl -sSL --write-out "HTTP_STATUS:%{http_code}" -X POST -H "X-API-Key: $CASTAI_API_TOKEN" "$CASTAI_API_URL/v1/kubernetes/external-clusters/impersonation-service-account" -d '{"provider":"aks"}') | |
| RESPONSE_STATUS=$(echo "$RESPONSE" | tr -d '\n' | sed -e 's/.*HTTP_STATUS://') | |
| RESPONSE_BODY=$(echo "$RESPONSE" | sed -e 's/HTTP_STATUS\:.*//g') | |
| if [[ $RESPONSE_STATUS != "200" ]]; then | |
| echo "Failed to create service account for impersonation" | |
| echo $RESPONSE_BODY | |
| exit 1 | |
| fi | |
| # Parse the service_account field from the JSON response using jq | |
| CASTAI_SERVICE_ACCOUNT_ID=$(echo "$RESPONSE_BODY" | jq -r '.id') | |
| if [[ "$CASTAI_SERVICE_ACCOUNT_ID" == "" || "$CASTAI_SERVICE_ACCOUNT_ID" == "null" ]]; then | |
| echo "Failed to create service account for impersonation" | |
| echo $RESPONSE_BODY | |
| exit 1 | |
| fi | |
| echo "Cast.ai service account id: $CASTAI_SERVICE_ACCOUNT_ID" | |
| if [[ -n $USE_MANAGED_IDENTITY ]]; then | |
| SUBSCRIPTION_JSON=$(az account list --query "[?id=='${SUBSCRIPTION_ID}'].{name:name,tenantId:tenantId}[0]") | |
| if [[ -z "$SUBSCRIPTION_JSON" || "$SUBSCRIPTION_JSON" == "null" ]]; then | |
| echo "Error: subscription not found" | |
| exit 1 | |
| fi | |
| TENANT_ID=$(echo "$SUBSCRIPTION_JSON" | jq -r '.tenantId') | |
| echo "Using tenantId: $TENANT_ID" | |
| echo "Checking if castai resource group $RESOURCE_GROUP exists..." | |
| if az group exists --name "$RESOURCE_GROUP" | grep -q true; then | |
| echo "Resource group $RESOURCE_GROUP already exists." | |
| RESOURCE_GROUP_JSON=$(az group show --name "$RESOURCE_GROUP") | |
| else | |
| echo "Creating castai resource group $RESOURCE_GROUP..." | |
| if ! RESOURCE_GROUP_JSON=$(az group create \ | |
| --location "${REGION:-westeurope}" \ | |
| --name "$RESOURCE_GROUP"); then | |
| fatal "Failed to create resource group $RESOURCE_GROUP" >&2 | |
| fi | |
| fi | |
| IDENTITY_JSON=$(az identity show --name "$IDENTITY_NAME" --resource-group "$RESOURCE_GROUP" 2>/dev/null || true) | |
| if [[ -z "$IDENTITY_JSON" || "$IDENTITY_JSON" == "null" ]]; then | |
| echo "Creating managed identity $IDENTITY_NAME..." | |
| IDENTITY_JSON=$(az identity create --name "$IDENTITY_NAME" --resource-group "$RESOURCE_GROUP") | |
| else | |
| echo "Found existing managed identity $IDENTITY_NAME" | |
| fi | |
| CLIENT_ID=$(echo "$IDENTITY_JSON" | jq -r '.clientId') | |
| SP_ID=$(echo "$IDENTITY_JSON" | jq -r '.principalId') | |
| IDENTITY_ID=$(echo "$IDENTITY_JSON" | jq -r '.id') | |
| echo "Managed identity info:" | |
| echo " Name: $IDENTITY_NAME" | |
| echo " Federated credential name: $FEDERATED_CRED_NAME" | |
| echo " Resource group: $RESOURCE_GROUP" | |
| echo " ClientId: $CLIENT_ID" | |
| echo " PrincipalId: $SP_ID" | |
| echo " IdentityId: $IDENTITY_ID" | |
| echo " Subject: $CASTAI_SERVICE_ACCOUNT_ID" | |
| FEDERATED_CREDS_JSON=$(az identity federated-credential show --name $FEDERATED_CRED_NAME --identity-name $IDENTITY_NAME --resource-group=$RESOURCE_GROUP 2>/dev/null || true) | |
| if [[ -z "$FEDERATED_CREDS_JSON" || "$FEDERATED_CREDS_JSON" == "null" ]]; then | |
| echo "Creating federated-credential $FEDERATED_CRED_NAME..." | |
| FEDERATED_CREDS_JSON=$(az identity federated-credential create \ | |
| --name "$FEDERATED_CRED_NAME" \ | |
| --identity-name "$IDENTITY_NAME" \ | |
| --resource-group "$RESOURCE_GROUP" \ | |
| --issuer "https://accounts.google.com" \ | |
| --subject "$CASTAI_SERVICE_ACCOUNT_ID" \ | |
| --audiences "api://AzureADTokenExchange") | |
| if [ $? -ne 0 ]; then | |
| fatal "Failed to create federated credential for $FEDERATED_CRED_NAME" | |
| fi | |
| else | |
| echo "Found existing managed identity $FEDERATED_CRED_NAME" | |
| fi | |
| FEFERATION_ID=$(echo $FEDERATED_CREDS_JSON | jq -r '.id') | |
| if [[ $FEFERATION_ID == "" ]]; then | |
| fatal "Failed to create federated credential for $FEDERATED_CRED_NAME" | |
| fi | |
| CREDENTIALS='{ | |
| "subscriptionId": "'"$SUBSCRIPTION_ID"'", | |
| "tenantId": "'"$TENANT_ID"'", | |
| "clientId": "'"$CLIENT_ID"'", | |
| "federationId": "'"$FEFERATION_ID"'" | |
| }' | |
| else | |
| PARAMETERS_FILE="parameters-${CASTAI_CLUSTER_ID}.json" | |
| echo "Creating $PARAMETERS_FILE file" | |
| cat >"$PARAMETERS_FILE" <<EOF | |
| { | |
| "name": "AzureAccessForCASTAI-$CASTAI_CLUSTER_ID", | |
| "issuer": "https://accounts.google.com", | |
| "subject": "$CASTAI_SERVICE_ACCOUNT_ID", | |
| "description": "Azure Access for CAST AI $CASTAI_CLUSTER_ID", | |
| "audiences": [ | |
| "api://AzureADTokenExchange" | |
| ] | |
| } | |
| EOF | |
| echo "Checking if federated credential exists for app $APP_ID" | |
| FED_CREDENTIALS=$(az ad app federated-credential list --id "$APP_ID" --query "[].id" -o tsv) | |
| for FED_CRED_ID in $FED_CREDENTIALS; do | |
| echo "Removing federated credential ID: $FED_CRED_ID..." | |
| az ad app federated-credential delete --federated-credential-id "$FED_CRED_ID" --id "$APP_ID" | |
| done | |
| FEDERATED_CREDS=$(az ad app federated-credential create --id $APP_ID --parameters $PARAMETERS_FILE) | |
| if [ $? -ne 0 ]; then | |
| fatal "Failed to create federated credential for app $APP_ID, $FEDERATED_CREDS" | |
| fi | |
| FEFERATION_ID=$(echo $FEDERATED_CREDS | jq -r '.id') | |
| if [[ $FEFERATION_ID == "" ]]; then | |
| fatal "Failed to create federated credential for app $APP_ID, $FEDERATED_CREDS" | |
| fi | |
| echo "Federated credential created with ID: $FEFERATION_ID" | |
| rm -f "$PARAMETERS_FILE" | |
| CREDENTIALS='{ | |
| "subscriptionId": "'"$SUBSCRIPTION_ID"'", | |
| "tenantId": "'"$(echo "$SUBSCRIPTION" | jq -r '.tenantId')"'", | |
| "clientId": "'"$APP_ID"'", | |
| "federationId": "'"$FEFERATION_ID"'" | |
| }' | |
| fi | |
| else | |
| SECRET_NAME="${CLUSTER_NAME}-castai" | |
| SECRET_KEY_ID=$(echo $APP | jq -r --arg SECRET_NAME "$SECRET_NAME" '.passwordCredentials[0] | select(.displayName==$SECRET_NAME) | .keyId') | |
| if [[ $SECRET_KEY_ID != "" ]]; then | |
| echo "Removing app old secret: '${SECRET_NAME}'" | |
| az rest -m POST -u "${GRAPH_URL}/v1.0/applications/${APP_OBJECT_ID}/removePassword" --headers Content-Type=application/json -b "{'keyId':'${SECRET_KEY_ID}'}" | |
| fi | |
| function create_app_secret() { | |
| echo "Creating app secret: '${SECRET_NAME}'" | |
| if ! APP_SECRET=$(az rest -m POST -u "${GRAPH_URL}/v1.0/applications/${APP_OBJECT_ID}/addPassword" --headers Content-Type=application/json -b "{'passwordCredential':{'displayName':'${SECRET_NAME}','endDateTime':'2199-01-01T11:11:11.111Z'}}" 2>&1); then | |
| echo "ERROR: Failed to create app secret '${SECRET_NAME}' for application ${APP_OBJECT_ID}" >&2 | |
| echo "ERROR: Azure API response: ${APP_SECRET}" >&2 | |
| return 1 | |
| fi | |
| APP_SECRET=$(echo "$APP_SECRET" | jq -r '.secretText') | |
| if [[ -z "$APP_SECRET" || "$APP_SECRET" == "null" ]]; then | |
| echo "ERROR: App secret '${SECRET_NAME}' was created but returned empty or null value" >&2 | |
| echo "ERROR: Raw response value: '${APP_SECRET}'" >&2 | |
| return 1 | |
| fi | |
| return 0 | |
| } | |
| fi | |
| if [[ -z "$USE_MANAGED_IDENTITY" ]]; then | |
| function get_or_create_service_principal() { | |
| SP_ID=$(az rest -m GET -u "${GRAPH_URL}/v1.0/servicePrincipals\?\$filter=appId eq '$APP_ID'" --headers ConsistencyLevel=eventual 2>&1 | jq -r '.value[0].id') | |
| if [[ $SP_ID == "" || $SP_ID == "null" ]]; then | |
| echo "Creating service principal" | |
| if ! SP_ID=$(az rest -m POST -u "${GRAPH_URL}/v1.0/servicePrincipals" --headers Content-Type=application/json -b "{'appId':'${APP_ID}'}" 2>&1); then | |
| echo "Failed to create service principal, will retry..." | |
| return 1 | |
| fi | |
| SP_ID=$(echo "$SP_ID" | jq -r '.id') | |
| if [[ -z "$SP_ID" || "$SP_ID" == "null" ]]; then | |
| echo "Service principal ID is empty or null, will retry..." | |
| return 1 | |
| fi | |
| else | |
| echo "Using existing service principal: '${SP_ID}'" | |
| fi | |
| return 0 | |
| } | |
| if ! retry get_or_create_service_principal; then | |
| fatal "Failed to get or create service principal after $RETRY_TIMES attempts" | |
| fi | |
| fi | |
| if [[ -n $USE_MANAGED_IDENTITY ]]; then | |
| echo "Assigning role '$ROLE_NAME' to '$IDENTITY_NAME'" | |
| else | |
| echo "Assigning role to '$APP_NAME' app" | |
| fi | |
| i=0 | |
| for scope in "${SCOPES[@]}"; do | |
| assignment_name=${CASTAI_CLUSTER_ID:0:32}000$i | |
| echo "assignment name: $assignment_name" | |
| retry az role assignment create \ | |
| --assignee-object-id "$SP_ID" \ | |
| --assignee-principal-type "ServicePrincipal" \ | |
| --scope "$scope" \ | |
| --role $ROLE_NAME -o none \ | |
| --name "$assignment_name" | |
| if [ $? -ne 0 ]; then | |
| fatal "Failed to assign scope \"$scope\" to $ROLE_NAME as ServicePrincipal. Please check your permissions if you have enough rights to access aforementioned scope." | |
| fi | |
| i=$(expr $i + 1) | |
| done | |
| echo "--------------------------------------------------------------------------------" | |
| echo "Your generated credentials:" | |
| echo $CREDENTIALS | |
| echo "Installing autoscaler cluster components" | |
| if [[ $INSTALL_POD_PINNER = "true" ]]; then | |
| echo "Installing castai-pod-pinner." | |
| helm upgrade -i castai-pod-pinner castai-helm/castai-pod-pinner -n castai-agent \ | |
| --set castai.apiURL=$CASTAI_API_URL \ | |
| --set castai.grpcURL=$CASTAI_GRPC_URL \ | |
| --set castai.apiKey=$CASTAI_API_TOKEN \ | |
| --set castai.clusterID=$CASTAI_CLUSTER_ID \ | |
| --set replicaCount=0 | |
| echo "Finished installing castai-pod-pinner." | |
| fi | |
| if [[ $INSTALL_NVIDIA_DEVICE_PLUGIN = "true" ]]; then | |
| echo "Installing NVIDIA device plugin" | |
| kubectl apply -f - <<EOF | |
| apiVersion: apps/v1 | |
| kind: DaemonSet | |
| metadata: | |
| name: nvidia-device-plugin-daemonset | |
| namespace: kube-system | |
| spec: | |
| selector: | |
| matchLabels: | |
| name: nvidia-device-plugin-ds | |
| updateStrategy: | |
| type: RollingUpdate | |
| template: | |
| metadata: | |
| labels: | |
| name: nvidia-device-plugin-ds | |
| spec: | |
| tolerations: | |
| - key: nvidia.com/gpu | |
| operator: Exists | |
| effect: NoSchedule | |
| # Required to allow nvidia device plugin pods to run on nodes created by ai-enabler | |
| - key: "scheduling.cast.ai/node-template" | |
| value: "llms-by-castai" | |
| effect: "NoSchedule" | |
| # Mark this pod as a critical add-on; when enabled, the critical add-on | |
| # scheduler reserves resources for critical add-on pods so that they can | |
| # be rescheduled after a failure. | |
| # See https://kubernetes.io/docs/tasks/administer-cluster/guaranteed-scheduling-critical-addon-pods/ | |
| priorityClassName: "system-node-critical" | |
| containers: | |
| - image: nvcr.io/nvidia/k8s-device-plugin:v0.16.1 | |
| name: nvidia-device-plugin-ctr | |
| env: | |
| - name: FAIL_ON_INIT_ERROR | |
| value: "false" | |
| securityContext: | |
| allowPrivilegeEscalation: false | |
| capabilities: | |
| drop: ["ALL"] | |
| volumeMounts: | |
| - name: device-plugin | |
| mountPath: /var/lib/kubelet/device-plugins | |
| volumes: | |
| - name: device-plugin | |
| hostPath: | |
| path: /var/lib/kubelet/device-plugins | |
| EOF | |
| fi | |
| echo "Sending credentials to CAST AI console..." | |
| API_URL="${CASTAI_API_URL}/v1/kubernetes/external-clusters/${CASTAI_CLUSTER_ID}" | |
| BODY=$(jq -c -n --arg CREDENTIALS "$CREDENTIALS" '{credentials:$CREDENTIALS}') | |
| function update_cluster() { | |
| RESPONSE=$(curl -sSL --write-out "HTTP_STATUS:%{http_code}" -X POST -H "X-API-Key: ${CASTAI_API_TOKEN}" -d "${BODY}" $API_URL) | |
| RESPONSE_STATUS=$(echo "$RESPONSE" | tr -d '\n' | sed -e 's/.*HTTP_STATUS://') | |
| RESPONSE_BODY=$(echo "$RESPONSE" | sed -e 's/HTTP_STATUS\:.*//g') | |
| if [[ $RESPONSE_STATUS == "401" ]]; then | |
| RESPONSE_BODY="401 Unauthorized" | |
| return 0 | |
| fi | |
| if [[ $RESPONSE_STATUS != "200" ]]; then | |
| if [[ $(grep -c "Failed to refresh the Token for request" <<<$RESPONSE_BODY) -gt 0 ]]; then | |
| echo "Request failed, will retry after $RETRY_SEC seconds ..." | |
| return 1 | |
| fi | |
| if [[ "$RESPONSE_BODY_OLD" == "$RESPONSE_BODY" ]]; then | |
| echo "Request failed again, will retry after $RETRY_SEC seconds ..." | |
| else | |
| echo "Request failed with error: $RESPONSE_BODY" | |
| echo "Will retry after $RETRY_SEC seconds ..." | |
| fi | |
| RESPONSE_BODY_OLD=$RESPONSE_BODY | |
| return 1 | |
| fi | |
| } | |
| RETRY_SEC=10 RETRY_TIMES=30 retry update_cluster | |
| if [[ $RESPONSE_STATUS == "200" ]]; then | |
| echo "Successfully sent." | |
| else | |
| echo "Couldn't save credentials to CAST AI console. Try pasting the credentials to the console manually." | |
| echo $RESPONSE_BODY | |
| exit 1 | |
| fi | |
| } | |
| function enable_ai_optimizer_proxy() { | |
| echo "Installing AI Optimizer Proxy" | |
| echo "Installing castai-ai-optimizer-proxy." | |
| helm upgrade -i castai-ai-optimizer-proxy castai-helm/castai-ai-optimizer-proxy -n castai-agent \ | |
| --set castai.apiKey=$CASTAI_API_TOKEN \ | |
| --set castai.clusterID=$CASTAI_CLUSTER_ID \ | |
| --set castai.apiURL=$CASTAI_API_URL \ | |
| --set createNamespace=true | |
| echo "Finished installing castai-ai-optimizer-proxy." | |
| } | |
| helm repo add castai-helm https://castai.github.io/helm-charts | |
| helm repo update castai-helm | |
| enable_base_components | |
| if [[ $INSTALL_AUTOSCALER = "true" ]]; then | |
| enable_autoscaler_agent | |
| fi | |
| if [[ $INSTALL_AUTOSCALER = "true" || $INSTALL_WORKLOAD_AUTOSCALER = "true" || ($INSTALL_KENT = "true" && $INSTALL_EVICTOR = "true") ]]; then | |
| REPOSITORY="${REPOSITORY:-us-docker.pkg.dev/castai-hub/library}" | |
| SCALE_OPTS="--set replicaCount=-0" | |
| if [[ ($INSTALL_KENT = "true" && $INSTALL_EVICTOR = "true") || $INSTALL_WORKLOAD_AUTOSCALER = "true" ]]; then | |
| SCALE_OPTS="" | |
| fi | |
| echo "Installing castai-evictor." | |
| helm upgrade -i castai-evictor castai-helm/castai-evictor -n castai-agent $SCALE_OPTS \ | |
| --set image.repository=$REPOSITORY/evictor | |
| echo "Finished installing castai-evictor." | |
| fi | |
| if [[ $INSTALL_SECURITY_AGENT = "true" || "$INSTALL_NETFLOW_EXPORTER" == "true" ]]; then | |
| K8S_PROVIDER="aks" | |
| if [ -z $CASTAI_KVISOR_GRPC_URL ] || [ -z $CASTAI_API_URL ] || [ -z $CASTAI_CLUSTER_ID ]; then | |
| echo "CASTAI_KVISOR_GRPC_URL, CASTAI_API_URL or CASTAI_CLUSTER_ID variables were not provided" | |
| exit 1 | |
| fi | |
| if [ -z $K8S_PROVIDER ]; then | |
| echo "K8S_PROVIDER is not provided" | |
| exit 1 | |
| fi | |
| value_overrides="--set castai.grpcAddr=$CASTAI_KVISOR_GRPC_URL \ | |
| --set castai.apiKey=$CASTAI_API_TOKEN \ | |
| --set castai.clusterID=$CASTAI_CLUSTER_ID" | |
| if [[ $INSTALL_SECURITY_AGENT = "true" ]]; then | |
| value_overrides="$value_overrides \ | |
| --set controller.extraArgs.kube-linter-enabled=true \ | |
| --set controller.extraArgs.image-scan-enabled=true \ | |
| --set controller.extraArgs.kube-bench-enabled=true \ | |
| --set controller.extraArgs.kube-bench-cloud-provider=$K8S_PROVIDER" | |
| fi | |
| if [[ $INSTALL_NETFLOW_EXPORTER = "true" ]]; then | |
| value_overrides="$value_overrides \ | |
| --set agent.enabled=true \ | |
| --set agent.extraArgs.netflow-enabled=true" | |
| if helm status castai-egressd -n castai-agent >/dev/null 2>&1; then | |
| echo "Uninstalling castai-egressd (Replaced by new castai-kvisor netflow collection)." | |
| helm uninstall castai-egressd -n castai-agent | |
| echo "Finished uninstalling castai-egressd." | |
| fi | |
| fi | |
| echo "Installing castai-kvisor." | |
| helm upgrade -i castai-kvisor castai-helm/castai-kvisor -n castai-agent --reset-then-reuse-values \ | |
| $value_overrides | |
| echo "Finished installing castai-kvisor." | |
| fi | |
| if [[ $INSTALL_AI_OPTIMIZER_PROXY = "true" ]]; then | |
| enable_ai_optimizer_proxy | |
| fi | |
| if [[ $INSTALL_GPU_METRICS_EXPORTER = "true" ]]; then | |
| K8S_PROVIDER="aks" | |
| #!/bin/bash | |
| ###################################################################################################### | |
| # This script installs the gpu-metrics-exporter chart from the CAST AI helm repository. # | |
| # It checks the cluster for the presence of dcgm-exporter and nv-hostengine and configures the # | |
| # gpu-metrics-exporter chart accordingly. # | |
| # If both dcgm-exporter and nv-hostengine are present, it configures the chart to use nv-hostengine. # | |
| # If only dcgm-exporter is present, it configures the chart to use it. # | |
| # If neither is present, it deploys a new dcgm-exporter with an embedded nv-hostengine. # | |
| # The script requires the following environment variables to be set: # | |
| # CASTAI_API_TOKEN - the API token for the CAST AI API # | |
| # CASTAI_CLUSTER_ID - the ID of the CAST AI cluster # | |
| # K8S_PROVIDER - the provider of the Kubernetes cluster (e.g. eks, gke, aks) # | |
| # The script also requires the helm command to be installed. # | |
| ###################################################################################################### | |
| set -e | |
| # Constants used throughout the script | |
| DCGM_EXPORTER_COMMAND_SUBSTRING="dcgm-exporter" | |
| NV_HOSTENGINE_COMMAND_SUBSTRING="nv-hostengine" | |
| DCGM_EXPORTER_IMAGES=("nvcr.io/nvidia/k8s/dcgm-exporter" "nvidia/dcgm-exporter", "nvidia/gke-dcgm-exporter") | |
| DCGM_IMAGES=("nvcr.io/nvidia/cloud-native/dcgm" "nvidia/dcgm") | |
| CASTAI_AGENT_NAMESPACE="castai-agent" | |
| CASTAI_GPU_METRICS_EXPORTER_DAEMONSET="castai-gpu-metrics-exporter" | |
| CASTAI_API_URL="${CASTAI_API_URL:-https://api.cast.ai}" | |
| CASTWARE_CASTAI_API_URL="${CASTWARE_CASTAI_API_URL:-$CASTAI_API_URL}" | |
| # Global vars populated by the functions | |
| # Which daemon set is running dcgm-exporter | |
| DCGM_EXPORTER_DAEMONSET="" | |
| DCGM_EXPORTER_NAMESPACE="" | |
| # Which ds is running nv-hostengine | |
| NV_HOSTENGINE_DAEMONSET="" | |
| NV_HOSTENGINE_NAMESPACE="" | |
| #### Functions start here #### | |
| # check_pods_command - check the command of a given ds if it contains dcgm-exporter or nv-hostengine | |
| # results are stored in the global vars DCGM_EXPORTER_DAEMONSET and NV_HOSTENGINE_DAEMONSET | |
| check_daemonset_command() { | |
| namespace=$1 | |
| ds=$2 | |
| all_container_commands=$(kubectl -n $namespace get daemonsets -o=jsonpath-as-json='{$.spec.template.spec.containers[*].command[*]}' $ds) | |
| for image in $(echo $all_container_commands | tr " " "\n"); do | |
| if [[ $image == *$DCGM_EXPORTER_COMMAND_SUBSTRING* ]]; then | |
| DCGM_EXPORTER_DAEMONSET=$ds | |
| DCGM_EXPORTER_NAMESPACE=$namespace | |
| elif [[ $image == *$NV_HOSTENGINE_COMMAND_SUBSTRING* ]]; then | |
| NV_HOSTENGINE_DAEMONSET=$ds | |
| NV_HOSTENGINE_NAMESPACE=$namespace | |
| fi | |
| done | |
| } | |
| # check_pod_args - check the arguments of a given ds if it contains dcgm-exporter | |
| # results are stored in the global vars DCGM_EXPORTER_DAEMONSET | |
| check_pod_args() { | |
| namespace=$1 | |
| ds=$2 | |
| all_container_commands=$(kubectl -n $namespace get daemonsets -o=jsonpath-as-json='{$.spec.template.spec.containers[*].args[*]}' $ds) | |
| for image in $(echo $all_container_commands | tr " " "\n"); do | |
| if [[ $image == *$DCGM_EXPORTER_COMMAND_SUBSTRING* ]]; then | |
| DCGM_EXPORTER_DAEMONSET=$ds | |
| DCGM_EXPORTER_NAMESPACE=$namespace | |
| fi | |
| done | |
| } | |
| # check_daemonset_image - check the image of a given ds if it is for dcgm-exporter or dcgm | |
| check_daemonset_image() { | |
| namespace=$1 | |
| ds=$2 | |
| all_container_images=$(kubectl -n $namespace get daemonsets -o=jsonpath-as-json='{$.spec.template.spec.containers[*].image}' $ds) | |
| for image in $(echo $all_container_images | tr " " "\n"); do | |
| for required_image in "${DCGM_EXPORTER_IMAGES[@]}"; do | |
| if [[ $image == *$required_image* ]]; then | |
| DCGM_EXPORTER_DAEMONSET=$ds | |
| DCGM_EXPORTER_NAMESPACE=$namespace | |
| fi | |
| done | |
| for required_image in "${DCGM_IMAGES[@]}"; do | |
| if [[ $image == *$required_image* ]]; then | |
| NV_HOSTENGINE_DAEMONSET=$ds | |
| NV_HOSTENGINE_NAMESPACE=$namespace | |
| fi | |
| done | |
| done | |
| } | |
| # check_all_daemonsets_in_namespace - check all daemonsets in a given namespace whether they contain dcgm-exporter or nv-hostengine | |
| # results are stored in the global vars DCGM_EXPORTER_DAEMONSET and NV_HOSTENGINE_DAEMONSET | |
| check_all_daemonsets_in_namespace() { | |
| namespace=$1 | |
| all_ds=$(kubectl get daemonsets -n $namespace --ignore-not-found | cut -d ' ' -f 1) | |
| all_ds=($all_ds) | |
| num_ds=${#all_ds[@]} | |
| [[ ! -z "$DEBUG" ]] && echo " Found $num_ds daemonsets" | |
| for ds in "${all_ds[@]:1}"; do | |
| if [[ $ds = "$CASTAI_GPU_METRICS_EXPORTER_DAEMONSET" ]]; then | |
| # Skip our own daemonset | |
| continue | |
| fi | |
| [[ ! -z "$DEBUG" ]] && echo " Checking daemonset $ds" | |
| # if any of the global vars are empty, we check by image first | |
| if [[ -z "$DCGM_EXPORTER_DAEMONSET" ]] || [[ -z "$NV_HOSTENGINE_DAEMONSET" ]]; then | |
| check_daemonset_image $namespace $ds | |
| fi | |
| # if any of the global vars are still empty, we check by command | |
| if [[ -z "$DCGM_EXPORTER_DAEMONSET" ]] || [[ -z "$NV_HOSTENGINE_DAEMONSET" ]]; then | |
| check_daemonset_command $namespace $ds | |
| fi | |
| # dcgm command can be in args because command is /bin/bash -c $args | |
| if [[ -z "$DCGM_EXPORTER_DAEMONSET" ]]; then | |
| check_pod_args $namespace $ds | |
| # we found both dcgm-exporter and nv-hostengine, no need to look further | |
| fi | |
| if [[ ! -z "$DCGM_EXPORTER_DAEMONSET" ]] && [[ ! -z "$NV_HOSTENGINE_DAEMONSET" ]]; then | |
| return | |
| fi | |
| done | |
| } | |
| # unquote_string - remove quotes from a string if they are at the start or end | |
| unquote_string() { | |
| local str=$1 | |
| temp="${str%\"}" | |
| temp="${temp#\"}" | |
| echo $temp | |
| } | |
| # find_dcgm_exporter_label_value - find the label value of the dcgm-exporter daemonset | |
| # the label value is used to find the service name of the dcgm-exporter | |
| find_dcgm_exporter_label_value() { | |
| local namespace=$1 | |
| local ds=$2 | |
| label_value=$(kubectl -n $namespace get daemonsets -o=jsonpath-as-json='{$.metadata.labels.app\.kubernetes\.io\/name}' $ds | tr -d ' []\n') | |
| label_value=$(unquote_string $label_value) | |
| if [[ ! -z $label_value ]]; then | |
| echo "app.kubernetes.io/name:$label_value" | |
| else | |
| label_value=$(kubectl -n $namespace get daemonsets -o=jsonpath-as-json='{$.metadata.labels.app}' $ds | tr -d ' []\n') | |
| label_value=$(unquote_string $label_value) | |
| if [[ ! -z "$label_value" ]]; then | |
| echo "app:$label_value" | |
| fi | |
| fi | |
| } | |
| # find_dcgm_exporter_svc - find the service name of the dcgm-exporter | |
| find_dcgm_exporter_svc() { | |
| local namespace=$1 | |
| local dcgm_exporter_label=$2 | |
| dcgm_exporter_label=$(echo $dcgm_exporter_label | tr ':' '=') | |
| svc_count=$(kubectl -n $namespace get svc -l $dcgm_exporter_label -o=jsonpath='{.items | length}' 2>/dev/null || echo "0") | |
| if [ "$svc_count" -gt "0" ]; then | |
| # Only access items[0] if there are items | |
| svc=$(kubectl -n $namespace get svc -l $dcgm_exporter_label -o=jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "") | |
| else | |
| svc="" | |
| fi | |
| echo $svc | |
| } | |
| #### Functions end here ##### | |
| echo "Installing castai-gpu-metrics-exporter." | |
| if [ -z $CASTAI_API_TOKEN ] || [ -z $CASTAI_API_URL ] || [ -z $CASTAI_CLUSTER_ID ] || [ -z $K8S_PROVIDER ]; then | |
| echo "CASTAI_API_TOKEN, CASTAI_API_URL, CASTAI_CLUSTER_ID, K8S_PROVIDER variables were not provided" | |
| exit 1 | |
| fi | |
| # determine which components need to be installed and how to configure them | |
| echo "Checking presence of dcgm-exporter and nv-hostengine in the cluster." | |
| echo "Iterating through daemonsets in all namespaces." | |
| echo "Will take a few seconds per daemon set. Might take a few minutes" | |
| echo "Set the DEBUG environment variable to any value to see more details." | |
| [[ ! -z "$DEBUG" ]] && echo "Going through all daemon sets in all namespaces.\n" | |
| all_namespaces=$(kubectl get namespaces | cut -d ' ' -f 1) | |
| all_namespaces=($all_namespaces) | |
| num_namespaces=${#all_namespaces[@]} | |
| current_ns=0 | |
| for ns in "${all_namespaces[@]:1}"; do | |
| let current_ns=current_ns+1 | |
| [[ ! -z "$DEBUG" ]] && echo "$current_ns/$num_namespaces: Checking namespace $ns" | |
| check_all_daemonsets_in_namespace $ns | |
| if [[ ! -z "$DCGM_EXPORTER_DAEMONSET" ]] && [[ ! -z "$NV_HOSTENGINE_DAEMONSET" ]]; then | |
| break | |
| fi | |
| done | |
| echo "" | |
| value_overrides="--set castai.apiUrl=$CASTWARE_CASTAI_API_URL \ | |
| --set castai.clusterId=$CASTAI_CLUSTER_ID \ | |
| --set castai.apiKey=$CASTAI_API_TOKEN \ | |
| --set provider=$K8S_PROVIDER" | |
| # if we found nv-hostengine, we need to configure the dcgm-exporter container in the gpu-metrics-exporter chart | |
| # to connect to the 5555 port of the node. | |
| if [ ! -z $NV_HOSTENGINE_DAEMONSET ]; then | |
| echo "Found nv-hostengine, configuring gpu-metrics-exporter to use it" | |
| value_overrides="$value_overrides \ | |
| --set dcgmExporter.enabled=true \ | |
| --set dcgmExporter.useExternalHostEngine=true" | |
| # if nv-hostengine does not exist but DCGM exporter exists, then we don't deploy a new DCGM exporter just | |
| # configure our gpu-metrics-exporter to find the existing DCGM exporter by scanning the labels | |
| elif [ ! -z $DCGM_EXPORTER_DAEMONSET ]; then | |
| echo "Found dcgm-exporter with an embedded nv-hostengine, configuring gpu-metrics-exporter to use it" | |
| dcgm_label=$(find_dcgm_exporter_label_value $DCGM_EXPORTER_NAMESPACE $DCGM_EXPORTER_DAEMONSET) | |
| [[ ! -z "$DEBUG" ]] && echo "Discovered DCGM-exporter label: $dcgm_label" | |
| dcgm_service_name=$(find_dcgm_exporter_svc $DCGM_EXPORTER_NAMESPACE $dcgm_label) | |
| [[ ! -z "$DEBUG" ]] && echo "Discovered DCGM-exporter service name: $dcgm_service_name" | |
| if [ ! -z $dcgm_service_name ]; then | |
| value_overrides="$value_overrides \ | |
| --set dcgmExporter.enabled=false \ | |
| --set gpuMetricsExporter.config.DCGM_HOST=$dcgm_service_name.$DCGM_EXPORTER_NAMESPACE.svc.cluster.local." | |
| elif [ ! -z $dcgm_label ]; then | |
| value_overrides="$value_overrides \ | |
| --set dcgmExporter.enabled=false \ | |
| --set gpuMetricsExporter.config.DCGM_LABELS=$dcgm_label" | |
| else | |
| echo "Could not find the service name of the dcgm-exporter or a app name label. Please check the dcgm-exporter daemonset." | |
| exit 1 | |
| fi | |
| else | |
| echo "DCGM exporter and nv-hostengine not found. Deploying a new DCGM exporter with an embedded nv-hostengine." | |
| fi | |
| helm upgrade -i castai-gpu-metrics-exporter castai-helm/gpu-metrics-exporter -n castai-agent \ | |
| $value_overrides | |
| echo "Finished installing castai-gpu-metrics-exporter." | |
| fi | |
| if [[ $INSTALL_WORKLOAD_AUTOSCALER = "true" ]]; then | |
| K8S_PROVIDER="aks" | |
| WORKLOAD_AUTOSCALER_CONFIG_SOURCE="castai-cluster-controller" | |
| WORKLOAD_AUTOSCALER_CHART=${WORKLOAD_AUTOSCALER_CHART:-"castai-helm/castai-workload-autoscaler"} | |
| WORKLOAD_AUTOSCALER_EXPORTER_CHART=${WORKLOAD_AUTOSCALER_EXPORTER_CHART:-"castai-helm/castai-workload-autoscaler-exporter"} | |
| WORKLOAD_AUTOSCALER_EXPORTER_TELEMETRY_URL=${WORKLOAD_AUTOSCALER_EXPORTER_TELEMETRY_URL:-"telemetry.prod-master.cast.ai:443"} | |
| bare_header() { | |
| HEADER_COLOR="6" # cyan | |
| if tput bold &>/dev/null && tput sgr0 &>/dev/null; then | |
| tput bold | |
| if tput setaf "$HEADER_COLOR" &>/dev/null; then | |
| tput setaf "$HEADER_COLOR" | |
| fi | |
| echo "█ $@" | |
| tput sgr0 | |
| else | |
| echo "█ $@" | |
| fi | |
| } | |
| success() { | |
| HEADER_COLOR="2" # green | |
| if tput bold &>/dev/null && tput sgr0 &>/dev/null; then | |
| tput bold | |
| if tput setaf "$HEADER_COLOR" &>/dev/null; then | |
| tput setaf "$HEADER_COLOR" | |
| fi | |
| echo "✓ $@" | |
| tput sgr0 | |
| else | |
| echo "✓ $@" | |
| fi | |
| } | |
| header() { | |
| echo | |
| bare_header $@ | |
| } | |
| install_metrics_server() { | |
| if ! kubectl get --raw /apis/metrics.k8s.io >/dev/null 2>&1; then | |
| header "Installing Kubernetes metrics-server." | |
| kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml | |
| success "Finished installing Kubernetes metrics-server." | |
| header "Waiting for Kubernetes metrics-server to be ready." | |
| INSECURE_TLS_HANDLED=0 | |
| INTERNAL_NETWORKING_HANDLED=0 | |
| while ! kubectl -nkube-system wait "--for=condition=Ready" pod -l k8s-app=metrics-server --timeout=5s >/dev/null 2>&1; do | |
| # Handle the case when --kubelet-insecure-tls is required, i.e. local machines or Linode | |
| if [[ "$INSECURE_TLS_HANDLED" == "0" && "$(kubectl -nkube-system logs -l k8s-app=metrics-server 2>&1 | grep 'cannot validate certificate')" != "" ]]; then | |
| header "Enabling self-signed certificate support in Kubernetes metrics-server." | |
| kubectl -nkube-system patch deployment metrics-server --type json -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--kubelet-insecure-tls"}]' -n kube-system | |
| header "Waiting for Kubernetes metrics-server to be ready after changes." | |
| INSECURE_TLS_HANDLED=1 | |
| elif [[ "$INTERNAL_NETWORKING_HANDLED" == "0" && "$(kubectl -nkube-system logs -l k8s-app=metrics-server 2>&1 | grep 'dial tcp')" != "" ]]; then | |
| header "Enabling InternalIP as an address preference in Kubernetes metrics-server." | |
| kubectl -nkube-system patch deployment metrics-server --type json -p='[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP"}]' -n kube-system | |
| header "Waiting for Kubernetes metrics-server to be ready after changes." | |
| INTERNAL_NETWORKING_HANDLED=1 | |
| fi | |
| sleep 5 | |
| done | |
| success "Kubernetes metrics-server is ready." | |
| fi | |
| } | |
| install_workload_autoscaler() { | |
| header "Installing castai-workload-autoscaler." | |
| helm upgrade --reset-then-reuse-values -i castai-workload-autoscaler -n castai-agent $WORKLOAD_AUTOSCALER_EXTRA_HELM_OPTS \ | |
| --set castai.apiKeySecretRef="$WORKLOAD_AUTOSCALER_CONFIG_SOURCE" \ | |
| --set castai.configMapRef="$WORKLOAD_AUTOSCALER_CONFIG_SOURCE" \ | |
| --set castai.telemetryURL="$WORKLOAD_AUTOSCALER_EXPORTER_TELEMETRY_URL" \ | |
| "$WORKLOAD_AUTOSCALER_CHART" | |
| success "Finished installing castai-workload-autoscaler." | |
| } | |
| install_workload_autoscaler_metrics_exporter() { | |
| header "Installing castai-workload-autoscaler-exporter." | |
| # --reset-then-reuse-values prevents overwriting current datasource configuration | |
| helm upgrade --reset-then-reuse-values -i castai-workload-autoscaler-exporter -n castai-agent $EXTRA_CME_HELM_OPTS \ | |
| --set castai.apiKeySecretRef="$WORKLOAD_AUTOSCALER_CONFIG_SOURCE" \ | |
| --set castai.configMapRef="$WORKLOAD_AUTOSCALER_CONFIG_SOURCE" \ | |
| --set castai.telemetryURL="$WORKLOAD_AUTOSCALER_EXPORTER_TELEMETRY_URL" \ | |
| "$WORKLOAD_AUTOSCALER_EXPORTER_CHART" | |
| success "Finished installing castai-workload-autoscaler-exporter." | |
| } | |
| test_workload_autoscaler_logs() { | |
| echo -e "Test of castai-workload-autoscaler has failed. See: https://docs.cast.ai/docs/workload-autoscaling-overview#failed-helm-test-hooks\n" | |
| kubectl logs -n castai-agent pod/test-castai-workload-autoscaler-verification | |
| exit 1 | |
| } | |
| test_workload_autoscaler() { | |
| header "Testing castai-workload-autoscaler." | |
| trap test_workload_autoscaler_logs INT TERM ERR | |
| kubectl rollout status deployment/castai-workload-autoscaler -n castai-agent --timeout=300s | |
| helm test castai-workload-autoscaler -n castai-agent | |
| success "Finished testing castai-workload-autoscaler." | |
| } | |
| main() { | |
| install_metrics_server | |
| install_workload_autoscaler_metrics_exporter | |
| install_workload_autoscaler | |
| test_workload_autoscaler | |
| } | |
| main | |
| fi | |
| if [[ $INSTALL_POD_MUTATOR = "true" ]]; then | |
| #!/bin/bash | |
| set -e | |
| # Check if required commands are installed | |
| if ! [ -x "$(command -v kubectl)" ]; then | |
| echo "Error: kubectl is not installed." | |
| exit 1 | |
| fi | |
| if ! [ -x "$(command -v helm)" ]; then | |
| echo "Error: helm is not installed." | |
| exit 1 | |
| fi | |
| if ! [ -x "$(command -v jq)" ]; then | |
| echo "Error: jq is not installed." | |
| exit 1 | |
| fi | |
| CAST_NS=castai-agent | |
| CAST_AGENT_MIN_VERSION="castai-agent-0.111.0" | |
| echo "Starting installation of castai-pod-mutator." | |
| # Check if CASTAI_API_URL is set | |
| if [ -z "${CASTAI_API_URL}" ]; then | |
| echo "Please provide CASTAI_API_URL." | |
| exit 1 | |
| fi | |
| CASTWARE_CASTAI_API_URL="${CASTWARE_CASTAI_API_URL:-$CASTAI_API_URL}" | |
| # Check if CASTAI_API_TOKEN is set | |
| if [ -z "${CASTAI_API_TOKEN}" ]; then | |
| echo "Please provide CASTAI_API_TOKEN. You can create it in the CAST AI Console." | |
| exit 1 | |
| fi | |
| # Check if CASTAI_CLUSTER_ID is set | |
| if [ -z "${CASTAI_CLUSTER_ID}" ]; then | |
| echo "Please provide CASTAI_CLUSTER_ID. You can check it in the CAST AI Console." | |
| exit 1 | |
| fi | |
| # Check if CAST AI namespace exists | |
| if ! kubectl get namespace $CAST_NS >>/dev/null 2>&1; then | |
| echo "CAST AI namespace not found. Please make sure your cluster is fully onboarded first." | |
| exit 1 | |
| fi | |
| if ! helm status cluster-controller -n $CAST_NS >>/dev/null 2>&1; then | |
| echo "CAST AI cluster-controller not found. Please make sure latest version is installed https://docs.cast.ai/docs/cluster-controller" | |
| exit 1 | |
| fi | |
| if ! kubectl get deployment castai-agent -n $CAST_NS >>/dev/null 2>&1; then | |
| echo "CAST AI agent not found. Please install the latest version using the CAST AI Console UI or Helm: https://docs.cast.ai/docs/helm-charts" | |
| exit 1 | |
| fi | |
| INSTALLED=$(kubectl get deployment castai-agent -n $CAST_NS -o jsonpath='{.metadata.labels.helm\.sh/chart}' 2>/dev/null) | |
| if [ -z "$INSTALLED" ]; then | |
| echo "Error: Could not determine CAST AI agent version." | |
| exit 1 | |
| fi | |
| INSTALLED_VERSION="${INSTALLED##*-}" | |
| REQUIRED_VERSION="${CAST_AGENT_MIN_VERSION##*-}" | |
| if [ "$(printf '%s\n%s\n' "$REQUIRED_VERSION" "$INSTALLED_VERSION" | sort -V | head -n1)" != "$REQUIRED_VERSION" ]; then | |
| echo "CAST AI agent version '${REQUIRED_VERSION}' or greater is required. Current version '${INSTALLED_VERSION}'. Please upgrade using the CAST AI Console UI or Helm: https://docs.cast.ai/docs/helm-charts" | |
| exit 1 | |
| fi | |
| echo "Adding helm repository for CAST AI charts." | |
| helm repo add castai-helm https://castai.github.io/helm-charts | |
| helm repo update castai-helm | |
| echo "Installing castai-pod-mutator." | |
| helm upgrade -i pod-mutator --create-namespace -n $CAST_NS \ | |
| --set castai.apiUrl=${CASTWARE_CASTAI_API_URL} \ | |
| --set castai.apiKey=${CASTAI_API_TOKEN} \ | |
| --set castai.clusterID=${CASTAI_CLUSTER_ID} \ | |
| castai-helm/castai-pod-mutator | |
| fi | |
| if [[ $INSTALL_OMNI = "true" ]]; then | |
| RESPONSE=$(curl -sSL --write-out "HTTP_STATUS:%{http_code}" -X POST -H "X-API-Key: $CASTAI_API_TOKEN" "$CASTAI_API_URL/omni-provisioner/v1beta/organizations/$CASTAI_ORGANIZATION_ID/clusters/$CASTAI_CLUSTER_ID:onboard") | |
| RESPONSE_STATUS=$(echo "$RESPONSE" | tr -d '\n' | sed -e 's/.*HTTP_STATUS://') | |
| RESPONSE_BODY=$(echo "$RESPONSE" | sed -e 's/HTTP_STATUS\:.*//g') | |
| if [[ $RESPONSE_STATUS != "200" ]]; then | |
| echo "Failed to get Omni onboarding script. HTTP status: $RESPONSE_STATUS" | |
| echo $RESPONSE_BODY | |
| exit 1 | |
| fi | |
| bash -c "$(echo $RESPONSE_BODY | jq -r '.script')" | |
| fi | |
| echo "Scaling castai-agent:" | |
| kubectl scale deployments/castai-agent --replicas=2 --namespace castai-agent |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment