mdrakiburrahman

{
   "eventTime":"2026-01-09T23:18:22.634Z",
   "producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark",
   "schemaURL":"https://openlineage.io/spec/2-0-2/OpenLineage.json#/$defs/RunEvent",
   "eventType":"START",
   "run":{
      "runId":"019ba50d-c8ab-798b-86ae-e437366a5a3f",
      "facets":{
         "parent":{
            "_producer":"https://github.com/OpenLineage/OpenLineage/tree/1.23.0/integration/spark",

mdrakiburrahman

Table "columnar"."log" {
    "id" varchar [pk]
    "event_year_date" varchar
    "region" varchar
    "service_name" varchar
    "time_utc" datetime2
    "observed_time_utc" datetime2
    "scope_name" varchar
    "scope_version" varchar
    "trace_id" varchar

mdrakiburrahman

package microsoft.opentelemetrycollector.otlp.json.v1

import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.types.StructType

// @formatter:off
/** Represents a schema for a Logs payload.
  *
  * @param spark
  *   The SparkSession for schema inference.

mdrakiburrahman

IFS='|' read -ra JAR_ARRAY <<<"$SYNAPSE_WORKSPACE_PACKAGE_UPLOAD_PIPE_DELIMITED"
for JAR in "${JAR_ARRAY[@]}"; do

    export PACKAGE_NAME=$(echo "${JAR}" | awk -F'/' '{print $NF}')
    echo "Checking if package ${PACKAGE_NAME} exists in ${SYNAPSE_WORKSPACE_NAME}"
    export PACKAGE_ID_EXISTING=$(az synapse workspace-package show --workspace-name "$SYNAPSE_WORKSPACE_NAME" --name "$PACKAGE_NAME" --query "id" -o tsv)
    export PACKAGE_EXISTS=false
    if [ -z "$PACKAGE_ID_EXISTING" ]; then
        echo "Package ${PACKAGE_NAME} does not exist in ${SYNAPSE_WORKSPACE_NAME}."
    else

mdrakiburrahman

Pictures and stuff.

mdrakiburrahman

graph TD

  subgraph SqlServerInstancePrimary1
    createEndpoint1
    grantConnectOnEndpoint1
    createAvailabilityGroup1
  end

  subgraph SqlServerInstanceSecondary2
    createEndpoint2

mdrakiburrahman

{
    "id": "/subscriptions/d4aa0040-6651-443d-a078-728ce72a87ab/resourcegroups/rakirahman.me/providers/Microsoft.Cdn/profiles/rakirahman/endpoints/rakirahman",
    "type": "Microsoft.Cdn/profiles/endpoints",
    "name": "rakirahman",
    "location": "Global",
    "tags": {},
    "properties": {
        "hostName": "rakirahman.azureedge.net",
        "originHostHeader": "rakirahman.z9.web.core.windows.net",
        "originPath": null,

mdrakiburrahman

#!/bin/bash

# Continue on error
set +e

# Params
resource_group="arcdata-ci-direct"
days_to_keep=7

# Get current date and calculate 7 days ago in seconds

mdrakiburrahman

# Calculate Eviction Signals - Current State vs Threshold:
#
#  https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#eviction-signals
#  https://medium.com/kubernetes-tutorials/efficient-node-out-of-resource-management-in-kubernetes-67f158da6e59
#
NODES=($(kubectl get nodes -o jsonpath='{.items[*].metadata.name}'))
NODE_COUNT=${#NODES[@]}

while true;
do

mdrakiburrahman

##########################################################
# APPLY
##########################################################
# Apply workflow
#
kubectl apply -f sqlinstance-ha.yaml

# Get primary
# 
kubectl get pods -l=role.ag.mssql.microsoft.com/chaos-bc-chaos-bc=primary -n ns1669830131581917320