philerooski

"""
A script which uploads validation results and a data validation
report to S3 for the FitbitSleepLogs data type. This was run in 
Glue 4.0 while specifying --additional-python-modules great_expectations==0.18.11,boto3==1.24.70
"""
import json
import logging
import os
import subprocess
import sys

philerooski

"""
Run this script from within the unzipped directory `JMV_fitbit_dta`

Download zipped data here: https://www.synapse.org/Synapse:syn62667431
"""
import pandas as pd
import json
import os

philerooski

### Does not work

expectation_suite_name = "my_expectation_suite"
checkpoint_name = "my_checkpoint"

context = gx.get_context()

# # Initialize expectation suite
def init_expectation_suite():
    expectation_suite = context.add_expectation_suite(

philerooski

import os
import random
import logging
import argparse
import toml
import snowflake.connector
from atlassian import Confluence

# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')

philerooski

"""
Load snapshot data from Snowflake stage into tables.

This script processes prefixes from PREFIX_LIST table, derives table names,
creates tables using INFER_SCHEMA, and logs all operations to LOAD_LOG.

See `--help` for optional parameter `--only-affected`
"""

import snowflake.connector

philerooski

"""
Analyze errors from the LOAD_LOG table.

This script queries the LOAD_LOG table for failed operations,
categorizes errors by type, and groups data types by error category.

This is a complementary script to https://gist.github.com/philerooski/a740b25f066f1ad205344637160aa969
"""

import snowflake.connector

philerooski

"""
Load RDS snapshot data from S3 via Snowflake external stage into tables.

This script supports two modes:
1. Bootstrap mode (--bootstrap-stack): Creates a new schema, external stage,
   file format, and grants privileges before loading data.
2. Manual mode: Loads data into an existing schema with pre-configured stage.

The script dynamically discovers all data types from the S3 stage URL,
creates tables using INFER_SCHEMA from Parquet files, loads the data,

philerooski

#!/usr/bin/env python3
"""
Download all form data from a specific form group to local directory.
"""

import argparse
import sys
import json
import tempfile
import shutil

philerooski

"""
Verify snapshot data load integrity.

This script checks:
1. LOAD_LOG table for any errors or anomalies
2. Compares tables between PROD_576 and PROD_568 schemas
3. Validates record counts (PROD_576 should have more records)

Usage:
    python verify_snapshot_load.py --database SYNAPSE_RDS_SNAPSHOT \\

philerooski

USE ROLE USERADMIN;

-- Copied from admin/users.sql lines 165-254 for isolated testing.
-- Set DEFAULT_SECONDARY_ROLES based on user type and role access.
-- A user is treated as an analyst if ALL of the following are true:
--   1) user type is not SERVICE
--   2) user name does not contain 'service' (case-insensitive)
--   3) user is not granted any of these roles:
--      DATA_ENGINEER, ACCOUNTADMIN, SYSADMIN, SECURITYADMIN, USERADMIN
-- Analysts get DEFAULT_SECONDARY_ROLES=('ALL').