Skip to content

Instantly share code, notes, and snippets.

View vvgsrk's full-sized avatar

Venkata Gowri Sai Rakesh Kumar Varanasi vvgsrk

View GitHub Profile
@vvgsrk
vvgsrk / generate_row_rank_using_event_ts.sql
Created April 25, 2022 12:45
Generate row rank using event timestamp
{%- macro generate_row_rank_using_event_ts(model, keycolumn) -%}
{%- set columns = adapter.get_columns_in_relation(model) -%}
WITH latest_data_changes AS
(
SELECT
ROW_NUMBER() OVER (
PARTITION BY {{ keycolumn }}
ORDER BY __event_ts DESC
) AS row_rank,
*
@vvgsrk
vvgsrk / generate_row_rank_using_event_ts_incremental.sql
Created April 25, 2022 19:35
Generate row rank using event timestamp column with DBT incremental materialization
{#
- this macro creates incremental tables based
ON history DATA - it uses is_incremental() macro TO wrap valid SQL that filters THE rows - Filter will ONLY be applied
ON an incremental run - it uses unique_key PARAMETER which ensures THE existing ROW IS updated IN target TABLE #}
{%- macro generate_row_rank_using_event_ts_incremental(model, keycolumn) -%}
{%- set columns = adapter.get_columns_in_relation(model) -%}
{{ config(materialized = var('incremental'), unique_key = '__uuid') }}
@vvgsrk
vvgsrk / src_product_management_product_history.sql
Created April 26, 2022 10:48
Product management product history query in DBT Source layer
WITH source AS (
SELECT
*
FROM
{{ source(
'product_management',
'product_history'
) }}
),
renamed AS (
@vvgsrk
vvgsrk / src_product_management_product_latest.sql
Created April 26, 2022 11:17
Product management product latest model in DBT Source layer
{{ generate_row_rank_using_event_ts(ref('src_product_management_product_history'), 'product_id') }}
@vvgsrk
vvgsrk / src_product_management_product_latest_incremental
Created April 26, 2022 11:21
Product management product latest incremental model in DBT source layer
{{ generate_row_rank_using_event_ts_incremental(ref('src_product_management_product_history'), 'product_id') }}
@vvgsrk
vvgsrk / compiled_version_of_src_product_management_product_latest_incremental.sql
Last active April 26, 2022 11:39
Compiled version of src_product_management_product_latest_incremental.sql
WITH ranked_data AS (
SELECT
ROW_NUMBER() over (
PARTITION BY product_id
ORDER BY
__event_ts DESC
) AS row_rank,
*
FROM dev_edw_dbt_incremental_model_test.dbt_src_product_management.src_product_management_product_history
)
@vvgsrk
vvgsrk / product_latest.sql
Created April 26, 2022 19:16
Product latest info
SELECT *
FROM {{ ref('src_product_management_product_latest_incremental') }}
@vvgsrk
vvgsrk / transient_incremental_table.sql
Last active April 26, 2022 19:32
Incremental table query on snowflake
create or replace transient table dev_edw_dbt_incremental_model_test.dbt_src_product_management.src_product_management_product_latest_incremental as
(
WITH ranked_data AS (
SELECT
ROW_NUMBER() over (
PARTITION BY product_id
ORDER BY
__event_ts DESC
) AS row_rank,
*
create or replace temporary table dev_edw_dbt_incremental_model_test.dbt_src_product_management.src_product_management_product_latest_incremental__dbt_tmp as
(
WITH ranked_data AS (
SELECT
ROW_NUMBER() over (
PARTITION BY product_id
ORDER BY
__event_ts DESC
) AS row_rank,
*
@vvgsrk
vvgsrk / merge_into_incremental_table.sql
Created April 26, 2022 19:53
Merge statement for incremental table
merge into dev_edw_dbt_incremental_model_test.dbt_src_product_management.src_product_management_product_latest_incremental as DBT_INTERNAL_DEST
using dev_edw_dbt_incremental_model_test.dbt_src_product_management.src_product_management_product_latest_incremental__dbt_tmp as DBT_INTERNAL_SOURCE
on
DBT_INTERNAL_SOURCE.__uuid = DBT_INTERNAL_DEST.__uuid
when matched then update set
"PRODUCT_ID" = DBT_INTERNAL_SOURCE."PRODUCT_ID","PRODUCT_NAME" = DBT_INTERNAL_SOURCE."PRODUCT_NAME","PRICE_PER_UNIT" = DBT_INTERNAL_SOURCE."PRICE_PER_UNIT","BASIC_UNIT" = DBT_INTERNAL_SOURCE."BASIC_UNIT","IS_STOCK_LIMITED" = DBT_INTERNAL_SOURCE."IS_STOCK_LIMITED","IS_ACTIVE_FOR_SALE" = DBT_INTERNAL_SOURCE."IS_ACTIVE_FOR_SALE","__EVENT_TS" = DBT_INTERNAL_SOURCE."__EVENT_TS","__LOAD_TS" = DBT_INTERNAL_SOURCE."__LOAD_TS","__UUID" = DBT_INTERNAL_SOURCE."__UUID"
when not matched then insert
("PRODUCT_ID", "PRODUCT_NAME", "PRICE_PER_UNIT", "BASIC_UNIT", "IS_STOCK_LIMITED", "IS_