patrikbraborec

   def main(self):
       config = Config(self.args.config)
       for table in config.tables:
           self.recreate_table(table)
           for org in config.organizations:
               if table.org_level:
                   # Some endpoints are organization-level, e.g. users exist in organization, not in each repo
                   file_name = CSV_FILE_PATH_TMPL_ORG.format(table_name=table.name, org_name=org.name)
                   self.store_data(table, file_name)
               else:

patrikbraborec

with users as (
 select to_json("item") as item_json from cicd_input_stage.users
),
 
final as (
 select
   CAST(json_extract_path_text(item_json, 'id') as INT) as user_id,
   CAST(json_extract_path_text(item_json, 'html_url') as TEXT) as url,
   CAST(json_extract_path_text(item_json, 'login') as TEXT) as login
 from users

patrikbraborec

version: 2

models:
 - name: users
   columns:
     - name: user_id
       tests:
         - dbt_constraints.primary_key

patrikbraborec

from gooddata_sdk import GoodDataSdk
import os
import sys
 
host = os.environ["GOODDATA_HOST"]
token = os.environ["GOODDATA_TOKEN"]
staging_workspace_id = os.environ["STAGING_WORKSPACE_ID"]
 
sdk = GoodDataSdk.create(host, token)
 

patrikbraborec

from gooddata_sdk import GoodDataSdk, CatalogWorkspace
import os
 
host = os.environ["GOODDATA_HOST"]
token = os.environ["GOODDATA_TOKEN"]
staging_workspace_id = os.environ["STAGING_WORKSPACE_ID"]
production_workspace_id = os.environ["PRODUCTION_WORKSPACE_ID"]
 
sdk = GoodDataSdk.create(host, token)
 

patrikbraborec

stages:
 # pre-merge
 - extract_load
 - transform
 - analytics_staging
 # post-merge
 - analytics_prod

patrikbraborec

extract_load:
 stage: extract_load
 image: python:latest
 script:
   - cd "$CI_PROJECT_DIR/extract_load"
   - pip install -r requirements.txt
   - ./extract.py
   - ./load.py
 rules:
   - if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "main"'

patrikbraborec

gooddata_staging:
 stage: analytics_staging
 image: python:latest
 script:
   - cd "$CI_PROJECT_DIR/analytics"
   - pip install -r requirements.txt
 
   # Deliver into staging workspace
   # Data source properties are stored in Gitlab ENV vars
   # If admin changes them, he has to run this job manually (RUN_ALL_JOBS=true) to deliver new DS definition

patrikbraborec

import http.client
import json
import os

token = os.environ["RAPID_API_TOKEN"]
stock_symbol = os.environ["STOCK_SYMBOL"]
conn = http.client.HTTPSConnection("meteostat.p.rapidapi.com")

headers = {
    'X-RapidAPI-Key': token,

patrikbraborec

import http.client
import json
import os
 
token = os.environ["RAPID_API_TOKEN"]
stock_symbol = os.environ["STOCK_SYMBOL"]
conn = http.client.HTTPSConnection("alpha-vantage.p.rapidapi.com")
 
headers = {
   'X-RapidAPI-Key': "4fc37563d6msh7b9fb7f0417a5f4p1caa4ejsnf36924701105",