Created
August 28, 2026 17:51
-
-
Save lassebenni/acaa5f82c778b2b5974e28cbabefaf59 to your computer and use it in GitHub Desktop.
Code Explainer: Week 15 ingestion Python package
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <!DOCTYPE html> | |
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <title>Code Explainer: Ingestion Python package</title> | |
| <style> | |
| * { margin: 0; padding: 0; box-sizing: border-box; } | |
| body { | |
| background: #0d1117; | |
| display: flex; | |
| justify-content: center; | |
| align-items: flex-start; | |
| min-height: 100vh; | |
| font-family: 'SF Pro Text', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; | |
| padding: 12px; | |
| color: #c9d1d9; | |
| } | |
| .explainer-container { | |
| width: 100%; | |
| max-width: 860px; | |
| background: #161b22; | |
| border-radius: 10px; | |
| overflow: hidden; | |
| box-shadow: 0 16px 36px rgba(0,0,0,0.6); | |
| border: 1px solid #30363d; | |
| display: flex; | |
| flex-direction: column; | |
| } | |
| .titlebar { | |
| background: #0d1117; | |
| padding: 10px 16px; | |
| display: flex; | |
| align-items: center; | |
| gap: 8px; | |
| border-bottom: 1px solid #21262d; | |
| user-select: none; | |
| } | |
| .dot { width: 12px; height: 12px; border-radius: 50%; } | |
| .dot.r { background: #ff5f56; } | |
| .dot.y { background: #ffbd2e; } | |
| .dot.g { background: #27c93f; } | |
| .titlebar .title { | |
| color: #8b949e; | |
| font-size: 13px; | |
| flex: 1; | |
| text-align: center; | |
| font-family: 'SF Mono', 'Fira Code', monospace; | |
| font-weight: 500; | |
| } | |
| .code-body { | |
| padding: 14px 0; | |
| font-family: 'SF Mono', 'Fira Code', 'Menlo', monospace; | |
| font-size: 12px; | |
| line-height: 1.55; | |
| overflow-x: auto; | |
| background: #0d1117; | |
| max-height: 380px; | |
| overflow-y: auto; | |
| } | |
| .code-line { | |
| display: flex; | |
| padding: 2px 16px; | |
| cursor: pointer; | |
| border-left: 3px solid transparent; | |
| transition: background 0.15s ease; | |
| } | |
| .code-line.annotated { border-left-color: #1f6feb; background: rgba(31,111,235,0.06); } | |
| .code-line.annotated:hover { background: rgba(31,111,235,0.22); } | |
| .code-line.annotated.active { background: rgba(56,139,253,0.28); border-left-color: #58a6ff; } | |
| .line-num { width: 40px; color: #484f58; text-align: right; padding-right: 16px; flex-shrink: 0; } | |
| .line-content { white-space: pre; flex: 1; } | |
| .file-sep { | |
| padding: 8px 16px; | |
| color: #8b949e; | |
| font-size: 11px; | |
| text-transform: uppercase; | |
| letter-spacing: 0.04em; | |
| border-top: 1px solid #21262d; | |
| border-bottom: 1px solid #21262d; | |
| background: #161b22; | |
| } | |
| .kwd { color: #ff7b72; font-weight: 600; } | |
| .fn { color: #d2a8ff; } | |
| .str { color: #a5d6ff; } | |
| .cmnt { color: #8b949e; font-style: italic; } | |
| .var { color: #ffa657; } | |
| .explanation-dock { | |
| background: #161b22; | |
| border-top: 1px solid #30363d; | |
| padding: 16px 20px; | |
| min-height: 120px; | |
| } | |
| .badge { | |
| font-size: 11px; | |
| text-transform: uppercase; | |
| font-weight: 700; | |
| padding: 2px 8px; | |
| border-radius: 12px; | |
| } | |
| .badge.key { background: rgba(255,166,87,0.15); color: #ffa657; border: 1px solid rgba(255,166,87,0.3); } | |
| .badge.delta { background: rgba(56,139,253,0.15); color: #58a6ff; border: 1px solid rgba(56,139,253,0.3); } | |
| .badge.dbt { background: rgba(46,160,67,0.15); color: #3fb950; border: 1px solid rgba(46,160,67,0.3); } | |
| .dock-header { display: flex; align-items: center; gap: 10px; margin-bottom: 6px; } | |
| .dock-title { font-size: 14px; font-weight: 600; color: #f0f6fc; } | |
| .dock-body { font-size: 13px; line-height: 1.5; color: #c9d1d9; } | |
| .hint-text { font-size: 13px; color: #8b949e; } | |
| </style> | |
| </head> | |
| <body> | |
| <div class="explainer-container"> | |
| <div class="titlebar"> | |
| <span class="dot r"></span><span class="dot y"></span><span class="dot g"></span> | |
| <span class="title">data/src/ingestion — fetch · validate · SOURCE_NAME</span> | |
| </div> | |
| <div class="code-body"> | |
| <div class="file-sep">ingest.py</div> | |
| <div class="code-line annotated" data-key="fetch" onclick="selectLine(this,'fetch')" onmouseenter="selectLine(this,'fetch')"><span class="line-num">20</span><span class="line-content"><span class="kwd">def</span> <span class="fn">fetch_raw</span>(url: <span class="kwd">str</span>) -> list[Any]:</span></div> | |
| <div class="code-line annotated" data-key="fetch" onclick="selectLine(this,'fetch')" onmouseenter="selectLine(this,'fetch')"><span class="line-num">23</span><span class="line-content"> response = requests.<span class="fn">get</span>(url, timeout=REQUEST_TIMEOUT_SECONDS)</span></div> | |
| <div class="code-line annotated" data-key="unwrap" onclick="selectLine(this,'unwrap')" onmouseenter="selectLine(this,'unwrap')"><span class="line-num">26</span><span class="line-content"> <span class="cmnt"># Some sources wrap rows in {"data": [...]}, others return a list.</span></span></div> | |
| <div class="code-line annotated" data-key="unwrap" onclick="selectLine(this,'unwrap')" onmouseenter="selectLine(this,'unwrap')"><span class="line-num">27</span><span class="line-content"> records = payload.<span class="fn">get</span>(<span class="str">"data"</span>, payload) <span class="kwd">if</span> <span class="fn">isinstance</span>(payload, dict) <span class="kwd">else</span> payload</span></div> | |
| <div class="code-line"><span class="line-num">28</span><span class="line-content"></span></div> | |
| <div class="code-line annotated" data-key="parse" onclick="selectLine(this,'parse')" onmouseenter="selectLine(this,'parse')"><span class="line-num">34</span><span class="line-content"><span class="kwd">def</span> <span class="fn">parse_records</span>(records: list[Any]) -> tuple[list[Posting], <span class="kwd">int</span>]:</span></div> | |
| <div class="code-line annotated" data-key="parse" onclick="selectLine(this,'parse')" onmouseenter="selectLine(this,'parse')"><span class="line-num">45</span><span class="line-content"> parsed.<span class="fn">append</span>(Posting.<span class="fn">model_validate</span>(record))</span></div> | |
| <div class="code-line annotated" data-key="reject" onclick="selectLine(this,'reject')" onmouseenter="selectLine(this,'reject')"><span class="line-num">46</span><span class="line-content"> <span class="kwd">except</span> ValidationError <span class="kwd">as</span> exc:</span></div> | |
| <div class="code-line annotated" data-key="reject" onclick="selectLine(this,'reject')" onmouseenter="selectLine(this,'reject')"><span class="line-num">47</span><span class="line-content"> rejected += 1</span></div> | |
| <div class="file-sep">pipeline.py</div> | |
| <div class="code-line annotated" data-key="source_name" onclick="selectLine(this,'source_name')" onmouseenter="selectLine(this,'source_name')"><span class="line-num">38</span><span class="line-content"><span class="var">SOURCE_NAME</span> = <span class="str">"postings"</span> <span class="cmnt"># team A often "events" — not an .env key</span></span></div> | |
| <div class="code-line annotated" data-key="config" onclick="selectLine(this,'config')" onmouseenter="selectLine(this,'config')"><span class="line-num">73</span><span class="line-content"> source_api_url=<span class="fn">required</span>(<span class="str">"SOURCE_API_URL"</span>),</span></div> | |
| <div class="code-line annotated" data-key="config" onclick="selectLine(this,'config')" onmouseenter="selectLine(this,'config')"><span class="line-num">78</span><span class="line-content"> landing_container=os.<span class="fn">getenv</span>(<span class="str">"LANDING_CONTAINER"</span>, PRODUCTION_CONTAINER),</span></div> | |
| <div class="code-line annotated" data-key="config" onclick="selectLine(this,'config')" onmouseenter="selectLine(this,'config')"><span class="line-num">79</span><span class="line-content"> landing_prefix=os.<span class="fn">getenv</span>(<span class="str">"LANDING_PREFIX"</span>, PRODUCTION_PREFIX),</span></div> | |
| <div class="code-line annotated" data-key="run" onclick="selectLine(this,'run')" onmouseenter="selectLine(this,'run')"><span class="line-num">92</span><span class="line-content"> records = <span class="fn">fetch_raw</span>(config.source_api_url)</span></div> | |
| <div class="code-line annotated" data-key="run" onclick="selectLine(this,'run')" onmouseenter="selectLine(this,'run')"><span class="line-num">93</span><span class="line-content"> parsed, rejected = <span class="fn">parse_records</span>(records)</span></div> | |
| <div class="code-line annotated" data-key="land" onclick="selectLine(this,'land')" onmouseenter="selectLine(this,'land')"><span class="line-num">108</span><span class="line-content"> path = <span class="fn">blob_path</span>(SOURCE_NAME, run_date, config.landing_prefix)</span></div> | |
| <div class="code-line annotated" data-key="land" onclick="selectLine(this,'land')" onmouseenter="selectLine(this,'land')"><span class="line-num">120</span><span class="line-content"> landed = <span class="fn">land_raw_json</span>(…, records=records, container=config.landing_container)</span></div> | |
| </div> | |
| <div class="explanation-dock"> | |
| <div class="hint-text" id="hint">💡 Hover or tap blue lines. This is Layer 1: the package Path A, Mode 1, and the ACA image all run.</div> | |
| <div id="dockContent" style="display:none;"> | |
| <div class="dock-header"> | |
| <span class="badge key" id="dockBadge">key</span> | |
| <span class="dock-title" id="dockTitle"></span> | |
| </div> | |
| <div class="dock-body" id="dockBody"></div> | |
| </div> | |
| </div> | |
| </div> | |
| <script> | |
| const explanations = { | |
| fetch: { | |
| type: 'delta', badge: 'HTTP', | |
| title: 'fetch_raw — rewrite this for your API', | |
| body: 'GET the feed URL. On Path A / Mode 1 the URL comes from data/.env. Inside ACA it comes from job env (set by CI), not your laptop .env. Pagination and headers live here when your source needs them.' | |
| }, | |
| unwrap: { | |
| type: 'key', badge: 'Shape', | |
| title: 'Nested data vs bare list', | |
| body: 'Arbeitnow-style APIs wrap rows in {"data": [...]}. Others return a list. Your source may use "results" or another key — change this line when you swap feeds.' | |
| }, | |
| parse: { | |
| type: 'dbt', badge: 'Pydantic', | |
| title: 'parse_records → Posting.model_validate', | |
| body: 'Gate every row against models.py. Valid rows prove the shape; cleaning and joins still belong in dbt staging, not here.' | |
| }, | |
| reject: { | |
| type: 'key', badge: 'Partial fail', | |
| title: 'One bad row must not kill the batch', | |
| body: 'Count and skip ValidationError rows. Landing still writes the raw list the API sent — parsing is a gate, not a transform.' | |
| }, | |
| source_name: { | |
| type: 'key', badge: 'Constant', | |
| title: 'SOURCE_NAME is not an .env key', | |
| body: 'Landing subfolder under your prefix: postings (B/C) or events (A). Keep the name stable so staging read_files paths do not thrash. Change only with a matching dbt PR.' | |
| }, | |
| config: { | |
| type: 'delta', badge: 'Env', | |
| title: 'load_config reads process environment', | |
| body: 'Same Config on laptop and in the container. Laptop: dotenv from data/.env. ACA: Azure job env from deploy-dev / deploy-prod. SOURCE_API_URL on your laptop never reaches Mode 2.' | |
| }, | |
| run: { | |
| type: 'delta', badge: 'Orchestration', | |
| title: 'run() — fetch then validate', | |
| body: 'Empty parsed batch raises. Rejected rows can still land as raw JSON so Catalog Explorer stays inspectable.' | |
| }, | |
| land: { | |
| type: 'key', badge: 'Landing', | |
| title: 'blob_path + land_raw_json', | |
| body: 'Path includes SOURCE_NAME and LANDING_PREFIX (your name, aca-dev, or raw). Container is LANDING_CONTAINER (dev vs prod). dbt later reads via LANDING_PATH / Volumes — not this Python return value.' | |
| } | |
| }; | |
| function selectLine(el, key) { | |
| const exp = explanations[key]; | |
| if (!exp) return; | |
| document.querySelectorAll('.code-line').forEach(l => l.classList.remove('active')); | |
| document.querySelectorAll(`.code-line[data-key="${key}"]`).forEach(l => l.classList.add('active')); | |
| document.getElementById('hint').style.display = 'none'; | |
| document.getElementById('dockContent').style.display = 'block'; | |
| document.getElementById('dockBadge').textContent = exp.badge; | |
| document.getElementById('dockBadge').className = 'badge ' + exp.type; | |
| document.getElementById('dockTitle').textContent = exp.title; | |
| document.getElementById('dockBody').textContent = exp.body; | |
| } | |
| </script> | |
| </body> | |
| </html> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment