Created
August 28, 2026 17:50
-
-
Save lassebenni/0fb5a415c09322717b165944c7f91d80 to your computer and use it in GitHub Desktop.
Code Explainer: read_files and schemaHints (Week 15)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <!DOCTYPE html> | |
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <title>Code Explainer: read_files and schemaHints</title> | |
| <style> | |
| * { margin: 0; padding: 0; box-sizing: border-box; } | |
| body { | |
| background: #0d1117; | |
| display: flex; | |
| justify-content: center; | |
| align-items: flex-start; | |
| min-height: 100vh; | |
| font-family: 'SF Pro Text', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; | |
| padding: 12px; | |
| color: #c9d1d9; | |
| } | |
| .explainer-container { | |
| width: 100%; | |
| max-width: 860px; | |
| background: #161b22; | |
| border-radius: 10px; | |
| overflow: hidden; | |
| box-shadow: 0 16px 36px rgba(0,0,0,0.6); | |
| border: 1px solid #30363d; | |
| display: flex; | |
| flex-direction: column; | |
| } | |
| .titlebar { | |
| background: #0d1117; | |
| padding: 10px 16px; | |
| display: flex; | |
| align-items: center; | |
| gap: 8px; | |
| border-bottom: 1px solid #21262d; | |
| user-select: none; | |
| } | |
| .dot { width: 12px; height: 12px; border-radius: 50%; } | |
| .dot.r { background: #ff5f56; } | |
| .dot.y { background: #ffbd2e; } | |
| .dot.g { background: #27c93f; } | |
| .titlebar .title { | |
| color: #8b949e; | |
| font-size: 13px; | |
| flex: 1; | |
| text-align: center; | |
| font-family: 'SF Mono', 'Fira Code', monospace; | |
| font-weight: 500; | |
| } | |
| .code-body { | |
| padding: 14px 0; | |
| font-family: 'SF Mono', 'Fira Code', 'Menlo', monospace; | |
| font-size: 12px; | |
| line-height: 1.55; | |
| overflow-x: auto; | |
| background: #0d1117; | |
| max-height: 420px; | |
| overflow-y: auto; | |
| } | |
| .code-line { | |
| display: flex; | |
| padding: 2px 16px; | |
| cursor: pointer; | |
| transition: background 0.15s ease, border-color 0.15s ease; | |
| user-select: text; | |
| border-left: 3px solid transparent; | |
| } | |
| .code-line:hover { background: #161b22; } | |
| .code-line.annotated { | |
| border-left-color: #1f6feb; | |
| background: rgba(31, 111, 235, 0.06); | |
| } | |
| .code-line.annotated:hover { background: rgba(31, 111, 235, 0.22); } | |
| .code-line.annotated.active { | |
| background: rgba(56, 139, 253, 0.28); | |
| border-left-color: #58a6ff; | |
| } | |
| .line-num { | |
| width: 36px; | |
| color: #484f58; | |
| text-align: right; | |
| padding-right: 16px; | |
| user-select: none; | |
| flex-shrink: 0; | |
| } | |
| .line-content { white-space: pre; flex: 1; } | |
| .kwd { color: #ff7b72; font-weight: 600; } | |
| .fn { color: #d2a8ff; } | |
| .str { color: #a5d6ff; } | |
| .cmnt { color: #8b949e; font-style: italic; } | |
| .var { color: #ffa657; } | |
| .jja { color: #79c0ff; font-weight: 600; } | |
| .explanation-dock { | |
| background: #161b22; | |
| border-top: 1px solid #30363d; | |
| padding: 16px 20px; | |
| min-height: 120px; | |
| display: flex; | |
| flex-direction: column; | |
| justify-content: center; | |
| } | |
| .dock-header { display: flex; align-items: center; gap: 10px; margin-bottom: 6px; } | |
| .badge { | |
| font-size: 11px; | |
| text-transform: uppercase; | |
| font-weight: 700; | |
| padding: 2px 8px; | |
| border-radius: 12px; | |
| letter-spacing: 0.5px; | |
| } | |
| .badge.key { background: rgba(255, 166, 87, 0.15); color: #ffa657; border: 1px solid rgba(255, 166, 87, 0.3); } | |
| .badge.delta { background: rgba(56, 139, 253, 0.15); color: #58a6ff; border: 1px solid rgba(56, 139, 253, 0.3); } | |
| .badge.dbt { background: rgba(46, 160, 67, 0.15); color: #3fb950; border: 1px solid rgba(46, 160, 67, 0.3); } | |
| .badge.jinja { background: rgba(210, 168, 255, 0.15); color: #d2a8ff; border: 1px solid rgba(210, 168, 255, 0.3); } | |
| .dock-title { font-size: 14px; font-weight: 600; color: #f0f6fc; } | |
| .dock-body { font-size: 13px; line-height: 1.5; color: #c9d1d9; } | |
| .hint-text { display: flex; align-items: center; gap: 8px; font-size: 13px; color: #8b949e; } | |
| </style> | |
| </head> | |
| <body> | |
| <div class="explainer-container" id="container"> | |
| <div class="titlebar"> | |
| <span class="dot r"></span><span class="dot y"></span><span class="dot g"></span> | |
| <span class="title">data/dbt/models/staging/stg_postings.sql</span> | |
| </div> | |
| <div class="code-body" id="codeBody"> | |
| <div class="code-line"><span class="line-num">17</span><span class="line-content"><span class="kwd">with</span></span></div> | |
| <div class="code-line annotated" data-key="source" onclick="selectLine(this,'source')" onmouseenter="selectLine(this,'source')"><span class="line-num">18</span><span class="line-content"> source <span class="kwd">as</span> (</span></div> | |
| <div class="code-line annotated" data-key="meta" onclick="selectLine(this,'meta')" onmouseenter="selectLine(this,'meta')"><span class="line-num">30</span><span class="line-content"> *,</span></div> | |
| <div class="code-line annotated" data-key="meta" onclick="selectLine(this,'meta')" onmouseenter="selectLine(this,'meta')"><span class="line-num">31</span><span class="line-content"> _metadata.file_path <span class="kwd">as</span> source_file,</span></div> | |
| <div class="code-line annotated" data-key="meta" onclick="selectLine(this,'meta')" onmouseenter="selectLine(this,'meta')"><span class="line-num">32</span><span class="line-content"> _metadata.file_modification_time <span class="kwd">as</span> ingested_at</span></div> | |
| <div class="code-line"><span class="line-num">33</span><span class="line-content"> <span class="kwd">from</span></span></div> | |
| <div class="code-line annotated" data-key="read" onclick="selectLine(this,'read')" onmouseenter="selectLine(this,'read')"><span class="line-num">34</span><span class="line-content"> <span class="fn">read_files</span>(</span></div> | |
| <div class="code-line annotated" data-key="path" onclick="selectLine(this,'path')" onmouseenter="selectLine(this,'path')"><span class="line-num">37</span><span class="line-content"> <span class="str">'</span><span class="jja">{{</span> <span class="fn">var</span>(<span class="str">"landing_path"</span>) <span class="jja">}}</span><span class="str">/postings'</span>,</span></div> | |
| <div class="code-line annotated" data-key="format" onclick="selectLine(this,'format')" onmouseenter="selectLine(this,'format')"><span class="line-num">38</span><span class="line-content"> format => <span class="str">'json'</span>,</span></div> | |
| <div class="code-line annotated" data-key="hints" onclick="selectLine(this,'hints')" onmouseenter="selectLine(this,'hints')"><span class="line-num">39</span><span class="line-content"> schemahints</span></div> | |
| <div class="code-line annotated" data-key="hints" onclick="selectLine(this,'hints')" onmouseenter="selectLine(this,'hints')"><span class="line-num">40</span><span class="line-content"> => <span class="str">'slug string, title string, …, created_at bigint'</span></span></div> | |
| <div class="code-line annotated" data-key="read" onclick="selectLine(this,'read')" onmouseenter="selectLine(this,'read')"><span class="line-num">41</span><span class="line-content"> )</span></div> | |
| <div class="code-line"><span class="line-num">43</span><span class="line-content"> ),</span></div> | |
| <div class="code-line annotated" data-key="renamed" onclick="selectLine(this,'renamed')" onmouseenter="selectLine(this,'renamed')"><span class="line-num">45</span><span class="line-content"> renamed <span class="kwd">as</span> (</span></div> | |
| <div class="code-line annotated" data-key="rename_cols" onclick="selectLine(this,'rename_cols')" onmouseenter="selectLine(this,'rename_cols')"><span class="line-num">51</span><span class="line-content"> slug <span class="kwd">as</span> posting_id,</span></div> | |
| <div class="code-line annotated" data-key="cast" onclick="selectLine(this,'cast')" onmouseenter="selectLine(this,'cast')"><span class="line-num">63</span><span class="line-content"> <span class="fn">timestamp_seconds</span>(created_at) <span class="kwd">as</span> posted_at,</span></div> | |
| <div class="code-line"><span class="line-num">72</span><span class="line-content"> <span class="kwd">from</span> source</span></div> | |
| <div class="code-line"><span class="line-num">74</span><span class="line-content"> ),</span></div> | |
| <div class="code-line annotated" data-key="dedupe" onclick="selectLine(this,'dedupe')" onmouseenter="selectLine(this,'dedupe')"><span class="line-num">76</span><span class="line-content"> deduplicated <span class="kwd">as</span> (</span></div> | |
| <div class="code-line annotated" data-key="dedupe" onclick="selectLine(this,'dedupe')" onmouseenter="selectLine(this,'dedupe')"><span class="line-num">91</span><span class="line-content"> <span class="kwd">select</span> *</span></div> | |
| <div class="code-line annotated" data-key="dedupe" onclick="selectLine(this,'dedupe')" onmouseenter="selectLine(this,'dedupe')"><span class="line-num">92</span><span class="line-content"> <span class="kwd">from</span> renamed</span></div> | |
| <div class="code-line annotated" data-key="dedupe" onclick="selectLine(this,'dedupe')" onmouseenter="selectLine(this,'dedupe')"><span class="line-num">93</span><span class="line-content"> <span class="kwd">qualify</span></span></div> | |
| <div class="code-line annotated" data-key="dedupe" onclick="selectLine(this,'dedupe')" onmouseenter="selectLine(this,'dedupe')"><span class="line-num">94</span><span class="line-content"> <span class="fn">row_number</span>() <span class="kwd">over</span> (<span class="kwd">partition by</span> posting_id <span class="kwd">order by</span> ingested_at <span class="kwd">desc</span>) = 1</span></div> | |
| <div class="code-line"><span class="line-num">96</span><span class="line-content"> )</span></div> | |
| <div class="code-line"><span class="line-num">98</span><span class="line-content"><span class="kwd">select</span> *</span></div> | |
| <div class="code-line"><span class="line-num">99</span><span class="line-content"><span class="kwd">from</span> deduplicated</span></div> | |
| </div> | |
| <div class="explanation-dock" id="dock"> | |
| <div class="hint-text" id="hint"><span>💡</span><span>Hover or tap any blue-accented line to see what it does.</span></div> | |
| <div id="dockContent" style="display:none;"> | |
| <div class="dock-header"> | |
| <span class="badge key" id="dockBadge">key</span> | |
| <span class="dock-title" id="dockTitle">Title</span> | |
| </div> | |
| <div class="dock-body" id="dockBody">Body</div> | |
| </div> | |
| </div> | |
| </div> | |
| <script> | |
| const explanations = { | |
| source: { | |
| type: 'dbt', | |
| badge: 'Staging CTE', | |
| title: 'source is not a dbt source()', | |
| body: 'There is no source() table for the landing folder. This CTE is the first model that touches the files. Lineage in dbt docs starts here.' | |
| }, | |
| meta: { | |
| type: 'key', | |
| badge: 'Provenance', | |
| title: '_metadata from read_files', | |
| body: 'Databricks attaches file path and modification time to every row. That tells you which JSON file a bad row came from when one day looks wrong.' | |
| }, | |
| read: { | |
| type: 'delta', | |
| badge: 'Databricks', | |
| title: 'read_files over a volume path', | |
| body: 'Reads every JSON file under the path. No CREATE TABLE for raw data, and a new day\'s file is picked up without editing this model.' | |
| }, | |
| path: { | |
| type: 'jinja', | |
| badge: 'Path shape', | |
| title: 'landing_path + /postings', | |
| body: 'Teams B/C: LANDING_PATH is the parent only; staging appends /postings (matches SOURCE_NAME in pipeline.py). Team A often puts /events in LANDING_PATH and matches that in SQL.' | |
| }, | |
| format: { | |
| type: 'delta', | |
| badge: 'Format', | |
| title: 'format => json', | |
| body: 'Each landed file is the raw API payload. Keep that shape on disk; clean and cast in staging, not in the ingest package.' | |
| }, | |
| hints: { | |
| type: 'key', | |
| badge: 'schemaHints', | |
| title: 'Pin types to source field names', | |
| body: 'Use the names the API sends (slug, created_at), not your cleaned mart names. Without hints, Spark guesses types from files and a "1" vs 1 drift can break casts downstream.' | |
| }, | |
| renamed: { | |
| type: 'dbt', | |
| badge: 'Rename CTE', | |
| title: 'Clean names for the rest of the project', | |
| body: 'Downstream models should not depend on whatever the API called a field. Rename once here.' | |
| }, | |
| rename_cols: { | |
| type: 'dbt', | |
| badge: 'Grain key', | |
| title: 'slug → posting_id', | |
| body: 'Stable business key for unique tests and for the backend. Prefer keeping this name when you swap sources so publish and contracts stay stable.' | |
| }, | |
| cast: { | |
| type: 'key', | |
| badge: 'Date cast', | |
| title: 'Match the bytes on disk', | |
| body: 'Arbeitnow sends Unix seconds → timestamp_seconds. If your feed sends ISO strings, change this cast. Leaving the wrong cast after a source swap breaks dbt or produces nonsense dates.' | |
| }, | |
| dedupe: { | |
| type: 'key', | |
| badge: 'Dedupe', | |
| title: 'qualify keeps the newest row', | |
| body: 'read_files reads every file in the folder. Open postings reappear daily. Without this, unique tests on posting_id fail even when the data is fine.' | |
| } | |
| }; | |
| function selectLine(el, key) { | |
| if (!explanations[key]) return; | |
| document.querySelectorAll('.code-line').forEach(l => l.classList.remove('active')); | |
| document.querySelectorAll(`.code-line[data-key="${key}"]`).forEach(l => l.classList.add('active')); | |
| const exp = explanations[key]; | |
| document.getElementById('hint').style.display = 'none'; | |
| document.getElementById('dockContent').style.display = 'block'; | |
| document.getElementById('dockBadge').textContent = exp.badge; | |
| document.getElementById('dockBadge').className = 'badge ' + exp.type; | |
| document.getElementById('dockTitle').textContent = exp.title; | |
| document.getElementById('dockBody').textContent = exp.body; | |
| } | |
| </script> | |
| </body> | |
| </html> |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment