noklam

"""
This script creates an IPython extension to load Kedro-related variables in
local scope.
"""

from __future__ import annotations

import inspect
import logging
import os

noklam

def _prepare_function_body(func: Callable) -> str:
    # https://stackoverflow.com/questions/38050649/getting-a-python-functions-source-code-without-the-definition-lines
    all_source_lines = inspect.getsourcelines(func)[0]
    # Remove any decorators
    func_lines = dropwhile(lambda x: x.startswith("@"), all_source_lines)
    line: str = next(func_lines).strip()
    if not line.startswith("def "):
        return line.rsplit(",")[0].strip()
    # Handle functions that are not one-liners
    first_line = next(func_lines)

noklam

def show_install(show_nvidia_smi: bool = False):
    "Print user's setup information"

    import kedro, kedro_datasets, platform, os, sys

    rep = []
    opt_mods = []

    rep.append(["=== Software ===", None])
    rep.append(["python", platform.python_version()])

noklam

"""``SequentialRunner`` is an ``AbstractRunner`` implementation. It can be
used to run the ``Pipeline`` in a sequential manner using a topological sort
of provided nodes.
"""

from collections import Counter
from itertools import chain
from typing import Any, Dict, Set

from kedro.io import DataCatalog

noklam

# Experiment 2 - Simplified Merge Config Case

%%writefile base.yml
# conf/local/parameters.yaml
client:
  url: http://${host}:${server.port}/
  server_port: ${server.port}


%%writefile local.yml

noklam

# Here you can define all your data sets by using simple YAML syntax.
#
# Documentation for this file format can be found in "The Data Catalog"
# Link: https://kedro.readthedocs.io/en/stable/data/data_catalog.html
#
# We support interacting with a variety of data stores including local file systems, cloud, network and HDFS
#
# An example data set definition can look as follows:
#
#bikes:

noklam

<div id="app">

noklam

# TRAINS - Example of LightGBM integration
#
import lightgbm as lgb
import pandas as pd
from sklearn.metrics import mean_squared_error

from trains import Task

task = Task.init(project_name="examples", task_name="LIGHTgbm")

noklam

# # Using pandas merge_as_of (similar to a merge join) to do inexact join (join the nearest key instead)
import pandas as pd

# %%
table_a = pd.DataFrame([('2020-01-01'), ('2020-01-03'), ('2020-01-06')],
                       columns=['PK'])
table_b = pd.DataFrame([('2020-01-01', 'A'), ('2020-01-02', 'A'),
                        ('2020-01-04', 'B'), ('2020-01-05', 'B')],
                       columns=['FK', 'Category'])
# %%

noklam