Skip to content

Instantly share code, notes, and snippets.

import prefect
from prefect import Flow, task, Task, case, Parameter
from prefect.environments import LocalEnvironment
from prefect.environments.storage import S3
from prefect.tasks.prefect import FlowRunTask
from prefect.tasks.control_flow import merge
from prefect.engine.results import PrefectResult
@task
import prefect
from prefect import Flow, task, Task, case, Parameter
from prefect.environments import LocalEnvironment
from prefect.environments.storage import S3
from prefect.tasks.prefect import FlowRunTask
from prefect.tasks.control_flow import merge
from prefect.engine.results import PrefectResult
@task
@gryBox
gryBox / build_flow.py
Created July 17, 2020 19:24
merge flows
def build_a_flow(
flow: Flow,
flows_to_add: list=[],
params_to_remove: dict={},
visualize: bool=False
):
"""
A function to build a flow using update and merge
"""
@gryBox
gryBox / working_register.py
Last active July 2, 2020 15:25
Docker Prefect
import os
from prefect import Flow
#from prefect.environments.storage import Docker
from prefect.tasks.shell import ShellTask
from prefect.tasks.secrets.base import PrefectSecret
import boto3
import docker
import base64
from prefect import Flow, Task, Parameter
import copy
def remove_middle_params(G: "Flow", params_to_remove: dict) -> Flow:
"""
Removes a parameter from a flow and reassigns its downstream task to a new upstream task. This is useful after merging two flows where
a parameter was only acting as conduit between two disparate flows.