This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from prefect import Flow, Task, Parameter | |
import copy | |
def remove_middle_params(G: "Flow", params_to_remove: dict) -> Flow: | |
""" | |
Removes a parameter from a flow and reassigns its downstream task to a new upstream task. This is useful after merging two flows where | |
a parameter was only acting as conduit between two disparate flows. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
from prefect import Flow | |
#from prefect.environments.storage import Docker | |
from prefect.tasks.shell import ShellTask | |
from prefect.tasks.secrets.base import PrefectSecret | |
import boto3 | |
import docker | |
import base64 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def build_a_flow( | |
flow: Flow, | |
flows_to_add: list=[], | |
params_to_remove: dict={}, | |
visualize: bool=False | |
): | |
""" | |
A function to build a flow using update and merge | |
""" | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import prefect | |
from prefect import Flow, task, Task, case, Parameter | |
from prefect.environments import LocalEnvironment | |
from prefect.environments.storage import S3 | |
from prefect.tasks.prefect import FlowRunTask | |
from prefect.tasks.control_flow import merge | |
from prefect.engine.results import PrefectResult | |
@task |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import prefect | |
from prefect import Flow, task, Task, case, Parameter | |
from prefect.environments import LocalEnvironment | |
from prefect.environments.storage import S3 | |
from prefect.tasks.prefect import FlowRunTask | |
from prefect.tasks.control_flow import merge | |
from prefect.engine.results import PrefectResult | |
@task |