hamelsmu

from metaflow import FlowSpec, step
url = "https://raw.githubusercontent.com/Netflix/metaflow/master/metaflow/tutorials/02-statistics/movies.csv"
local_path = "./movies.csv"
class CSVFlow(FlowSpec):
    
    @step
    def start(self):
        self.next(self.get_csv_from_web)
        
    @step

hamelsmu

-- Access this query at https://console.cloud.google.com/bigquery?sq=235037502967:a71a4b32d74442558a2739b581064e5f

SELECT url, title, body
FROM(
SELECT url, title, body
     , RANK() OVER (PARTITION BY SUBSTR(body, 75, 125) ORDER BY url) as count_body_beg
FROM(
SELECT url, title, body
   , RANK() OVER (PARTITION BY SUBSTR(body, 50, 100) ORDER BY url) as count_body_beg
FROM(

hamelsmu

<DocSection type="class" name="ExampleError" module="test_lib.example" link="test_lib/example.py#L155">
    <SigArgSection>
        <SigArg name="msg" type="str" default="foo"/>
        <SigArg name="code" type="str" default="foo"/>
    </SigArgSection>
    <Description summary="Exceptions are documented in the same way as classes." extended_summary="The __init__ method may be documented in either the class level\ndocstring, or as a docstring on the __init__ method itself.\n\nEither form is acceptable, but the two should not be mixed. Choose one\nconvention to document the __init__ method and be consistent with it." />
    <ParamSection name="Parameters">
        <Parameter name="msg" type="str" desc="Human readable string describing the exception." />
        <Parameter name="code" type=":obj:`int`, optional" desc="Numeric error code." />
    </ParamSection>

hamelsmu

#From nbdev.export2html
class ExecuteShowDocPreprocessor(ExecutePreprocessor):
    "An `ExecutePreprocessor` that only executes `show_doc` and `import` cells"
    def preprocess_cell(self, cell, resources, index):
        if not check_re(cell, _re_notebook2script):
            if check_re(cell, _re_show_doc):
                return super().preprocess_cell(cell, resources, index)
            elif check_re_multi(cell, [_re_import, _re_lib_import.re]):
                if check_re_multi(cell, [_re_export, 'show_doc', '^\s*#\s*import']):
#                     r = list(filter(_non_comment_code, cell['source'].split('\n')))

hamelsmu

Fastai Notes

• Fastai Notes
  • DataLoaders
  • DataBlocks
  • Training
    • Interpetability
• Cleaning

hamelsmu

@patch
def predict_batch(self:Learner, item, rm_type_tfms=None, with_input=False):
    dl = self.dls.test_dl(item, rm_type_tfms=rm_type_tfms, num_workers=0)
    inp,preds,_,dec_preds = self.get_preds(dl=dl, with_input=True, with_decoded=True)
    i = getattr(self.dls, 'n_inp', -1)
    inp = (inp,) if i==1 else tuplify(inp)
    dec_inp, nm = zip(*self.dls.decode_batch(inp + tuplify(dec_preds)))
    res = preds,nm,dec_preds
    if with_input: res = (dec_inp,) + res
    return res

hamelsmu

from metaflow import step, current, FlowSpec, Parameter, card
from mymodel import train_model

class NBFlow(FlowSpec):
    "A toy example of using the notebook card."

    @step
    def start(self):
        # Train a model, save the results in `model_results`
        self.model_results = train_model(...)

hamelsmu

Feedback on @pytorch_parallel

Related to [metaflow #907](Netflix/metaflow#907 and the related docs draft

1. Developer Ergonimics

To use the feature, the user must learn a brand new way of doing foreach. This adds a high degree of congntive load, as the user must remember that for this particular use case and this use case only, they need to use self.next(..., num_parallel=...).

The api makes it very unclear where the parallelization is happening. For example, pytorch_lightning.trainer has an argument gpus=-1, which means that it will use all available gpus. In this case, what does num_parallel add to this? The user has lots of cognitive overload to have to reason about where and what kind of parallelization is happening.

hamelsmu

import os
from azure.storage.blob import BlobServiceClient

blob_service_client = BlobServiceClient.from_connection_string(os.getenv('AZURE_STORAGE_CONNECTION_STRING'))

with open('gh_repo_topics.parquet', 'rb') as data:
    blob_client = container_client.upload_blob(name="sample_data/gh_repo_topics.parquet", data=data, overwrite=True)

####### Downloading files ##########
f = container_client.download_blob("sample_data/gh_repo_topics.parquet")

hamelsmu