hamelsmu

#From nbdev.export2html
class ExecuteShowDocPreprocessor(ExecutePreprocessor):
    "An `ExecutePreprocessor` that only executes `show_doc` and `import` cells"
    def preprocess_cell(self, cell, resources, index):
        if not check_re(cell, _re_notebook2script):
            if check_re(cell, _re_show_doc):
                return super().preprocess_cell(cell, resources, index)
            elif check_re_multi(cell, [_re_import, _re_lib_import.re]):
                if check_re_multi(cell, [_re_export, 'show_doc', '^\s*#\s*import']):
#                     r = list(filter(_non_comment_code, cell['source'].split('\n')))

hamelsmu

Fastai Notes

• Fastai Notes
  • DataLoaders
  • DataBlocks
  • Training
    • Interpetability
• Cleaning

hamelsmu

@patch
def predict_batch(self:Learner, item, rm_type_tfms=None, with_input=False):
    dl = self.dls.test_dl(item, rm_type_tfms=rm_type_tfms, num_workers=0)
    inp,preds,_,dec_preds = self.get_preds(dl=dl, with_input=True, with_decoded=True)
    i = getattr(self.dls, 'n_inp', -1)
    inp = (inp,) if i==1 else tuplify(inp)
    dec_inp, nm = zip(*self.dls.decode_batch(inp + tuplify(dec_preds)))
    res = preds,nm,dec_preds
    if with_input: res = (dec_inp,) + res
    return res

hamelsmu

from metaflow import step, current, FlowSpec, Parameter, card
from mymodel import train_model

class NBFlow(FlowSpec):
    "A toy example of using the notebook card."

    @step
    def start(self):
        # Train a model, save the results in `model_results`
        self.model_results = train_model(...)

hamelsmu

Feedback on @pytorch_parallel

Related to [metaflow #907](Netflix/metaflow#907 and the related docs draft

1. Developer Ergonimics

To use the feature, the user must learn a brand new way of doing foreach. This adds a high degree of congntive load, as the user must remember that for this particular use case and this use case only, they need to use self.next(..., num_parallel=...).

The api makes it very unclear where the parallelization is happening. For example, pytorch_lightning.trainer has an argument gpus=-1, which means that it will use all available gpus. In this case, what does num_parallel add to this? The user has lots of cognitive overload to have to reason about where and what kind of parallelization is happening.

hamelsmu

import os
from azure.storage.blob import BlobServiceClient

blob_service_client = BlobServiceClient.from_connection_string(os.getenv('AZURE_STORAGE_CONNECTION_STRING'))

with open('gh_repo_topics.parquet', 'rb') as data:
    blob_client = container_client.upload_blob(name="sample_data/gh_repo_topics.parquet", data=data, overwrite=True)

####### Downloading files ##########
f = container_client.download_blob("sample_data/gh_repo_topics.parquet")

hamelsmu

hamelsmu

hamelsmu

export IMAGE_FAMILY="pytorch-latest-gpu"
export ZONE="us-west1-b"
export INSTANCE_NAME="dl-rig"

gcloud compute instances create $INSTANCE_NAME \
  --zone=$ZONE \
  --image-family=$IMAGE_FAMILY \
  --image-project=deeplearning-platform-release \
  --maintenance-policy=TERMINATE \
  --accelerator="type=nvidia-tesla-v100,count=2" \

hamelsmu