stephanie-wang

import ray
import dask
import dask.dataframe as dd
import json
import pandas as pd
import numpy as np
from ray.util.dask import ray_dask_get
import os.path
import csv
import fastparquet

stephanie-wang

import ray
from ray.util.dask import ray_dask_get

import dask
import dask.dataframe as dd 
import pandas as pd
import numpy as np

dask.config.set(scheduler=ray_dask_get) # Sets Ray as the default backend.

stephanie-wang

import ray
import numpy as np


@ray.remote
def map(data, npartitions):
    outputs = [list() for _ in range(npartitions)]
    for row in data:
        outputs[int(row * npartitions)].append(row)
    return tuple(sorted(output) for output in outputs)

stephanie-wang

stephanie-wang

import numpy as np
import ray
import os


@ray.remote
def f():
    return np.random.rand(1000_000_000 // 8)

stephanie-wang

(MemoryMonitorActor pid=1328) 148	10.8GiB	/home/ray/anaconda3/lib/python3.7/site-packages/ray/core/src/ray/gcs/gcs_server --log_dir=/tmp/ray/s
(MemoryMonitorActor pid=1328) 1252	1.24GiB	python stress_tests/test_threaded_actors.py --test-runtime 3600 --kill-interval_s 60
(MemoryMonitorActor pid=1328) 173	0.87GiB	/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/dash
(MemoryMonitorActor pid=1328) 250	0.12GiB	/home/ray/anaconda3/bin/python -u /home/ray/anaconda3/lib/python3.7/site-packages/ray/dashboard/agen
(MemoryMonitorActor pid=1328) 64	0.09GiB	/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/anyscale session web_terminal_server --deploy
(MemoryMonitorActor pid=1328) 384	0.09GiB	/home/ray/anaconda3/bin/python /home/ray/anaconda3/bin/anyscale session auth_start
(MemoryMonitorActor pid=1328) 1328	0.06GiB	ray::MemoryMonitorActor.run()
(MemoryMonitorActor pid=132

stephanie-wang

import ray
import pandas as pd
import numpy as np
import time
import builtins
from typing import Any, Generic, List, Callable, Union, Tuple, Iterable
import os
import psutil
import resource

stephanie-wang

import ray


@ray.remote
def map(start, end, boundaries):
    vals = list(range(start, end))
    partitions = []

    prev_bound = 0
    for next_bound in boundaries:

stephanie-wang

"""
Runs single-node microbenchmarks for reading and preprocessing an image dataset
from local disk and (for supported data loaders) cloud storage. Pass a --data-root,
--parquet-data-root, --tf-data-root, and/or --mosaic-data-root pointing to the
dataset directory. Throughputs are written to `output.csv` in total images/s.
"""
import ray
import torch
import torchvision
import os

stephanie-wang

import ray
import time


@ray.remote(num_cpus=0)
class Counter:
    def __init__(self):
        self.num_f = 0
        self.num_g = 0