Skip to content

Instantly share code, notes, and snippets.

@vicyap

vicyap/example.py

Created February 8, 2022 22:01
Show Gist options
  • Save vicyap/1c0966b274cb89da0515810080368160 to your computer and use it in GitHub Desktop.
Save vicyap/1c0966b274cb89da0515810080368160 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
example.py
import logging
import ray
@ray.remote
def task():
import time
time.sleep(30)
ray.init("ray://mycluster.internal:10001", logging_level=logging.DEBUG)
ray.get([task.remote() for _ in range(50)])
Raw
ray-client.log
2022-02-08 15:50:35,219 DEBUG worker.py:314 -- client gRPC channel state change: ChannelConnectivity.IDLE
2022-02-08 15:50:35,422 DEBUG worker.py:314 -- client gRPC channel state change: ChannelConnectivity.CONNECTING
2022-02-08 15:50:35,928 DEBUG worker.py:314 -- client gRPC channel state change: ChannelConnectivity.READY
2022-02-08 15:50:35,928 DEBUG worker.py:660 -- Pinging server.
2022-02-08 15:50:38,273 DEBUG worker.py:532 -- Retaining ffffffffffffffffffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:38,274 DEBUG worker.py:459 -- Scheduling name: "get_dashboard_url"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\001\000\000\000"
options {
pickled_options: "\200\004\225)\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224K\000\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:38,344 DEBUG worker.py:532 -- Retaining b996d537eda8ba31ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:38,985 DEBUG worker.py:527 -- Releasing b996d537eda8ba31ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,056 DEBUG worker.py:532 -- Retaining ffffffffffffffffffffffffffffffffffffffff0500000002000000
2022-02-08 15:50:39,056 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,056 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,057 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,057 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,057 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,058 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,058 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,058 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,058 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,059 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,059 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,059 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,059 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,060 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,060 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,060 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,061 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,061 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,061 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,062 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,062 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,062 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,062 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,062 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,063 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,063 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,063 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,063 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,064 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,064 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,064 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,064 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,065 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,065 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,065 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,065 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,066 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,066 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,066 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,066 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,066 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,067 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,067 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,067 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,067 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,067 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,068 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,068 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,068 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,068 DEBUG worker.py:459 -- Scheduling name: "task"
payload_id: "\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\005\000\000\000\002\000\000\000"
options {
pickled_options: "\200\004\225(\001\000\000\000\000\000\000}\224(\214\013num_returns\224N\214\010num_cpus\224N\214\010num_gpus\224N\214\006memory\224N\214\023object_store_memory\224N\214\020accelerator_type\224N\214\tresources\224N\214\013max_retries\224N\214\020retry_exceptions\224N\214\017placement_group\224\214\007default\224\214\034placement_group_bundle_index\224J\377\377\377\377\214#placement_group_capture_child_tasks\224N\214\013runtime_env\224N\214\004name\224\214\000\224\214\023scheduling_strategy\224Nu."
}
baseline_options {
pickled_options: "\200\004\225\235\000\000\000\000\000\000\000}\224(\214\010num_cpus\224K\001\214\010num_gpus\224N\214\tmax_calls\224K\000\214\013max_retries\224K\003\214\tresources\224N\214\020accelerator_type\224N\214\013num_returns\224K\001\214\006memory\224N\214\013runtime_env\224N\214\023scheduling_strategy\224Nu."
}
2022-02-08 15:50:39,131 DEBUG worker.py:532 -- Retaining e413db02c7a52895ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,131 DEBUG worker.py:532 -- Retaining 421b46655a0e20bdffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,132 DEBUG worker.py:532 -- Retaining 59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,132 DEBUG worker.py:532 -- Retaining 1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,133 DEBUG worker.py:532 -- Retaining 2c7016ae20dc760bffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,133 DEBUG worker.py:532 -- Retaining 8778541c095e32a1ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,134 DEBUG worker.py:532 -- Retaining ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,136 DEBUG worker.py:532 -- Retaining 69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,137 DEBUG worker.py:532 -- Retaining f13f2849d3e23e8effffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,137 DEBUG worker.py:532 -- Retaining d60e35309d5d500affffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,212 DEBUG worker.py:532 -- Retaining 80ae01584c090e42ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,212 DEBUG worker.py:532 -- Retaining 4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,212 DEBUG worker.py:532 -- Retaining 89822183d62eb606ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,213 DEBUG worker.py:532 -- Retaining ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,213 DEBUG worker.py:532 -- Retaining 8b36e77bbf694769ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,213 DEBUG worker.py:532 -- Retaining 1714485ad688d855ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,213 DEBUG worker.py:532 -- Retaining a28bfffad44ccecaffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,213 DEBUG worker.py:532 -- Retaining a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,213 DEBUG worker.py:532 -- Retaining 8ccd3e54e584583bffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,213 DEBUG worker.py:532 -- Retaining f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,214 DEBUG worker.py:532 -- Retaining 7e993a2f27da930fffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,214 DEBUG worker.py:532 -- Retaining 8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,214 DEBUG worker.py:532 -- Retaining bf60743f0229cd0dffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,214 DEBUG worker.py:532 -- Retaining e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,214 DEBUG worker.py:532 -- Retaining 7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,214 DEBUG worker.py:532 -- Retaining a07e5c87bf456562ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,215 DEBUG worker.py:532 -- Retaining f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,215 DEBUG worker.py:532 -- Retaining 954755a24622080fffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,215 DEBUG worker.py:532 -- Retaining 99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,215 DEBUG worker.py:532 -- Retaining 22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,215 DEBUG worker.py:532 -- Retaining 8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,215 DEBUG worker.py:532 -- Retaining dc9bb175087dae14ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,216 DEBUG worker.py:532 -- Retaining 9ea80f565708140effffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,216 DEBUG worker.py:532 -- Retaining 2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,216 DEBUG worker.py:532 -- Retaining e47d633fe602977dffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,216 DEBUG worker.py:532 -- Retaining 73a727873e8d133affffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,216 DEBUG worker.py:532 -- Retaining 4804a13e788229cbffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,216 DEBUG worker.py:532 -- Retaining 8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,216 DEBUG worker.py:532 -- Retaining e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,217 DEBUG worker.py:532 -- Retaining 3d5a78e5375defbbffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,217 DEBUG worker.py:532 -- Retaining 3df4e4a69944735dffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,217 DEBUG worker.py:532 -- Retaining ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,217 DEBUG worker.py:532 -- Retaining 4bf3d0acca90c31effffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,217 DEBUG worker.py:532 -- Retaining 923ce73a223a406dffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,217 DEBUG worker.py:532 -- Retaining 782164c2351a5b45ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,217 DEBUG worker.py:532 -- Retaining a079b90de7430202ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,218 DEBUG worker.py:532 -- Retaining 42e57f55d595ae78ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,326 DEBUG worker.py:532 -- Retaining 920d783927583c19ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,327 DEBUG worker.py:532 -- Retaining 935981ffd30971baffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:39,327 DEBUG worker.py:532 -- Retaining 398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000
2022-02-08 15:50:41,426 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:50:43,521 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:50:45,619 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:50:47,714 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:50:49,814 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:50:51,910 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:50:54,006 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:50:56,106 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:50:58,201 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:00,295 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:02,399 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:04,494 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:06,588 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:08,691 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:10,789 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:12,879 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:14,982 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:17,079 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:19,171 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:21,271 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:23,368 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:25,461 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:27,561 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:29,659 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:31,753 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:33,857 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:35,952 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:38,044 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:40,343 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:42,422 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:44,491 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:46,568 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:48,639 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:50,708 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:52,776 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:54,847 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:56,926 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:51:59,029 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:52:01,133 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:52:03,222 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:52:05,321 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:52:07,418 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:52:09,512 DEBUG worker.py:364 -- Internal retry for get [ClientObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ClientObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ClientObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ClientObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ClientObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ClientObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ClientObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ClientObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ClientObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ClientObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ClientObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ClientObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ClientObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ClientObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ClientObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ClientObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ClientObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ClientObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ClientObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ClientObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ClientObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ClientObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ClientObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ClientObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ClientObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ClientObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ClientObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ClientObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ClientObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ClientObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ClientObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ClientObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ClientObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ClientObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ClientObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ClientObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ClientObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ClientObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ClientObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ClientObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ClientObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ClientObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 15:52:11,287 DEBUG dataclient.py:119 -- Got unawaited response connection_cleanup {
}
2022-02-08 15:52:11,352 DEBUG dataclient.py:110 -- Shutting down data channel.
Raw
ray-operator.log
======== Autoscaler status: 2022-02-08 13:50:36.446033 ========
Node status
---------------------------------------------------------------
Healthy:
1 head
1 wkr-15cpu30g-ondemand
1 wkr-7cpu14g-spot
Pending:
(no pending nodes)
Recent failures:
(no failures)
Resources
---------------------------------------------------------------
Usage:
0.0/22.0 CPU
0.00/35.700 GiB memory
0.00/14.142 GiB object_store_memory
Demands:
(no resource demands)
ray-py38-cu112,karpenter:2022-02-08 13:50:36,512 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status_legacy' b"Cluster status: 2 nodes\n - MostDelayedHeartbeats: {'10.16.87.89': 0.5632271766662598, '10.16.109.160': 0.5631787776947021, '10.16.73.237': 0.5631287097930908}\n - NodeIdleSeconds: Min=0 Mean=54 Max=136\n - ResourceUsage: 0.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory\n - TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0\nWorker node types:\n - wkr-15cpu30g-ondemand: 1\n - wkr-7cpu14g-spot: 1" True None
ray-py38-cu112,karpenter:2022-02-08 13:50:36,514 DEBUG legacy_info_string.py:26 -- Cluster status: 2 nodes
- MostDelayedHeartbeats: {'10.16.87.89': 0.5632271766662598, '10.16.109.160': 0.5631787776947021, '10.16.73.237': 0.5631287097930908}
- NodeIdleSeconds: Min=0 Mean=54 Max=136
- ResourceUsage: 0.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
Worker node types:
- wkr-15cpu30g-ondemand: 1
- wkr-7cpu14g-spot: 1
ray-py38-cu112,karpenter:2022-02-08 13:50:36,675 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:50:36,724 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:50:36,772 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:50:36,809 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:50:37,035 DEBUG resource_demand_scheduler.py:201 -- Cluster resources: [{'object_store_memory': 2053644202.0, 'memory': 5261334937.0, 'node:10.16.109.160': 1.0}, {'memory': 22548578304.0, 'node:10.16.73.237': 1.0, 'CPU': 15.0, 'object_store_memory': 8973386956.0}, {'object_store_memory': 4157303193.0, 'CPU': 7.0, 'memory': 10522669875.0, 'node:10.16.87.89': 1.0}]
ray-py38-cu112,karpenter:2022-02-08 13:50:37,035 DEBUG resource_demand_scheduler.py:202 -- Node counts: defaultdict(<class 'int'>, {'head': 1, 'wkr-15cpu30g-ondemand': 1, 'wkr-7cpu14g-spot': 1})
ray-py38-cu112,karpenter:2022-02-08 13:50:37,035 DEBUG resource_demand_scheduler.py:219 -- Placement group demands: []
ray-py38-cu112,karpenter:2022-02-08 13:50:37,036 DEBUG resource_demand_scheduler.py:283 -- Resource demands: []
ray-py38-cu112,karpenter:2022-02-08 13:50:37,036 DEBUG resource_demand_scheduler.py:284 -- Unfulfilled demands: []
ray-py38-cu112,karpenter:2022-02-08 13:50:37,036 DEBUG resource_demand_scheduler.py:292 -- Final unfulfilled: []
ray-py38-cu112,karpenter:2022-02-08 13:50:37,128 DEBUG resource_demand_scheduler.py:317 -- Node requests: {}
ray-py38-cu112,karpenter:2022-02-08 13:50:37,220 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status' b'{"load_metrics_report": {"usage": {"object_store_memory": [86.0, 15184334437.0], "node:10.16.109.160": [0.0, 1.0], "memory": [0.0, 38332583116.0], "node:10.16.73.237": [0.0, 1.0], "CPU": [0.0, 22.0], "node:10.16.87.89": [0.0, 1.0]}, "resource_demand": [], "pg_demand": [], "request_demand": [], "node_types": [[{"memory": 5261334937.0, "object_store_memory": 2053644288.0, "node:10.16.109.160": 1.0}, 1], [{"CPU": 15.0, "node:10.16.73.237": 1.0, "memory": 22548578304.0, "object_store_memory": 8973386956.0}, 1], [{"memory": 10522669875.0, "node:10.16.87.89": 1.0, "object_store_memory": 4157303193.0, "CPU": 7.0}, 1]], "head_ip": null}, "time": 1644357035.8843253, "monitor_pid": 15133, "autoscaler_report": {"active_nodes": {"head": 1, "wkr-15cpu30g-ondemand": 1, "wkr-7cpu14g-spot": 1}, "pending_nodes": [], "pending_launches": {}, "failed_nodes": []}}' True None
ray-py38-cu112,karpenter:2022-02-08 13:50:42,223 INFO monitor.py:521 -- Logging raw resource message pulled from GCS.
ray-py38-cu112,karpenter:2022-02-08 13:50:42,223 INFO monitor.py:522 -- batch {
node_id: "\341\262\255\300\327\014\331\367\350\354\025\275\257\225\210[\022.\276\026\260\305 :f`\264\243"
resources_available {
key: "memory"
value: 10522669875.0
}
resources_available {
key: "node:10.16.87.89"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 4157303193.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 7.0
}
resources_total {
key: "memory"
value: 10522669875.0
}
resources_total {
key: "node:10.16.87.89"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 4157303193.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
resource_demands {
shape {
key: "CPU"
value: 1.0
}
num_ready_requests_queued: 1
}
}
node_manager_address: "10.16.87.89"
}
batch {
node_id: "\240u}D\242\243\030\206PG\251\364\213\306\202y\306f\375\254\251\03294\020\024!\263"
resources_available {
key: "memory"
value: 5261334937.0
}
resources_available {
key: "node:10.16.109.160"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 2053643576.0
}
resources_available_changed: true
resources_total {
key: "memory"
value: 5261334937.0
}
resources_total {
key: "node:10.16.109.160"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 2053644288.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.109.160"
}
batch {
node_id: "\267-\201\317\372\212\311\207\271\240\261\277\311\356\363^\243\357\325\362\345\271\024?j\034\\m"
resources_available {
key: "memory"
value: 22548578304.0
}
resources_available {
key: "node:10.16.73.237"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 8973386956.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 15.0
}
resources_total {
key: "memory"
value: 22548578304.0
}
resources_total {
key: "node:10.16.73.237"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 8973386956.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.73.237"
}
resource_load_by_shape {
resource_demands {
shape {
key: "CPU"
value: 1.0
}
num_ready_requests_queued: 1
}
}
placement_group_load {
}
ray-py38-cu112,karpenter:2022-02-08 13:50:42,223 INFO monitor.py:523 -- Done logging raw resource message.
ray-py38-cu112,karpenter:2022-02-08 13:50:42,224 DEBUG gcs_utils.py:228 -- internal_kv_get b'autoscaler_resource_request' None
ray-py38-cu112,karpenter:2022-02-08 13:50:42,849 INFO autoscaler.py:327 --
======== Autoscaler status: 2022-02-08 13:50:42.849600 ========
Node status
---------------------------------------------------------------
Healthy:
1 head
1 wkr-15cpu30g-ondemand
1 wkr-7cpu14g-spot
Pending:
(no pending nodes)
Recent failures:
(no failures)
Resources
---------------------------------------------------------------
Usage:
22.0/22.0 CPU
0.00/35.700 GiB memory
0.00/14.142 GiB object_store_memory
Demands:
{'CPU': 1.0}: 1+ pending tasks/actors
ray-py38-cu112,karpenter:2022-02-08 13:50:42,928 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status_legacy' b"Cluster status: 2 nodes\n - MostDelayedHeartbeats: {'10.16.87.89': 0.625971794128418, '10.16.109.160': 0.6258735656738281, '10.16.73.237': 0.62577223777771}\n - NodeIdleSeconds: Min=0 Mean=0 Max=0\n - ResourceUsage: 22.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory\n - TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0\nWorker node types:\n - wkr-15cpu30g-ondemand: 1\n - wkr-7cpu14g-spot: 1" True None
ray-py38-cu112,karpenter:2022-02-08 13:50:42,929 DEBUG legacy_info_string.py:26 -- Cluster status: 2 nodes
- MostDelayedHeartbeats: {'10.16.87.89': 0.625971794128418, '10.16.109.160': 0.6258735656738281, '10.16.73.237': 0.62577223777771}
- NodeIdleSeconds: Min=0 Mean=0 Max=0
- ResourceUsage: 22.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
Worker node types:
- wkr-15cpu30g-ondemand: 1
- wkr-7cpu14g-spot: 1
ray-py38-cu112,karpenter:2022-02-08 13:50:43,104 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:50:43,155 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:50:43,207 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:50:43,238 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:50:43,496 DEBUG resource_demand_scheduler.py:201 -- Cluster resources: [{'object_store_memory': 2053643576.0, 'memory': 5261334937.0, 'node:10.16.109.160': 1.0}, {'object_store_memory': 8973386956.0, 'node:10.16.73.237': 1.0, 'memory': 22548578304.0, 'CPU': 0.0}, {'node:10.16.87.89': 1.0, 'object_store_memory': 4157303193.0, 'memory': 10522669875.0, 'CPU': 0.0}]
ray-py38-cu112,karpenter:2022-02-08 13:50:43,496 DEBUG resource_demand_scheduler.py:202 -- Node counts: defaultdict(<class 'int'>, {'head': 1, 'wkr-15cpu30g-ondemand': 1, 'wkr-7cpu14g-spot': 1})
ray-py38-cu112,karpenter:2022-02-08 13:50:43,496 DEBUG resource_demand_scheduler.py:219 -- Placement group demands: []
ray-py38-cu112,karpenter:2022-02-08 13:50:43,496 DEBUG resource_demand_scheduler.py:283 -- Resource demands: [{'CPU': 1.0}]
ray-py38-cu112,karpenter:2022-02-08 13:50:43,496 DEBUG resource_demand_scheduler.py:284 -- Unfulfilled demands: [{'CPU': 1.0}]
ray-py38-cu112,karpenter:2022-02-08 13:50:43,497 DEBUG resource_demand_scheduler.py:292 -- Final unfulfilled: []
ray-py38-cu112,karpenter:2022-02-08 13:50:43,624 DEBUG resource_demand_scheduler.py:317 -- Node requests: {'wkr-7cpu14g-spot': 1}
ray-py38-cu112,karpenter:2022-02-08 13:50:43,624 INFO autoscaler.py:1216 -- StandardAutoscaler: Queue 1 new nodes for launch
ray-py38-cu112,karpenter:2022-02-08 13:50:43,629 INFO node_launcher.py:123 -- NodeLauncher0: Got 1 nodes to launch.
ray-py38-cu112,karpenter:2022-02-08 13:50:43,630 INFO node_launcher.py:123 -- NodeLauncher0: Launching 1 nodes, type wkr-7cpu14g-spot.
ray-py38-cu112,karpenter:2022-02-08 13:50:43,630 INFO node_provider.py:145 -- KubernetesNodeProvider: calling create_namespaced_pod (count=1).
ray-py38-cu112,karpenter:2022-02-08 13:50:43,751 INFO monitor.py:386 -- :event_summary:Adding 1 nodes of type wkr-7cpu14g-spot.
ray-py38-cu112,karpenter:2022-02-08 13:50:43,752 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status' b'{"load_metrics_report": {"usage": {"memory": [0.0, 38332583116.0], "object_store_memory": [712.0, 15184334437.0], "node:10.16.109.160": [0.0, 1.0], "node:10.16.73.237": [0.0, 1.0], "CPU": [22.0, 22.0], "node:10.16.87.89": [0.0, 1.0]}, "resource_demand": [[{"CPU": 1.0}, 1]], "pg_demand": [], "request_demand": [], "node_types": [[{"memory": 5261334937.0, "object_store_memory": 2053644288.0, "node:10.16.109.160": 1.0}, 1], [{"CPU": 15.0, "node:10.16.73.237": 1.0, "memory": 22548578304.0, "object_store_memory": 8973386956.0}, 1], [{"CPU": 7.0, "node:10.16.87.89": 1.0, "object_store_memory": 4157303193.0, "memory": 10522669875.0}, 1]], "head_ip": null}, "time": 1644357042.2267685, "monitor_pid": 15133, "autoscaler_report": {"active_nodes": {"head": 1, "wkr-15cpu30g-ondemand": 1, "wkr-7cpu14g-spot": 1}, "pending_nodes": [], "pending_launches": {}, "failed_nodes": []}}' True None
ray-py38-cu112,karpenter:2022-02-08 13:50:48,759 INFO monitor.py:521 -- Logging raw resource message pulled from GCS.
ray-py38-cu112,karpenter:2022-02-08 13:50:48,759 INFO monitor.py:522 -- batch {
node_id: "\341\262\255\300\327\014\331\367\350\354\025\275\257\225\210[\022.\276\026\260\305 :f`\264\243"
resources_available {
key: "memory"
value: 10522669875.0
}
resources_available {
key: "node:10.16.87.89"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 4157303193.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 7.0
}
resources_total {
key: "memory"
value: 10522669875.0
}
resources_total {
key: "node:10.16.87.89"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 4157303193.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
resource_demands {
shape {
key: "CPU"
value: 1.0
}
num_ready_requests_queued: 1
}
}
node_manager_address: "10.16.87.89"
}
batch {
node_id: "\240u}D\242\243\030\206PG\251\364\213\306\202y\306f\375\254\251\03294\020\024!\263"
resources_available {
key: "memory"
value: 5261334937.0
}
resources_available {
key: "node:10.16.109.160"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 2053643576.0
}
resources_available_changed: true
resources_total {
key: "memory"
value: 5261334937.0
}
resources_total {
key: "node:10.16.109.160"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 2053644288.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.109.160"
}
batch {
node_id: "\267-\201\317\372\212\311\207\271\240\261\277\311\356\363^\243\357\325\362\345\271\024?j\034\\m"
resources_available {
key: "memory"
value: 22548578304.0
}
resources_available {
key: "node:10.16.73.237"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 8973386956.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 15.0
}
resources_total {
key: "memory"
value: 22548578304.0
}
resources_total {
key: "node:10.16.73.237"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 8973386956.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.73.237"
}
resource_load_by_shape {
resource_demands {
shape {
key: "CPU"
value: 1.0
}
num_ready_requests_queued: 1
}
}
placement_group_load {
}
ray-py38-cu112,karpenter:2022-02-08 13:50:48,759 INFO monitor.py:523 -- Done logging raw resource message.
ray-py38-cu112,karpenter:2022-02-08 13:50:48,760 DEBUG gcs_utils.py:228 -- internal_kv_get b'autoscaler_resource_request' None
ray-py38-cu112,karpenter:2022-02-08 13:50:49,645 INFO autoscaler.py:327 --
======== Autoscaler status: 2022-02-08 13:50:49.645072 ========
Node status
---------------------------------------------------------------
Healthy:
1 head
1 wkr-15cpu30g-ondemand
1 wkr-7cpu14g-spot
Pending:
None: wkr-7cpu14g-spot, uninitialized
Recent failures:
(no failures)
Resources
---------------------------------------------------------------
Usage:
22.0/22.0 CPU
0.00/35.700 GiB memory
0.00/14.142 GiB object_store_memory
Demands:
{'CPU': 1.0}: 1+ pending tasks/actors
ray-py38-cu112,karpenter:2022-02-08 13:50:49,776 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status_legacy' b"Cluster status: 3 nodes\n - MostDelayedHeartbeats: {'10.16.87.89': 0.8852829933166504, '10.16.109.160': 0.88521409034729, '10.16.73.237': 0.8851485252380371}\n - NodeIdleSeconds: Min=0 Mean=0 Max=0\n - ResourceUsage: 22.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory\n - TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0\nWorker node types:\n - wkr-15cpu30g-ondemand: 1\n - wkr-7cpu14g-spot: 2" True None
ray-py38-cu112,karpenter:2022-02-08 13:50:49,777 DEBUG legacy_info_string.py:26 -- Cluster status: 3 nodes
- MostDelayedHeartbeats: {'10.16.87.89': 0.8852829933166504, '10.16.109.160': 0.88521409034729, '10.16.73.237': 0.8851485252380371}
- NodeIdleSeconds: Min=0 Mean=0 Max=0
- ResourceUsage: 22.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
Worker node types:
- wkr-15cpu30g-ondemand: 1
- wkr-7cpu14g-spot: 2
ray-py38-cu112,karpenter:2022-02-08 13:50:50,091 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:50:50,153 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-czw85 is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:50:50,191 DEBUG autoscaler.py:606 -- ray-py38-cu112-wkr-7cpu14g--spot-czw85: Starting new thread runner.
ray-py38-cu112,karpenter:2022-02-08 13:50:50,213 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:50:50,251 INFO autoscaler.py:1165 -- Creating new (spawn_updater) updater thread for node ray-py38-cu112-wkr-7cpu14g--spot-czw85.
ray-py38-cu112,karpenter:2022-02-08 13:50:50,326 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:50:50,361 INFO command_runner.py:179 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-czw85: Running kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)'
ray-py38-cu112,karpenter:2022-02-08 13:50:50,397 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
Unable to use a TTY - input is not a terminal or the right kind of file
Error from server: no preferred addresses found; known addresses: []
ray-py38-cu112,karpenter:2022-02-08 13:50:50,680 DEBUG resource_demand_scheduler.py:201 -- Cluster resources: [{'memory': 5261334937.0, 'object_store_memory': 2053643576.0, 'node:10.16.109.160': 1.0}, {'object_store_memory': 8973386956.0, 'node:10.16.73.237': 1.0, 'memory': 22548578304.0, 'CPU': 0.0}, {'CPU': 7.0, 'GPU': 0, 'memory': 10522669875.0, 'object_store_memory': 4157303193.0}, {'memory': 10522669875.0, 'object_store_memory': 4157303193.0, 'node:10.16.87.89': 1.0, 'CPU': 0.0}]
ray-py38-cu112,karpenter:2022-02-08 13:50:50,680 DEBUG resource_demand_scheduler.py:202 -- Node counts: defaultdict(<class 'int'>, {'head': 1, 'wkr-15cpu30g-ondemand': 1, 'wkr-7cpu14g-spot': 2})
ray-py38-cu112,karpenter:2022-02-08 13:50:50,680 DEBUG resource_demand_scheduler.py:219 -- Placement group demands: []
ray-py38-cu112,karpenter:2022-02-08 13:50:50,681 DEBUG resource_demand_scheduler.py:283 -- Resource demands: [{'CPU': 1.0}]
ray-py38-cu112,karpenter:2022-02-08 13:50:50,681 DEBUG resource_demand_scheduler.py:284 -- Unfulfilled demands: []
ray-py38-cu112,karpenter:2022-02-08 13:50:50,681 DEBUG resource_demand_scheduler.py:292 -- Final unfulfilled: []
ray-py38-cu112,karpenter:2022-02-08 13:50:50,793 DEBUG resource_demand_scheduler.py:317 -- Node requests: {}
ray-py38-cu112,karpenter:2022-02-08 13:50:50,914 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status' b'{"load_metrics_report": {"usage": {"memory": [0.0, 38332583116.0], "node:10.16.109.160": [0.0, 1.0], "object_store_memory": [712.0, 15184334437.0], "CPU": [22.0, 22.0], "node:10.16.73.237": [0.0, 1.0], "node:10.16.87.89": [0.0, 1.0]}, "resource_demand": [[{"CPU": 1.0}, 1]], "pg_demand": [], "request_demand": [], "node_types": [[{"memory": 5261334937.0, "node:10.16.109.160": 1.0, "object_store_memory": 2053644288.0}, 1], [{"CPU": 15.0, "object_store_memory": 8973386956.0, "memory": 22548578304.0, "node:10.16.73.237": 1.0}, 1], [{"CPU": 7.0, "node:10.16.87.89": 1.0, "object_store_memory": 4157303193.0, "memory": 10522669875.0}, 1]], "head_ip": null}, "time": 1644357048.7615864, "monitor_pid": 15133, "autoscaler_report": {"active_nodes": {"head": 1, "wkr-15cpu30g-ondemand": 1, "wkr-7cpu14g-spot": 1}, "pending_nodes": [[null, "wkr-7cpu14g-spot", "waiting-for-ssh"]], "pending_launches": {}, "failed_nodes": []}}' True None
ray-py38-cu112,karpenter:2022-02-08 13:50:55,580 INFO command_runner.py:179 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-czw85: Running kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)'
Unable to use a TTY - input is not a terminal or the right kind of file
Error from server: no preferred addresses found; known addresses: []
ray-py38-cu112,karpenter:2022-02-08 13:50:55,922 INFO monitor.py:521 -- Logging raw resource message pulled from GCS.
ray-py38-cu112,karpenter:2022-02-08 13:50:55,922 INFO monitor.py:522 -- batch {
node_id: "\341\262\255\300\327\014\331\367\350\354\025\275\257\225\210[\022.\276\026\260\305 :f`\264\243"
resources_available {
key: "memory"
value: 10522669875.0
}
resources_available {
key: "node:10.16.87.89"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 4157303193.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 7.0
}
resources_total {
key: "memory"
value: 10522669875.0
}
resources_total {
key: "node:10.16.87.89"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 4157303193.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
resource_demands {
shape {
key: "CPU"
value: 1.0
}
num_ready_requests_queued: 1
}
}
node_manager_address: "10.16.87.89"
}
batch {
node_id: "\240u}D\242\243\030\206PG\251\364\213\306\202y\306f\375\254\251\03294\020\024!\263"
resources_available {
key: "memory"
value: 5261334937.0
}
resources_available {
key: "node:10.16.109.160"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 2053643576.0
}
resources_available_changed: true
resources_total {
key: "memory"
value: 5261334937.0
}
resources_total {
key: "node:10.16.109.160"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 2053644288.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.109.160"
}
batch {
node_id: "\267-\201\317\372\212\311\207\271\240\261\277\311\356\363^\243\357\325\362\345\271\024?j\034\\m"
resources_available {
key: "memory"
value: 22548578304.0
}
resources_available {
key: "node:10.16.73.237"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 8973386956.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 15.0
}
resources_total {
key: "memory"
value: 22548578304.0
}
resources_total {
key: "node:10.16.73.237"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 8973386956.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.73.237"
}
resource_load_by_shape {
resource_demands {
shape {
key: "CPU"
value: 1.0
}
num_ready_requests_queued: 1
}
}
placement_group_load {
}
ray-py38-cu112,karpenter:2022-02-08 13:50:55,922 INFO monitor.py:523 -- Done logging raw resource message.
ray-py38-cu112,karpenter:2022-02-08 13:50:55,923 DEBUG gcs_utils.py:228 -- internal_kv_get b'autoscaler_resource_request' None
ray-py38-cu112,karpenter:2022-02-08 13:50:56,610 INFO autoscaler.py:327 --
======== Autoscaler status: 2022-02-08 13:50:56.610615 ========
Node status
---------------------------------------------------------------
Healthy:
1 head
1 wkr-15cpu30g-ondemand
1 wkr-7cpu14g-spot
Pending:
None: wkr-7cpu14g-spot, waiting-for-ssh
Recent failures:
(no failures)
Resources
---------------------------------------------------------------
Usage:
22.0/22.0 CPU
0.00/35.700 GiB memory
0.00/14.142 GiB object_store_memory
Demands:
{'CPU': 1.0}: 1+ pending tasks/actors
ray-py38-cu112,karpenter:2022-02-08 13:50:56,716 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status_legacy' b"Cluster status: 3 nodes (1 updating)\n - MostDelayedHeartbeats: {'10.16.87.89': 0.6882650852203369, '10.16.109.160': 0.6881914138793945, '10.16.73.237': 0.6881265640258789}\n - NodeIdleSeconds: Min=0 Mean=0 Max=0\n - ResourceUsage: 22.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory\n - TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0\nWorker node types:\n - wkr-15cpu30g-ondemand: 1\n - wkr-7cpu14g-spot: 2" True None
ray-py38-cu112,karpenter:2022-02-08 13:50:56,717 DEBUG legacy_info_string.py:26 -- Cluster status: 3 nodes (1 updating)
- MostDelayedHeartbeats: {'10.16.87.89': 0.6882650852203369, '10.16.109.160': 0.6881914138793945, '10.16.73.237': 0.6881265640258789}
- NodeIdleSeconds: Min=0 Mean=0 Max=0
- ResourceUsage: 22.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
Worker node types:
- wkr-15cpu30g-ondemand: 1
- wkr-7cpu14g-spot: 2
ray-py38-cu112,karpenter:2022-02-08 13:50:56,994 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:50:57,052 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:50:57,114 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:50:57,147 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:50:57,408 DEBUG resource_demand_scheduler.py:201 -- Cluster resources: [{'node:10.16.109.160': 1.0, 'memory': 5261334937.0, 'object_store_memory': 2053643576.0}, {'node:10.16.73.237': 1.0, 'memory': 22548578304.0, 'object_store_memory': 8973386956.0, 'CPU': 0.0}, {'CPU': 7.0, 'GPU': 0, 'memory': 10522669875.0, 'object_store_memory': 4157303193.0}, {'object_store_memory': 4157303193.0, 'node:10.16.87.89': 1.0, 'memory': 10522669875.0, 'CPU': 0.0}]
ray-py38-cu112,karpenter:2022-02-08 13:50:57,408 DEBUG resource_demand_scheduler.py:202 -- Node counts: defaultdict(<class 'int'>, {'head': 1, 'wkr-15cpu30g-ondemand': 1, 'wkr-7cpu14g-spot': 2})
ray-py38-cu112,karpenter:2022-02-08 13:50:57,409 DEBUG resource_demand_scheduler.py:219 -- Placement group demands: []
ray-py38-cu112,karpenter:2022-02-08 13:50:57,409 DEBUG resource_demand_scheduler.py:283 -- Resource demands: [{'CPU': 1.0}]
ray-py38-cu112,karpenter:2022-02-08 13:50:57,409 DEBUG resource_demand_scheduler.py:284 -- Unfulfilled demands: []
ray-py38-cu112,karpenter:2022-02-08 13:50:57,409 DEBUG resource_demand_scheduler.py:292 -- Final unfulfilled: []
ray-py38-cu112,karpenter:2022-02-08 13:50:57,533 DEBUG resource_demand_scheduler.py:317 -- Node requests: {}
ray-py38-cu112,karpenter:2022-02-08 13:50:57,651 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status' b'{"load_metrics_report": {"usage": {"memory": [0.0, 38332583116.0], "node:10.16.109.160": [0.0, 1.0], "object_store_memory": [712.0, 15184334437.0], "node:10.16.73.237": [0.0, 1.0], "CPU": [22.0, 22.0], "node:10.16.87.89": [0.0, 1.0]}, "resource_demand": [[{"CPU": 1.0}, 1]], "pg_demand": [], "request_demand": [], "node_types": [[{"memory": 5261334937.0, "node:10.16.109.160": 1.0, "object_store_memory": 2053644288.0}, 1], [{"node:10.16.73.237": 1.0, "object_store_memory": 8973386956.0, "memory": 22548578304.0, "CPU": 15.0}, 1], [{"memory": 10522669875.0, "node:10.16.87.89": 1.0, "object_store_memory": 4157303193.0, "CPU": 7.0}, 1]], "head_ip": null}, "time": 1644357055.9247482, "monitor_pid": 15133, "autoscaler_report": {"active_nodes": {"head": 1, "wkr-15cpu30g-ondemand": 1, "wkr-7cpu14g-spot": 1}, "pending_nodes": [[null, "wkr-7cpu14g-spot", "waiting-for-ssh"]], "pending_launches": {}, "failed_nodes": []}}' True None
ray-py38-cu112,karpenter:2022-02-08 13:51:00,774 INFO command_runner.py:179 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-czw85: Running kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)'
Unable to use a TTY - input is not a terminal or the right kind of file
Error from server: no preferred addresses found; known addresses: []
ray-py38-cu112,karpenter:2022-02-08 13:51:02,658 INFO monitor.py:521 -- Logging raw resource message pulled from GCS.
ray-py38-cu112,karpenter:2022-02-08 13:51:02,658 INFO monitor.py:522 -- batch {
node_id: "\341\262\255\300\327\014\331\367\350\354\025\275\257\225\210[\022.\276\026\260\305 :f`\264\243"
resources_available {
key: "memory"
value: 10522669875.0
}
resources_available {
key: "node:10.16.87.89"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 4157303193.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 7.0
}
resources_total {
key: "memory"
value: 10522669875.0
}
resources_total {
key: "node:10.16.87.89"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 4157303193.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
resource_demands {
shape {
key: "CPU"
value: 1.0
}
num_ready_requests_queued: 1
}
}
node_manager_address: "10.16.87.89"
}
batch {
node_id: "\240u}D\242\243\030\206PG\251\364\213\306\202y\306f\375\254\251\03294\020\024!\263"
resources_available {
key: "memory"
value: 5261334937.0
}
resources_available {
key: "node:10.16.109.160"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 2053643576.0
}
resources_available_changed: true
resources_total {
key: "memory"
value: 5261334937.0
}
resources_total {
key: "node:10.16.109.160"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 2053644288.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.109.160"
}
batch {
node_id: "\267-\201\317\372\212\311\207\271\240\261\277\311\356\363^\243\357\325\362\345\271\024?j\034\\m"
resources_available {
key: "memory"
value: 22548578304.0
}
resources_available {
key: "node:10.16.73.237"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 8973386956.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 15.0
}
resources_total {
key: "memory"
value: 22548578304.0
}
resources_total {
key: "node:10.16.73.237"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 8973386956.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.73.237"
}
resource_load_by_shape {
resource_demands {
shape {
key: "CPU"
value: 1.0
}
num_ready_requests_queued: 1
}
}
placement_group_load {
}
ray-py38-cu112,karpenter:2022-02-08 13:51:02,658 INFO monitor.py:523 -- Done logging raw resource message.
ray-py38-cu112,karpenter:2022-02-08 13:51:02,659 DEBUG gcs_utils.py:228 -- internal_kv_get b'autoscaler_resource_request' None
ray-py38-cu112,karpenter:2022-02-08 13:51:03,480 INFO autoscaler.py:327 --
======== Autoscaler status: 2022-02-08 13:51:03.480507 ========
Node status
---------------------------------------------------------------
Healthy:
1 head
1 wkr-15cpu30g-ondemand
1 wkr-7cpu14g-spot
Pending:
None: wkr-7cpu14g-spot, waiting-for-ssh
Recent failures:
(no failures)
Resources
---------------------------------------------------------------
Usage:
22.0/22.0 CPU
0.00/35.700 GiB memory
0.00/14.142 GiB object_store_memory
Demands:
{'CPU': 1.0}: 1+ pending tasks/actors
ray-py38-cu112,karpenter:2022-02-08 13:51:03,617 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status_legacy' b"Cluster status: 3 nodes (1 updating)\n - MostDelayedHeartbeats: {'10.16.87.89': 0.8220739364624023, '10.16.109.160': 0.8220000267028809, '10.16.73.237': 0.8219308853149414}\n - NodeIdleSeconds: Min=0 Mean=0 Max=0\n - ResourceUsage: 22.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory\n - TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0\nWorker node types:\n - wkr-15cpu30g-ondemand: 1\n - wkr-7cpu14g-spot: 2" True None
ray-py38-cu112,karpenter:2022-02-08 13:51:03,619 DEBUG legacy_info_string.py:26 -- Cluster status: 3 nodes (1 updating)
- MostDelayedHeartbeats: {'10.16.87.89': 0.8220739364624023, '10.16.109.160': 0.8220000267028809, '10.16.73.237': 0.8219308853149414}
- NodeIdleSeconds: Min=0 Mean=0 Max=0
- ResourceUsage: 22.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
Worker node types:
- wkr-15cpu30g-ondemand: 1
- wkr-7cpu14g-spot: 2
ray-py38-cu112,karpenter:2022-02-08 13:51:03,935 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:04,021 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:04,083 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:04,116 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:04,399 DEBUG resource_demand_scheduler.py:201 -- Cluster resources: [{'memory': 5261334937.0, 'object_store_memory': 2053643576.0, 'node:10.16.109.160': 1.0}, {'object_store_memory': 8973386956.0, 'node:10.16.73.237': 1.0, 'memory': 22548578304.0, 'CPU': 0.0}, {'CPU': 7.0, 'GPU': 0, 'memory': 10522669875.0, 'object_store_memory': 4157303193.0}, {'node:10.16.87.89': 1.0, 'object_store_memory': 4157303193.0, 'memory': 10522669875.0, 'CPU': 0.0}]
ray-py38-cu112,karpenter:2022-02-08 13:51:04,399 DEBUG resource_demand_scheduler.py:202 -- Node counts: defaultdict(<class 'int'>, {'head': 1, 'wkr-15cpu30g-ondemand': 1, 'wkr-7cpu14g-spot': 2})
ray-py38-cu112,karpenter:2022-02-08 13:51:04,399 DEBUG resource_demand_scheduler.py:219 -- Placement group demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:04,400 DEBUG resource_demand_scheduler.py:283 -- Resource demands: [{'CPU': 1.0}]
ray-py38-cu112,karpenter:2022-02-08 13:51:04,400 DEBUG resource_demand_scheduler.py:284 -- Unfulfilled demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:04,400 DEBUG resource_demand_scheduler.py:292 -- Final unfulfilled: []
ray-py38-cu112,karpenter:2022-02-08 13:51:04,503 DEBUG resource_demand_scheduler.py:317 -- Node requests: {}
ray-py38-cu112,karpenter:2022-02-08 13:51:04,608 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status' b'{"load_metrics_report": {"usage": {"object_store_memory": [712.0, 15184334437.0], "node:10.16.109.160": [0.0, 1.0], "memory": [0.0, 38332583116.0], "node:10.16.73.237": [0.0, 1.0], "CPU": [22.0, 22.0], "node:10.16.87.89": [0.0, 1.0]}, "resource_demand": [[{"CPU": 1.0}, 1]], "pg_demand": [], "request_demand": [], "node_types": [[{"memory": 5261334937.0, "object_store_memory": 2053644288.0, "node:10.16.109.160": 1.0}, 1], [{"CPU": 15.0, "node:10.16.73.237": 1.0, "memory": 22548578304.0, "object_store_memory": 8973386956.0}, 1], [{"CPU": 7.0, "node:10.16.87.89": 1.0, "object_store_memory": 4157303193.0, "memory": 10522669875.0}, 1]], "head_ip": null}, "time": 1644357062.6609094, "monitor_pid": 15133, "autoscaler_report": {"active_nodes": {"head": 1, "wkr-15cpu30g-ondemand": 1, "wkr-7cpu14g-spot": 1}, "pending_nodes": [[null, "wkr-7cpu14g-spot", "waiting-for-ssh"]], "pending_launches": {}, "failed_nodes": []}}' True None
ray-py38-cu112,karpenter:2022-02-08 13:51:05,953 INFO command_runner.py:179 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-czw85: Running kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)'
Unable to use a TTY - input is not a terminal or the right kind of file
Error from server: no preferred addresses found; known addresses: []
ray-py38-cu112,karpenter:2022-02-08 13:51:09,615 INFO monitor.py:521 -- Logging raw resource message pulled from GCS.
ray-py38-cu112,karpenter:2022-02-08 13:51:09,616 INFO monitor.py:522 -- batch {
node_id: "\341\262\255\300\327\014\331\367\350\354\025\275\257\225\210[\022.\276\026\260\305 :f`\264\243"
resources_available {
key: "memory"
value: 10522669875.0
}
resources_available {
key: "node:10.16.87.89"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 4157303193.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 7.0
}
resources_total {
key: "memory"
value: 10522669875.0
}
resources_total {
key: "node:10.16.87.89"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 4157303193.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
resource_demands {
shape {
key: "CPU"
value: 1.0
}
num_ready_requests_queued: 1
}
}
node_manager_address: "10.16.87.89"
}
batch {
node_id: "\240u}D\242\243\030\206PG\251\364\213\306\202y\306f\375\254\251\03294\020\024!\263"
resources_available {
key: "memory"
value: 5261334937.0
}
resources_available {
key: "node:10.16.109.160"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 2053643576.0
}
resources_available_changed: true
resources_total {
key: "memory"
value: 5261334937.0
}
resources_total {
key: "node:10.16.109.160"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 2053644288.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.109.160"
}
batch {
node_id: "\267-\201\317\372\212\311\207\271\240\261\277\311\356\363^\243\357\325\362\345\271\024?j\034\\m"
resources_available {
key: "memory"
value: 22548578304.0
}
resources_available {
key: "node:10.16.73.237"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 8973386956.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 15.0
}
resources_total {
key: "memory"
value: 22548578304.0
}
resources_total {
key: "node:10.16.73.237"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 8973386956.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.73.237"
}
resource_load_by_shape {
resource_demands {
shape {
key: "CPU"
value: 1.0
}
num_ready_requests_queued: 1
}
}
placement_group_load {
}
ray-py38-cu112,karpenter:2022-02-08 13:51:09,616 INFO monitor.py:523 -- Done logging raw resource message.
ray-py38-cu112,karpenter:2022-02-08 13:51:09,616 DEBUG gcs_utils.py:228 -- internal_kv_get b'autoscaler_resource_request' None
ray-py38-cu112,karpenter:2022-02-08 13:51:10,396 INFO autoscaler.py:327 --
======== Autoscaler status: 2022-02-08 13:51:10.395891 ========
Node status
---------------------------------------------------------------
Healthy:
1 head
1 wkr-15cpu30g-ondemand
1 wkr-7cpu14g-spot
Pending:
None: wkr-7cpu14g-spot, waiting-for-ssh
Recent failures:
(no failures)
Resources
---------------------------------------------------------------
Usage:
22.0/22.0 CPU
0.00/35.700 GiB memory
0.00/14.142 GiB object_store_memory
Demands:
{'CPU': 1.0}: 1+ pending tasks/actors
ray-py38-cu112,karpenter:2022-02-08 13:51:10,534 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status_legacy' b"Cluster status: 3 nodes (1 updating)\n - MostDelayedHeartbeats: {'10.16.87.89': 0.7798099517822266, '10.16.109.160': 0.7797341346740723, '10.16.73.237': 0.7796516418457031}\n - NodeIdleSeconds: Min=0 Mean=0 Max=0\n - ResourceUsage: 22.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory\n - TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0\nWorker node types:\n - wkr-15cpu30g-ondemand: 1\n - wkr-7cpu14g-spot: 2" True None
ray-py38-cu112,karpenter:2022-02-08 13:51:10,535 DEBUG legacy_info_string.py:26 -- Cluster status: 3 nodes (1 updating)
- MostDelayedHeartbeats: {'10.16.87.89': 0.7798099517822266, '10.16.109.160': 0.7797341346740723, '10.16.73.237': 0.7796516418457031}
- NodeIdleSeconds: Min=0 Mean=0 Max=0
- ResourceUsage: 22.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
Worker node types:
- wkr-15cpu30g-ondemand: 1
- wkr-7cpu14g-spot: 2
ray-py38-cu112,karpenter:2022-02-08 13:51:10,798 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:10,843 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:10,889 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:10,918 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:11,120 INFO command_runner.py:179 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-czw85: Running kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)'
ray-py38-cu112,karpenter:2022-02-08 13:51:11,244 DEBUG resource_demand_scheduler.py:201 -- Cluster resources: [{'object_store_memory': 2053643576.0, 'memory': 5261334937.0, 'node:10.16.109.160': 1.0}, {'node:10.16.73.237': 1.0, 'object_store_memory': 8973386956.0, 'memory': 22548578304.0, 'CPU': 0.0}, {'CPU': 7.0, 'GPU': 0, 'memory': 10522669875.0, 'object_store_memory': 4157303193.0}, {'node:10.16.87.89': 1.0, 'memory': 10522669875.0, 'object_store_memory': 4157303193.0, 'CPU': 0.0}]
ray-py38-cu112,karpenter:2022-02-08 13:51:11,244 DEBUG resource_demand_scheduler.py:202 -- Node counts: defaultdict(<class 'int'>, {'head': 1, 'wkr-15cpu30g-ondemand': 1, 'wkr-7cpu14g-spot': 2})
ray-py38-cu112,karpenter:2022-02-08 13:51:11,244 DEBUG resource_demand_scheduler.py:219 -- Placement group demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:11,245 DEBUG resource_demand_scheduler.py:283 -- Resource demands: [{'CPU': 1.0}]
ray-py38-cu112,karpenter:2022-02-08 13:51:11,245 DEBUG resource_demand_scheduler.py:284 -- Unfulfilled demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:11,245 DEBUG resource_demand_scheduler.py:292 -- Final unfulfilled: []
Unable to use a TTY - input is not a terminal or the right kind of file
Error from server: no preferred addresses found; known addresses: []
ray-py38-cu112,karpenter:2022-02-08 13:51:11,389 DEBUG resource_demand_scheduler.py:317 -- Node requests: {}
ray-py38-cu112,karpenter:2022-02-08 13:51:11,529 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status' b'{"load_metrics_report": {"usage": {"node:10.16.109.160": [0.0, 1.0], "memory": [0.0, 38332583116.0], "object_store_memory": [712.0, 15184334437.0], "CPU": [22.0, 22.0], "node:10.16.73.237": [0.0, 1.0], "node:10.16.87.89": [0.0, 1.0]}, "resource_demand": [[{"CPU": 1.0}, 1]], "pg_demand": [], "request_demand": [], "node_types": [[{"memory": 5261334937.0, "node:10.16.109.160": 1.0, "object_store_memory": 2053644288.0}, 1], [{"CPU": 15.0, "object_store_memory": 8973386956.0, "memory": 22548578304.0, "node:10.16.73.237": 1.0}, 1], [{"memory": 10522669875.0, "node:10.16.87.89": 1.0, "object_store_memory": 4157303193.0, "CPU": 7.0}, 1]], "head_ip": null}, "time": 1644357069.6180155, "monitor_pid": 15133, "autoscaler_report": {"active_nodes": {"head": 1, "wkr-15cpu30g-ondemand": 1, "wkr-7cpu14g-spot": 1}, "pending_nodes": [[null, "wkr-7cpu14g-spot", "waiting-for-ssh"]], "pending_launches": {}, "failed_nodes": []}}' True None
ray-py38-cu112,karpenter:2022-02-08 13:51:16,309 INFO command_runner.py:179 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-czw85: Running kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)'
Unable to use a TTY - input is not a terminal or the right kind of file
Error from server: no preferred addresses found; known addresses: []
ray-py38-cu112,karpenter:2022-02-08 13:51:16,536 INFO monitor.py:521 -- Logging raw resource message pulled from GCS.
ray-py38-cu112,karpenter:2022-02-08 13:51:16,536 INFO monitor.py:522 -- batch {
node_id: "\341\262\255\300\327\014\331\367\350\354\025\275\257\225\210[\022.\276\026\260\305 :f`\264\243"
resources_available {
key: "memory"
value: 10522669875.0
}
resources_available {
key: "node:10.16.87.89"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 4157303193.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 7.0
}
resources_total {
key: "memory"
value: 10522669875.0
}
resources_total {
key: "node:10.16.87.89"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 4157303193.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
resource_demands {
shape {
key: "CPU"
value: 1.0
}
num_ready_requests_queued: 1
}
}
node_manager_address: "10.16.87.89"
}
batch {
node_id: "\240u}D\242\243\030\206PG\251\364\213\306\202y\306f\375\254\251\03294\020\024!\263"
resources_available {
key: "memory"
value: 5261334937.0
}
resources_available {
key: "node:10.16.109.160"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 2053643576.0
}
resources_available_changed: true
resources_total {
key: "memory"
value: 5261334937.0
}
resources_total {
key: "node:10.16.109.160"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 2053644288.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.109.160"
}
batch {
node_id: "\267-\201\317\372\212\311\207\271\240\261\277\311\356\363^\243\357\325\362\345\271\024?j\034\\m"
resources_available {
key: "memory"
value: 22548578304.0
}
resources_available {
key: "node:10.16.73.237"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 8973386956.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 15.0
}
resources_total {
key: "memory"
value: 22548578304.0
}
resources_total {
key: "node:10.16.73.237"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 8973386956.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.73.237"
}
resource_load_by_shape {
resource_demands {
shape {
key: "CPU"
value: 1.0
}
num_ready_requests_queued: 1
}
}
placement_group_load {
}
ray-py38-cu112,karpenter:2022-02-08 13:51:16,537 INFO monitor.py:523 -- Done logging raw resource message.
ray-py38-cu112,karpenter:2022-02-08 13:51:16,537 DEBUG gcs_utils.py:228 -- internal_kv_get b'autoscaler_resource_request' None
ray-py38-cu112,karpenter:2022-02-08 13:51:17,169 INFO autoscaler.py:327 --
======== Autoscaler status: 2022-02-08 13:51:17.168948 ========
Node status
---------------------------------------------------------------
Healthy:
1 head
1 wkr-15cpu30g-ondemand
1 wkr-7cpu14g-spot
Pending:
None: wkr-7cpu14g-spot, waiting-for-ssh
Recent failures:
(no failures)
Resources
---------------------------------------------------------------
Usage:
22.0/22.0 CPU
0.00/35.700 GiB memory
0.00/14.142 GiB object_store_memory
Demands:
{'CPU': 1.0}: 1+ pending tasks/actors
ray-py38-cu112,karpenter:2022-02-08 13:51:17,262 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status_legacy' b"Cluster status: 3 nodes (1 updating)\n - MostDelayedHeartbeats: {'10.16.87.89': 0.6318743228912354, '10.16.109.160': 0.6317973136901855, '10.16.73.237': 0.6316561698913574}\n - NodeIdleSeconds: Min=0 Mean=0 Max=0\n - ResourceUsage: 22.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory\n - TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0\nWorker node types:\n - wkr-15cpu30g-ondemand: 1\n - wkr-7cpu14g-spot: 2" True None
ray-py38-cu112,karpenter:2022-02-08 13:51:17,263 DEBUG legacy_info_string.py:26 -- Cluster status: 3 nodes (1 updating)
- MostDelayedHeartbeats: {'10.16.87.89': 0.6318743228912354, '10.16.109.160': 0.6317973136901855, '10.16.73.237': 0.6316561698913574}
- NodeIdleSeconds: Min=0 Mean=0 Max=0
- ResourceUsage: 22.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
Worker node types:
- wkr-15cpu30g-ondemand: 1
- wkr-7cpu14g-spot: 2
ray-py38-cu112,karpenter:2022-02-08 13:51:17,511 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:17,554 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:17,595 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:17,623 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:17,873 DEBUG resource_demand_scheduler.py:201 -- Cluster resources: [{'node:10.16.109.160': 1.0, 'memory': 5261334937.0, 'object_store_memory': 2053643576.0}, {'memory': 22548578304.0, 'node:10.16.73.237': 1.0, 'object_store_memory': 8973386956.0, 'CPU': 0.0}, {'CPU': 7.0, 'GPU': 0, 'memory': 10522669875.0, 'object_store_memory': 4157303193.0}, {'memory': 10522669875.0, 'object_store_memory': 4157303193.0, 'node:10.16.87.89': 1.0, 'CPU': 0.0}]
ray-py38-cu112,karpenter:2022-02-08 13:51:17,873 DEBUG resource_demand_scheduler.py:202 -- Node counts: defaultdict(<class 'int'>, {'head': 1, 'wkr-15cpu30g-ondemand': 1, 'wkr-7cpu14g-spot': 2})
ray-py38-cu112,karpenter:2022-02-08 13:51:17,873 DEBUG resource_demand_scheduler.py:219 -- Placement group demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:17,873 DEBUG resource_demand_scheduler.py:283 -- Resource demands: [{'CPU': 1.0}]
ray-py38-cu112,karpenter:2022-02-08 13:51:17,873 DEBUG resource_demand_scheduler.py:284 -- Unfulfilled demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:17,873 DEBUG resource_demand_scheduler.py:292 -- Final unfulfilled: []
ray-py38-cu112,karpenter:2022-02-08 13:51:17,980 DEBUG resource_demand_scheduler.py:317 -- Node requests: {}
ray-py38-cu112,karpenter:2022-02-08 13:51:18,089 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status' b'{"load_metrics_report": {"usage": {"memory": [0.0, 38332583116.0], "object_store_memory": [712.0, 15184334437.0], "node:10.16.109.160": [0.0, 1.0], "CPU": [22.0, 22.0], "node:10.16.73.237": [0.0, 1.0], "node:10.16.87.89": [0.0, 1.0]}, "resource_demand": [[{"CPU": 1.0}, 1]], "pg_demand": [], "request_demand": [], "node_types": [[{"memory": 5261334937.0, "object_store_memory": 2053644288.0, "node:10.16.109.160": 1.0}, 1], [{"CPU": 15.0, "object_store_memory": 8973386956.0, "memory": 22548578304.0, "node:10.16.73.237": 1.0}, 1], [{"CPU": 7.0, "node:10.16.87.89": 1.0, "object_store_memory": 4157303193.0, "memory": 10522669875.0}, 1]], "head_ip": null}, "time": 1644357076.5392773, "monitor_pid": 15133, "autoscaler_report": {"active_nodes": {"head": 1, "wkr-15cpu30g-ondemand": 1, "wkr-7cpu14g-spot": 1}, "pending_nodes": [[null, "wkr-7cpu14g-spot", "waiting-for-ssh"]], "pending_launches": {}, "failed_nodes": []}}' True None
ray-py38-cu112,karpenter:2022-02-08 13:51:21,484 INFO command_runner.py:179 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-czw85: Running kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)'
Unable to use a TTY - input is not a terminal or the right kind of file
Error from server: no preferred addresses found; known addresses: []
2022-02-08 13:48:56,460 INFO updater.py:314 -- SSH still not available (Exit Status 1): kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-qf6zw -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)', retrying in 5 seconds.
2022-02-08 13:49:01,617 INFO updater.py:314 -- SSH still not available (Exit Status 1): kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-qf6zw -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)', retrying in 5 seconds.
2022-02-08 13:49:06,857 INFO updater.py:314 -- SSH still not available (Exit Status 1): kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-qf6zw -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)', retrying in 5 seconds.
2022-02-08 13:49:12,029 INFO updater.py:314 -- SSH still not available (Exit Status 1): kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-qf6zw -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)', retrying in 5 seconds.
2022-02-08 13:49:17,217 INFO updater.py:314 -- SSH still not available (Exit Status 1): kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-qf6zw -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)', retrying in 5 seconds.
2022-02-08 13:49:22,407 INFO updater.py:314 -- SSH still not available (Exit Status 1): kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-qf6zw -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)', retrying in 5 seconds.
2022-02-08 13:49:27,564 INFO updater.py:314 -- SSH still not available (Exit Status 1): kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-qf6zw -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)', retrying in 5 seconds.
2022-02-08 13:49:32,775 INFO updater.py:314 -- SSH still not available (Exit Status 1): kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-qf6zw -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)', retrying in 5 seconds.
2022-02-08 13:49:38,094 INFO updater.py:314 -- SSH still not available (Exit Status 1): kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-qf6zw -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)', retrying in 5 seconds.
2022-02-08 13:49:43,259 INFO updater.py:314 -- SSH still not available (Exit Status 1): kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-qf6zw -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)', retrying in 5 seconds.
2022-02-08 13:49:48,419 INFO updater.py:314 -- SSH still not available (Exit Status 1): kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-qf6zw -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)', retrying in 5 seconds.
2022-02-08 13:49:53,613 INFO updater.py:314 -- SSH still not available (Exit Status 1): kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-qf6zw -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)', retrying in 5 seconds.
2022-02-08 13:49:58,890 INFO updater.py:314 -- SSH still not available (Exit Status 1): kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-qf6zw -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)', retrying in 5 seconds.
2022-02-08 13:50:04,687 SUCC updater.py:279 -- Success.
2022-02-08 13:50:04,688 INFO log_timer.py:30 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-qf6zw: Got remote shell [LogTimer=349031ms]
2022-02-08 13:50:04,728 INFO updater.py:374 -- Updating cluster configuration. [hash=0583f296a9e95cee648f39db3bf2330c60a73c5f]
2022-02-08 13:50:04,799 INFO updater.py:380 -- New status: syncing-files
2022-02-08 13:50:04,800 INFO updater.py:238 -- [2/7] Processing file mounts
2022-02-08 13:50:04,800 INFO updater.py:256 -- [3/7] No worker file mounts to sync
2022-02-08 13:50:04,868 INFO updater.py:391 -- New status: setting-up
2022-02-08 13:50:04,868 INFO updater.py:434 -- [4/7] No initialization commands to run.
2022-02-08 13:50:04,868 INFO updater.py:439 -- [5/7] Initalizing command runner
2022-02-08 13:50:04,868 INFO updater.py:485 -- [6/7] No setup commands to run.
2022-02-08 13:50:04,869 INFO updater.py:489 -- [7/7] Starting the Ray runtime
2022-02-08 13:50:09,465 INFO log_timer.py:30 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-qf6zw: Ray start commands succeeded [LogTimer=4597ms]
2022-02-08 13:50:09,466 INFO log_timer.py:30 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-qf6zw: Applied config 0583f296a9e95cee648f39db3bf2330c60a73c5f [LogTimer=353899ms]
2022-02-08 13:50:09,523 INFO updater.py:187 -- New status: up-to-date
2022-02-08 13:50:50,340 INFO updater.py:323 -- New status: waiting-for-ssh
2022-02-08 13:50:50,340 INFO updater.py:261 -- [1/7] Waiting for SSH to become available
2022-02-08 13:50:50,340 INFO updater.py:265 -- Running `uptime` as a test.
2022-02-08 13:50:50,553 INFO updater.py:314 -- SSH still not available (Exit Status 1): kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)', retrying in 5 seconds.
2022-02-08 13:50:55,752 INFO updater.py:314 -- SSH still not available (Exit Status 1): kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)', retrying in 5 seconds.
2022-02-08 13:51:00,930 INFO updater.py:314 -- SSH still not available (Exit Status 1): kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)', retrying in 5 seconds.
2022-02-08 13:51:06,094 INFO updater.py:314 -- SSH still not available (Exit Status 1): kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)', retrying in 5 seconds.
2022-02-08 13:51:11,272 INFO updater.py:314 -- SSH still not available (Exit Status 1): kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)', retrying in 5 seconds.
2022-02-08 13:51:16,462 INFO updater.py:314 -- SSH still not available (Exit Status 1): kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)', retrying in 5 seconds.
ray-py38-cu112,karpenter:2022-02-08 13:51:23,096 INFO monitor.py:521 -- Logging raw resource message pulled from GCS.
ray-py38-cu112,karpenter:2022-02-08 13:51:23,097 INFO monitor.py:522 -- batch {
node_id: "\341\262\255\300\327\014\331\367\350\354\025\275\257\225\210[\022.\276\026\260\305 :f`\264\243"
resources_available {
key: "memory"
value: 10522669875.0
}
resources_available {
key: "node:10.16.87.89"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 4157303193.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 7.0
}
resources_total {
key: "memory"
value: 10522669875.0
}
resources_total {
key: "node:10.16.87.89"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 4157303193.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
resource_demands {
shape {
key: "CPU"
value: 1.0
}
num_ready_requests_queued: 1
}
}
node_manager_address: "10.16.87.89"
}
batch {
node_id: "\240u}D\242\243\030\206PG\251\364\213\306\202y\306f\375\254\251\03294\020\024!\263"
resources_available {
key: "memory"
value: 5261334937.0
}
resources_available {
key: "node:10.16.109.160"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 2053643576.0
}
resources_available_changed: true
resources_total {
key: "memory"
value: 5261334937.0
}
resources_total {
key: "node:10.16.109.160"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 2053644288.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.109.160"
}
batch {
node_id: "\267-\201\317\372\212\311\207\271\240\261\277\311\356\363^\243\357\325\362\345\271\024?j\034\\m"
resources_available {
key: "memory"
value: 22548578304.0
}
resources_available {
key: "node:10.16.73.237"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 8973386956.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 15.0
}
resources_total {
key: "memory"
value: 22548578304.0
}
resources_total {
key: "node:10.16.73.237"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 8973386956.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.73.237"
}
resource_load_by_shape {
resource_demands {
shape {
key: "CPU"
value: 1.0
}
num_ready_requests_queued: 1
}
}
placement_group_load {
}
ray-py38-cu112,karpenter:2022-02-08 13:51:23,097 INFO monitor.py:523 -- Done logging raw resource message.
ray-py38-cu112,karpenter:2022-02-08 13:51:23,097 DEBUG gcs_utils.py:228 -- internal_kv_get b'autoscaler_resource_request' None
ray-py38-cu112,karpenter:2022-02-08 13:51:23,813 INFO autoscaler.py:327 --
======== Autoscaler status: 2022-02-08 13:51:23.813376 ========
Node status
---------------------------------------------------------------
Healthy:
1 head
1 wkr-15cpu30g-ondemand
1 wkr-7cpu14g-spot
Pending:
None: wkr-7cpu14g-spot, waiting-for-ssh
Recent failures:
(no failures)
Resources
---------------------------------------------------------------
Usage:
22.0/22.0 CPU
0.00/35.700 GiB memory
0.00/14.142 GiB object_store_memory
Demands:
{'CPU': 1.0}: 1+ pending tasks/actors
ray-py38-cu112,karpenter:2022-02-08 13:51:23,942 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status_legacy' b"Cluster status: 3 nodes (1 updating)\n - MostDelayedHeartbeats: {'10.16.87.89': 0.7160403728485107, '10.16.109.160': 0.7159502506256104, '10.16.73.237': 0.715864896774292}\n - NodeIdleSeconds: Min=0 Mean=0 Max=0\n - ResourceUsage: 22.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory\n - TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0\nWorker node types:\n - wkr-15cpu30g-ondemand: 1\n - wkr-7cpu14g-spot: 2" True None
ray-py38-cu112,karpenter:2022-02-08 13:51:23,943 DEBUG legacy_info_string.py:26 -- Cluster status: 3 nodes (1 updating)
- MostDelayedHeartbeats: {'10.16.87.89': 0.7160403728485107, '10.16.109.160': 0.7159502506256104, '10.16.73.237': 0.715864896774292}
- NodeIdleSeconds: Min=0 Mean=0 Max=0
- ResourceUsage: 22.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
Worker node types:
- wkr-15cpu30g-ondemand: 1
- wkr-7cpu14g-spot: 2
ray-py38-cu112,karpenter:2022-02-08 13:51:24,275 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:24,322 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:24,374 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:24,405 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:24,660 DEBUG resource_demand_scheduler.py:201 -- Cluster resources: [{'memory': 5261334937.0, 'node:10.16.109.160': 1.0, 'object_store_memory': 2053643576.0}, {'memory': 22548578304.0, 'object_store_memory': 8973386956.0, 'node:10.16.73.237': 1.0, 'CPU': 0.0}, {'CPU': 7.0, 'GPU': 0, 'memory': 10522669875.0, 'object_store_memory': 4157303193.0}, {'node:10.16.87.89': 1.0, 'memory': 10522669875.0, 'object_store_memory': 4157303193.0, 'CPU': 0.0}]
ray-py38-cu112,karpenter:2022-02-08 13:51:24,660 DEBUG resource_demand_scheduler.py:202 -- Node counts: defaultdict(<class 'int'>, {'head': 1, 'wkr-15cpu30g-ondemand': 1, 'wkr-7cpu14g-spot': 2})
ray-py38-cu112,karpenter:2022-02-08 13:51:24,660 DEBUG resource_demand_scheduler.py:219 -- Placement group demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:24,660 DEBUG resource_demand_scheduler.py:283 -- Resource demands: [{'CPU': 1.0}]
ray-py38-cu112,karpenter:2022-02-08 13:51:24,660 DEBUG resource_demand_scheduler.py:284 -- Unfulfilled demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:24,660 DEBUG resource_demand_scheduler.py:292 -- Final unfulfilled: []
ray-py38-cu112,karpenter:2022-02-08 13:51:24,775 DEBUG resource_demand_scheduler.py:317 -- Node requests: {}
ray-py38-cu112,karpenter:2022-02-08 13:51:24,884 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status' b'{"load_metrics_report": {"usage": {"node:10.16.109.160": [0.0, 1.0], "object_store_memory": [712.0, 15184334437.0], "memory": [0.0, 38332583116.0], "node:10.16.73.237": [0.0, 1.0], "CPU": [22.0, 22.0], "node:10.16.87.89": [0.0, 1.0]}, "resource_demand": [[{"CPU": 1.0}, 1]], "pg_demand": [], "request_demand": [], "node_types": [[{"memory": 5261334937.0, "node:10.16.109.160": 1.0, "object_store_memory": 2053644288.0}, 1], [{"node:10.16.73.237": 1.0, "object_store_memory": 8973386956.0, "memory": 22548578304.0, "CPU": 15.0}, 1], [{"CPU": 7.0, "node:10.16.87.89": 1.0, "object_store_memory": 4157303193.0, "memory": 10522669875.0}, 1]], "head_ip": null}, "time": 1644357083.0993624, "monitor_pid": 15133, "autoscaler_report": {"active_nodes": {"head": 1, "wkr-15cpu30g-ondemand": 1, "wkr-7cpu14g-spot": 1}, "pending_nodes": [[null, "wkr-7cpu14g-spot", "waiting-for-ssh"]], "pending_launches": {}, "failed_nodes": []}}' True None
ray-py38-cu112,karpenter:2022-02-08 13:51:26,640 INFO command_runner.py:179 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-czw85: Running kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)'
Unable to use a TTY - input is not a terminal or the right kind of file
Error from server: no preferred addresses found; known addresses: []
ray-py38-cu112,karpenter:2022-02-08 13:51:29,892 INFO monitor.py:521 -- Logging raw resource message pulled from GCS.
ray-py38-cu112,karpenter:2022-02-08 13:51:29,892 INFO monitor.py:522 -- batch {
node_id: "\341\262\255\300\327\014\331\367\350\354\025\275\257\225\210[\022.\276\026\260\305 :f`\264\243"
resources_available {
key: "memory"
value: 10522669875.0
}
resources_available {
key: "node:10.16.87.89"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 4157303193.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 7.0
}
resources_total {
key: "memory"
value: 10522669875.0
}
resources_total {
key: "node:10.16.87.89"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 4157303193.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
resource_demands {
shape {
key: "CPU"
value: 1.0
}
num_ready_requests_queued: 1
}
}
node_manager_address: "10.16.87.89"
}
batch {
node_id: "\240u}D\242\243\030\206PG\251\364\213\306\202y\306f\375\254\251\03294\020\024!\263"
resources_available {
key: "memory"
value: 5261334937.0
}
resources_available {
key: "node:10.16.109.160"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 2053643576.0
}
resources_available_changed: true
resources_total {
key: "memory"
value: 5261334937.0
}
resources_total {
key: "node:10.16.109.160"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 2053644288.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.109.160"
}
batch {
node_id: "\267-\201\317\372\212\311\207\271\240\261\277\311\356\363^\243\357\325\362\345\271\024?j\034\\m"
resources_available {
key: "memory"
value: 22548578304.0
}
resources_available {
key: "node:10.16.73.237"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 8973386956.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 15.0
}
resources_total {
key: "memory"
value: 22548578304.0
}
resources_total {
key: "node:10.16.73.237"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 8973386956.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.73.237"
}
resource_load_by_shape {
resource_demands {
shape {
key: "CPU"
value: 1.0
}
num_ready_requests_queued: 1
}
}
placement_group_load {
}
ray-py38-cu112,karpenter:2022-02-08 13:51:29,893 INFO monitor.py:523 -- Done logging raw resource message.
ray-py38-cu112,karpenter:2022-02-08 13:51:29,893 DEBUG gcs_utils.py:228 -- internal_kv_get b'autoscaler_resource_request' None
ray-py38-cu112,karpenter:2022-02-08 13:51:30,509 INFO autoscaler.py:327 --
======== Autoscaler status: 2022-02-08 13:51:30.509337 ========
Node status
---------------------------------------------------------------
Healthy:
1 head
1 wkr-15cpu30g-ondemand
1 wkr-7cpu14g-spot
Pending:
None: wkr-7cpu14g-spot, waiting-for-ssh
Recent failures:
(no failures)
Resources
---------------------------------------------------------------
Usage:
22.0/22.0 CPU
0.00/35.700 GiB memory
0.00/14.142 GiB object_store_memory
Demands:
{'CPU': 1.0}: 1+ pending tasks/actors
ray-py38-cu112,karpenter:2022-02-08 13:51:30,596 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status_legacy' b"Cluster status: 3 nodes (1 updating)\n - MostDelayedHeartbeats: {'10.16.87.89': 0.6164615154266357, '10.16.109.160': 0.6163904666900635, '10.16.73.237': 0.6163249015808105}\n - NodeIdleSeconds: Min=0 Mean=0 Max=0\n - ResourceUsage: 22.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory\n - TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0\nWorker node types:\n - wkr-15cpu30g-ondemand: 1\n - wkr-7cpu14g-spot: 2" True None
ray-py38-cu112,karpenter:2022-02-08 13:51:30,597 DEBUG legacy_info_string.py:26 -- Cluster status: 3 nodes (1 updating)
- MostDelayedHeartbeats: {'10.16.87.89': 0.6164615154266357, '10.16.109.160': 0.6163904666900635, '10.16.73.237': 0.6163249015808105}
- NodeIdleSeconds: Min=0 Mean=0 Max=0
- ResourceUsage: 22.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
Worker node types:
- wkr-15cpu30g-ondemand: 1
- wkr-7cpu14g-spot: 2
ray-py38-cu112,karpenter:2022-02-08 13:51:30,830 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:30,871 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:30,915 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:30,946 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:31,198 DEBUG resource_demand_scheduler.py:201 -- Cluster resources: [{'object_store_memory': 2053643576.0, 'memory': 5261334937.0, 'node:10.16.109.160': 1.0}, {'memory': 22548578304.0, 'node:10.16.73.237': 1.0, 'object_store_memory': 8973386956.0, 'CPU': 0.0}, {'CPU': 7.0, 'GPU': 0, 'memory': 10522669875.0, 'object_store_memory': 4157303193.0}, {'memory': 10522669875.0, 'object_store_memory': 4157303193.0, 'node:10.16.87.89': 1.0, 'CPU': 0.0}]
ray-py38-cu112,karpenter:2022-02-08 13:51:31,199 DEBUG resource_demand_scheduler.py:202 -- Node counts: defaultdict(<class 'int'>, {'head': 1, 'wkr-15cpu30g-ondemand': 1, 'wkr-7cpu14g-spot': 2})
ray-py38-cu112,karpenter:2022-02-08 13:51:31,199 DEBUG resource_demand_scheduler.py:219 -- Placement group demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:31,199 DEBUG resource_demand_scheduler.py:283 -- Resource demands: [{'CPU': 1.0}]
ray-py38-cu112,karpenter:2022-02-08 13:51:31,199 DEBUG resource_demand_scheduler.py:284 -- Unfulfilled demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:31,199 DEBUG resource_demand_scheduler.py:292 -- Final unfulfilled: []
ray-py38-cu112,karpenter:2022-02-08 13:51:31,303 DEBUG resource_demand_scheduler.py:317 -- Node requests: {}
ray-py38-cu112,karpenter:2022-02-08 13:51:31,411 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status' b'{"load_metrics_report": {"usage": {"node:10.16.109.160": [0.0, 1.0], "memory": [0.0, 38332583116.0], "object_store_memory": [712.0, 15184334437.0], "CPU": [22.0, 22.0], "node:10.16.73.237": [0.0, 1.0], "node:10.16.87.89": [0.0, 1.0]}, "resource_demand": [[{"CPU": 1.0}, 1]], "pg_demand": [], "request_demand": [], "node_types": [[{"memory": 5261334937.0, "node:10.16.109.160": 1.0, "object_store_memory": 2053644288.0}, 1], [{"CPU": 15.0, "object_store_memory": 8973386956.0, "memory": 22548578304.0, "node:10.16.73.237": 1.0}, 1], [{"memory": 10522669875.0, "node:10.16.87.89": 1.0, "object_store_memory": 4157303193.0, "CPU": 7.0}, 1]], "head_ip": null}, "time": 1644357089.895082, "monitor_pid": 15133, "autoscaler_report": {"active_nodes": {"head": 1, "wkr-15cpu30g-ondemand": 1, "wkr-7cpu14g-spot": 1}, "pending_nodes": [[null, "wkr-7cpu14g-spot", "waiting-for-ssh"]], "pending_launches": {}, "failed_nodes": []}}' True None
ray-py38-cu112,karpenter:2022-02-08 13:51:31,791 INFO command_runner.py:179 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-czw85: Running kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)'
Unable to use a TTY - input is not a terminal or the right kind of file
Error from server: no preferred addresses found; known addresses: []
ray-py38-cu112,karpenter:2022-02-08 13:51:36,415 INFO monitor.py:521 -- Logging raw resource message pulled from GCS.
ray-py38-cu112,karpenter:2022-02-08 13:51:36,415 INFO monitor.py:522 -- batch {
node_id: "\341\262\255\300\327\014\331\367\350\354\025\275\257\225\210[\022.\276\026\260\305 :f`\264\243"
resources_available {
key: "memory"
value: 10522669875.0
}
resources_available {
key: "node:10.16.87.89"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 4157303193.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 7.0
}
resources_total {
key: "memory"
value: 10522669875.0
}
resources_total {
key: "node:10.16.87.89"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 4157303193.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
resource_demands {
shape {
key: "CPU"
value: 1.0
}
num_ready_requests_queued: 1
}
}
node_manager_address: "10.16.87.89"
}
batch {
node_id: "\240u}D\242\243\030\206PG\251\364\213\306\202y\306f\375\254\251\03294\020\024!\263"
resources_available {
key: "memory"
value: 5261334937.0
}
resources_available {
key: "node:10.16.109.160"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 2053643576.0
}
resources_available_changed: true
resources_total {
key: "memory"
value: 5261334937.0
}
resources_total {
key: "node:10.16.109.160"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 2053644288.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.109.160"
}
batch {
node_id: "\267-\201\317\372\212\311\207\271\240\261\277\311\356\363^\243\357\325\362\345\271\024?j\034\\m"
resources_available {
key: "memory"
value: 22548578304.0
}
resources_available {
key: "node:10.16.73.237"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 8973386956.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 15.0
}
resources_total {
key: "memory"
value: 22548578304.0
}
resources_total {
key: "node:10.16.73.237"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 8973386956.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.73.237"
}
resource_load_by_shape {
resource_demands {
shape {
key: "CPU"
value: 1.0
}
num_ready_requests_queued: 1
}
}
placement_group_load {
}
ray-py38-cu112,karpenter:2022-02-08 13:51:36,416 INFO monitor.py:523 -- Done logging raw resource message.
ray-py38-cu112,karpenter:2022-02-08 13:51:36,417 DEBUG gcs_utils.py:228 -- internal_kv_get b'autoscaler_resource_request' None
ray-py38-cu112,karpenter:2022-02-08 13:51:36,952 INFO command_runner.py:179 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-czw85: Running kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)'
Unable to use a TTY - input is not a terminal or the right kind of file
Error from server: error dialing backend: remote error: tls: internal error
ray-py38-cu112,karpenter:2022-02-08 13:51:37,184 INFO autoscaler.py:327 --
======== Autoscaler status: 2022-02-08 13:51:37.183649 ========
Node status
---------------------------------------------------------------
Healthy:
1 head
1 wkr-15cpu30g-ondemand
1 wkr-7cpu14g-spot
Pending:
None: wkr-7cpu14g-spot, waiting-for-ssh
Recent failures:
(no failures)
Resources
---------------------------------------------------------------
Usage:
22.0/22.0 CPU
0.00/35.700 GiB memory
0.00/14.142 GiB object_store_memory
Demands:
{'CPU': 1.0}: 1+ pending tasks/actors
ray-py38-cu112,karpenter:2022-02-08 13:51:37,313 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status_legacy' b"Cluster status: 3 nodes (1 updating)\n - MostDelayedHeartbeats: {'10.16.87.89': 0.7682485580444336, '10.16.109.160': 0.7680857181549072, '10.16.73.237': 0.7679226398468018}\n - NodeIdleSeconds: Min=0 Mean=0 Max=0\n - ResourceUsage: 22.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory\n - TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0\nWorker node types:\n - wkr-15cpu30g-ondemand: 1\n - wkr-7cpu14g-spot: 2" True None
ray-py38-cu112,karpenter:2022-02-08 13:51:37,314 DEBUG legacy_info_string.py:26 -- Cluster status: 3 nodes (1 updating)
- MostDelayedHeartbeats: {'10.16.87.89': 0.7682485580444336, '10.16.109.160': 0.7680857181549072, '10.16.73.237': 0.7679226398468018}
- NodeIdleSeconds: Min=0 Mean=0 Max=0
- ResourceUsage: 22.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
Worker node types:
- wkr-15cpu30g-ondemand: 1
- wkr-7cpu14g-spot: 2
ray-py38-cu112,karpenter:2022-02-08 13:51:37,634 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:37,693 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:37,765 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:37,792 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:38,121 DEBUG resource_demand_scheduler.py:201 -- Cluster resources: [{'node:10.16.109.160': 1.0, 'object_store_memory': 2053643576.0, 'memory': 5261334937.0}, {'object_store_memory': 8973386956.0, 'memory': 22548578304.0, 'node:10.16.73.237': 1.0, 'CPU': 0.0}, {'CPU': 7.0, 'GPU': 0, 'memory': 10522669875.0, 'object_store_memory': 4157303193.0}, {'node:10.16.87.89': 1.0, 'object_store_memory': 4157303193.0, 'memory': 10522669875.0, 'CPU': 0.0}]
ray-py38-cu112,karpenter:2022-02-08 13:51:38,121 DEBUG resource_demand_scheduler.py:202 -- Node counts: defaultdict(<class 'int'>, {'head': 1, 'wkr-15cpu30g-ondemand': 1, 'wkr-7cpu14g-spot': 2})
ray-py38-cu112,karpenter:2022-02-08 13:51:38,121 DEBUG resource_demand_scheduler.py:219 -- Placement group demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:38,121 DEBUG resource_demand_scheduler.py:283 -- Resource demands: [{'CPU': 1.0}]
ray-py38-cu112,karpenter:2022-02-08 13:51:38,121 DEBUG resource_demand_scheduler.py:284 -- Unfulfilled demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:38,122 DEBUG resource_demand_scheduler.py:292 -- Final unfulfilled: []
ray-py38-cu112,karpenter:2022-02-08 13:51:38,275 DEBUG resource_demand_scheduler.py:317 -- Node requests: {}
ray-py38-cu112,karpenter:2022-02-08 13:51:38,396 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status' b'{"load_metrics_report": {"usage": {"node:10.16.109.160": [0.0, 1.0], "object_store_memory": [712.0, 15184334437.0], "memory": [0.0, 38332583116.0], "CPU": [22.0, 22.0], "node:10.16.73.237": [0.0, 1.0], "node:10.16.87.89": [0.0, 1.0]}, "resource_demand": [[{"CPU": 1.0}, 1]], "pg_demand": [], "request_demand": [], "node_types": [[{"memory": 5261334937.0, "node:10.16.109.160": 1.0, "object_store_memory": 2053644288.0}, 1], [{"CPU": 15.0, "node:10.16.73.237": 1.0, "memory": 22548578304.0, "object_store_memory": 8973386956.0}, 1], [{"CPU": 7.0, "node:10.16.87.89": 1.0, "object_store_memory": 4157303193.0, "memory": 10522669875.0}, 1]], "head_ip": null}, "time": 1644357096.4217975, "monitor_pid": 15133, "autoscaler_report": {"active_nodes": {"head": 1, "wkr-15cpu30g-ondemand": 1, "wkr-7cpu14g-spot": 1}, "pending_nodes": [[null, "wkr-7cpu14g-spot", "waiting-for-ssh"]], "pending_launches": {}, "failed_nodes": []}}' True None
ray-py38-cu112,karpenter:2022-02-08 13:51:42,166 INFO command_runner.py:179 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-czw85: Running kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)'
Unable to use a TTY - input is not a terminal or the right kind of file
error: unable to upgrade connection: container not found ("ray-node")
ray-py38-cu112,karpenter:2022-02-08 13:51:43,403 INFO monitor.py:521 -- Logging raw resource message pulled from GCS.
ray-py38-cu112,karpenter:2022-02-08 13:51:43,403 INFO monitor.py:522 -- batch {
node_id: "\341\262\255\300\327\014\331\367\350\354\025\275\257\225\210[\022.\276\026\260\305 :f`\264\243"
resources_available {
key: "CPU"
value: 2.0
}
resources_available {
key: "memory"
value: 10522669875.0
}
resources_available {
key: "node:10.16.87.89"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 4157303193.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 7.0
}
resources_total {
key: "memory"
value: 10522669875.0
}
resources_total {
key: "node:10.16.87.89"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 4157303193.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.87.89"
}
batch {
node_id: "\240u}D\242\243\030\206PG\251\364\213\306\202y\306f\375\254\251\03294\020\024!\263"
resources_available {
key: "memory"
value: 5261334937.0
}
resources_available {
key: "node:10.16.109.160"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 2053643576.0
}
resources_available_changed: true
resources_total {
key: "memory"
value: 5261334937.0
}
resources_total {
key: "node:10.16.109.160"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 2053644288.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.109.160"
}
batch {
node_id: "\267-\201\317\372\212\311\207\271\240\261\277\311\356\363^\243\357\325\362\345\271\024?j\034\\m"
resources_available {
key: "CPU"
value: 14.0
}
resources_available {
key: "memory"
value: 22548578304.0
}
resources_available {
key: "node:10.16.73.237"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 8973386956.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 15.0
}
resources_total {
key: "memory"
value: 22548578304.0
}
resources_total {
key: "node:10.16.73.237"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 8973386956.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.73.237"
}
placement_group_load {
}
ray-py38-cu112,karpenter:2022-02-08 13:51:43,403 INFO monitor.py:523 -- Done logging raw resource message.
ray-py38-cu112,karpenter:2022-02-08 13:51:43,404 DEBUG gcs_utils.py:228 -- internal_kv_get b'autoscaler_resource_request' None
ray-py38-cu112,karpenter:2022-02-08 13:51:44,166 INFO autoscaler.py:327 --
======== Autoscaler status: 2022-02-08 13:51:44.165917 ========
Node status
---------------------------------------------------------------
Healthy:
1 head
1 wkr-15cpu30g-ondemand
1 wkr-7cpu14g-spot
Pending:
None: wkr-7cpu14g-spot, waiting-for-ssh
Recent failures:
(no failures)
Resources
---------------------------------------------------------------
Usage:
6.0/22.0 CPU
0.00/35.700 GiB memory
0.00/14.142 GiB object_store_memory
Demands:
(no resource demands)
ray-py38-cu112,karpenter:2022-02-08 13:51:44,268 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status_legacy' b"Cluster status: 3 nodes (1 updating)\n - MostDelayedHeartbeats: {'10.16.87.89': 0.7620394229888916, '10.16.109.160': 0.7619814872741699, '10.16.73.237': 0.7619214057922363}\n - NodeIdleSeconds: Min=0 Mean=0 Max=0\n - ResourceUsage: 6.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory\n - TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0\nWorker node types:\n - wkr-15cpu30g-ondemand: 1\n - wkr-7cpu14g-spot: 2" True None
ray-py38-cu112,karpenter:2022-02-08 13:51:44,269 DEBUG legacy_info_string.py:26 -- Cluster status: 3 nodes (1 updating)
- MostDelayedHeartbeats: {'10.16.87.89': 0.7620394229888916, '10.16.109.160': 0.7619814872741699, '10.16.73.237': 0.7619214057922363}
- NodeIdleSeconds: Min=0 Mean=0 Max=0
- ResourceUsage: 6.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
Worker node types:
- wkr-15cpu30g-ondemand: 1
- wkr-7cpu14g-spot: 2
ray-py38-cu112,karpenter:2022-02-08 13:51:44,611 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:44,706 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:44,787 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:44,837 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:45,200 DEBUG resource_demand_scheduler.py:201 -- Cluster resources: [{'node:10.16.109.160': 1.0, 'object_store_memory': 2053643576.0, 'memory': 5261334937.0}, {'node:10.16.73.237': 1.0, 'memory': 22548578304.0, 'object_store_memory': 8973386956.0, 'CPU': 14.0}, {'CPU': 7.0, 'GPU': 0, 'memory': 10522669875.0, 'object_store_memory': 4157303193.0}, {'memory': 10522669875.0, 'node:10.16.87.89': 1.0, 'object_store_memory': 4157303193.0, 'CPU': 2.0}]
ray-py38-cu112,karpenter:2022-02-08 13:51:45,200 DEBUG resource_demand_scheduler.py:202 -- Node counts: defaultdict(<class 'int'>, {'head': 1, 'wkr-15cpu30g-ondemand': 1, 'wkr-7cpu14g-spot': 2})
ray-py38-cu112,karpenter:2022-02-08 13:51:45,200 DEBUG resource_demand_scheduler.py:219 -- Placement group demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:45,201 DEBUG resource_demand_scheduler.py:283 -- Resource demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:45,201 DEBUG resource_demand_scheduler.py:284 -- Unfulfilled demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:45,201 DEBUG resource_demand_scheduler.py:292 -- Final unfulfilled: []
ray-py38-cu112,karpenter:2022-02-08 13:51:45,337 DEBUG resource_demand_scheduler.py:317 -- Node requests: {}
ray-py38-cu112,karpenter:2022-02-08 13:51:45,477 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status' b'{"load_metrics_report": {"usage": {"object_store_memory": [712.0, 15184334437.0], "node:10.16.109.160": [0.0, 1.0], "memory": [0.0, 38332583116.0], "node:10.16.73.237": [0.0, 1.0], "CPU": [6.0, 22.0], "node:10.16.87.89": [0.0, 1.0]}, "resource_demand": [], "pg_demand": [], "request_demand": [], "node_types": [[{"memory": 5261334937.0, "object_store_memory": 2053644288.0, "node:10.16.109.160": 1.0}, 1], [{"CPU": 15.0, "node:10.16.73.237": 1.0, "memory": 22548578304.0, "object_store_memory": 8973386956.0}, 1], [{"memory": 10522669875.0, "node:10.16.87.89": 1.0, "object_store_memory": 4157303193.0, "CPU": 7.0}, 1]], "head_ip": null}, "time": 1644357103.4057875, "monitor_pid": 15133, "autoscaler_report": {"active_nodes": {"head": 1, "wkr-15cpu30g-ondemand": 1, "wkr-7cpu14g-spot": 1}, "pending_nodes": [[null, "wkr-7cpu14g-spot", "waiting-for-ssh"]], "pending_launches": {}, "failed_nodes": []}}' True None
ray-py38-cu112,karpenter:2022-02-08 13:51:47,391 INFO command_runner.py:179 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-czw85: Running kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)'
Unable to use a TTY - input is not a terminal or the right kind of file
error: unable to upgrade connection: container not found ("ray-node")
ray-py38-cu112,karpenter:2022-02-08 13:51:50,485 INFO monitor.py:521 -- Logging raw resource message pulled from GCS.
ray-py38-cu112,karpenter:2022-02-08 13:51:50,485 INFO monitor.py:522 -- batch {
node_id: "\341\262\255\300\327\014\331\367\350\354\025\275\257\225\210[\022.\276\026\260\305 :f`\264\243"
resources_available {
key: "CPU"
value: 2.0
}
resources_available {
key: "memory"
value: 10522669875.0
}
resources_available {
key: "node:10.16.87.89"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 4157303193.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 7.0
}
resources_total {
key: "memory"
value: 10522669875.0
}
resources_total {
key: "node:10.16.87.89"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 4157303193.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.87.89"
}
batch {
node_id: "\240u}D\242\243\030\206PG\251\364\213\306\202y\306f\375\254\251\03294\020\024!\263"
resources_available {
key: "memory"
value: 5261334937.0
}
resources_available {
key: "node:10.16.109.160"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 2053643576.0
}
resources_available_changed: true
resources_total {
key: "memory"
value: 5261334937.0
}
resources_total {
key: "node:10.16.109.160"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 2053644288.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.109.160"
}
batch {
node_id: "\267-\201\317\372\212\311\207\271\240\261\277\311\356\363^\243\357\325\362\345\271\024?j\034\\m"
resources_available {
key: "CPU"
value: 14.0
}
resources_available {
key: "memory"
value: 22548578304.0
}
resources_available {
key: "node:10.16.73.237"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 8973386956.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 15.0
}
resources_total {
key: "memory"
value: 22548578304.0
}
resources_total {
key: "node:10.16.73.237"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 8973386956.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.73.237"
}
placement_group_load {
}
ray-py38-cu112,karpenter:2022-02-08 13:51:50,485 INFO monitor.py:523 -- Done logging raw resource message.
ray-py38-cu112,karpenter:2022-02-08 13:51:50,486 DEBUG gcs_utils.py:228 -- internal_kv_get b'autoscaler_resource_request' None
ray-py38-cu112,karpenter:2022-02-08 13:51:51,288 INFO autoscaler.py:327 --
======== Autoscaler status: 2022-02-08 13:51:51.288144 ========
Node status
---------------------------------------------------------------
Healthy:
1 head
1 wkr-15cpu30g-ondemand
1 wkr-7cpu14g-spot
Pending:
None: wkr-7cpu14g-spot, waiting-for-ssh
Recent failures:
(no failures)
Resources
---------------------------------------------------------------
Usage:
6.0/22.0 CPU
0.00/35.700 GiB memory
0.00/14.142 GiB object_store_memory
Demands:
(no resource demands)
ray-py38-cu112,karpenter:2022-02-08 13:51:51,424 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status_legacy' b"Cluster status: 3 nodes (1 updating)\n - MostDelayedHeartbeats: {'10.16.87.89': 0.8024759292602539, '10.16.109.160': 0.8024218082427979, '10.16.73.237': 0.8023653030395508}\n - NodeIdleSeconds: Min=0 Mean=0 Max=0\n - ResourceUsage: 6.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory\n - TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0\nWorker node types:\n - wkr-15cpu30g-ondemand: 1\n - wkr-7cpu14g-spot: 2" True None
ray-py38-cu112,karpenter:2022-02-08 13:51:51,425 DEBUG legacy_info_string.py:26 -- Cluster status: 3 nodes (1 updating)
- MostDelayedHeartbeats: {'10.16.87.89': 0.8024759292602539, '10.16.109.160': 0.8024218082427979, '10.16.73.237': 0.8023653030395508}
- NodeIdleSeconds: Min=0 Mean=0 Max=0
- ResourceUsage: 6.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
Worker node types:
- wkr-15cpu30g-ondemand: 1
- wkr-7cpu14g-spot: 2
ray-py38-cu112,karpenter:2022-02-08 13:51:51,699 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:51,769 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:51,818 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:51,938 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:52,339 DEBUG resource_demand_scheduler.py:201 -- Cluster resources: [{'object_store_memory': 2053643576.0, 'memory': 5261334937.0, 'node:10.16.109.160': 1.0}, {'CPU': 14.0, 'object_store_memory': 8973386956.0, 'memory': 22548578304.0, 'node:10.16.73.237': 1.0}, {'CPU': 7.0, 'GPU': 0, 'memory': 10522669875.0, 'object_store_memory': 4157303193.0}, {'object_store_memory': 4157303193.0, 'node:10.16.87.89': 1.0, 'memory': 10522669875.0, 'CPU': 2.0}]
ray-py38-cu112,karpenter:2022-02-08 13:51:52,339 DEBUG resource_demand_scheduler.py:202 -- Node counts: defaultdict(<class 'int'>, {'head': 1, 'wkr-15cpu30g-ondemand': 1, 'wkr-7cpu14g-spot': 2})
ray-py38-cu112,karpenter:2022-02-08 13:51:52,339 DEBUG resource_demand_scheduler.py:219 -- Placement group demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:52,339 DEBUG resource_demand_scheduler.py:283 -- Resource demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:52,339 DEBUG resource_demand_scheduler.py:284 -- Unfulfilled demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:52,339 DEBUG resource_demand_scheduler.py:292 -- Final unfulfilled: []
ray-py38-cu112,karpenter:2022-02-08 13:51:52,499 DEBUG resource_demand_scheduler.py:317 -- Node requests: {}
ray-py38-cu112,karpenter:2022-02-08 13:51:52,556 INFO command_runner.py:179 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-czw85: Running kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)'
ray-py38-cu112,karpenter:2022-02-08 13:51:52,669 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status' b'{"load_metrics_report": {"usage": {"object_store_memory": [712.0, 15184334437.0], "node:10.16.109.160": [0.0, 1.0], "memory": [0.0, 38332583116.0], "CPU": [6.0, 22.0], "node:10.16.73.237": [0.0, 1.0], "node:10.16.87.89": [0.0, 1.0]}, "resource_demand": [], "pg_demand": [], "request_demand": [], "node_types": [[{"memory": 5261334937.0, "object_store_memory": 2053644288.0, "node:10.16.109.160": 1.0}, 1], [{"CPU": 15.0, "object_store_memory": 8973386956.0, "memory": 22548578304.0, "node:10.16.73.237": 1.0}, 1], [{"memory": 10522669875.0, "node:10.16.87.89": 1.0, "object_store_memory": 4157303193.0, "CPU": 7.0}, 1]], "head_ip": null}, "time": 1644357110.487624, "monitor_pid": 15133, "autoscaler_report": {"active_nodes": {"head": 1, "wkr-15cpu30g-ondemand": 1, "wkr-7cpu14g-spot": 1}, "pending_nodes": [[null, "wkr-7cpu14g-spot", "waiting-for-ssh"]], "pending_launches": {}, "failed_nodes": []}}' True None
Unable to use a TTY - input is not a terminal or the right kind of file
error: unable to upgrade connection: container not found ("ray-node")
ray-py38-cu112,karpenter:2022-02-08 13:51:57,700 INFO monitor.py:521 -- Logging raw resource message pulled from GCS.
ray-py38-cu112,karpenter:2022-02-08 13:51:57,700 INFO monitor.py:522 -- batch {
node_id: "\341\262\255\300\327\014\331\367\350\354\025\275\257\225\210[\022.\276\026\260\305 :f`\264\243"
resources_available {
key: "CPU"
value: 2.0
}
resources_available {
key: "memory"
value: 10522669875.0
}
resources_available {
key: "node:10.16.87.89"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 4157303193.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 7.0
}
resources_total {
key: "memory"
value: 10522669875.0
}
resources_total {
key: "node:10.16.87.89"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 4157303193.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.87.89"
}
batch {
node_id: "\240u}D\242\243\030\206PG\251\364\213\306\202y\306f\375\254\251\03294\020\024!\263"
resources_available {
key: "memory"
value: 5261334937.0
}
resources_available {
key: "node:10.16.109.160"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 2053643576.0
}
resources_available_changed: true
resources_total {
key: "memory"
value: 5261334937.0
}
resources_total {
key: "node:10.16.109.160"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 2053644288.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.109.160"
}
batch {
node_id: "\267-\201\317\372\212\311\207\271\240\261\277\311\356\363^\243\357\325\362\345\271\024?j\034\\m"
resources_available {
key: "CPU"
value: 14.0
}
resources_available {
key: "memory"
value: 22548578304.0
}
resources_available {
key: "node:10.16.73.237"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 8973386956.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 15.0
}
resources_total {
key: "memory"
value: 22548578304.0
}
resources_total {
key: "node:10.16.73.237"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 8973386956.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.73.237"
}
placement_group_load {
}
ray-py38-cu112,karpenter:2022-02-08 13:51:57,700 INFO monitor.py:523 -- Done logging raw resource message.
ray-py38-cu112,karpenter:2022-02-08 13:51:57,701 DEBUG gcs_utils.py:228 -- internal_kv_get b'autoscaler_resource_request' None
ray-py38-cu112,karpenter:2022-02-08 13:51:57,787 INFO command_runner.py:179 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-czw85: Running kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)'
Unable to use a TTY - input is not a terminal or the right kind of file
error: unable to upgrade connection: container not found ("ray-node")
ray-py38-cu112,karpenter:2022-02-08 13:51:58,593 INFO autoscaler.py:327 --
======== Autoscaler status: 2022-02-08 13:51:58.593641 ========
Node status
---------------------------------------------------------------
Healthy:
1 head
1 wkr-15cpu30g-ondemand
1 wkr-7cpu14g-spot
Pending:
None: wkr-7cpu14g-spot, waiting-for-ssh
Recent failures:
(no failures)
Resources
---------------------------------------------------------------
Usage:
6.0/22.0 CPU
0.00/35.700 GiB memory
0.00/14.142 GiB object_store_memory
Demands:
(no resource demands)
ray-py38-cu112,karpenter:2022-02-08 13:51:58,731 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status_legacy' b"Cluster status: 3 nodes (1 updating)\n - MostDelayedHeartbeats: {'10.16.87.89': 0.8931031227111816, '10.16.109.160': 0.8930330276489258, '10.16.73.237': 0.8929681777954102}\n - NodeIdleSeconds: Min=0 Mean=0 Max=0\n - ResourceUsage: 6.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory\n - TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0\nWorker node types:\n - wkr-15cpu30g-ondemand: 1\n - wkr-7cpu14g-spot: 2" True None
ray-py38-cu112,karpenter:2022-02-08 13:51:58,732 DEBUG legacy_info_string.py:26 -- Cluster status: 3 nodes (1 updating)
- MostDelayedHeartbeats: {'10.16.87.89': 0.8931031227111816, '10.16.109.160': 0.8930330276489258, '10.16.73.237': 0.8929681777954102}
- NodeIdleSeconds: Min=0 Mean=0 Max=0
- ResourceUsage: 6.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
Worker node types:
- wkr-15cpu30g-ondemand: 1
- wkr-7cpu14g-spot: 2
ray-py38-cu112,karpenter:2022-02-08 13:51:59,112 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:59,181 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:59,239 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:59,273 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:51:59,549 DEBUG resource_demand_scheduler.py:201 -- Cluster resources: [{'memory': 5261334937.0, 'object_store_memory': 2053643576.0, 'node:10.16.109.160': 1.0}, {'CPU': 14.0, 'node:10.16.73.237': 1.0, 'memory': 22548578304.0, 'object_store_memory': 8973386956.0}, {'CPU': 7.0, 'GPU': 0, 'memory': 10522669875.0, 'object_store_memory': 4157303193.0}, {'object_store_memory': 4157303193.0, 'memory': 10522669875.0, 'node:10.16.87.89': 1.0, 'CPU': 2.0}]
ray-py38-cu112,karpenter:2022-02-08 13:51:59,549 DEBUG resource_demand_scheduler.py:202 -- Node counts: defaultdict(<class 'int'>, {'head': 1, 'wkr-15cpu30g-ondemand': 1, 'wkr-7cpu14g-spot': 2})
ray-py38-cu112,karpenter:2022-02-08 13:51:59,549 DEBUG resource_demand_scheduler.py:219 -- Placement group demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:59,549 DEBUG resource_demand_scheduler.py:283 -- Resource demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:59,549 DEBUG resource_demand_scheduler.py:284 -- Unfulfilled demands: []
ray-py38-cu112,karpenter:2022-02-08 13:51:59,549 DEBUG resource_demand_scheduler.py:292 -- Final unfulfilled: []
ray-py38-cu112,karpenter:2022-02-08 13:51:59,673 DEBUG resource_demand_scheduler.py:317 -- Node requests: {}
ray-py38-cu112,karpenter:2022-02-08 13:51:59,798 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status' b'{"load_metrics_report": {"usage": {"node:10.16.109.160": [0.0, 1.0], "object_store_memory": [712.0, 15184334437.0], "memory": [0.0, 38332583116.0], "CPU": [6.0, 22.0], "node:10.16.73.237": [0.0, 1.0], "node:10.16.87.89": [0.0, 1.0]}, "resource_demand": [], "pg_demand": [], "request_demand": [], "node_types": [[{"memory": 5261334937.0, "node:10.16.109.160": 1.0, "object_store_memory": 2053644288.0}, 1], [{"CPU": 15.0, "object_store_memory": 8973386956.0, "memory": 22548578304.0, "node:10.16.73.237": 1.0}, 1], [{"memory": 10522669875.0, "node:10.16.87.89": 1.0, "object_store_memory": 4157303193.0, "CPU": 7.0}, 1]], "head_ip": null}, "time": 1644357117.7022595, "monitor_pid": 15133, "autoscaler_report": {"active_nodes": {"head": 1, "wkr-15cpu30g-ondemand": 1, "wkr-7cpu14g-spot": 1}, "pending_nodes": [[null, "wkr-7cpu14g-spot", "waiting-for-ssh"]], "pending_launches": {}, "failed_nodes": []}}' True None
ray-py38-cu112,karpenter:2022-02-08 13:52:03,061 INFO command_runner.py:179 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-czw85: Running kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)'
Unable to use a TTY - input is not a terminal or the right kind of file
error: unable to upgrade connection: container not found ("ray-node")
ray-py38-cu112,karpenter:2022-02-08 13:52:04,805 INFO monitor.py:521 -- Logging raw resource message pulled from GCS.
ray-py38-cu112,karpenter:2022-02-08 13:52:04,806 INFO monitor.py:522 -- batch {
node_id: "\341\262\255\300\327\014\331\367\350\354\025\275\257\225\210[\022.\276\026\260\305 :f`\264\243"
resources_available {
key: "CPU"
value: 2.0
}
resources_available {
key: "memory"
value: 10522669875.0
}
resources_available {
key: "node:10.16.87.89"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 4157303193.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 7.0
}
resources_total {
key: "memory"
value: 10522669875.0
}
resources_total {
key: "node:10.16.87.89"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 4157303193.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.87.89"
}
batch {
node_id: "\240u}D\242\243\030\206PG\251\364\213\306\202y\306f\375\254\251\03294\020\024!\263"
resources_available {
key: "memory"
value: 5261334937.0
}
resources_available {
key: "node:10.16.109.160"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 2053643576.0
}
resources_available_changed: true
resources_total {
key: "memory"
value: 5261334937.0
}
resources_total {
key: "node:10.16.109.160"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 2053644288.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.109.160"
}
batch {
node_id: "\267-\201\317\372\212\311\207\271\240\261\277\311\356\363^\243\357\325\362\345\271\024?j\034\\m"
resources_available {
key: "CPU"
value: 14.0
}
resources_available {
key: "memory"
value: 22548578304.0
}
resources_available {
key: "node:10.16.73.237"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 8973386956.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 15.0
}
resources_total {
key: "memory"
value: 22548578304.0
}
resources_total {
key: "node:10.16.73.237"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 8973386956.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.73.237"
}
placement_group_load {
}
ray-py38-cu112,karpenter:2022-02-08 13:52:04,806 INFO monitor.py:523 -- Done logging raw resource message.
ray-py38-cu112,karpenter:2022-02-08 13:52:04,806 DEBUG gcs_utils.py:228 -- internal_kv_get b'autoscaler_resource_request' None
ray-py38-cu112,karpenter:2022-02-08 13:52:05,571 INFO autoscaler.py:327 --
======== Autoscaler status: 2022-02-08 13:52:05.571011 ========
Node status
---------------------------------------------------------------
Healthy:
1 head
1 wkr-15cpu30g-ondemand
1 wkr-7cpu14g-spot
Pending:
None: wkr-7cpu14g-spot, waiting-for-ssh
Recent failures:
(no failures)
Resources
---------------------------------------------------------------
Usage:
6.0/22.0 CPU
0.00/35.700 GiB memory
0.00/14.142 GiB object_store_memory
Demands:
(no resource demands)
ray-py38-cu112,karpenter:2022-02-08 13:52:05,695 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status_legacy' b"Cluster status: 3 nodes (1 updating)\n - MostDelayedHeartbeats: {'10.16.87.89': 0.7647335529327393, '10.16.109.160': 0.7646734714508057, '10.16.73.237': 0.7646000385284424}\n - NodeIdleSeconds: Min=0 Mean=0 Max=0\n - ResourceUsage: 6.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory\n - TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0\nWorker node types:\n - wkr-15cpu30g-ondemand: 1\n - wkr-7cpu14g-spot: 2" True None
ray-py38-cu112,karpenter:2022-02-08 13:52:05,697 DEBUG legacy_info_string.py:26 -- Cluster status: 3 nodes (1 updating)
- MostDelayedHeartbeats: {'10.16.87.89': 0.7647335529327393, '10.16.109.160': 0.7646734714508057, '10.16.73.237': 0.7646000385284424}
- NodeIdleSeconds: Min=0 Mean=0 Max=0
- ResourceUsage: 6.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
Worker node types:
- wkr-15cpu30g-ondemand: 1
- wkr-7cpu14g-spot: 2
ray-py38-cu112,karpenter:2022-02-08 13:52:05,989 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:52:06,044 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:52:06,118 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:52:06,150 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:52:06,457 DEBUG resource_demand_scheduler.py:201 -- Cluster resources: [{'node:10.16.109.160': 1.0, 'object_store_memory': 2053643576.0, 'memory': 5261334937.0}, {'memory': 22548578304.0, 'node:10.16.73.237': 1.0, 'CPU': 14.0, 'object_store_memory': 8973386956.0}, {'CPU': 7.0, 'GPU': 0, 'memory': 10522669875.0, 'object_store_memory': 4157303193.0}, {'node:10.16.87.89': 1.0, 'CPU': 2.0, 'object_store_memory': 4157303193.0, 'memory': 10522669875.0}]
ray-py38-cu112,karpenter:2022-02-08 13:52:06,457 DEBUG resource_demand_scheduler.py:202 -- Node counts: defaultdict(<class 'int'>, {'head': 1, 'wkr-15cpu30g-ondemand': 1, 'wkr-7cpu14g-spot': 2})
ray-py38-cu112,karpenter:2022-02-08 13:52:06,457 DEBUG resource_demand_scheduler.py:219 -- Placement group demands: []
ray-py38-cu112,karpenter:2022-02-08 13:52:06,457 DEBUG resource_demand_scheduler.py:283 -- Resource demands: []
ray-py38-cu112,karpenter:2022-02-08 13:52:06,457 DEBUG resource_demand_scheduler.py:284 -- Unfulfilled demands: []
ray-py38-cu112,karpenter:2022-02-08 13:52:06,457 DEBUG resource_demand_scheduler.py:292 -- Final unfulfilled: []
ray-py38-cu112,karpenter:2022-02-08 13:52:06,576 DEBUG resource_demand_scheduler.py:317 -- Node requests: {}
ray-py38-cu112,karpenter:2022-02-08 13:52:06,715 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status' b'{"load_metrics_report": {"usage": {"memory": [0.0, 38332583116.0], "object_store_memory": [712.0, 15184334437.0], "node:10.16.109.160": [0.0, 1.0], "node:10.16.73.237": [0.0, 1.0], "CPU": [6.0, 22.0], "node:10.16.87.89": [0.0, 1.0]}, "resource_demand": [], "pg_demand": [], "request_demand": [], "node_types": [[{"memory": 5261334937.0, "object_store_memory": 2053644288.0, "node:10.16.109.160": 1.0}, 1], [{"node:10.16.73.237": 1.0, "object_store_memory": 8973386956.0, "memory": 22548578304.0, "CPU": 15.0}, 1], [{"memory": 10522669875.0, "node:10.16.87.89": 1.0, "object_store_memory": 4157303193.0, "CPU": 7.0}, 1]], "head_ip": null}, "time": 1644357124.8085408, "monitor_pid": 15133, "autoscaler_report": {"active_nodes": {"head": 1, "wkr-15cpu30g-ondemand": 1, "wkr-7cpu14g-spot": 1}, "pending_nodes": [[null, "wkr-7cpu14g-spot", "waiting-for-ssh"]], "pending_launches": {}, "failed_nodes": []}}' True None
ray-py38-cu112,karpenter:2022-02-08 13:52:08,219 INFO command_runner.py:179 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-czw85: Running kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)'
Unable to use a TTY - input is not a terminal or the right kind of file
error: unable to upgrade connection: container not found ("ray-node")
ray-py38-cu112,karpenter:2022-02-08 13:52:11,720 INFO monitor.py:521 -- Logging raw resource message pulled from GCS.
ray-py38-cu112,karpenter:2022-02-08 13:52:11,720 INFO monitor.py:522 -- batch {
node_id: "\341\262\255\300\327\014\331\367\350\354\025\275\257\225\210[\022.\276\026\260\305 :f`\264\243"
resources_available {
key: "CPU"
value: 7.0
}
resources_available {
key: "memory"
value: 10522669875.0
}
resources_available {
key: "node:10.16.87.89"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 4157303193.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 7.0
}
resources_total {
key: "memory"
value: 10522669875.0
}
resources_total {
key: "node:10.16.87.89"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 4157303193.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.87.89"
}
batch {
node_id: "\240u}D\242\243\030\206PG\251\364\213\306\202y\306f\375\254\251\03294\020\024!\263"
resources_available {
key: "memory"
value: 5261334937.0
}
resources_available {
key: "node:10.16.109.160"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 2053643576.0
}
resources_available_changed: true
resources_total {
key: "memory"
value: 5261334937.0
}
resources_total {
key: "node:10.16.109.160"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 2053644288.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.109.160"
}
batch {
node_id: "\267-\201\317\372\212\311\207\271\240\261\277\311\356\363^\243\357\325\362\345\271\024?j\034\\m"
resources_available {
key: "CPU"
value: 15.0
}
resources_available {
key: "memory"
value: 22548578304.0
}
resources_available {
key: "node:10.16.73.237"
value: 1.0
}
resources_available {
key: "object_store_memory"
value: 8973386956.0
}
resources_available_changed: true
resources_total {
key: "CPU"
value: 15.0
}
resources_total {
key: "memory"
value: 22548578304.0
}
resources_total {
key: "node:10.16.73.237"
value: 1.0
}
resources_total {
key: "object_store_memory"
value: 8973386956.0
}
resource_load {
key: "CPU"
value: 1.0
}
resource_load_by_shape {
}
node_manager_address: "10.16.73.237"
}
placement_group_load {
}
ray-py38-cu112,karpenter:2022-02-08 13:52:11,720 INFO monitor.py:523 -- Done logging raw resource message.
ray-py38-cu112,karpenter:2022-02-08 13:52:11,721 DEBUG gcs_utils.py:228 -- internal_kv_get b'autoscaler_resource_request' None
ray-py38-cu112,karpenter:2022-02-08 13:52:12,428 INFO autoscaler.py:327 --
======== Autoscaler status: 2022-02-08 13:52:12.428237 ========
Node status
---------------------------------------------------------------
Healthy:
1 head
1 wkr-15cpu30g-ondemand
1 wkr-7cpu14g-spot
Pending:
None: wkr-7cpu14g-spot, waiting-for-ssh
Recent failures:
(no failures)
Resources
---------------------------------------------------------------
Usage:
0.0/22.0 CPU
0.00/35.700 GiB memory
0.00/14.142 GiB object_store_memory
Demands:
(no resource demands)
ray-py38-cu112,karpenter:2022-02-08 13:52:12,562 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status_legacy' b"Cluster status: 3 nodes (1 updating)\n - MostDelayedHeartbeats: {'10.16.87.89': 0.7074546813964844, '10.16.109.160': 0.7073965072631836, '10.16.73.237': 0.7072863578796387}\n - NodeIdleSeconds: Min=0 Mean=5 Max=7\n - ResourceUsage: 0.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory\n - TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0\nWorker node types:\n - wkr-15cpu30g-ondemand: 1\n - wkr-7cpu14g-spot: 2" True None
ray-py38-cu112,karpenter:2022-02-08 13:52:12,563 DEBUG legacy_info_string.py:26 -- Cluster status: 3 nodes (1 updating)
- MostDelayedHeartbeats: {'10.16.87.89': 0.7074546813964844, '10.16.109.160': 0.7073965072631836, '10.16.73.237': 0.7072863578796387}
- NodeIdleSeconds: Min=0 Mean=5 Max=7
- ResourceUsage: 0.0/22.0 CPU, 0.0 GiB/35.7 GiB memory, 0.0 GiB/14.14 GiB object_store_memory
- TimeSinceLastHeartbeat: Min=0 Mean=0 Max=0
Worker node types:
- wkr-15cpu30g-ondemand: 1
- wkr-7cpu14g-spot: 2
ray-py38-cu112,karpenter:2022-02-08 13:52:12,846 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:52:12,893 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:52:12,944 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-15cpu30g--ondemand-4qj4q is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:52:12,975 DEBUG autoscaler.py:1210 -- ray-py38-cu112-wkr-7cpu14g--spot-qf6zw is not being updated and passes config check (can_update=True).
ray-py38-cu112,karpenter:2022-02-08 13:52:13,252 DEBUG resource_demand_scheduler.py:201 -- Cluster resources: [{'memory': 5261334937.0, 'node:10.16.109.160': 1.0, 'object_store_memory': 2053643576.0}, {'object_store_memory': 8973386956.0, 'memory': 22548578304.0, 'CPU': 15.0, 'node:10.16.73.237': 1.0}, {'CPU': 7.0, 'GPU': 0, 'memory': 10522669875.0, 'object_store_memory': 4157303193.0}, {'object_store_memory': 4157303193.0, 'CPU': 7.0, 'memory': 10522669875.0, 'node:10.16.87.89': 1.0}]
ray-py38-cu112,karpenter:2022-02-08 13:52:13,253 DEBUG resource_demand_scheduler.py:202 -- Node counts: defaultdict(<class 'int'>, {'head': 1, 'wkr-15cpu30g-ondemand': 1, 'wkr-7cpu14g-spot': 2})
ray-py38-cu112,karpenter:2022-02-08 13:52:13,253 DEBUG resource_demand_scheduler.py:219 -- Placement group demands: []
ray-py38-cu112,karpenter:2022-02-08 13:52:13,253 DEBUG resource_demand_scheduler.py:283 -- Resource demands: []
ray-py38-cu112,karpenter:2022-02-08 13:52:13,253 DEBUG resource_demand_scheduler.py:284 -- Unfulfilled demands: []
ray-py38-cu112,karpenter:2022-02-08 13:52:13,253 DEBUG resource_demand_scheduler.py:292 -- Final unfulfilled: []
ray-py38-cu112,karpenter:2022-02-08 13:52:13,380 INFO command_runner.py:179 -- NodeUpdater: ray-py38-cu112-wkr-7cpu14g--spot-czw85: Running kubectl -n karpenter exec -it ray-py38-cu112-wkr-7cpu14g--spot-czw85 -- bash --login -c -i 'true && source ~/.bashrc && export OMP_NUM_THREADS=1 PYTHONWARNINGS=ignore && (uptime)'
ray-py38-cu112,karpenter:2022-02-08 13:52:13,396 DEBUG resource_demand_scheduler.py:317 -- Node requests: {}
Unable to use a TTY - input is not a terminal or the right kind of file
error: unable to upgrade connection: container not found ("ray-node")
ray-py38-cu112,karpenter:2022-02-08 13:52:13,649 DEBUG gcs_utils.py:245 -- internal_kv_put b'__autoscaling_status' b'{"load_metrics_report": {"usage": {"node:10.16.109.160": [0.0, 1.0], "memory": [0.0, 38332583116.0], "object_store_memory": [712.0, 15184334437.0], "node:10.16.73.237": [0.0, 1.0], "CPU": [0.0, 22.0], "node:10.16.87.89": [0.0, 1.0]}, "resource_demand": [], "pg_demand": [], "request_demand": [], "node_types": [[{"memory": 5261334937.0, "node:10.16.109.160": 1.0, "object_store_memory": 2053644288.0}, 1], [{"CPU": 15.0, "node:10.16.73.237": 1.0, "memory": 22548578304.0, "object_store_memory": 8973386956.0}, 1], [{"CPU": 7.0, "node:10.16.87.89": 1.0, "object_store_memory": 4157303193.0, "memory": 10522669875.0}, 1]], "head_ip": null}, "time": 1644357131.7231796, "monitor_pid": 15133, "autoscaler_report": {"active_nodes": {"head": 1, "wkr-15cpu30g-ondemand": 1, "wkr-7cpu14g-spot": 1}, "pending_nodes": [[null, "wkr-7cpu14g-spot", "waiting-for-ssh"]], "pending_launches": {}, "failed_nodes": []}}' True None
Raw
ray_client_server_23000.err
INFO:ray.util.client.server.server:Starting Ray Client server on 0.0.0.0:23000
INFO:ray.util.client.server.logservicer:New logs connection established. Total clients: 1
2022-02-08 13:42:32,133 INFO worker.py:852 -- Connecting to existing Ray cluster at address: 10.16.109.160:6379
Raw
ray_client_server_23004.err
INFO:ray.util.client.server.server:Starting Ray Client server on 0.0.0.0:23004
INFO:ray.util.client.server.logservicer:New logs connection established. Total clients: 1
2022-02-08 13:50:37,642 INFO worker.py:852 -- Connecting to existing Ray cluster at address: 10.16.109.160:6379
2022-02-08 13:50:37,643 DEBUG gcs_utils.py:238 -- internal_kv_get b'session_name' session
2022-02-08 13:50:37,643 DEBUG gcs_utils.py:238 -- internal_kv_get b'temp_dir' session
2022-02-08 13:50:37,644 DEBUG gcs_utils.py:238 -- internal_kv_get b'session_dir' session
2022-02-08 13:50:37,747 DEBUG gcs_utils.py:238 -- internal_kv_get b'webui:url' dashboard
2022-02-08 13:50:37,959 DEBUG gcs_utils.py:238 -- internal_kv_get b'Exports:\x00\x00\x00\x00\x00\x00\x00\x01' fun
2022-02-08 13:50:37,960 DEBUG gcs_utils.py:253 -- internal_kv_put b"Lock:FunctionsToRun:\x1d\xda\xb5\x98\xa3\x9072\x84\xda'\xd2\xa4<S\xf9\xb8r?\x88v;\xe0\xf6'l;\xec" b'1' False fun
2022-02-08 13:50:37,960 DEBUG gcs_utils.py:238 -- internal_kv_get b"FunctionsToRun:\x1d\xda\xb5\x98\xa3\x9072\x84\xda'\xd2\xa4<S\xf9\xb8r?\x88v;\xe0\xf6'l;\xec" fun
2022-02-08 13:50:37,961 DEBUG gcs_utils.py:238 -- internal_kv_get b'tracing_startup_hook' tracing
2022-02-08 13:50:37,961 DEBUG gcs_utils.py:238 -- internal_kv_get b'__autoscaling_error' None
2022-02-08 13:50:37,961 DEBUG gcs_utils.py:238 -- internal_kv_get b'Exports:\x00\x00\x00\x00\x00\x00\x00\x02' fun
2022-02-08 13:50:37,962 DEBUG gcs_utils.py:238 -- internal_kv_get b"RemoteFunction:\x01\x00\x00\x00:\xcc\x86\xf1D\xd3\xbf\xf1\xa7\x07\xb5\xa2UZ\xf4\xdc\xe9&*\x86\x85\xcb\x88'\nT\x8e\x7f\xc2" fun
2022-02-08 13:50:37,963 DEBUG gcs_utils.py:238 -- internal_kv_get b"RemoteFunction:\x01\x00\x00\x00:\xcc\x86\xf1D\xd3\xbf\xf1\xa7\x07\xb5\xa2UZ\xf4\xdc\xe9&*\x86\x85\xcb\x88'\nT\x8e\x7f\xc2" fun
2022-02-08 13:50:37,963 DEBUG gcs_utils.py:238 -- internal_kv_get b'Exports:\x00\x00\x00\x00\x00\x00\x00\x03' fun
2022-02-08 13:50:37,963 DEBUG gcs_utils.py:238 -- internal_kv_get b"RemoteFunction:\x02\x00\x00\x00:'\x83\xc0#u\xe8Lp{\x00\x9e\xe3\xca\x17\xce\xa58D\xc2<m\xbc\n\xf9\xb5\xd2\x11\x0b" fun
2022-02-08 13:50:37,964 DEBUG gcs_utils.py:238 -- internal_kv_get b"RemoteFunction:\x02\x00\x00\x00:'\x83\xc0#u\xe8Lp{\x00\x9e\xe3\xca\x17\xce\xa58D\xc2<m\xbc\n\xf9\xb5\xd2\x11\x0b" fun
2022-02-08 13:50:37,964 DEBUG gcs_utils.py:238 -- internal_kv_get b'Exports:\x00\x00\x00\x00\x00\x00\x00\x04' fun
2022-02-08 13:50:37,964 DEBUG gcs_utils.py:238 -- internal_kv_get b'RemoteFunction:\x02\x00\x00\x00:\x98PH\x1er\x08D\xf4l+\xa2\xf2\x16J\x9c\xdb0QB\xc2\xb0y>)Z\x9bF\xc6' fun
2022-02-08 13:50:37,964 DEBUG gcs_utils.py:238 -- internal_kv_get b'RemoteFunction:\x02\x00\x00\x00:\x98PH\x1er\x08D\xf4l+\xa2\xf2\x16J\x9c\xdb0QB\xc2\xb0y>)Z\x9bF\xc6' fun
2022-02-08 13:50:37,965 DEBUG gcs_utils.py:238 -- internal_kv_get b'Exports:\x00\x00\x00\x00\x00\x00\x00\x05' fun
2022-02-08 13:50:37,965 DEBUG gcs_utils.py:238 -- internal_kv_get b'RemoteFunction:\x03\x00\x00\x00:u\xb7C\xb2T7\xb1z%O\xa8H\xe3q\xec:\xd7\xa8\x9f\xba0wUjD\xb8#\x93' fun
2022-02-08 13:50:37,965 DEBUG gcs_utils.py:238 -- internal_kv_get b'RemoteFunction:\x03\x00\x00\x00:u\xb7C\xb2T7\xb1z%O\xa8H\xe3q\xec:\xd7\xa8\x9f\xba0wUjD\xb8#\x93' fun
2022-02-08 13:50:37,965 DEBUG gcs_utils.py:238 -- internal_kv_get b'Exports:\x00\x00\x00\x00\x00\x00\x00\x06' fun
2022-02-08 13:50:37,966 DEBUG gcs_utils.py:238 -- internal_kv_get b'RemoteFunction:\x03\x00\x00\x00:\xb8\xbe\xb67\xae\xa6\xec\xcc\n\xf6\xd5\x9b6\xc4\x05D\xab\xcc\x19\xd2\x92\xfa:\xe3\x9d?\xab\x92' fun
2022-02-08 13:50:37,966 DEBUG gcs_utils.py:238 -- internal_kv_get b'RemoteFunction:\x03\x00\x00\x00:\xb8\xbe\xb67\xae\xa6\xec\xcc\n\xf6\xd5\x9b6\xc4\x05D\xab\xcc\x19\xd2\x92\xfa:\xe3\x9d?\xab\x92' fun
2022-02-08 13:50:37,966 DEBUG gcs_utils.py:238 -- internal_kv_get b'Exports:\x00\x00\x00\x00\x00\x00\x00\x07' fun
2022-02-08 13:50:37,967 DEBUG gcs_utils.py:238 -- internal_kv_get b'RemoteFunction:\x04\x00\x00\x00:\xb7\xc1B\xeec#;""!-\x94S\xd6\xa4\xcf\xff\xf1\x03\xa4\xfd\x9fm\xfa\x01\x02\xc16' fun
2022-02-08 13:50:37,967 DEBUG gcs_utils.py:238 -- internal_kv_get b'RemoteFunction:\x04\x00\x00\x00:\xb7\xc1B\xeec#;""!-\x94S\xd6\xa4\xcf\xff\xf1\x03\xa4\xfd\x9fm\xfa\x01\x02\xc16' fun
2022-02-08 13:50:37,967 DEBUG gcs_utils.py:238 -- internal_kv_get b'Exports:\x00\x00\x00\x00\x00\x00\x00\x08' fun
2022-02-08 13:50:37,967 DEBUG gcs_utils.py:238 -- internal_kv_get b'RemoteFunction:\x04\x00\x00\x00:\xa9\xb7\x98\xe5\x97\xcb\x04\xbb\xbe\x1a\x8e\x80JS\xe4\x19\xa3\xf7I\xd8d3\x91=\xaeH\xa7.' fun
2022-02-08 13:50:37,968 DEBUG gcs_utils.py:238 -- internal_kv_get b'RemoteFunction:\x04\x00\x00\x00:\xa9\xb7\x98\xe5\x97\xcb\x04\xbb\xbe\x1a\x8e\x80JS\xe4\x19\xa3\xf7I\xd8d3\x91=\xaeH\xa7.' fun
2022-02-08 13:50:37,968 DEBUG gcs_utils.py:238 -- internal_kv_get b'Exports:\x00\x00\x00\x00\x00\x00\x00\t' fun
2022-02-08 13:50:38,079 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357038.0797954}' True healthcheck
2022-02-08 13:50:38,282 DEBUG server.py:455 -- put: ObjectRef(ffffffffffffffffffffffffffffffffffffffff0500000001000000)
2022-02-08 13:50:38,407 DEBUG server.py:496 -- schedule: get_dashboard_url FUNCTION
2022-02-08 13:50:38,409 DEBUG gcs_utils.py:278 -- internal_kv_exists b'RemoteFunction:\x05\x00\x00\x00:6\t\xd7\x06\xbfM\xaf\x96\x11\x12\xe8\xadNF\xef\xf8}m\xe9\x07\x81\xc6_\xf1\xcf\xf3~\xd7' fun
2022-02-08 13:50:38,410 DEBUG gcs_utils.py:253 -- internal_kv_put b'RemoteFunction:\x05\x00\x00\x00:6\t\xd7\x06\xbfM\xaf\x96\x11\x12\xe8\xadNF\xef\xf8}m\xe9\x07\x81\xc6_\xf1\xcf\xf3~\xd7' b'\x80\x05\x95\x00\x01\x00\x00\x00\x00\x00\x00}\x94(\x8c\x06job_id\x94C\x04\x05\x00\x00\x00\x94\x8c\x0bfunction_id\x94C\x1c6\t\xd7\x06\xbfM\xaf\x96\x11\x12\xe8\xadNF\xef\xf8}m\xe9\x07\x81\xc6_\xf1\xcf\xf3~\xd7\x94\x8c\rfunction_name\x94\x8c\x1cray.worker.get_dashboard_url\x94\x8c\x06module\x94\x8c\nray.worker\x94\x8c\x08function\x94C/\x80\x05\x95$\x00\x00\x00\x00\x00\x00\x00\x8c\nray.worker\x94\x8c\x11get_dashboard_url\x94\x93\x94.\x94\x8c\x14collision_identifier\x94C\x14\x9au~\x16\xa3\xc4j{W\x12\xbaBa\xfd\xe5\x13]@\xa6Z\x94\x8c\tmax_calls\x94K\x00u.' True fun
2022-02-08 13:50:38,410 DEBUG gcs_utils.py:253 -- internal_kv_put b'Exports:\x00\x00\x00\x00\x00\x00\x00\t' b'RemoteFunction:\x05\x00\x00\x00:6\t\xd7\x06\xbfM\xaf\x96\x11\x12\xe8\xadNF\xef\xf8}m\xe9\x07\x81\xc6_\xf1\xcf\xf3~\xd7' False fun
2022-02-08 13:50:38,416 DEBUG gcs_utils.py:238 -- internal_kv_get b'Exports:\x00\x00\x00\x00\x00\x00\x00\t' fun
2022-02-08 13:50:38,416 DEBUG gcs_utils.py:238 -- internal_kv_get b'RemoteFunction:\x05\x00\x00\x00:6\t\xd7\x06\xbfM\xaf\x96\x11\x12\xe8\xadNF\xef\xf8}m\xe9\x07\x81\xc6_\xf1\xcf\xf3~\xd7' fun
2022-02-08 13:50:38,416 DEBUG gcs_utils.py:238 -- internal_kv_get b'RemoteFunction:\x05\x00\x00\x00:6\t\xd7\x06\xbfM\xaf\x96\x11\x12\xe8\xadNF\xef\xf8}m\xe9\x07\x81\xc6_\xf1\xcf\xf3~\xd7' fun
2022-02-08 13:50:38,417 DEBUG gcs_utils.py:238 -- internal_kv_get b'Exports:\x00\x00\x00\x00\x00\x00\x00\n' fun
2022-02-08 13:50:38,478 DEBUG server.py:415 -- get: [ObjectRef(b996d537eda8ba31ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:50:39,081 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357039.081425}' True healthcheck
2022-02-08 13:50:39,118 DEBUG server.py:258 -- Releasing object b996d537eda8ba31ffffffffffffffffffffffff0500000001000000 for 03f832479f1c4a34b6076b30b782564c
2022-02-08 13:50:39,120 DEBUG server.py:455 -- put: ObjectRef(ffffffffffffffffffffffffffffffffffffffff0500000002000000)
2022-02-08 13:50:39,194 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,195 DEBUG gcs_utils.py:278 -- internal_kv_exists b'RemoteFunction:\x05\x00\x00\x00:\x1b@\xfe\x95"\xfb\xbb\xaf"\xde\x9b\xc6R\xd6p=\x1e\x96\xd0iK9\xd7\xeb4p\xca?' fun
2022-02-08 13:50:39,196 DEBUG gcs_utils.py:253 -- internal_kv_put b'RemoteFunction:\x05\x00\x00\x00:\x1b@\xfe\x95"\xfb\xbb\xaf"\xde\x9b\xc6R\xd6p=\x1e\x96\xd0iK9\xd7\xeb4p\xca?' b'\x80\x05\x95\xb7\x02\x00\x00\x00\x00\x00\x00}\x94(\x8c\x06job_id\x94C\x04\x05\x00\x00\x00\x94\x8c\x0bfunction_id\x94C\x1c\x1b@\xfe\x95"\xfb\xbb\xaf"\xde\x9b\xc6R\xd6p=\x1e\x96\xd0iK9\xd7\xeb4p\xca?\x94\x8c\rfunction_name\x94\x8c\r__main__.task\x94\x8c\x06module\x94\x8c\x08__main__\x94\x8c\x08function\x94B\xf4\x01\x00\x00\x80\x05\x95\xe9\x01\x00\x00\x00\x00\x00\x00\x8c\x1bray.cloudpickle.cloudpickle\x94\x8c\r_builtin_type\x94\x93\x94\x8c\nLambdaType\x94\x85\x94R\x94(h\x02\x8c\x08CodeType\x94\x85\x94R\x94(K\x00K\x00K\x00K\x01K\x03KCC\x16d\x01d\x00l\x00}\x00|\x00\xa0\x01d\x02\xa1\x01\x01\x00d\x00S\x00\x94NK\x00K\x1e\x87\x94\x8c\x04time\x94\x8c\x05sleep\x94\x86\x94h\x0b\x85\x94\x8c\x16test_ray_autoscaler.py\x94\x8c\x04task\x94K\x06C\x04\x00\x02\x08\x01\x94))t\x94R\x94}\x94(\x8c\x0b__package__\x94N\x8c\x08__name__\x94\x8c\x08__main__\x94\x8c\x08__file__\x94h\x0fuNNNt\x94R\x94\x8c ray.cloudpickle.cloudpickle_fast\x94\x8c\x12_function_setstate\x94\x93\x94h\x1a}\x94}\x94(h\x16h\x10\x8c\x0c__qualname__\x94h\x10\x8c\x0f__annotations__\x94}\x94\x8c\x0e__kwdefaults__\x94N\x8c\x0c__defaults__\x94N\x8c\n__module__\x94h\x17\x8c\x07__doc__\x94N\x8c\x0b__closure__\x94N\x8c\x17_cloudpickle_submodules\x94]\x94\x8c\x0b__globals__\x94}\x94u\x86\x94\x86R0.\x94\x8c\x14collision_identifier\x94C\x14Bc\xf0\x7f\xb1\x0b\xb0q\x14t^rQs,\x9c\x89\xa1>\x90\x94\x8c\tmax_calls\x94K\x00u.' True fun
2022-02-08 13:50:39,196 DEBUG gcs_utils.py:253 -- internal_kv_put b'Exports:\x00\x00\x00\x00\x00\x00\x00\n' b'RemoteFunction:\x05\x00\x00\x00:\x1b@\xfe\x95"\xfb\xbb\xaf"\xde\x9b\xc6R\xd6p=\x1e\x96\xd0iK9\xd7\xeb4p\xca?' False fun
2022-02-08 13:50:39,197 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,198 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,199 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,199 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,199 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,200 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,201 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,201 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,202 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,203 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,203 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,204 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,205 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,205 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,206 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,206 DEBUG gcs_utils.py:238 -- internal_kv_get b'Exports:\x00\x00\x00\x00\x00\x00\x00\n' fun
2022-02-08 13:50:39,207 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,207 DEBUG gcs_utils.py:238 -- internal_kv_get b'RemoteFunction:\x05\x00\x00\x00:\x1b@\xfe\x95"\xfb\xbb\xaf"\xde\x9b\xc6R\xd6p=\x1e\x96\xd0iK9\xd7\xeb4p\xca?' fun
2022-02-08 13:50:39,207 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,208 DEBUG gcs_utils.py:238 -- internal_kv_get b'RemoteFunction:\x05\x00\x00\x00:\x1b@\xfe\x95"\xfb\xbb\xaf"\xde\x9b\xc6R\xd6p=\x1e\x96\xd0iK9\xd7\xeb4p\xca?' fun
2022-02-08 13:50:39,208 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,208 DEBUG gcs_utils.py:238 -- internal_kv_get b'Exports:\x00\x00\x00\x00\x00\x00\x00\x0b' fun
2022-02-08 13:50:39,209 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,210 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,210 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,210 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,211 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,211 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,211 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,261 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,262 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,262 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,263 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,263 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,264 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,265 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,265 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,266 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,267 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,267 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,268 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,268 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,269 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,270 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,270 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,271 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,271 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,271 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,272 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,272 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,273 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,273 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,274 DEBUG server.py:496 -- schedule: task FUNCTION
2022-02-08 13:50:39,462 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:50:40,083 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357040.0830033}' True healthcheck
2022-02-08 13:50:41,084 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357041.0847073}' True healthcheck
2022-02-08 13:50:41,561 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:50:42,086 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357042.0863354}' True healthcheck
2022-02-08 13:50:43,088 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357043.0879946}' True healthcheck
2022-02-08 13:50:43,655 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:50:44,089 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357044.0896952}' True healthcheck
2022-02-08 13:50:45,091 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357045.0913877}' True healthcheck
2022-02-08 13:50:45,753 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:50:46,093 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357046.093129}' True healthcheck
2022-02-08 13:50:47,094 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357047.0947254}' True healthcheck
2022-02-08 13:50:47,850 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:50:48,096 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357048.0964208}' True healthcheck
2022-02-08 13:50:49,097 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357049.0974252}' True healthcheck
2022-02-08 13:50:49,950 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:50:50,099 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357050.0991216}' True healthcheck
2022-02-08 13:50:51,100 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357051.1008706}' True healthcheck
2022-02-08 13:50:52,048 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:50:52,102 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357052.1025543}' True healthcheck
2022-02-08 13:50:53,103 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357053.1037118}' True healthcheck
2022-02-08 13:50:54,104 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357054.1046782}' True healthcheck
2022-02-08 13:50:54,141 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:50:55,105 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357055.1054382}' True healthcheck
2022-02-08 13:50:56,107 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357056.1072292}' True healthcheck
2022-02-08 13:50:56,242 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:50:57,108 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357057.1089017}' True healthcheck
2022-02-08 13:50:58,110 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357058.1106565}' True healthcheck
2022-02-08 13:50:58,335 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:50:59,112 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357059.1120412}' True healthcheck
2022-02-08 13:51:00,113 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357060.1134322}' True healthcheck
2022-02-08 13:51:00,430 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:01,115 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357061.1151748}' True healthcheck
2022-02-08 13:51:02,116 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357062.116888}' True healthcheck
2022-02-08 13:51:02,536 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:03,118 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357063.1186047}' True healthcheck
2022-02-08 13:51:04,120 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357064.1202877}' True healthcheck
2022-02-08 13:51:04,631 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:05,122 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357065.122095}' True healthcheck
2022-02-08 13:51:06,123 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357066.123873}' True healthcheck
2022-02-08 13:51:06,723 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:07,125 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357067.125436}' True healthcheck
2022-02-08 13:51:08,127 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357068.1272101}' True healthcheck
2022-02-08 13:51:08,828 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:09,128 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357069.128927}' True healthcheck
2022-02-08 13:51:10,130 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357070.1299262}' True healthcheck
2022-02-08 13:51:10,924 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:11,131 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357071.1316936}' True healthcheck
2022-02-08 13:51:12,132 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357072.1327286}' True healthcheck
2022-02-08 13:51:13,017 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:13,133 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357073.13343}' True healthcheck
2022-02-08 13:51:14,135 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357074.135108}' True healthcheck
2022-02-08 13:51:15,117 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:15,136 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357075.1369152}' True healthcheck
2022-02-08 13:51:16,138 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357076.1386511}' True healthcheck
2022-02-08 13:51:17,140 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357077.140261}' True healthcheck
2022-02-08 13:51:17,216 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:18,141 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357078.1412382}' True healthcheck
2022-02-08 13:51:19,143 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357079.1429455}' True healthcheck
2022-02-08 13:51:19,309 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:20,144 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357080.1447217}' True healthcheck
2022-02-08 13:51:21,146 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357081.1463833}' True healthcheck
2022-02-08 13:51:21,407 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:22,148 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357082.1481593}' True healthcheck
2022-02-08 13:51:23,149 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357083.1494265}' True healthcheck
2022-02-08 13:51:23,505 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:24,151 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357084.1511066}' True healthcheck
2022-02-08 13:51:25,152 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357085.1527991}' True healthcheck
2022-02-08 13:51:25,622 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:26,154 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357086.1545494}' True healthcheck
2022-02-08 13:51:27,156 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357087.1562557}' True healthcheck
2022-02-08 13:51:27,696 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:28,157 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357088.157432}' True healthcheck
2022-02-08 13:51:29,159 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357089.1591353}' True healthcheck
2022-02-08 13:51:29,794 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:30,160 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357090.1608267}' True healthcheck
2022-02-08 13:51:31,162 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357091.1625624}' True healthcheck
2022-02-08 13:51:31,923 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:32,164 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357092.164219}' True healthcheck
2022-02-08 13:51:33,165 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357093.1654282}' True healthcheck
2022-02-08 13:51:33,992 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:34,167 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357094.1670928}' True healthcheck
2022-02-08 13:51:35,168 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357095.168842}' True healthcheck
2022-02-08 13:51:36,086 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:36,170 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357096.170623}' True healthcheck
2022-02-08 13:51:37,171 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357097.171786}' True healthcheck
2022-02-08 13:51:38,173 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357098.1734266}' True healthcheck
2022-02-08 13:51:38,210 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:39,175 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357099.175154}' True healthcheck
2022-02-08 13:51:40,176 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357100.1767547}' True healthcheck
2022-02-08 13:51:40,486 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:41,178 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357101.178541}' True healthcheck
2022-02-08 13:51:42,180 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357102.180268}' True healthcheck
2022-02-08 13:51:42,557 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:43,182 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357103.182001}' True healthcheck
2022-02-08 13:51:44,183 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357104.183719}' True healthcheck
2022-02-08 13:51:44,627 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:45,185 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357105.18529}' True healthcheck
2022-02-08 13:51:46,187 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357106.1870265}' True healthcheck
2022-02-08 13:51:46,703 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:47,188 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357107.1887314}' True healthcheck
2022-02-08 13:51:48,190 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357108.1904845}' True healthcheck
2022-02-08 13:51:48,774 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:49,191 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357109.1915662}' True healthcheck
2022-02-08 13:51:50,193 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357110.1933048}' True healthcheck
2022-02-08 13:51:50,842 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:51,195 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357111.1951027}' True healthcheck
2022-02-08 13:51:52,196 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357112.196824}' True healthcheck
2022-02-08 13:51:52,912 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:53,198 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357113.1985543}' True healthcheck
2022-02-08 13:51:54,200 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357114.2003024}' True healthcheck
2022-02-08 13:51:54,984 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:55,201 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357115.201894}' True healthcheck
2022-02-08 13:51:56,203 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357116.2031744}' True healthcheck
2022-02-08 13:51:57,062 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:57,204 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357117.2048137}' True healthcheck
2022-02-08 13:51:58,206 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357118.2064755}' True healthcheck
2022-02-08 13:51:59,166 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:51:59,208 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357119.2081573}' True healthcheck
2022-02-08 13:52:00,209 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357120.2097392}' True healthcheck
2022-02-08 13:52:01,211 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357121.2114668}' True healthcheck
2022-02-08 13:52:01,271 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:52:02,213 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357122.2130978}' True healthcheck
2022-02-08 13:52:03,214 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357123.2146468}' True healthcheck
2022-02-08 13:52:03,357 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:52:04,216 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357124.2163482}' True healthcheck
2022-02-08 13:52:05,217 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357125.217795}' True healthcheck
2022-02-08 13:52:05,458 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:52:06,219 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357126.2193294}' True healthcheck
2022-02-08 13:52:07,220 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357127.2205176}' True healthcheck
2022-02-08 13:52:07,554 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:52:08,222 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357128.2221947}' True healthcheck
2022-02-08 13:52:09,223 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357129.2238345}' True healthcheck
2022-02-08 13:52:09,671 DEBUG server.py:415 -- get: [ObjectRef(e413db02c7a52895ffffffffffffffffffffffff0500000001000000), ObjectRef(421b46655a0e20bdffffffffffffffffffffffff0500000001000000), ObjectRef(59199d3d5ecaf69fffffffffffffffffffffffff0500000001000000), ObjectRef(1af6c2b24d0711dcffffffffffffffffffffffff0500000001000000), ObjectRef(2c7016ae20dc760bffffffffffffffffffffffff0500000001000000), ObjectRef(8778541c095e32a1ffffffffffffffffffffffff0500000001000000), ObjectRef(ebc9a99fb1df8bceffffffffffffffffffffffff0500000001000000), ObjectRef(69fbfba8ec168ad6ffffffffffffffffffffffff0500000001000000), ObjectRef(f13f2849d3e23e8effffffffffffffffffffffff0500000001000000), ObjectRef(d60e35309d5d500affffffffffffffffffffffff0500000001000000), ObjectRef(80ae01584c090e42ffffffffffffffffffffffff0500000001000000), ObjectRef(4a6e19c6868bf012ffffffffffffffffffffffff0500000001000000), ObjectRef(89822183d62eb606ffffffffffffffffffffffff0500000001000000), ObjectRef(ff7c52bc56e68d56ffffffffffffffffffffffff0500000001000000), ObjectRef(8b36e77bbf694769ffffffffffffffffffffffff0500000001000000), ObjectRef(1714485ad688d855ffffffffffffffffffffffff0500000001000000), ObjectRef(a28bfffad44ccecaffffffffffffffffffffffff0500000001000000), ObjectRef(a026f7a0f3d9e3eaffffffffffffffffffffffff0500000001000000), ObjectRef(8ccd3e54e584583bffffffffffffffffffffffff0500000001000000), ObjectRef(f03b9e06fff74cfdffffffffffffffffffffffff0500000001000000), ObjectRef(7e993a2f27da930fffffffffffffffffffffffff0500000001000000), ObjectRef(8f8acb2c8fc54221ffffffffffffffffffffffff0500000001000000), ObjectRef(bf60743f0229cd0dffffffffffffffffffffffff0500000001000000), ObjectRef(e25c3bd851bc2ccdffffffffffffffffffffffff0500000001000000), ObjectRef(7f37878b47b16bf8ffffffffffffffffffffffff0500000001000000), ObjectRef(a07e5c87bf456562ffffffffffffffffffffffff0500000001000000), ObjectRef(f48c4b9b8a29f3dbffffffffffffffffffffffff0500000001000000), ObjectRef(954755a24622080fffffffffffffffffffffffff0500000001000000), ObjectRef(99e0ae1cefa3277fffffffffffffffffffffffff0500000001000000), ObjectRef(22ecb5884ded3d47ffffffffffffffffffffffff0500000001000000), ObjectRef(8a589d9f038ea7bcffffffffffffffffffffffff0500000001000000), ObjectRef(dc9bb175087dae14ffffffffffffffffffffffff0500000001000000), ObjectRef(9ea80f565708140effffffffffffffffffffffff0500000001000000), ObjectRef(2fe95a5584cf8db1ffffffffffffffffffffffff0500000001000000), ObjectRef(e47d633fe602977dffffffffffffffffffffffff0500000001000000), ObjectRef(73a727873e8d133affffffffffffffffffffffff0500000001000000), ObjectRef(4804a13e788229cbffffffffffffffffffffffff0500000001000000), ObjectRef(8fc7a1d8480dd377ffffffffffffffffffffffff0500000001000000), ObjectRef(e6033a33bbbdce86ffffffffffffffffffffffff0500000001000000), ObjectRef(3d5a78e5375defbbffffffffffffffffffffffff0500000001000000), ObjectRef(3df4e4a69944735dffffffffffffffffffffffff0500000001000000), ObjectRef(ddbfe4cd9a409f09ffffffffffffffffffffffff0500000001000000), ObjectRef(4bf3d0acca90c31effffffffffffffffffffffff0500000001000000), ObjectRef(923ce73a223a406dffffffffffffffffffffffff0500000001000000), ObjectRef(782164c2351a5b45ffffffffffffffffffffffff0500000001000000), ObjectRef(a079b90de7430202ffffffffffffffffffffffff0500000001000000), ObjectRef(42e57f55d595ae78ffffffffffffffffffffffff0500000001000000), ObjectRef(920d783927583c19ffffffffffffffffffffffff0500000001000000), ObjectRef(935981ffd30971baffffffffffffffffffffffff0500000001000000), ObjectRef(398c6a9a6eab89d3ffffffffffffffffffffffff0500000001000000)]
2022-02-08 13:52:10,225 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357130.2254834}' True healthcheck
2022-02-08 13:52:11,227 DEBUG gcs_utils.py:253 -- internal_kv_put b'ray_client_server' b'{"time": 1644357131.227155}' True healthcheck
2022-02-08 13:52:11,354 DEBUG dataservicer.py:225 -- Stream is broken with client 03f832479f1c4a34b6076b30b782564c
2022-02-08 13:52:11,354 DEBUG dataservicer.py:240 -- Cleanup was requested, cleaning up immediately.
2022-02-08 13:52:11,354 DEBUG server.py:299 -- Released all 52 objects for client 03f832479f1c4a34b6076b30b782564c
2022-02-08 13:52:11,354 DEBUG server.py:303 -- Releasing client with no actors: 03f832479f1c4a34b6076b30b782564c
2022-02-08 13:52:11,374 DEBUG dataservicer.py:264 -- Removed client 03f832479f1c4a34b6076b30b782564c, remaining=0
2022-02-08 13:52:11,374 DEBUG dataservicer.py:273 -- Shutting down ray.
2022-02-08 13:52:16,235 INFO server.py:849 -- 25 idle checks before shutdown.
2022-02-08 13:52:21,242 INFO server.py:849 -- 20 idle checks before shutdown.
2022-02-08 13:52:26,250 INFO server.py:849 -- 15 idle checks before shutdown.
2022-02-08 13:52:31,257 INFO server.py:849 -- 10 idle checks before shutdown.
2022-02-08 13:52:36,265 INFO server.py:849 -- 5 idle checks before shutdown.
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment