Created
February 3, 2024 18:20
-
-
Save ranchodeluxe/5bb51c826a2034f37b6ab02865215d28 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Worker: severity: ERROR timestamp { seconds: 1706984149 nanos: 336367845 } message: "OSError: [Errno 22] Bad Request [while running \'Create|OpenWithKerchunk|FilterVars|WriteCombinedReference|ValidateDatasetDimensions/OpenWithKerchunk/Open with Kerchunk\']\nTraceback (most recent call last):\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 113, in _error_wrapper\n return await func(*args, **kwargs)\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/aiobotocore/client.py\", line 371, in _make_api_call\n raise error_class(parsed_response, operation_name)\nbotocore.exceptions.ClientError: An error occurred (400) when calling the HeadObject operation: Bad Request\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"apache_beam/runners/common.py\", line 1435, in apache_beam.runners.common.DoFnRunner.process\n File \"apache_beam/runners/common.py\", line 851, in apache_beam.runners.common.PerWindowInvoker.invoke_process\n File \"apache_beam/runners/common.py\", line 997, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/transforms/core.py\", line 1961, in <lambda>\n wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/pangeo_forge_recipes/transforms.py\", line 104, in wrapper\n result = func(item, *args, **kwargs)\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/pangeo_forge_recipes/openers.py\", line 165, in open_with_kerchunk\n h5chunks = SingleHdf5ToZarr(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/kerchunk/hdf.py\", line 83, in __init__\n self.input_file = fs.open(path, \"rb\")\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py\", line 1241, in open\n f = self._open(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 659, in _open\n return S3File(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 2066, in __init__\n super().__init__(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py\", line 1597, in __init__\n self.size = self.details[\"size\"]\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py\", line 1610, in details\n self._details = self.fs.info(self.path)\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py\", line 121, in wrapper\n return sync(self.loop, func, *args, **kwargs)\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py\", line 106, in sync\n raise return_result\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py\", line 61, in _runner\n result[0] = await coro\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 1271, in _info\n out = await self._call_s3(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 348, in _call_s3\n return await _error_wrapper(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 140, in _error_wrapper\n raise err\nOSError: [Errno 22] Bad Request" trace: "Traceback (most recent call last):\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 113, in _error_wrapper\n return await func(*args, **kwargs)\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/aiobotocore/client.py\", line 371, in _make_api_call\n raise error_class(parsed_response, operation_name)\nbotocore.exceptions.ClientError: An error occurred (400) when calling the HeadObject operation: Bad Request\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"apache_beam/runners/common.py\", line 1435, in apache_beam.runners.common.DoFnRunner.process\n File \"apache_beam/runners/common.py\", line 851, in apache_beam.runners.common.PerWindowInvoker.invoke_process\n File \"apache_beam/runners/common.py\", line 997, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/transforms/core.py\", line 1961, in <lambda>\n wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/pangeo_forge_recipes/transforms.py\", line 104, in wrapper\n result = func(item, *args, **kwargs)\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/pangeo_forge_recipes/openers.py\", line 165, in open_with_kerchunk\n h5chunks = SingleHdf5ToZarr(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/kerchunk/hdf.py\", line 83, in __init__\n self.input_file = fs.open(path, \"rb\")\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py\", line 1241, in open\n f = self._open(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 659, in _open\n return S3File(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 2066, in __init__\n super().__init__(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py\", line 1597, in __init__\n self.size = self.details[\"size\"]\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py\", line 1610, in details\n self._details = self.fs.info(self.path)\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py\", line 121, in wrapper\n return sync(self.loop, func, *args, **kwargs)\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py\", line 106, in sync\n raise return_result\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py\", line 61, in _runner\n result[0] = await coro\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 1271, in _info\n out = await self._call_s3(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 348, in _call_s3\n return await _error_wrapper(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 140, in _error_wrapper\n raise err\nOSError: [Errno 22] Bad Request\n" instruction_id: "bundle_2" transform_id: "Create|OpenWithKerchunk|FilterVars|WriteCombinedReference|ValidateDatasetDimensions/OpenWithKerchunk/Open with Kerchunk" log_location: "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py:237" thread: "Thread-13" | |
| Traceback (most recent call last): | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 113, in _error_wrapper | |
| return await func(*args, **kwargs) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/aiobotocore/client.py", line 371, in _make_api_call | |
| raise error_class(parsed_response, operation_name) | |
| botocore.exceptions.ClientError: An error occurred (400) when calling the HeadObject operation: Bad Request | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "apache_beam/runners/common.py", line 1435, in apache_beam.runners.common.DoFnRunner.process | |
| File "apache_beam/runners/common.py", line 851, in apache_beam.runners.common.PerWindowInvoker.invoke_process | |
| File "apache_beam/runners/common.py", line 997, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/transforms/core.py", line 1961, in <lambda> | |
| wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)] | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/pangeo_forge_recipes/transforms.py", line 104, in wrapper | |
| result = func(item, *args, **kwargs) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/pangeo_forge_recipes/openers.py", line 165, in open_with_kerchunk | |
| h5chunks = SingleHdf5ToZarr( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/kerchunk/hdf.py", line 83, in __init__ | |
| self.input_file = fs.open(path, "rb") | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py", line 1241, in open | |
| f = self._open( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 659, in _open | |
| return S3File( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 2066, in __init__ | |
| super().__init__( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py", line 1597, in __init__ | |
| self.size = self.details["size"] | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py", line 1610, in details | |
| self._details = self.fs.info(self.path) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py", line 121, in wrapper | |
| return sync(self.loop, func, *args, **kwargs) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py", line 106, in sync | |
| raise return_result | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py", line 61, in _runner | |
| result[0] = await coro | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 1271, in _info | |
| out = await self._call_s3( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 348, in _call_s3 | |
| return await _error_wrapper( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 140, in _error_wrapper | |
| raise err | |
| OSError: [Errno 22] Bad Request | |
| During handling of the above exception, another exception occurred: | |
| Traceback (most recent call last): | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py", line 297, in _execute | |
| response = task() | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py", line 372, in <lambda> | |
| lambda: self.create_worker().do_instruction(request), request) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py", line 625, in do_instruction | |
| return getattr(self, request_type)( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py", line 663, in process_bundle | |
| bundle_processor.process_bundle(instruction_id)) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1056, in process_bundle | |
| input_op_by_transform_id[element.transform_id].process_encoded( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py", line 237, in process_encoded | |
| self.output(decoded_value) | |
| File "apache_beam/runners/worker/operations.py", line 570, in apache_beam.runners.worker.operations.Operation.output | |
| File "apache_beam/runners/worker/operations.py", line 572, in apache_beam.runners.worker.operations.Operation.output | |
| File "apache_beam/runners/worker/operations.py", line 263, in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive | |
| File "apache_beam/runners/worker/operations.py", line 266, in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive | |
| File "apache_beam/runners/worker/operations.py", line 953, in apache_beam.runners.worker.operations.DoOperation.process | |
| File "apache_beam/runners/worker/operations.py", line 954, in apache_beam.runners.worker.operations.DoOperation.process | |
| File "apache_beam/runners/common.py", line 1437, in apache_beam.runners.common.DoFnRunner.process | |
| File "apache_beam/runners/common.py", line 1526, in apache_beam.runners.common.DoFnRunner._reraise_augmented | |
| File "apache_beam/runners/common.py", line 1435, in apache_beam.runners.common.DoFnRunner.process | |
| File "apache_beam/runners/common.py", line 636, in apache_beam.runners.common.SimpleInvoker.invoke_process | |
| File "apache_beam/runners/common.py", line 1621, in apache_beam.runners.common._OutputHandler.handle_process_outputs | |
| File "apache_beam/runners/common.py", line 1734, in apache_beam.runners.common._OutputHandler._write_value_to_tag | |
| File "apache_beam/runners/worker/operations.py", line 266, in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive | |
| File "apache_beam/runners/worker/operations.py", line 953, in apache_beam.runners.worker.operations.DoOperation.process | |
| File "apache_beam/runners/worker/operations.py", line 954, in apache_beam.runners.worker.operations.DoOperation.process | |
| File "apache_beam/runners/common.py", line 1437, in apache_beam.runners.common.DoFnRunner.process | |
| File "apache_beam/runners/common.py", line 1526, in apache_beam.runners.common.DoFnRunner._reraise_augmented | |
| File "apache_beam/runners/common.py", line 1435, in apache_beam.runners.common.DoFnRunner.process | |
| File "apache_beam/runners/common.py", line 636, in apache_beam.runners.common.SimpleInvoker.invoke_process | |
| File "apache_beam/runners/common.py", line 1621, in apache_beam.runners.common._OutputHandler.handle_process_outputs | |
| File "apache_beam/runners/common.py", line 1734, in apache_beam.runners.common._OutputHandler._write_value_to_tag | |
| File "apache_beam/runners/worker/operations.py", line 266, in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive | |
| File "apache_beam/runners/worker/operations.py", line 953, in apache_beam.runners.worker.operations.DoOperation.process | |
| File "apache_beam/runners/worker/operations.py", line 954, in apache_beam.runners.worker.operations.DoOperation.process | |
| File "apache_beam/runners/common.py", line 1437, in apache_beam.runners.common.DoFnRunner.process | |
| File "apache_beam/runners/common.py", line 1526, in apache_beam.runners.common.DoFnRunner._reraise_augmented | |
| File "apache_beam/runners/common.py", line 1435, in apache_beam.runners.common.DoFnRunner.process | |
| File "apache_beam/runners/common.py", line 636, in apache_beam.runners.common.SimpleInvoker.invoke_process | |
| File "apache_beam/runners/common.py", line 1621, in apache_beam.runners.common._OutputHandler.handle_process_outputs | |
| File "apache_beam/runners/common.py", line 1734, in apache_beam.runners.common._OutputHandler._write_value_to_tag | |
| File "apache_beam/runners/worker/operations.py", line 266, in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive | |
| File "apache_beam/runners/worker/operations.py", line 953, in apache_beam.runners.worker.operations.DoOperation.process | |
| File "apache_beam/runners/worker/operations.py", line 954, in apache_beam.runners.worker.operations.DoOperation.process | |
| File "apache_beam/runners/common.py", line 1437, in apache_beam.runners.common.DoFnRunner.process | |
| File "apache_beam/runners/common.py", line 1547, in apache_beam.runners.common.DoFnRunner._reraise_augmented | |
| File "apache_beam/runners/common.py", line 1435, in apache_beam.runners.common.DoFnRunner.process | |
| File "apache_beam/runners/common.py", line 851, in apache_beam.runners.common.PerWindowInvoker.invoke_process | |
| File "apache_beam/runners/common.py", line 997, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/transforms/core.py", line 1961, in <lambda> | |
| wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)] | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/pangeo_forge_recipes/transforms.py", line 104, in wrapper | |
| result = func(item, *args, **kwargs) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/pangeo_forge_recipes/openers.py", line 165, in open_with_kerchunk | |
| h5chunks = SingleHdf5ToZarr( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/kerchunk/hdf.py", line 83, in __init__ | |
| self.input_file = fs.open(path, "rb") | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py", line 1241, in open | |
| f = self._open( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 659, in _open | |
| return S3File( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 2066, in __init__ | |
| super().__init__( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py", line 1597, in __init__ | |
| self.size = self.details["size"] | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py", line 1610, in details | |
| self._details = self.fs.info(self.path) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py", line 121, in wrapper | |
| return sync(self.loop, func, *args, **kwargs) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py", line 106, in sync | |
| raise return_result | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py", line 61, in _runner | |
| result[0] = await coro | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 1271, in _info | |
| out = await self._call_s3( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 348, in _call_s3 | |
| return await _error_wrapper( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 140, in _error_wrapper | |
| raise err | |
| RuntimeError: OSError: [Errno 22] Bad Request [while running 'Create|OpenWithKerchunk|FilterVars|WriteCombinedReference|ValidateDatasetDimensions/OpenWithKerchunk/Open with Kerchunk'] | |
| Worker: severity: ERROR timestamp { seconds: 1706984149 nanos: 496680259 } message: "Error processing instruction bundle_2. Original traceback is\nTraceback (most recent call last):\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 113, in _error_wrapper\n return await func(*args, **kwargs)\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/aiobotocore/client.py\", line 371, in _make_api_call\n raise error_class(parsed_response, operation_name)\nbotocore.exceptions.ClientError: An error occurred (400) when calling the HeadObject operation: Bad Request\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File \"apache_beam/runners/common.py\", line 1435, in apache_beam.runners.common.DoFnRunner.process\n File \"apache_beam/runners/common.py\", line 851, in apache_beam.runners.common.PerWindowInvoker.invoke_process\n File \"apache_beam/runners/common.py\", line 997, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/transforms/core.py\", line 1961, in <lambda>\n wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/pangeo_forge_recipes/transforms.py\", line 104, in wrapper\n result = func(item, *args, **kwargs)\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/pangeo_forge_recipes/openers.py\", line 165, in open_with_kerchunk\n h5chunks = SingleHdf5ToZarr(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/kerchunk/hdf.py\", line 83, in __init__\n self.input_file = fs.open(path, \"rb\")\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py\", line 1241, in open\n f = self._open(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 659, in _open\n return S3File(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 2066, in __init__\n super().__init__(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py\", line 1597, in __init__\n self.size = self.details[\"size\"]\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py\", line 1610, in details\n self._details = self.fs.info(self.path)\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py\", line 121, in wrapper\n return sync(self.loop, func, *args, **kwargs)\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py\", line 106, in sync\n raise return_result\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py\", line 61, in _runner\n result[0] = await coro\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 1271, in _info\n out = await self._call_s3(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 348, in _call_s3\n return await _error_wrapper(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 140, in _error_wrapper\n raise err\nOSError: [Errno 22] Bad Request\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py\", line 297, in _execute\n response = task()\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py\", line 372, in <lambda>\n lambda: self.create_worker().do_instruction(request), request)\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py\", line 625, in do_instruction\n return getattr(self, request_type)(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py\", line 663, in process_bundle\n bundle_processor.process_bundle(instruction_id))\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py\", line 1056, in process_bundle\n input_op_by_transform_id[element.transform_id].process_encoded(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py\", line 237, in process_encoded\n self.output(decoded_value)\n File \"apache_beam/runners/worker/operations.py\", line 570, in apache_beam.runners.worker.operations.Operation.output\n File \"apache_beam/runners/worker/operations.py\", line 572, in apache_beam.runners.worker.operations.Operation.output\n File \"apache_beam/runners/worker/operations.py\", line 263, in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive\n File \"apache_beam/runners/worker/operations.py\", line 266, in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive\n File \"apache_beam/runners/worker/operations.py\", line 953, in apache_beam.runners.worker.operations.DoOperation.process\n File \"apache_beam/runners/worker/operations.py\", line 954, in apache_beam.runners.worker.operations.DoOperation.process\n File \"apache_beam/runners/common.py\", line 1437, in apache_beam.runners.common.DoFnRunner.process\n File \"apache_beam/runners/common.py\", line 1526, in apache_beam.runners.common.DoFnRunner._reraise_augmented\n File \"apache_beam/runners/common.py\", line 1435, in apache_beam.runners.common.DoFnRunner.process\n File \"apache_beam/runners/common.py\", line 636, in apache_beam.runners.common.SimpleInvoker.invoke_process\n File \"apache_beam/runners/common.py\", line 1621, in apache_beam.runners.common._OutputHandler.handle_process_outputs\n File \"apache_beam/runners/common.py\", line 1734, in apache_beam.runners.common._OutputHandler._write_value_to_tag\n File \"apache_beam/runners/worker/operations.py\", line 266, in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive\n File \"apache_beam/runners/worker/operations.py\", line 953, in apache_beam.runners.worker.operations.DoOperation.process\n File \"apache_beam/runners/worker/operations.py\", line 954, in apache_beam.runners.worker.operations.DoOperation.process\n File \"apache_beam/runners/common.py\", line 1437, in apache_beam.runners.common.DoFnRunner.process\n File \"apache_beam/runners/common.py\", line 1526, in apache_beam.runners.common.DoFnRunner._reraise_augmented\n File \"apache_beam/runners/common.py\", line 1435, in apache_beam.runners.common.DoFnRunner.process\n File \"apache_beam/runners/common.py\", line 636, in apache_beam.runners.common.SimpleInvoker.invoke_process\n File \"apache_beam/runners/common.py\", line 1621, in apache_beam.runners.common._OutputHandler.handle_process_outputs\n File \"apache_beam/runners/common.py\", line 1734, in apache_beam.runners.common._OutputHandler._write_value_to_tag\n File \"apache_beam/runners/worker/operations.py\", line 266, in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive\n File \"apache_beam/runners/worker/operations.py\", line 953, in apache_beam.runners.worker.operations.DoOperation.process\n File \"apache_beam/runners/worker/operations.py\", line 954, in apache_beam.runners.worker.operations.DoOperation.process\n File \"apache_beam/runners/common.py\", line 1437, in apache_beam.runners.common.DoFnRunner.process\n File \"apache_beam/runners/common.py\", line 1526, in apache_beam.runners.common.DoFnRunner._reraise_augmented\n File \"apache_beam/runners/common.py\", line 1435, in apache_beam.runners.common.DoFnRunner.process\n File \"apache_beam/runners/common.py\", line 636, in apache_beam.runners.common.SimpleInvoker.invoke_process\n File \"apache_beam/runners/common.py\", line 1621, in apache_beam.runners.common._OutputHandler.handle_process_outputs\n File \"apache_beam/runners/common.py\", line 1734, in apache_beam.runners.common._OutputHandler._write_value_to_tag\n File \"apache_beam/runners/worker/operations.py\", line 266, in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive\n File \"apache_beam/runners/worker/operations.py\", line 953, in apache_beam.runners.worker.operations.DoOperation.process\n File \"apache_beam/runners/worker/operations.py\", line 954, in apache_beam.runners.worker.operations.DoOperation.process\n File \"apache_beam/runners/common.py\", line 1437, in apache_beam.runners.common.DoFnRunner.process\n File \"apache_beam/runners/common.py\", line 1547, in apache_beam.runners.common.DoFnRunner._reraise_augmented\n File \"apache_beam/runners/common.py\", line 1435, in apache_beam.runners.common.DoFnRunner.process\n File \"apache_beam/runners/common.py\", line 851, in apache_beam.runners.common.PerWindowInvoker.invoke_process\n File \"apache_beam/runners/common.py\", line 997, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/transforms/core.py\", line 1961, in <lambda>\n wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)]\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/pangeo_forge_recipes/transforms.py\", line 104, in wrapper\n result = func(item, *args, **kwargs)\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/pangeo_forge_recipes/openers.py\", line 165, in open_with_kerchunk\n h5chunks = SingleHdf5ToZarr(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/kerchunk/hdf.py\", line 83, in __init__\n self.input_file = fs.open(path, \"rb\")\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py\", line 1241, in open\n f = self._open(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 659, in _open\n return S3File(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 2066, in __init__\n super().__init__(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py\", line 1597, in __init__\n self.size = self.details[\"size\"]\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py\", line 1610, in details\n self._details = self.fs.info(self.path)\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py\", line 121, in wrapper\n return sync(self.loop, func, *args, **kwargs)\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py\", line 106, in sync\n raise return_result\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py\", line 61, in _runner\n result[0] = await coro\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 1271, in _info\n out = await self._call_s3(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 348, in _call_s3\n return await _error_wrapper(\n File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 140, in _error_wrapper\n raise err\nRuntimeError: OSError: [Errno 22] Bad Request [while running \'Create|OpenWithKerchunk|FilterVars|WriteCombinedReference|ValidateDatasetDimensions/OpenWithKerchunk/Open with Kerchunk\']\n\n" instruction_id: "bundle_2" log_location: "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:301" thread: "Thread-13" | |
| Worker: severity: INFO timestamp { seconds: 1706984149 nanos: 668663024 } message: "No more requests from control plane" log_location: "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:274" thread: "MainThread" | |
| Worker: severity: INFO timestamp { seconds: 1706984149 nanos: 668795108 } message: "SDK Harness waiting for in-flight requests to complete" log_location: "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:275" thread: "MainThread" | |
| Worker: severity: INFO timestamp { seconds: 1706984149 nanos: 668847560 } message: "Closing all cached grpc data channels." log_location: "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/data_plane.py:803" thread: "MainThread" | |
| Worker: severity: INFO timestamp { seconds: 1706984149 nanos: 668896913 } message: "Closing all cached gRPC state handlers." log_location: "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:904" thread: "MainThread" | |
| Worker: severity: INFO timestamp { seconds: 1706984149 nanos: 669815063 } message: "Done consuming work." log_location: "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py:287" thread: "MainThread" | |
| Worker: severity: INFO timestamp { seconds: 1706984149 nanos: 669890165 } message: "Python sdk harness exiting." log_location: "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker_main.py:213" thread: "MainThread" | |
| Traceback (most recent call last): | |
| File "/srv/conda/envs/notebook/bin/pangeo-forge-runner", line 8, in <module> | |
| sys.exit(main()) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/pangeo_forge_runner/cli.py", line 28, in main | |
| app.start() | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/pangeo_forge_runner/cli.py", line 23, in start | |
| super().start() | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/traitlets/config/application.py", line 473, in start | |
| return self.subapp.start() | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/pangeo_forge_runner/commands/bake.py", line 326, in start | |
| pipeline.run() | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/pipeline.py", line 585, in run | |
| return self.runner.run_pipeline(self, self._options) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/direct/direct_runner.py", line 128, in run_pipeline | |
| return runner.run_pipeline(pipeline, options) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py", line 202, in run_pipeline | |
| self._latest_run_result = self.run_via_runner_api( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py", line 224, in run_via_runner_api | |
| return self.run_stages(stage_context, stages) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py", line 455, in run_stages | |
| bundle_results = self._execute_bundle( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py", line 783, in _execute_bundle | |
| self._run_bundle( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py", line 1020, in _run_bundle | |
| result, splits = bundle_manager.process_bundle( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/portability/fn_api_runner/fn_runner.py", line 1395, in process_bundle | |
| raise RuntimeError(result.error) | |
| RuntimeError: Traceback (most recent call last): | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 113, in _error_wrapper | |
| return await func(*args, **kwargs) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/aiobotocore/client.py", line 371, in _make_api_call | |
| raise error_class(parsed_response, operation_name) | |
| botocore.exceptions.ClientError: An error occurred (400) when calling the HeadObject operation: Bad Request | |
| The above exception was the direct cause of the following exception: | |
| Traceback (most recent call last): | |
| File "apache_beam/runners/common.py", line 1435, in apache_beam.runners.common.DoFnRunner.process | |
| File "apache_beam/runners/common.py", line 851, in apache_beam.runners.common.PerWindowInvoker.invoke_process | |
| File "apache_beam/runners/common.py", line 997, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/transforms/core.py", line 1961, in <lambda> | |
| wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)] | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/pangeo_forge_recipes/transforms.py", line 104, in wrapper | |
| result = func(item, *args, **kwargs) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/pangeo_forge_recipes/openers.py", line 165, in open_with_kerchunk | |
| h5chunks = SingleHdf5ToZarr( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/kerchunk/hdf.py", line 83, in __init__ | |
| self.input_file = fs.open(path, "rb") | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py", line 1241, in open | |
| f = self._open( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 659, in _open | |
| return S3File( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 2066, in __init__ | |
| super().__init__( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py", line 1597, in __init__ | |
| self.size = self.details["size"] | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py", line 1610, in details | |
| self._details = self.fs.info(self.path) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py", line 121, in wrapper | |
| return sync(self.loop, func, *args, **kwargs) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py", line 106, in sync | |
| raise return_result | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py", line 61, in _runner | |
| result[0] = await coro | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 1271, in _info | |
| out = await self._call_s3( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 348, in _call_s3 | |
| return await _error_wrapper( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 140, in _error_wrapper | |
| raise err | |
| OSError: [Errno 22] Bad Request | |
| During handling of the above exception, another exception occurred: | |
| Traceback (most recent call last): | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py", line 297, in _execute | |
| response = task() | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py", line 372, in <lambda> | |
| lambda: self.create_worker().do_instruction(request), request) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py", line 625, in do_instruction | |
| return getattr(self, request_type)( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/sdk_worker.py", line 663, in process_bundle | |
| bundle_processor.process_bundle(instruction_id)) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py", line 1056, in process_bundle | |
| input_op_by_transform_id[element.transform_id].process_encoded( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/runners/worker/bundle_processor.py", line 237, in process_encoded | |
| self.output(decoded_value) | |
| File "apache_beam/runners/worker/operations.py", line 570, in apache_beam.runners.worker.operations.Operation.output | |
| File "apache_beam/runners/worker/operations.py", line 572, in apache_beam.runners.worker.operations.Operation.output | |
| File "apache_beam/runners/worker/operations.py", line 263, in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive | |
| File "apache_beam/runners/worker/operations.py", line 266, in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive | |
| File "apache_beam/runners/worker/operations.py", line 953, in apache_beam.runners.worker.operations.DoOperation.process | |
| File "apache_beam/runners/worker/operations.py", line 954, in apache_beam.runners.worker.operations.DoOperation.process | |
| File "apache_beam/runners/common.py", line 1437, in apache_beam.runners.common.DoFnRunner.process | |
| File "apache_beam/runners/common.py", line 1526, in apache_beam.runners.common.DoFnRunner._reraise_augmented | |
| File "apache_beam/runners/common.py", line 1435, in apache_beam.runners.common.DoFnRunner.process | |
| File "apache_beam/runners/common.py", line 636, in apache_beam.runners.common.SimpleInvoker.invoke_process | |
| File "apache_beam/runners/common.py", line 1621, in apache_beam.runners.common._OutputHandler.handle_process_outputs | |
| File "apache_beam/runners/common.py", line 1734, in apache_beam.runners.common._OutputHandler._write_value_to_tag | |
| File "apache_beam/runners/worker/operations.py", line 266, in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive | |
| File "apache_beam/runners/worker/operations.py", line 953, in apache_beam.runners.worker.operations.DoOperation.process | |
| File "apache_beam/runners/worker/operations.py", line 954, in apache_beam.runners.worker.operations.DoOperation.process | |
| File "apache_beam/runners/common.py", line 1437, in apache_beam.runners.common.DoFnRunner.process | |
| File "apache_beam/runners/common.py", line 1526, in apache_beam.runners.common.DoFnRunner._reraise_augmented | |
| File "apache_beam/runners/common.py", line 1435, in apache_beam.runners.common.DoFnRunner.process | |
| File "apache_beam/runners/common.py", line 636, in apache_beam.runners.common.SimpleInvoker.invoke_process | |
| File "apache_beam/runners/common.py", line 1621, in apache_beam.runners.common._OutputHandler.handle_process_outputs | |
| File "apache_beam/runners/common.py", line 1734, in apache_beam.runners.common._OutputHandler._write_value_to_tag | |
| File "apache_beam/runners/worker/operations.py", line 266, in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive | |
| File "apache_beam/runners/worker/operations.py", line 953, in apache_beam.runners.worker.operations.DoOperation.process | |
| File "apache_beam/runners/worker/operations.py", line 954, in apache_beam.runners.worker.operations.DoOperation.process | |
| File "apache_beam/runners/common.py", line 1437, in apache_beam.runners.common.DoFnRunner.process | |
| File "apache_beam/runners/common.py", line 1526, in apache_beam.runners.common.DoFnRunner._reraise_augmented | |
| File "apache_beam/runners/common.py", line 1435, in apache_beam.runners.common.DoFnRunner.process | |
| File "apache_beam/runners/common.py", line 636, in apache_beam.runners.common.SimpleInvoker.invoke_process | |
| File "apache_beam/runners/common.py", line 1621, in apache_beam.runners.common._OutputHandler.handle_process_outputs | |
| File "apache_beam/runners/common.py", line 1734, in apache_beam.runners.common._OutputHandler._write_value_to_tag | |
| File "apache_beam/runners/worker/operations.py", line 266, in apache_beam.runners.worker.operations.SingletonElementConsumerSet.receive | |
| File "apache_beam/runners/worker/operations.py", line 953, in apache_beam.runners.worker.operations.DoOperation.process | |
| File "apache_beam/runners/worker/operations.py", line 954, in apache_beam.runners.worker.operations.DoOperation.process | |
| File "apache_beam/runners/common.py", line 1437, in apache_beam.runners.common.DoFnRunner.process | |
| File "apache_beam/runners/common.py", line 1547, in apache_beam.runners.common.DoFnRunner._reraise_augmented | |
| File "apache_beam/runners/common.py", line 1435, in apache_beam.runners.common.DoFnRunner.process | |
| File "apache_beam/runners/common.py", line 851, in apache_beam.runners.common.PerWindowInvoker.invoke_process | |
| File "apache_beam/runners/common.py", line 997, in apache_beam.runners.common.PerWindowInvoker._invoke_process_per_window | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/apache_beam/transforms/core.py", line 1961, in <lambda> | |
| wrapper = lambda x, *args, **kwargs: [fn(x, *args, **kwargs)] | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/pangeo_forge_recipes/transforms.py", line 104, in wrapper | |
| result = func(item, *args, **kwargs) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/pangeo_forge_recipes/openers.py", line 165, in open_with_kerchunk | |
| h5chunks = SingleHdf5ToZarr( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/kerchunk/hdf.py", line 83, in __init__ | |
| self.input_file = fs.open(path, "rb") | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py", line 1241, in open | |
| f = self._open( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 659, in _open | |
| return S3File( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 2066, in __init__ | |
| super().__init__( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py", line 1597, in __init__ | |
| self.size = self.details["size"] | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/spec.py", line 1610, in details | |
| self._details = self.fs.info(self.path) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py", line 121, in wrapper | |
| return sync(self.loop, func, *args, **kwargs) | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py", line 106, in sync | |
| raise return_result | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/fsspec/asyn.py", line 61, in _runner | |
| result[0] = await coro | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 1271, in _info | |
| out = await self._call_s3( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 348, in _call_s3 | |
| return await _error_wrapper( | |
| File "/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py", line 140, in _error_wrapper | |
| raise err | |
| RuntimeError: OSError: [Errno 22] Bad Request [while running 'Create|OpenWithKerchunk|FilterVars|WriteCombinedReference|ValidateDatasetDimensions/OpenWithKerchunk/Open with Kerchunk'] |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment