ranchodeluxe

from postal.expand import expand_address
import sys
import xxhash
import math
import pprint
import pandas as pd
from sqlalchemy import create_engine, text
from datetime import datetime, timedelta

DATABASE_URI = "postgresql://kaiizenadmin:[email protected]/kaiizen"

ranchodeluxe

from postal.expand import expand_address
import sys
import xxhash
import pandas as pd
from sqlalchemy import create_engine, text
from datetime import datetime, timedelta

DATABASE_URI = "postgresql://kaiizenadmin:[email protected]/kaiizen"
engine = create_engine(DATABASE_URI)

ranchodeluxe

from postal.expand import expand_address
import xxhash
from sqlalchemy import create_engine
import pandas as pd
import sys

DATABASE_URI = "postgresql://kaiizenadmin:[email protected]/kaiizen"
engine = create_engine(DATABASE_URI)

ranchodeluxe

          addr          | cherre_properties_current_pk |    pk     
------------------------+------------------------------+-----------
 9708 NW 86TH TER       |          6254075447118810691 | 317232748
 1840 QUEENS MEADOW LN  |         -3759370953727434638 | 411670888
 2 ARIANA DR            |         -8100103611089870304 | 504686232
 125 SPRINGFIELD CT     |            21506068986401567 | 155389880
 23 TERRACE PL          |          6191829075971316516 | 504233524
 2500 WOODFIELD WAY     |         -4593649634119494027 | 544301740
 7911 FOREST FLAME      |          4530592079101629405 | 503017212
 1451 DANIEL LN         |         -4478937597315041628 | 158381596

ranchodeluxe

{
    "cherre_properties_current_pk": -8001908294576819000,
    "arms_length_code": "Arms length transaction",
    "assessed_tax_year": 2023,
    "assessed_value_improvements": 18400,
    "assessed_value_land": 2800,
    "assessed_value_total": 21200,
    "basement_finished_sqft": null,
    "basement_sqft": null,
    "cherre_is_flip": false,

ranchodeluxe

d9d88ca43c44','level':'Error','levelId':4,'processId':28,'exceptionDetails':'Microsoft.Azure.WebJobs.Host.FunctionInvocationException : Exception while executing function: Functions.HrrrStactoolsCollection ---> Microsoft.Azure.WebJobs.Script.Workers.Rpc.RpcException : Result: Failure\nException: RuntimeError: Cannot find the ecCodes library\nStack:   File \'/azure-functions-host/workers/python/3.11/LINUX/X64/azure_functions_worker/dispatcher.py\', line 604, in _handle__invocation_request\n    call_result = await self._loop.run_in_executor(\n                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \'/usr/local/lib/python3.11/concurrent/futures/thread.py\', line 58, in run\n    result = self.fn(*self.args, **self.kwargs)\n             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \'/azure-functions-host/workers/python/3.11/LINUX/X64/azure_functions_worker/dispatcher.py\', line 933, in _run_sync_func\n    return ExtensionManager.get_sync_invocation_wrapper(context,\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

ranchodeluxe

import os
import pytest
import s3fs
import fsspec
import pickle
from multiprocessing import Process, Queue

fs = fsspec.filesystem('local')

ranchodeluxe

Thread 9160 (idle): "MainThread"
    wait (threading.py:331)
    _wait_once (grpc/_common.py:116)
    wait (grpc/_common.py:156)
    _next (grpc/_channel.py:959)
    __next__ (grpc/_channel.py:542)
    run (apache_beam/runners/worker/sdk_worker.py:262)
    main (apache_beam/runners/worker/sdk_worker_main.py:212)
    <module> (apache_beam/runners/worker/sdk_worker_main.py:367)
    _run_code (<frozen runpy>:88)

ranchodeluxe

• all runs below use LocalDirectRunner
• all runs below should attempt to run in under 60 minutes so we don't hit token expiry (only for EDL https/s3 auth)
• when we say cached "disabled" we mean you never set c.InputCacheStorage.fsspec_class in your config

Protected Bucket via EDL Temp Token

https://github.com/ranchodeluxe/staged-recipes/tree/gpm_zarr_failcase

Source	s3://gesdisc-cumulus-prod-protected	s3://gesdisc-cumulus-prod-protected	s3://gesdisc-cumulus-prod-protected

ranchodeluxe

Worker: severity: ERROR timestamp {   seconds: 1706984149   nanos: 336367845 } message: "OSError: [Errno 22] Bad Request [while running \'Create|OpenWithKerchunk|FilterVars|WriteCombinedReference|ValidateDatasetDimensions/OpenWithKerchunk/Open with Kerchunk\']\nTraceback (most recent call last):\n  File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/s3fs/core.py\", line 113, in _error_wrapper\n    return await func(*args, **kwargs)\n  File \"/srv/conda/envs/notebook/lib/python3.10/site-packages/aiobotocore/client.py\", line 371, in _make_api_call\n    raise error_class(parsed_response, operation_name)\nbotocore.exceptions.ClientError: An error occurred (400) when calling the HeadObject operation: Bad Request\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n  File \"apache_beam/runners/common.py\", line 1435, in apache_beam.runners.common.DoFnRunner.process\n  File \"apache_beam/runners/common.py\", line 851, in apache_beam.runners.common.P