lgray

# ==========================================================================
#  AIDA Detector description implementation
# --------------------------------------------------------------------------
# Copyright (C) Organisation europeenne pour la Recherche nucleaire (CERN)
# All rights reserved.
#
# For the licensing terms see $DD4hepINSTALL/LICENSE.
# For the list of contributors see $DD4hepINSTALL/doc/CREDITS.
#
# ==========================================================================

lgray

Every 1.0s: zpool iostat -vly 1 1                                                                                                                                                              godrick: Mon Sep 26 19:53:16 2022

                                                   capacity     operations     bandwidth    total_wait     disk_wait    syncq_wait    asyncq_wait  scrub   trim
pool                                             alloc   free   read  write   read  write   read  write   read  write   read  write   read  write   wait   wait
-----------------------------------------------  -----  -----  -----  -----  -----  -----  -----  -----  -----  -----  -----  -----  -----  -----  -----  -----
chrysalid                                        3.81T  10.7T     13    667  1.74M  28.3M    2ms    7ms    2ms  502us    2us    1us    2us    7ms      -      -
  mirror-0                                       1.91T  5.35T      2    295   383K  13.1M    2ms    8ms    2ms  464us    2us    1us      -    7ms      -      

lgray

import awkward
import uproot
import glob
from rich.progress import track
import multiprocessing
import concurrent.futures
import pathlib

def extract_file(c3FileName, keep):
    c3File = uproot.open(c3FileName)

lgray

import uproot
import awkward
import hist
import numpy as np
import os
import math

def get_root_parent(index, mcparticles):
         parents = mcparticles[index].member("parents").tolist()
         if len(parents) == 0:

lgray

(coffea-dev) lgray@dhcp-131-225-97-134 coffea % python -i spark_work.py
Setting default log level to "WARN".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
22/12/09 16:42:22 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
40 * {Muon_pt: var * float32, Muon_eta: var * float32, Muon_phi: var * float32, Muon_mass: var * float32, Muon_charge: var * int32, nMuon: int64}
[pyarrow.RecordBatch
Muon_pt: list<item: float not null> not null
  child 0, item: float not null
Muon_eta: list<item: float not null> not null
  child 0, item: float not null

lgray

import time

import awkward as ak
import dask_awkward as dak
import numpy as np

import os

from coffea.lookup_tools import extractor
from coffea.jetmet_tools import FactorizedJetCorrector

lgray

# import warnings
# warnings.filterwarnings("error")


import logging
import os
import time

from coffea import processor
from coffea.nanoevents import NanoAODSchema

lgray

from coffea.nanoevents import NanoEventsFactory, NanoAODSchema
from coffea.processor import accumulate
from distributed import Client
import dask
import dask_awkward as dak
import dask.array
from dask.diagnostics import ProgressBar


import awkward

lgray

class CustomDataGenerator(tf.keras.utils.Sequence):
    
    def __init__(self, 
                data_directory_path: str = "./",
                labels_directory_path: str = "./",
                is_directory_recursive: bool = False,
                file_type: str = "csv",
                data_format: str = "2D",
                batch_size: int = 32,
                file_count = None,

lgray

import numpy as np
from scipy.stats import uniform
import vector
import hist
from math import pi

def make_vector(rawvec):
    return vector.arr({"px": rawvec[:, 0], "py": rawvec[:, 1], "pz": rawvec[:, 2], "M": rawvec[:, 3]})
 
P_beam = 1000 # GeV