Ashvin Nihalani ashvinnihalani

Command: podman run --security-opt=no-new-privileges --cap-drop=ALL --security-opt label=type:nvidia_container_t --hooks-dir=/usr/share/containers/oci/hooks.d/ -v /home/nihalani/Downloads/val2017:/images -v /home/nihalani/Downloads/cocoface/:/annotations --rm -it --ipc=host odtk:latest

Instructions:

	------------------------ arguments ------------------------
	account_for_embedding_in_pipeline_split ......... False
	account_for_loss_in_pipeline_split .............. False
	accumulate_allreduce_grads_in_fp32 .............. True # Double check Good
	adam_beta1 ...................................... 0.9
	adam_beta2 ...................................... 0.95
	adam_eps ........................................ 1e-08
	add_bias_linear ................................. False
	add_position_embedding .......................... True # Double Check
	add_qkv_bias .................................... False

	from kubernetes import client, config

	def terminate_vcjob(vcjob: dict, namespace):
	vcjob_name = vcjob["metadata"]["name"]
	api_instance = client.CustomObjectsApi()

	try:
	vcjob["status"]["state"]["phase"] = "Aborted"
	api_instance.replace_namespaced_custom_object_status(
	group="batch.volcano.sh",

	min_lr=config.min_lr,
	decoupled_lr=config.decoupled_lr,
	decoupled_min_lr=config.decoupled_min_lr,
	)
	param_groups = list(filter(filter_fn, param_groups))
	param_groups.sort(key=lambda g: g.get("wd_mult", 1.0), reverse=True)

	buffers = {}
	for model_chunk_idx, model_chunk in enumerate(model_chunks):
	if hasattr(model_chunk, buffer_name):

	"""
	Launch like:

	OMP_NUM_THREADS=1 LOGLEVEL=WARNING torchrun --rdzv-backend static --master-addr ${VC_TRAIN_0_HOSTS:-$(hostname --fqdn)} \
	--node-rank $NODE_INDEX --nnodes $NNODES --nproc-per-node 8 /shared/workspace/rynli/check_nans_ckpt.py
	"""



	import os