Pablo San José pablosjv

	from collections import namedtuple

	from torch.utils.data import Dataset

	Tokens = namedtuple("Tokens", ["input_ids", "attention_mask"])


	class TokensDataset(Dataset):
	def __init__(self, iids, amask):
	self.input_ids = iids.to_numpy()

	#!/bin/sh

	spark-submit \
	--conf spark.executorEnv.YARN_CONTAINER_RUNTIME_TYPE=docker \
	--conf spark.executorEnv.YARN_CONTAINER_RUNTIME_DOCKER_CLIENT_CONFIG="hdfs:///user/hadoop/config.json" \
	--conf spark.executorEnv.YARN_CONTAINER_RUNTIME_DOCKER_IMAGE=${YOUR_DOCKER_IMAGE} \
	--conf spark.yarn.appMasterEnv.YARN_CONTAINER_RUNTIME_TYPE=docker \
	--conf spark.yarn.appMasterEnv.YARN_CONTAINER_RUNTIME_DOCKER_CLIENT_CONFIG="hdfs:///user/hadoop/config.json" \
	--conf spark.yarn.appMasterEnv.YARN_CONTAINER_RUNTIME_DOCKER_IMAGE=${YOUR_DOCKER_IMAGE} \
	s3://your-bucket/path/to/your/script.py