node_datasets_2024-08-17T15_42_00/knowledge_compliance_personally-identifiable-information_task.yaml

gistfile1.txt

[root@tyler-a100 generated]# cat node_datasets_2024-08-17T15_42_00/knowledge_compliance_personally-identifiable-information_task.yaml 
dataset_kwargs:
  data_files:
    test: /var/mnt/inststg1/instructlab/generated//node_datasets_2024-08-17T15_42_00/mmlubench_knowledge_compliance_personally-identifiable-information.jsonl
dataset_name: null
dataset_path: json
doc_to_choice: '{{[choices[0], choices[1], choices[2], choices[3]]}}'
doc_to_target: '{{answer}}'
doc_to_text: '{{question.strip()}}

  A. {{choices[0]}}

  B. {{choices[1]}}

  C. {{choices[2]}}

  D. {{choices[3]}}

  Answer:'
group: mmlu_pr
metric_list:
- aggregation: mean
  higher_is_better: 'true'
  metric: acc
output_type: multiple_choice
task: knowledge_compliance_personally-identifiable-information
test_split: test