| datasets = [ |
| [ |
| dict( |
| abbr='lukaemon_mmlu_college_chemistry', |
| eval_cfg=dict( |
| evaluator=dict( |
| type='opencompass.openicl.icl_evaluator.AccEvaluator')), |
| infer_cfg=dict( |
| ice_template=dict( |
| template=dict( |
| A= |
| '{input}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: A\n', |
| B= |
| '{input}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: B\n', |
| C= |
| '{input}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: C\n', |
| D='{input}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: D\n' |
| ), |
| type= |
| 'opencompass.openicl.icl_prompt_template.PromptTemplate'), |
| inferencer=dict( |
| type='opencompass.openicl.icl_inferencer.PPLInferencer'), |
| prompt_template=dict( |
| ice_token='</E>', |
| template=dict( |
| A= |
| 'The following are multiple choice questions (with answers) about college chemistry.\n\n</E>{input}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: A', |
| B= |
| 'The following are multiple choice questions (with answers) about college chemistry.\n\n</E>{input}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: B', |
| C= |
| 'The following are multiple choice questions (with answers) about college chemistry.\n\n</E>{input}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: C', |
| D='The following are multiple choice questions (with answers) about college chemistry.\n\n</E>{input}\nA. {A}\nB. {B}\nC. {C}\nD. {D}\nAnswer: D' |
| ), |
| type= |
| 'opencompass.openicl.icl_prompt_template.PromptTemplate'), |
| retriever=dict( |
| fix_id_list=[ |
| 0, |
| 1, |
| 2, |
| 3, |
| 4, |
| ], |
| type='opencompass.openicl.icl_retriever.FixKRetriever')), |
| name='college_chemistry', |
| path='./data/mmlu/', |
| reader_cfg=dict( |
| input_columns=[ |
| 'input', |
| 'A', |
| 'B', |
| 'C', |
| 'D', |
| ], |
| output_column='target', |
| train_split='dev'), |
| type='opencompass.datasets.MMLUDataset'), |
| ], |
| ] |
| eval = dict(runner=dict(task=dict())) |
| models = [ |
| dict( |
| abbr='my_api', |
| api_key='', |
| batch_size=8, |
| max_out_len=100, |
| max_seq_len=2048, |
| meta_template=dict(round=[ |
| dict(api_role='HUMAN', role='HUMAN'), |
| dict(api_role='BOT', generate=True, role='BOT'), |
| ]), |
| path='my_api', |
| run_cfg=dict(num_gpus=1, num_procs=1), |
| type='opencompass.models.my_api.MyAPIModel', |
| url='http://127.0.0.1:12345/testing'), |
| ] |
| work_dir = './outputs/default/20240306_164404' |
|
|
