diff --git "a/eval_milebench/DocVQA/pred_with_extracted.json" "b/eval_milebench/DocVQA/pred_with_extracted.json" new file mode 100644--- /dev/null +++ "b/eval_milebench/DocVQA/pred_with_extracted.json" @@ -0,0 +1,5304 @@ +[ + { + "sample_id": 0, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: what is the title for the name robert kauk?\nChoice list: \nA. VICE PRESIDENT, BRUSH BEET GROWERS ASS'N\nB. TREASURER, BRUSH BEET GROWERS ASS'N\nC. PRESIDENT, BRUSH BEET GROWERS ASS'N\nD. SECRETARY, BRUSH BEET GROWERS ASS'N\nAnswer with the option's letter from the given choices directly.", + "gt_response": "president brush beet growers ass'n", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "vice president brush beet growers ass'n", + "treasurer brush beet growers ass'n", + "president brush beet growers ass'n", + "secretary brush beet growers ass'n" + ], + "image_quantity_level": "Few", + "image": [ + "0.jpg", + "1.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 5, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: Which institution is mentioned at the top of the page?\nChoice list: \nA. MEDICAL RESEARCH FOUNDATION OF OREGON AND WASHINGTON\nB. MEDICAL RESEARCH FOUNDATION OF OREGON\nC. OREGON MEDICAL RESEARCH FOUNDATION\nD. MEDICAL RESEARCH ORGANIZATION OF OREGON\nAnswer with the option's letter from the given choices directly.", + "gt_response": "medical research foundation of oregon", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "medical research foundation of oregon and washington", + "medical research foundation of oregon", + "oregon medical research foundation", + "medical research organization of oregon" + ], + "image_quantity_level": "Few", + "image": [ + "19.jpg", + "20.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 7, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: who is the chairman of nutrition program, penn state\nChoice list: \nA. Jennifer D. Harris\nB. Helen A. guthrie\nC. Michael C. Thompson\nD. John B. Smith\nAnswer with the option's letter from the given choices directly.", + "gt_response": "helen a guthrie", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "jennifer d harris", + "helen a guthrie", + "michael c thompson", + "john b smith" + ], + "image_quantity_level": "Few", + "image": [ + "27.jpg", + "28.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 12, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: Who has the assignment for \"Plant Clean up\"?\nChoice list: \nA. Corsberg\nB. Corsberg Jr.\nC. Corsburg\nD. Corsber\nAnswer with the option's letter from the given choices directly.", + "gt_response": "corsberg", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "corsberg", + "corsberg jr", + "corsburg", + "corsber" + ], + "image_quantity_level": "Few", + "image": [ + "46.jpg", + "47.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 14, + "question": "For each group of pictures, there is a question. Please give a short answer to it. You must choose your answer from the Choice List. \nQuestion: To whom this is addressed\nChoice list: \nA. Ms. K.S. Houghton\nB. Dr.K.S.Houghton\nC. Mrs. K.S. Houghton\nD. Mr. K.S. Houghton\nAnswer with the option's letter from the given choices directly.", + "gt_response": "drkshoughton", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "ms ks houghton", + "drkshoughton", + "mrs ks houghton", + "mr ks houghton" + ], + "image_quantity_level": "Few", + "image": [ + "51.jpg", + "52.jpg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 15, + "question": "There is a question for each group of pictures. Please answer it in a short way. You must choose your answer from the Choice List. \nQuestion: What is the Direct materials cost (Items under $50) given in the voucher?\nChoice list: \nA. 118.15\nB. $118.50\nC. 118.14\nD. $118\nAnswer with the option's letter from the given choices directly.", + "gt_response": "118.14", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "118.15", + "$118.50", + "118.14", + "$118" + ], + "image_quantity_level": "Few", + "image": [ + "53.jpg", + "54.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 20, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: What is the place of meeting?\nChoice list: \nA. Charlotte\nB. Greensboro\nC. Raleigh\nD. Winston-Salem\nAnswer with the option's letter from the given choices directly.", + "gt_response": "winston salem", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "charlotte", + "greensboro", + "raleigh", + "winston salem" + ], + "image_quantity_level": "Few", + "image": [ + "74.jpg", + "75.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 23, + "question": "There is a question for each group of pictures. Please answer it in a short way. You must choose your answer from the Choice List. \nQuestion: what is the flight no. mentioned in the given page?\nChoice list: \nA. 408 ($48.30)\nB. 407 ($48.29)\nC. 407 ($48.30)\nD. 406 ($48.30)\nAnswer with the option's letter from the given choices directly.", + "gt_response": "407 $48.30", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "408 $48.30", + "407 $48.29", + "407 $48.30", + "406 $48.30" + ], + "image_quantity_level": "Few", + "image": [ + "86.jpg", + "87.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 34, + "question": "For each group of pictures, there is a question. Please give a short answer to it. You must choose your answer from the Choice List. \nQuestion: to whom this letter was written?\nChoice list: \nA. Tom\nB. Tim\nC. Alex\nD. John\nAnswer with the option's letter from the given choices directly.", + "gt_response": "tim", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "tom", + "tim", + "alex", + "john" + ], + "image_quantity_level": "Few", + "image": [ + "130.jpg", + "131.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 35, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: Who conducted the paper on the topic \" Iron in Human Nutrition \" on Friday?\nChoice list: \nA. Aaron C. Manson\nB. Richard D. Thompson\nC. Gilbert B. Forbes\nD. Gilbert B. Forsythe\nAnswer with the option's letter from the given choices directly.", + "gt_response": "gilbert b forbes", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "aaron c manson", + "richard d thompson", + "gilbert b forbes", + "gilbert b forsythe" + ], + "image_quantity_level": "Few", + "image": [ + "132.jpg", + "133.jpg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 36, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: When did Charles E. Hirsch promoted as Chief Engineer?\nChoice list: \nA. September 1, 1948\nB. September 1, 1950\nC. September 1, 1949\nD. August 1, 1949\nAnswer with the option's letter from the given choices directly.", + "gt_response": "september 1 1949", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "september 1 1948", + "september 1 1950", + "september 1 1949", + "august 1 1949" + ], + "image_quantity_level": "Few", + "image": [ + "134.jpg", + "135.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 40, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the Name?\nChoice list: \nA. Dr. Robert E. Shaw\nB. Dr. Richard E. Frank\nC. Dr. Robert E. Shank\nD. Mr. Robert E. Shank\nAnswer with the option's letter from the given choices directly.", + "gt_response": "dr robert e shank", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "dr robert e shaw", + "dr richard e frank", + "dr robert e shank", + "mr robert e shank" + ], + "image_quantity_level": "Few", + "image": [ + "150.jpg", + "151.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 42, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: When is the Third day?\nChoice list: \nA. Friday 22nd\nB. Monday 22nd\nC. Tuesday 22nd\nD. Wednesday 22nd\nAnswer with the option's letter from the given choices directly.", + "gt_response": "wednesday 22nd", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "friday 22nd", + "monday 22nd", + "tuesday 22nd", + "wednesday 22nd" + ], + "image_quantity_level": "Few", + "image": [ + "156.jpg", + "157.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 43, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: By how much has the cement industry reduced energy use?\nChoice list: \nA. nearly 30 percent\nB. about 35 percent\nC. almost 40 percent\nD. approximately 25 percent\nAnswer with the option's letter from the given choices directly.", + "gt_response": "nearly 30 percent", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "nearly 30 percent", + "about 35 percent", + "almost 40 percent", + "approximately 25 percent" + ], + "image_quantity_level": "Few", + "image": [ + "158.jpg", + "159.jpg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 45, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the total voucher amount?\nChoice list: \nA. $1,295.87\nB. \u20ac1,295.87\nC. \u00a51,295.87\nD. $1,295.88\nAnswer with the option's letter from the given choices directly.", + "gt_response": "$1295.87", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "$1295.87", + "\u20ac1295.87", + "\u00a51295.87", + "$1295.88" + ], + "image_quantity_level": "Few", + "image": [ + "165.jpg", + "166.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 46, + "question": "For each group of pictures, there is a question. Please give a short answer to it. You must choose your answer from the Choice List. \nQuestion: Who is the chairman of Environment and Public Works Committee?\nChoice list: \nA. Mitch McConnell\nB. Barack Obama\nC. Nancy Pelosi\nD. John Barrasso\nAnswer with the option's letter from the given choices directly.", + "gt_response": "john barrasso", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "mitch mcconnell", + "barack obama", + "nancy pelosi", + "john barrasso" + ], + "image_quantity_level": "Few", + "image": [ + "167.jpg", + "168.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 49, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: Who is giving the introductory remarks?\nChoice list: \nA. Dr. Dodge\nB. Dr. Dodgerson\nC. Mrs. Dodge\nD. Dr. Doug\nAnswer with the option's letter from the given choices directly.", + "gt_response": "dr dodge", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "dr dodge", + "dr dodgerson", + "mrs dodge", + "dr doug" + ], + "image_quantity_level": "Few", + "image": [ + "178.jpg", + "179.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 50, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the Timing for the Journal \"Journal of Managed Care Pharmacy\"?\nChoice list: \nA. 3Q99\nB. 3Q98\nC. 4Q98\nD. 2Q98\nAnswer with the option's letter from the given choices directly.", + "gt_response": "3q98", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "3q99", + "3q98", + "4q98", + "2q98" + ], + "image_quantity_level": "Few", + "image": [ + "180.jpg", + "181.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 1, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: How much Total Income?\nChoice list: \nA. $12,000 to $15,000\nB. $11,000 to $14,000\nC. $10,000 to $13,000\nD. $13,000 to $16,000\nAnswer with the option's letter from the given choices directly.", + "gt_response": "$12000 to $15000", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "$12000 to $15000", + "$11000 to $14000", + "$10000 to $13000", + "$13000 to $16000" + ], + "image_quantity_level": "Few", + "image": [ + "2.jpg", + "3.jpg", + "4.jpg", + "5.jpg", + "6.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 8, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the Net sales of Campbell Soup Company for the year 1965?\nChoice list: \nA. \u20ac712,785,000\nB. $712,785,000\nC. $712,785,001\nD. \u00a5712,785,000\nAnswer with the option's letter from the given choices directly.", + "gt_response": "$712785000", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "\u20ac712785000", + "$712785000", + "$712785001", + "\u00a5712785000" + ], + "image_quantity_level": "Few", + "image": [ + "29.jpg", + "30.jpg", + "31.jpg", + "32.jpg", + "33.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 11, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: When is the approval needed in order to meet overall project timing?\nChoice list: \nA. by 4/22/95\nB. by 4/20/95\nC. by 4/21/95\nD. by 4/23/95\nAnswer with the option's letter from the given choices directly.", + "gt_response": "by 4 21 95", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "by 4 22 95", + "by 4 20 95", + "by 4 21 95", + "by 4 23 95" + ], + "image_quantity_level": "Few", + "image": [ + "41.jpg", + "42.jpg", + "43.jpg", + "44.jpg", + "45.jpg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 16, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the product PO number?\nChoice list: \nA. #072799\nB. #0727981\nC. #072798\nD. #072797\nAnswer with the option's letter from the given choices directly.", + "gt_response": "#072798", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "#072799", + "#0727981", + "#072798", + "#072797" + ], + "image_quantity_level": "Few", + "image": [ + "55.jpg", + "56.jpg", + "57.jpg", + "58.jpg", + "59.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 19, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: what is the proposed grand total amount for the year 1976?\nChoice list: \nA. \u20ac 129,576\nB. $ 129,576\nC. $ 129,577\nD. \u00a5 129,576\nAnswer with the option's letter from the given choices directly.", + "gt_response": "$ 129576", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "\u20ac 129576", + "$ 129576", + "$ 129577", + "\u00a5 129576" + ], + "image_quantity_level": "Few", + "image": [ + "69.jpg", + "70.jpg", + "71.jpg", + "72.jpg", + "73.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 21, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the Profit after Taxation before Share of Results of Associates for 2008?\nChoice list: \nA. 5000.00\nB. 3178.17\nC. 800.00\nD. 3178.18\nAnswer with the option's letter from the given choices directly.", + "gt_response": "3178.17", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "5000.00", + "3178.17", + "800.00", + "3178.18" + ], + "image_quantity_level": "Few", + "image": [ + "76.jpg", + "77.jpg", + "78.jpg", + "79.jpg", + "80.jpg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 22, + "question": "Each group of pictures has a related question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: What is the project name written this sheet\nChoice list: \nA. Project \"NN\"\nB. Project \"AI\"\nC. Project \"CV\"\nD. Project \"ML\"\nAnswer with the option's letter from the given choices directly.", + "gt_response": "project nn", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "project nn", + "project ai", + "project cv", + "project ml" + ], + "image_quantity_level": "Few", + "image": [ + "81.jpg", + "82.jpg", + "83.jpg", + "84.jpg", + "85.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 24, + "question": "For each group of pictures, there is a question. Please give a short answer to it. You must choose your answer from the Choice List. \nQuestion: How much mg of potassium per 100 grams is contained in Kale (R)?\nChoice list: \nA. 401\nB. 402\nC. 400\nD. 403\nAnswer with the option's letter from the given choices directly.", + "gt_response": "401", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "401", + "402", + "400", + "403" + ], + "image_quantity_level": "Few", + "image": [ + "88.jpg", + "89.jpg", + "90.jpg", + "91.jpg", + "92.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 29, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the cost of the parking at hotel in daytime?\nChoice list: \nA. $10\nB. \u00a512\nC. $12\nD. \u20ac12\nAnswer with the option's letter from the given choices directly.", + "gt_response": "$12", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "$10", + "\u00a512", + "$12", + "\u20ac12" + ], + "image_quantity_level": "Few", + "image": [ + "108.jpg", + "109.jpg", + "110.jpg", + "111.jpg", + "112.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 2, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the title of Irene Karl?\nChoice list: \nA. Res. Asst.\nB. Res. Associate\nC. Research Associate\nD. Assistant Researcher\nAnswer with the option's letter from the given choices directly.", + "gt_response": "res asst", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "res asst", + "res associate", + "research associate", + "assistant researcher" + ], + "image_quantity_level": "Few", + "image": [ + "7.jpg", + "8.jpg", + "9.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 3, + "question": "Each group of pictures has a related question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: What is the End Date of Returns?\nChoice list: \nA. 10/10/98\nB. 10/10/99\nC. 10/10/97\nD. 10/10/96\nAnswer with the option's letter from the given choices directly.", + "gt_response": "10 10 97", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "10 10 98", + "10 10 99", + "10 10 97", + "10 10 96" + ], + "image_quantity_level": "Few", + "image": [ + "10.jpg", + "11.jpg", + "12.jpg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 10, + "question": "There is a question for each group of pictures. Please answer it in a short way. You must choose your answer from the Choice List. \nQuestion: what is the designation of Dr. F. James Levinson?\nChoice list: \nA. manager\nB. director\nC. supervisor\nD. executive\nAnswer with the option's letter from the given choices directly.", + "gt_response": "director", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "manager", + "director", + "supervisor", + "executive" + ], + "image_quantity_level": "Few", + "image": [ + "38.jpg", + "39.jpg", + "40.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 13, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the weight of Lactalbumin in Milk protein - 75g?\nChoice list: \nA. 7.60\nB. 7.25\nC. 8.50\nD. 6.90\nAnswer with the option's letter from the given choices directly.", + "gt_response": "7.60", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "7.60", + "7.25", + "8.50", + "6.90" + ], + "image_quantity_level": "Few", + "image": [ + "48.jpg", + "49.jpg", + "50.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 17, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: What is the Written in To Field?\nChoice list: \nA. All Boston Region Full-Time Employees\nB. All New York Region Full-Time Personnel\nC. All San Francisco Region Part-Time Personnel\nD. All Boston Region Full-Time Personnel\nAnswer with the option's letter from the given choices directly.", + "gt_response": "all boston region full time personnel", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "all boston region full time employees", + "all new york region full time personnel", + "all san francisco region part time personnel", + "all boston region full time personnel" + ], + "image_quantity_level": "Few", + "image": [ + "60.jpg", + "61.jpg", + "62.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 26, + "question": "There is a question for each group of pictures. Please answer it in a short way. You must choose your answer from the Choice List. \nQuestion: Who is the recipient?\nChoice list: \nA. K. P. Dubrovin\nB. J. P. Dubrovin\nC. K. P. Durovin\nD. K. P. Dubrovin Jr.\nAnswer with the option's letter from the given choices directly.", + "gt_response": "k p dubrovin", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "k p dubrovin", + "j p dubrovin", + "k p durovin", + "k p dubrovin jr" + ], + "image_quantity_level": "Few", + "image": [ + "97.jpg", + "98.jpg", + "99.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 31, + "question": "There is a question for each group of pictures. Please answer it in a short way. You must choose your answer from the Choice List. \nQuestion: what does camel crema deliver?\nChoice list: \nA. a creamy indulgent flavor that offers an intriguing and pleasurable smoking experience.\nB. a rich and smooth taste that provides a satisfying and enjoyable smoking session.\nC. a luxurious and palatable blend that delivers a captivating and delightful smoking experience.\nD. a velvety and luscious sensation that enhances the smoking ritual.\nAnswer with the option's letter from the given choices directly.", + "gt_response": "a creamy indulgent flavor that offers an intriguing and pleasurable smoking experience", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "a creamy indulgent flavor that offers an intriguing and pleasurable smoking experience", + "a rich and smooth taste that provides a satisfying and enjoyable smoking session", + "a luxurious and palatable blend that delivers a captivating and delightful smoking experience", + "a velvety and luscious sensation that enhances the smoking ritual" + ], + "image_quantity_level": "Few", + "image": [ + "117.jpg", + "118.jpg", + "119.jpg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 38, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: In table XIV how many total abnormalities in hospital K?\nChoice list: \nA. 62\nB. 63\nC. 60\nD. 64\nAnswer with the option's letter from the given choices directly.", + "gt_response": "63", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "62", + "63", + "60", + "64" + ], + "image_quantity_level": "Few", + "image": [ + "141.jpg", + "142.jpg", + "143.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 52, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: In table II what is the total period of observation of Man-years in hospital K?\nChoice list: \nA. 1,088.7\nB. 1,089.0\nC. 1,088.9\nD. 1,088.8\nAnswer with the option's letter from the given choices directly.", + "gt_response": "1088.8", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "1088.7", + "1089.0", + "1088.9", + "1088.8" + ], + "image_quantity_level": "Few", + "image": [ + "186.jpg", + "187.jpg", + "188.jpg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 54, + "question": "There is a question for each group of pictures. Please answer it in a short way. You must choose your answer from the Choice List. \nQuestion: what is the nickname given for thiamine?\nChoice list: \nA. energy vitamin\nB. victory vitamin\nC. morale vitamin\nD. optimism vitamin\nAnswer with the option's letter from the given choices directly.", + "gt_response": "morale vitamin", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "energy vitamin", + "victory vitamin", + "morale vitamin", + "optimism vitamin" + ], + "image_quantity_level": "Few", + "image": [ + "191.jpg", + "192.jpg", + "193.jpg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 4, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: Which Class is Peter H. Raven in?\nChoice list: \nA. II\nB. III\nC. I\nD. IV\nAnswer with the option's letter from the given choices directly.", + "gt_response": "ii", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "ii", + "iii", + "i", + "iv" + ], + "image_quantity_level": "Medium", + "image": [ + "13.jpg", + "14.jpg", + "15.jpg", + "16.jpg", + "17.jpg", + "18.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 6, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: Which day is October 10, 1968\nChoice list: \nA. Wednesday\nB. Monday\nC. Thursday\nD. Friday\nAnswer with the option's letter from the given choices directly.", + "gt_response": "thursday", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "wednesday", + "monday", + "thursday", + "friday" + ], + "image_quantity_level": "Medium", + "image": [ + "21.jpg", + "22.jpg", + "23.jpg", + "24.jpg", + "25.jpg", + "26.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 18, + "question": "There is a question for each group of pictures. Please answer it in a short way. You must choose your answer from the Choice List. \nQuestion: What is the name of the Store?\nChoice list: \nA. - Exxon Mobil Corporation\nB. Sunoco, Inc. (R&M)\nC. - BP plc\nD. - Chevron Corporation\nAnswer with the option's letter from the given choices directly.", + "gt_response": "sunoco inc r&m", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "exxon mobil corporation", + "sunoco inc r&m", + "bp plc", + "chevron corporation" + ], + "image_quantity_level": "Medium", + "image": [ + "63.jpg", + "64.jpg", + "65.jpg", + "66.jpg", + "67.jpg", + "68.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 32, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: what is the unit of pension for Actuarial Gains/(Losses) for the year ended in 2010?\nChoice list: \nA. 7.50\nB. 5.00\nC. 7.00\nD. 10.00\nAnswer with the option's letter from the given choices directly.", + "gt_response": "7.50", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "7.50", + "5.00", + "7.00", + "10.00" + ], + "image_quantity_level": "Medium", + "image": [ + "120.jpg", + "121.jpg", + "122.jpg", + "123.jpg", + "124.jpg", + "125.jpg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 39, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: what is title on the y axis of fig.1?\nChoice list: \nA. PLANT DRY WT(%)\nB. PLANT DRY CONTROL WT(%)\nC. PLANT DRY WT(% OF CONTROL)\nD. PLANT WEIGHT(%)\nAnswer with the option's letter from the given choices directly.", + "gt_response": "plant dry wt % of control", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "plant dry wt %", + "plant dry control wt %", + "plant dry wt % of control", + "plant weight %" + ], + "image_quantity_level": "Medium", + "image": [ + "144.jpg", + "145.jpg", + "146.jpg", + "147.jpg", + "148.jpg", + "149.jpg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 55, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the Percentage of loyalty according to CDF measures?\nChoice list: \nA. 35.9%\nB. 37.9%\nC. 36.9%\nD. 36%\nAnswer with the option's letter from the given choices directly.", + "gt_response": "36.9%", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "35.9%", + "37.9%", + "36.9%", + "36%" + ], + "image_quantity_level": "Medium", + "image": [ + "194.jpg", + "195.jpg", + "196.jpg", + "197.jpg", + "198.jpg", + "199.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 9, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: What is the email id of Mr. Hironao Ihara?\nChoice list: \nA. h.ihara@takeda.com\nB. hironaoihara@takeda.com\nC. hironao.ihara@takeda.com\nD. iharahironao@takeda.com\nAnswer with the option's letter from the given choices directly.", + "gt_response": "hironaoihara takedacom", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "hihara takedacom", + "hironaoihara takedacom", + "hironaoihara takedacom", + "iharahironao takedacom" + ], + "image_quantity_level": "Few", + "image": [ + "34.jpg", + "35.jpg", + "36.jpg", + "37.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 25, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What are the reporting dates?\nChoice list: \nA. 10.5-10/30-98\nB. 10-5-10/30-98\nC. 10/5-1030/98\nD. 10/5 -10/30/98\nAnswer with the option's letter from the given choices directly.", + "gt_response": "10 5 10 30 98", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "10.5 10 30 98", + "10 5 10 30 98", + "10 5 1030 98", + "10 5 10 30 98" + ], + "image_quantity_level": "Few", + "image": [ + "93.jpg", + "94.jpg", + "95.jpg", + "96.jpg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 27, + "question": "For each group of pictures, there is a question. Please give a short answer to it. You must choose your answer from the Choice List. \nQuestion: what is the date mentioned in the given page?\nChoice list: \nA. Thursday, November 1, 1974\nB. Friday, November 1, 1974\nC. Saturday, November 1, 1974\nD. Friday, October 1, 1974\nAnswer with the option's letter from the given choices directly.", + "gt_response": "friday november 1 1974", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "thursday november 1 1974", + "friday november 1 1974", + "saturday november 1 1974", + "friday october 1 1974" + ], + "image_quantity_level": "Few", + "image": [ + "100.jpg", + "101.jpg", + "102.jpg", + "103.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 28, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the first TARGET AUDIENCE mentioned?\nChoice list: \nA. BVS2 (AMM)\nB. BVS1 (PWM)\nC. BVS1 (ZMM)\nD. BVS1 (AMM)\nAnswer with the option's letter from the given choices directly.", + "gt_response": "bvs1 amm", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "bvs2 amm", + "bvs1 pwm", + "bvs1 zmm", + "bvs1 amm" + ], + "image_quantity_level": "Few", + "image": [ + "104.jpg", + "105.jpg", + "106.jpg", + "107.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 30, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: How much is the debt equity ratio?\nChoice list: \nA. 0.02\nB. 0.2\nC. 0.01\nD. 0.05\nAnswer with the option's letter from the given choices directly.", + "gt_response": "0.02", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "0.02", + "0.2", + "0.01", + "0.05" + ], + "image_quantity_level": "Few", + "image": [ + "113.jpg", + "114.jpg", + "115.jpg", + "116.jpg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 33, + "question": "Each group of pictures has a related question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: what is the date mentioned in the given page?\nChoice list: \nA. 2nd November 1976\nB. 2 November 1966\nC. 2 nov 1967\nD. 2nd November 1957\nAnswer with the option's letter from the given choices directly.", + "gt_response": "2 nov 1967", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "2nd november 1976", + "2 november 1966", + "2 nov 1967", + "2nd november 1957" + ], + "image_quantity_level": "Few", + "image": [ + "126.jpg", + "127.jpg", + "128.jpg", + "129.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 41, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: what is the cat.no?\nChoice list: \nA. 0612-80-505\nB. 0612-80-606\nC. 6192-80-505\nD. 0612-90-505\nAnswer with the option's letter from the given choices directly.", + "gt_response": "0612 80 505", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "0612 80 505", + "0612 80 606", + "6192 80 505", + "0612 90 505" + ], + "image_quantity_level": "Few", + "image": [ + "152.jpg", + "153.jpg", + "154.jpg", + "155.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 48, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the File No.?\nChoice list: \nA. 183-58361\nB. 500-85831\nC. 185-58361\nD. 185-58362\nAnswer with the option's letter from the given choices directly.", + "gt_response": "185 58361", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "183 58361", + "500 85831", + "185 58361", + "185 58362" + ], + "image_quantity_level": "Few", + "image": [ + "174.jpg", + "175.jpg", + "176.jpg", + "177.jpg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 53, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the name of reviewer #2 given in the form?\nChoice list: \nA. Coleman Bill\nB. Bill Coleman\nC. David Coleman\nD. Reviewer #2\nAnswer with the option's letter from the given choices directly.", + "gt_response": "bill coleman", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "coleman bill", + "bill coleman", + "david coleman", + "reviewer #2" + ], + "image_quantity_level": "Few", + "image": [ + "189.jpg", + "190.jpg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 56, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: What agreement is shown here?\nChoice list: \nA. DORAL EDLP RETAIL PRICING AGREEMENT\nB. DORAL EDLP RETAIL PRICING PARTNERSHIP AGREEMENT\nC. DORAL EDLP RETAIL PRICING LICENSING AGREEMENT\nD. DORAL EDLP RETAIL PRICING CONTRACT\nAnswer with the option's letter from the given choices directly.", + "gt_response": "doral edlp retail pricing agreement", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "doral edlp retail pricing agreement", + "doral edlp retail pricing partnership agreement", + "doral edlp retail pricing licensing agreement", + "doral edlp retail pricing contract" + ], + "image_quantity_level": "Few", + "image": [ + "200.jpg", + "201.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 67, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: what is the cost of dinner on dec. 2?\nChoice list: \nA. \u00a32.50\nB. $2.50\nC. $3.50\nD. 2.50\nAnswer with the option's letter from the given choices directly.", + "gt_response": "2.50", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "\u00a32.50", + "$2.50", + "$3.50", + "2.50" + ], + "image_quantity_level": "Few", + "image": [ + "241.jpg", + "242.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 68, + "question": "For each group of pictures, there is a question. Please give a short answer to it. You must choose your answer from the Choice List. \nQuestion: who has signed at the bottom of the letter?\nChoice list: \nA. Dr. M.G. Wagh (without the period)\nB. Dr. M.G. Wagh\nC. Ms. M.G. Wagh\nD. Mr. M.G. Wagh\nAnswer with the option's letter from the given choices directly.", + "gt_response": "dr mg wagh", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "dr mg wagh without the period", + "dr mg wagh", + "ms mg wagh", + "mr mg wagh" + ], + "image_quantity_level": "Few", + "image": [ + "243.jpg", + "244.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 73, + "question": "Each group of pictures relates to a question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: what is the price of iron analysis in atomic absorption methodology?\nChoice list: \nA. 8.50\nB. 9.00\nC. 7.50\nD. 8.00\nAnswer with the option's letter from the given choices directly.", + "gt_response": "8.00", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "8.50", + "9.00", + "7.50", + "8.00" + ], + "image_quantity_level": "Few", + "image": [ + "263.jpg", + "264.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 78, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: what is the total population under 21 years in 1950?\nChoice list: \nA. 139,131\nB. 139,123\nC. 139,133\nD. 139,132\nAnswer with the option's letter from the given choices directly.", + "gt_response": "139132", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "139131", + "139123", + "139133", + "139132" + ], + "image_quantity_level": "Few", + "image": [ + "283.jpg", + "284.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 82, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the annual total in the year 1995-96?\nChoice list: \nA. 1,713,000\nB. 2,000,000\nC. 1,700,000\nD. 1,713,001\nAnswer with the option's letter from the given choices directly.", + "gt_response": "1713000", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "1713000", + "2000000", + "1700000", + "1713001" + ], + "image_quantity_level": "Few", + "image": [ + "294.jpg", + "295.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 88, + "question": "Each group of pictures has a related question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: Which university is mentioned in the letter head?\nChoice list: \nA. ALBANY MEDICAL COLLEGE\nB. UNION UNIVERSITY\nC. THE ALBANY MEDICAL COLLEGE OF UNION UNIVERSITY\nD. THE UNION UNIVERSITY\nAnswer with the option's letter from the given choices directly.", + "gt_response": "the albany medical college of union university", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "albany medical college", + "union university", + "the albany medical college of union university", + "the union university" + ], + "image_quantity_level": "Few", + "image": [ + "315.jpg", + "316.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 92, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the Fullform of SCM?\nChoice list: \nA. State College of Management (SCM)\nB. Supply chain Management (SCM)\nC. Social capital measurement (SCM)\nD. System Configuration Manager (SCM)\nAnswer with the option's letter from the given choices directly.", + "gt_response": "supply chain management scm", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "state college of management scm", + "supply chain management scm", + "social capital measurement scm", + "system configuration manager scm" + ], + "image_quantity_level": "Few", + "image": [ + "333.jpg", + "334.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 94, + "question": "There is a question for each group of pictures. Please answer it in a short way. You must choose your answer from the Choice List. \nQuestion: What are the sizes of Turquoise or navy?\nChoice list: \nA. 10-16 and 141/2-241/2.\nB. 12-20 and 131/2-231/2.\nC. 12-20 and 141/2-241/2.\nD. 12-20 and 141/2-241/2.\nAnswer with the option's letter from the given choices directly.", + "gt_response": "12 20 and 141 2 241 2", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "10 16 and 141 2 241 2", + "12 20 and 131 2 231 2", + "12 20 and 141 2 241 2", + "12 20 and 141 2 241 2" + ], + "image_quantity_level": "Few", + "image": [ + "340.jpg", + "341.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 99, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the Name of the Foundation?\nChoice list: \nA. Nutrition Foundation of Sweden\nB. Swedish Nutrition Foundation\nC. International Nutrition Foundation\nD. Swedish Nutrition Institute\nAnswer with the option's letter from the given choices directly.", + "gt_response": "swedish nutrition foundation", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "nutrition foundation of sweden", + "swedish nutrition foundation", + "international nutrition foundation", + "swedish nutrition institute" + ], + "image_quantity_level": "Few", + "image": [ + "361.jpg", + "362.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 100, + "question": "Each group of pictures has a related question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: How much Mg/Day iron for 6-8 years?\nChoice list: \nA. 6\nB. 8\nC. 10\nD. 12\nAnswer with the option's letter from the given choices directly.", + "gt_response": "10", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "6", + "8", + "10", + "12" + ], + "image_quantity_level": "Few", + "image": [ + "363.jpg", + "364.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 101, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: to whom the letter was written?\nChoice list: \nA. members of the U.S. medical advisory committee\nB. members of the U.S. medical advisory council\nC. members of the U.S. medical advisory board\nD. members of the U.S. medical advisory panel\nAnswer with the option's letter from the given choices directly.", + "gt_response": "members of the us medical advisory committee", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "members of the us medical advisory committee", + "members of the us medical advisory council", + "members of the us medical advisory board", + "members of the us medical advisory panel" + ], + "image_quantity_level": "Few", + "image": [ + "365.jpg", + "366.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 102, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: what is the demo number mentioned?\nChoice list: \nA. 03760\nB. 03670\nC. 03765\nD. 03960\nAnswer with the option's letter from the given choices directly.", + "gt_response": "03760", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "03760", + "03670", + "03765", + "03960" + ], + "image_quantity_level": "Few", + "image": [ + "367.jpg", + "368.jpg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 104, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: What is the due date(outline)?\nChoice list: \nA. February 25, 2000\nB. March 25, 2002\nC. February 25, 2021\nD. February 25, 2002\nAnswer with the option's letter from the given choices directly.", + "gt_response": "february 25 2002", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "february 25 2000", + "march 25 2002", + "february 25 2021", + "february 25 2002" + ], + "image_quantity_level": "Few", + "image": [ + "375.jpg", + "376.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 107, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the emp. no. of Ivano N Rivers?\nChoice list: \nA. 751034\nB. 751035\nC. 751036\nD. 751037\nAnswer with the option's letter from the given choices directly.", + "gt_response": "751035", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "751034", + "751035", + "751036", + "751037" + ], + "image_quantity_level": "Few", + "image": [ + "386.jpg", + "387.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 108, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: In how many countries global company is operating?\nChoice list: \nA. more than 220\nB. between 200 and 220\nC. over 200\nD. less than 220\nAnswer with the option's letter from the given choices directly.", + "gt_response": "more than 220", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "more than 220", + "between 200 and 220", + "over 200", + "less than 220" + ], + "image_quantity_level": "Few", + "image": [ + "388.jpg", + "389.jpg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 112, + "question": "Each group of pictures has a related question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: What is the Order No.?\nChoice list: \nA. 32699\nB. 32708\nC. 32710\nD. 32709\nAnswer with the option's letter from the given choices directly.", + "gt_response": "32709", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "32699", + "32708", + "32710", + "32709" + ], + "image_quantity_level": "Few", + "image": [ + "402.jpg", + "403.jpg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 37, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the Page Number?\nChoice list: \nA. 5\nB. 4\nC. 3\nD. 6\nAnswer with the option's letter from the given choices directly.", + "gt_response": "4", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "5", + "4", + "3", + "6" + ], + "image_quantity_level": "Few", + "image": [ + "136.jpg", + "137.jpg", + "138.jpg", + "139.jpg", + "140.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 44, + "question": "Each group of pictures has a related question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: What was the amount of the total grants in 1962-63?\nChoice list: \nA. 586,950\nB. 586,950.50\nC. 586,900\nD. 586,590\nAnswer with the option's letter from the given choices directly.", + "gt_response": "586950", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "586950", + "586950.50", + "586900", + "586590" + ], + "image_quantity_level": "Few", + "image": [ + "160.jpg", + "161.jpg", + "162.jpg", + "163.jpg", + "164.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 47, + "question": "Each group of pictures relates to a question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: what is the amount of potassium mg./100g. for Gelatin, Plain?\nChoice list: \nA. 25\nB. 21\nC. 23\nD. 22\nAnswer with the option's letter from the given choices directly.", + "gt_response": "22", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "25", + "21", + "23", + "22" + ], + "image_quantity_level": "Few", + "image": [ + "169.jpg", + "170.jpg", + "171.jpg", + "172.jpg", + "173.jpg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 66, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: What is the brand code?\nChoice list: \nA. CF85\nB. CF86\nC. CF95\nD. EF85\nAnswer with the option's letter from the given choices directly.", + "gt_response": "cf85", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "cf85", + "cf86", + "cf95", + "ef85" + ], + "image_quantity_level": "Few", + "image": [ + "236.jpg", + "237.jpg", + "238.jpg", + "239.jpg", + "240.jpg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 72, + "question": "Each group of pictures relates to a question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: Which is the institution or agency of ALEXANDER, E. Russell, M.D.?\nChoice list: \nA. Johns Hopkins\nB. Harvard\nC. Washington\nD. Yale\nAnswer with the option's letter from the given choices directly.", + "gt_response": "washington", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "johns hopkins", + "harvard", + "washington", + "yale" + ], + "image_quantity_level": "Few", + "image": [ + "258.jpg", + "259.jpg", + "260.jpg", + "261.jpg", + "262.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 83, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: what is the month mentioned in the date of birth in the given form?\nChoice list: \nA. october\nB. august\nC. september\nD. november\nAnswer with the option's letter from the given choices directly.", + "gt_response": "september", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "october", + "august", + "september", + "november" + ], + "image_quantity_level": "Few", + "image": [ + "296.jpg", + "297.jpg", + "298.jpg", + "299.jpg", + "300.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 85, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the amount of potassium present in lentils, dry(R) as mentioned in the given page?\nChoice list: \nA. 1200\nB. 1300\nC. 1400\nD. 1150\nAnswer with the option's letter from the given choices directly.", + "gt_response": "1200", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "1200", + "1300", + "1400", + "1150" + ], + "image_quantity_level": "Few", + "image": [ + "304.jpg", + "305.jpg", + "306.jpg", + "307.jpg", + "308.jpg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 93, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the G/L account code mentioned in the voucher?\nChoice list: \nA. 04 08 1199 9143 821\nB. 05 08 1100 9143 821\nC. 04 08 1100 9143 821\nD. 04 08 1100 9143 822\nAnswer with the option's letter from the given choices directly.", + "gt_response": "04 08 1100 9143 821", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "04 08 1199 9143 821", + "05 08 1100 9143 821", + "04 08 1100 9143 821", + "04 08 1100 9143 822" + ], + "image_quantity_level": "Few", + "image": [ + "335.jpg", + "336.jpg", + "337.jpg", + "338.jpg", + "339.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 95, + "question": "Each group of pictures has a related question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: What is the name of the Hotel where the reservation is done?\nChoice list: \nA. Naples Grande Beach Resort\nB. Naples Beach Hotel and Tennis Club\nC. Naples Beach Resort & Spa\nD. Naples Beach Hotel & Golf Club\nAnswer with the option's letter from the given choices directly.", + "gt_response": "naples beach hotel & golf club", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "naples grande beach resort", + "naples beach hotel and tennis club", + "naples beach resort & spa", + "naples beach hotel & golf club" + ], + "image_quantity_level": "Few", + "image": [ + "342.jpg", + "343.jpg", + "344.jpg", + "345.jpg", + "346.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 59, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the address of the business office of the company?\nChoice list: \nA. 107 Depot Avenue, N. E., Knoxville, Tennessee\nB. 108 Depot Avenue, N. E., Knoxville, Tennessee\nC. 109 Depot Avenue, N. E., Knoxville, Tennessee\nD. 110 Depot Avenue, N. E., Knoxville, Tennessee\nAnswer with the option's letter from the given choices directly.", + "gt_response": "108 depot avenue n e knoxville tennessee", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "107 depot avenue n e knoxville tennessee", + "108 depot avenue n e knoxville tennessee", + "109 depot avenue n e knoxville tennessee", + "110 depot avenue n e knoxville tennessee" + ], + "image_quantity_level": "Few", + "image": [ + "212.jpg", + "213.jpg", + "214.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 60, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: How many pages are there?\nChoice list: \nA. 4\nB. 5\nC. 2\nD. 3\nAnswer with the option's letter from the given choices directly.", + "gt_response": "3", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "4", + "5", + "2", + "3" + ], + "image_quantity_level": "Few", + "image": [ + "215.jpg", + "216.jpg", + "217.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 61, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: in which state is Washington University at?\nChoice list: \nA. Missouri\nB. Washington State\nC. Washington D.C\nD. Washington\nAnswer with the option's letter from the given choices directly.", + "gt_response": "missouri", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "missouri", + "washington state", + "washington dc", + "washington" + ], + "image_quantity_level": "Few", + "image": [ + "218.jpg", + "219.jpg", + "220.jpg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 63, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: Which topic is handled at 1:30 PM?\nChoice list: \nA. Closing Remarks/Conclusion\nB. Panel Discussion/Debate\nC. Opening Remarks/Introduction\nD. Keynote Speech/Address\nAnswer with the option's letter from the given choices directly.", + "gt_response": "opening remarks introduction", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "closing remarks conclusion", + "panel discussion debate", + "opening remarks introduction", + "keynote speech address" + ], + "image_quantity_level": "Few", + "image": [ + "227.jpg", + "228.jpg", + "229.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 64, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the amount of total voucher as mentioned in the given form?\nChoice list: \nA. 1,517.00\nB. 1,516.86\nC. 1,516.85\nD. 1,516.87\nAnswer with the option's letter from the given choices directly.", + "gt_response": "1516.86", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "1517.00", + "1516.86", + "1516.85", + "1516.87" + ], + "image_quantity_level": "Few", + "image": [ + "230.jpg", + "231.jpg", + "232.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 65, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: When is the date of departure?\nChoice list: \nA. July 10th, 1949\nB. July 10th, 1950\nC. July 9th, 1950\nD. July 10th, 1951\nAnswer with the option's letter from the given choices directly.", + "gt_response": "july 10th 1950", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "july 10th 1949", + "july 10th 1950", + "july 9th 1950", + "july 10th 1951" + ], + "image_quantity_level": "Few", + "image": [ + "233.jpg", + "234.jpg", + "235.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 71, + "question": "There is a question for each group of pictures. Please answer it in a short way. You must choose your answer from the Choice List. \nQuestion: what is the office phone number?\nChoice list: \nA. 383-5411\nB. 281-2511\nC. 282-2512\nD. 282-2511\nAnswer with the option's letter from the given choices directly.", + "gt_response": "282 2511", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "383 5411", + "281 2511", + "282 2512", + "282 2511" + ], + "image_quantity_level": "Few", + "image": [ + "255.jpg", + "256.jpg", + "257.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 76, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is estimated population in poverty in Lewis?\nChoice list: \nA. 2,199\nB. 2,399\nC. 2,299\nD. 2,299.50\nAnswer with the option's letter from the given choices directly.", + "gt_response": "2299", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "2199", + "2399", + "2299", + "2299.50" + ], + "image_quantity_level": "Few", + "image": [ + "277.jpg", + "278.jpg", + "279.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 77, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the expiration date?\nChoice list: \nA. 09/30/97\nB. 10/01/97\nC. 09/30/98\nD. 09/29/97\nAnswer with the option's letter from the given choices directly.", + "gt_response": "09 30 97", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "09 30 97", + "10 01 97", + "09 30 98", + "09 29 97" + ], + "image_quantity_level": "Few", + "image": [ + "280.jpg", + "281.jpg", + "282.jpg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 79, + "question": "There is a question for each group of pictures. Please answer it in a short way. You must choose your answer from the Choice List. \nQuestion: what is the brand mentioned in the confirmation?\nChoice list: \nA. WINSTON-SALEM\nB. WINSTONE\nC. WESTON\nD. WINSTON\nAnswer with the option's letter from the given choices directly.", + "gt_response": "winston", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "winston salem", + "winstone", + "weston", + "winston" + ], + "image_quantity_level": "Few", + "image": [ + "285.jpg", + "286.jpg", + "287.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 57, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is shown in Y axis of the graph in Capital letters?\nChoice list: \nA. AVERAGE SALES\nB. SPECIFIC ACTIVITY\nC. PROFIT MARGIN\nD. TOTAL REVENUE\nAnswer with the option's letter from the given choices directly.", + "gt_response": "specific activity", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "average sales", + "specific activity", + "profit margin", + "total revenue" + ], + "image_quantity_level": "Medium", + "image": [ + "202.jpg", + "203.jpg", + "204.jpg", + "205.jpg", + "206.jpg", + "207.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 62, + "question": "Each group of pictures has a related question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: Who requested Response Code?\nChoice list: \nA. Dylan Huyett\nB. Dan Huyett\nC. David Huyett\nD. Drew Huyett\nAnswer with the option's letter from the given choices directly.", + "gt_response": "drew huyett", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "dylan huyett", + "dan huyett", + "david huyett", + "drew huyett" + ], + "image_quantity_level": "Medium", + "image": [ + "221.jpg", + "222.jpg", + "223.jpg", + "224.jpg", + "225.jpg", + "226.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 69, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is written within parentheses, just below the title?\nChoice list: \nA. Calories per meal\nB. Calories per serving\nC. Calories per week\nD. Calories per day\nAnswer with the option's letter from the given choices directly.", + "gt_response": "calories per day", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "calories per meal", + "calories per serving", + "calories per week", + "calories per day" + ], + "image_quantity_level": "Medium", + "image": [ + "245.jpg", + "246.jpg", + "247.jpg", + "248.jpg", + "249.jpg", + "250.jpg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 74, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the Institution?\nChoice list: \nA. Massachusetts Institute of Technology\nB. Harvard University\nC. William Marsh rice University\nD. Stanford University\nAnswer with the option's letter from the given choices directly.", + "gt_response": "william marsh rice university", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "massachusetts institute of technology", + "harvard university", + "william marsh rice university", + "stanford university" + ], + "image_quantity_level": "Medium", + "image": [ + "265.jpg", + "266.jpg", + "267.jpg", + "268.jpg", + "269.jpg", + "270.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 75, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the total amount mentioned in the given form?\nChoice list: \nA. 574.48\nB. 574.49\nC. 574.50\nD. 573.49\nAnswer with the option's letter from the given choices directly.", + "gt_response": "574.49", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "574.48", + "574.49", + "574.50", + "573.49" + ], + "image_quantity_level": "Medium", + "image": [ + "271.jpg", + "272.jpg", + "273.jpg", + "274.jpg", + "275.jpg", + "276.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 89, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: who is the whole-time director (from August 14,2009)?\nChoice list: \nA. Mr. R Raghavan\nB. Mr. R Ramamurthy\nC. Mr. R Rajmohan\nD. Mr. R Ramanujam\nAnswer with the option's letter from the given choices directly.", + "gt_response": "mr r ramamurthy", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "mr r raghavan", + "mr r ramamurthy", + "mr r rajmohan", + "mr r ramanujam" + ], + "image_quantity_level": "Medium", + "image": [ + "317.jpg", + "318.jpg", + "319.jpg", + "320.jpg", + "321.jpg", + "322.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 51, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: How much is the time and mileage charge given?\nChoice list: \nA. 22.8\nB. 22.50\nC. 22.80\nD. 22.85\nAnswer with the option's letter from the given choices directly.", + "gt_response": "22.85", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "22.8", + "22.50", + "22.80", + "22.85" + ], + "image_quantity_level": "Few", + "image": [ + "182.jpg", + "183.jpg", + "184.jpg", + "185.jpg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 58, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: \"wisdom and art,strength and wealth are of no avail if health is lacking \" was written by?\nChoice list: \nA. Aristotle\nB. Herophilus\nC. Hippocrates\nD. Socrates\nAnswer with the option's letter from the given choices directly.", + "gt_response": "herophilus", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "aristotle", + "herophilus", + "hippocrates", + "socrates" + ], + "image_quantity_level": "Few", + "image": [ + "208.jpg", + "209.jpg", + "210.jpg", + "211.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 70, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: what is the candidate name for promotion?\nChoice list: \nA. George M. Saunders\nB. George Murdoch Smith\nC. George Saunders\nD. George Murdoch Saunders\nAnswer with the option's letter from the given choices directly.", + "gt_response": "george murdoch saunders", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "george m saunders", + "george murdoch smith", + "george saunders", + "george murdoch saunders" + ], + "image_quantity_level": "Few", + "image": [ + "251.jpg", + "252.jpg", + "253.jpg", + "254.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 90, + "question": "For each group of pictures, there is a question. Please give a short answer to it. You must choose your answer from the Choice List. \nQuestion: What is the Timing of Conference Reminder?\nChoice list: \nA. 2:30 to 4:00 p.m\nB. 12:30 to 2:30 p.m\nC. 11:30 a.m to 2:00 p.m\nD. 1.30 to 3:00 p.m\nAnswer with the option's letter from the given choices directly.", + "gt_response": "1.30 to 3:00 pm", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "2:30 to 4:00 pm", + "12:30 to 2:30 pm", + "11:30 am to 2:00 pm", + "1.30 to 3:00 pm" + ], + "image_quantity_level": "Few", + "image": [ + "323.jpg", + "324.jpg", + "325.jpg", + "326.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 105, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: what is the heading at top of the page?\nChoice list: \nA. safety data sheet\nB. product data sheet\nC. material safety data sheet\nD. technical data sheet\nAnswer with the option's letter from the given choices directly.", + "gt_response": "material safety data sheet", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "safety data sheet", + "product data sheet", + "material safety data sheet", + "technical data sheet" + ], + "image_quantity_level": "Few", + "image": [ + "377.jpg", + "378.jpg", + "379.jpg", + "380.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 111, + "question": "There is a question for each group of pictures. Please answer it in a short way. You must choose your answer from the Choice List. \nQuestion: What kind of \"studies\" have ITC commenced for few of their products?\nChoice list: \nA. ecological impact studies\nB. life cycle assessment studies\nC. sustainability studies\nD. carbon footprint studies\nAnswer with the option's letter from the given choices directly.", + "gt_response": "life cycle assessment studies", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "ecological impact studies", + "life cycle assessment studies", + "sustainability studies", + "carbon footprint studies" + ], + "image_quantity_level": "Few", + "image": [ + "398.jpg", + "399.jpg", + "400.jpg", + "401.jpg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 119, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: How much is the hotel cost?\nChoice list: \nA. $ 10.00\nB. $10.01\nC. \u20ac10.00\nD. \u00a510.00\nAnswer with the option's letter from the given choices directly.", + "gt_response": "$ 10.00", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "$ 10.00", + "$10.01", + "\u20ac10.00", + "\u00a510.00" + ], + "image_quantity_level": "Few", + "image": [ + "424.jpg", + "425.jpg", + "426.jpg", + "427.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 113, + "question": "Each group of pictures relates to a question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: what is the emp. no. of herman w arnett jr?\nChoice list: \nA. i052008\nB. 052008JR\nC. 052008\nD. 052008H\nAnswer with the option's letter from the given choices directly.", + "gt_response": "052008", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "i052008", + "052008jr", + "052008", + "052008h" + ], + "image_quantity_level": "Few", + "image": [ + "404.jpg", + "405.jpg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 114, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: what is the account to be charged as mentioned in the given page?\nChoice list: \nA. PA72WBC44\nB. PA73WBC44\nC. PB73WBC44\nD. PA73WBC45\nAnswer with the option's letter from the given choices directly.", + "gt_response": "pa73wbc44", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "pa72wbc44", + "pa73wbc44", + "pb73wbc44", + "pa73wbc45" + ], + "image_quantity_level": "Few", + "image": [ + "406.jpg", + "407.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 116, + "question": "Each group of pictures has a related question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: what is the payee's name?\nChoice list: \nA. Robert E Shank Jr\nB. Rob E Shank\nC. Robert E. Shank\nD. Robert E. Shanks\nAnswer with the option's letter from the given choices directly.", + "gt_response": "robert e shank", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "robert e shank jr", + "rob e shank", + "robert e shank", + "robert e shanks" + ], + "image_quantity_level": "Few", + "image": [ + "414.jpg", + "415.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 117, + "question": "Each group of pictures has a related question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: what is the contact number of The Visiting Nurse Association of Greater St. Louis, St. Louis County ?\nChoice list: \nA. 314-533-9680\nB. 314-533-9685\nC. 314-533-9600\nD. 314-533-9681\nAnswer with the option's letter from the given choices directly.", + "gt_response": "314 533 9680", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "314 533 9680", + "314 533 9685", + "314 533 9600", + "314 533 9681" + ], + "image_quantity_level": "Few", + "image": [ + "416.jpg", + "417.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 129, + "question": "There is a question for each group of pictures. Please answer it in a short way. You must choose your answer from the Choice List. \nQuestion: What is the Days offer open?\nChoice list: \nA. 90\nB. 92\nC. 91\nD. 93\nAnswer with the option's letter from the given choices directly.", + "gt_response": "91", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "90", + "92", + "91", + "93" + ], + "image_quantity_level": "Few", + "image": [ + "470.jpg", + "471.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 130, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the date of arrival at Greensboro Airport?\nChoice list: \nA. Monday / January 4th\nB. SUNDAY / January 4th\nC. Saturday / January 4th\nD. Jan 4th\nAnswer with the option's letter from the given choices directly.", + "gt_response": "sunday january 4th", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "monday january 4th", + "sunday january 4th", + "saturday january 4th", + "jan 4th" + ], + "image_quantity_level": "Few", + "image": [ + "472.jpg", + "473.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 132, + "question": "There is a question for each group of pictures. Please answer it in a short way. You must choose your answer from the Choice List. \nQuestion: what is the general budget total for Medical Coordinator?\nChoice list: \nA. $2,500\nB. $3,000\nC. \u00a33,000\nD. \u20ac3,000\nAnswer with the option's letter from the given choices directly.", + "gt_response": "$3000", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "$2500", + "$3000", + "\u00a33000", + "\u20ac3000" + ], + "image_quantity_level": "Few", + "image": [ + "478.jpg", + "479.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 133, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: Which department is involved?\nChoice list: \nA. ELECTRICAL ENGINEERING DEPARTMENT\nB. BIOLOGY DEPARTMENT\nC. PHYSICS DEPARTMENT\nD. CHEMICAL DEPARTMENT\nAnswer with the option's letter from the given choices directly.", + "gt_response": "chemical department", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "electrical engineering department", + "biology department", + "physics department", + "chemical department" + ], + "image_quantity_level": "Few", + "image": [ + "480.jpg", + "481.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 140, + "question": "Each group of pictures relates to a question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: What is the date mentioned in this letter?\nChoice list: \nA. September 9, 1964\nB. September 9, 1963\nC. September 10, 1963\nD. September 8, 1963\nAnswer with the option's letter from the given choices directly.", + "gt_response": "september 9 1963", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "september 9 1964", + "september 9 1963", + "september 10 1963", + "september 8 1963" + ], + "image_quantity_level": "Few", + "image": [ + "513.jpg", + "514.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 143, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the name of heart association?\nChoice list: \nA. United States\nB. American\nC. International\nD. National\nAnswer with the option's letter from the given choices directly.", + "gt_response": "american", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "united states", + "american", + "international", + "national" + ], + "image_quantity_level": "Few", + "image": [ + "523.jpg", + "524.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 144, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the subject of the inter-office memorandum?\nChoice list: \nA. BILL SMITH\nB. LASZLO TOTH\nC. JANE DOE\nD. JOHN ADAMS\nAnswer with the option's letter from the given choices directly.", + "gt_response": "laszlo toth", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "bill smith", + "laszlo toth", + "jane doe", + "john adams" + ], + "image_quantity_level": "Few", + "image": [ + "525.jpg", + "526.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 148, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: when is the Report of Travel dated?\nChoice list: \nA. Sept. 20, 1971\nB. Sept. 19, 1971\nC. Oct. 20, 1971\nD. Aug. 20, 1971\nAnswer with the option's letter from the given choices directly.", + "gt_response": "sept 20 1971", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "sept 20 1971", + "sept 19 1971", + "oct 20 1971", + "aug 20 1971" + ], + "image_quantity_level": "Few", + "image": [ + "541.jpg", + "542.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 151, + "question": "Each group of pictures relates to a question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: what is the name of the university mentioned in the given form?\nChoice list: \nA. University of Washington\nB. Washington university\nC. George Washington University\nD. Washington state university\nAnswer with the option's letter from the given choices directly.", + "gt_response": "washington university", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "university of washington", + "washington university", + "george washington university", + "washington state university" + ], + "image_quantity_level": "Few", + "image": [ + "552.jpg", + "553.jpg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 154, + "question": "For each group of pictures, there is a question. Please give a short answer to it. You must choose your answer from the Choice List. \nQuestion: what is the contract no.?\nChoice list: \nA. DA-49-008-MD-1024\nB. DA-49-007-MD-1024\nC. DA-49-007-MD-1025\nD. DA-48-007-MD-1024\nAnswer with the option's letter from the given choices directly.", + "gt_response": "da 49 007 md 1024", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "da 49 008 md 1024", + "da 49 007 md 1024", + "da 49 007 md 1025", + "da 48 007 md 1024" + ], + "image_quantity_level": "Few", + "image": [ + "562.jpg", + "563.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 155, + "question": "There is a question for each group of pictures. Please answer it in a short way. You must choose your answer from the Choice List. \nQuestion: To whom is the confirmation addressed?\nChoice list: \nA. Kurt Henshaw\nB. Kirk Hinshore\nC. Kirk Hinshaw\nD. John Smith\nAnswer with the option's letter from the given choices directly.", + "gt_response": "kirk hinshaw", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "kurt henshaw", + "kirk hinshore", + "kirk hinshaw", + "john smith" + ], + "image_quantity_level": "Few", + "image": [ + "564.jpg", + "565.jpg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 157, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: What is the first entity of the \"TABLE 1:\"?\nChoice list: \nA. TABLE 2\nB. TABLE 1\nC. PM Code\nD. PM Code 1\nAnswer with the option's letter from the given choices directly.", + "gt_response": "pm code", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "table 2", + "table 1", + "pm code", + "pm code 1" + ], + "image_quantity_level": "Few", + "image": [ + "571.jpg", + "572.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 158, + "question": "There is a question for each group of pictures. Please answer it in a short way. You must choose your answer from the Choice List. \nQuestion: what is amount of Potassium mg./100g. of Figs (R)?\nChoice list: \nA. 280\nB. 300\nC. 290\nD. 310\nAnswer with the option's letter from the given choices directly.", + "gt_response": "290", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "280", + "300", + "290", + "310" + ], + "image_quantity_level": "Few", + "image": [ + "573.jpg", + "574.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 159, + "question": "Each group of pictures has a related question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: The number mentioned on the right of the leftside margin?\nChoice list: \nA. 314002840\nB. 314002839\nC. 314002837\nD. 314002838\nAnswer with the option's letter from the given choices directly.", + "gt_response": "314002838", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "314002840", + "314002839", + "314002837", + "314002838" + ], + "image_quantity_level": "Few", + "image": [ + "575.jpg", + "576.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 98, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: who is chief, Processing and Records Unit, PERS. MGMT. DIV.?\nChoice list: \nA. Marie D. Newman\nB. Mary C. Newman\nC. Marie C. Newton\nD. Marie C. Newman\nAnswer with the option's letter from the given choices directly.", + "gt_response": "marie c newman", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "marie d newman", + "mary c newman", + "marie c newton", + "marie c newman" + ], + "image_quantity_level": "Few", + "image": [ + "356.jpg", + "357.jpg", + "358.jpg", + "359.jpg", + "360.jpg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 106, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: What is the date on the form ?\nChoice list: \nA. september 27, 1987\nB. september 27, 1988\nC. september 27, 1989\nD. september 27, 1998\nAnswer with the option's letter from the given choices directly.", + "gt_response": "september 27 1989", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "september 27 1987", + "september 27 1988", + "september 27 1989", + "september 27 1998" + ], + "image_quantity_level": "Few", + "image": [ + "381.jpg", + "382.jpg", + "383.jpg", + "384.jpg", + "385.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 110, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the middle name mentioned in the given form?\nChoice list: \nA. Jenkins\nB. Williams\nC. Kirk\nD. Smith\nAnswer with the option's letter from the given choices directly.", + "gt_response": "kirk", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "jenkins", + "williams", + "kirk", + "smith" + ], + "image_quantity_level": "Few", + "image": [ + "393.jpg", + "394.jpg", + "395.jpg", + "396.jpg", + "397.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 123, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the Region Operations Office FAX?\nChoice list: \nA. 651-686-6016\nB. 656-686-6016\nC. 651-686-6036\nD. 651-686-6017\nAnswer with the option's letter from the given choices directly.", + "gt_response": "651 686 6016", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "651 686 6016", + "656 686 6016", + "651 686 6036", + "651 686 6017" + ], + "image_quantity_level": "Few", + "image": [ + "440.jpg", + "441.jpg", + "442.jpg", + "443.jpg", + "444.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 124, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the Total cost?\nChoice list: \nA. $14.55\nB. $15.55\nC. \u00a514.55\nD. \u20ac14.55\nAnswer with the option's letter from the given choices directly.", + "gt_response": "$14.55", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "$14.55", + "$15.55", + "\u00a514.55", + "\u20ac14.55" + ], + "image_quantity_level": "Few", + "image": [ + "445.jpg", + "446.jpg", + "447.jpg", + "448.jpg", + "449.jpg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 127, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: what is the EST response %?\nChoice list: \nA. 5.00%\nB. 6.00%\nC. 5.20%\nD. 4.80%\nAnswer with the option's letter from the given choices directly.", + "gt_response": "5.00%", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "5.00%", + "6.00%", + "5.20%", + "4.80%" + ], + "image_quantity_level": "Few", + "image": [ + "462.jpg", + "463.jpg", + "464.jpg", + "465.jpg", + "466.jpg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 137, + "question": "For each group of pictures, there is a question. Please give a short answer to it. You must choose your answer from the Choice List. \nQuestion: what is the name of the university mentioned in the given address?\nChoice list: \nA. george washington university\nB. james madison university\nC. benjamin franklin university\nD. thomas jefferson university\nAnswer with the option's letter from the given choices directly.", + "gt_response": "thomas jefferson university", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "george washington university", + "james madison university", + "benjamin franklin university", + "thomas jefferson university" + ], + "image_quantity_level": "Few", + "image": [ + "496.jpg", + "497.jpg", + "498.jpg", + "499.jpg", + "500.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 142, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the full form of GLP?\nChoice list: \nA. Good Laboratory Practice\nB. Good Laboratory Procedure\nC. Great Laboratory Practice\nD. Good Laboratory Policy\nAnswer with the option's letter from the given choices directly.", + "gt_response": "good laboratory practice", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "good laboratory practice", + "good laboratory procedure", + "great laboratory practice", + "good laboratory policy" + ], + "image_quantity_level": "Few", + "image": [ + "518.jpg", + "519.jpg", + "520.jpg", + "521.jpg", + "522.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 146, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: How many numbers of copies of the journal of nutrition?\nChoice list: \nA. 4300\nB. 4200\nC. 4500\nD. 4400\nAnswer with the option's letter from the given choices directly.", + "gt_response": "4400", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "4300", + "4200", + "4500", + "4400" + ], + "image_quantity_level": "Few", + "image": [ + "531.jpg", + "532.jpg", + "533.jpg", + "534.jpg", + "535.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 80, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: what is the program number?\nChoice list: \nA. 610270\nB. 601200\nC. 601279\nD. 601270\nAnswer with the option's letter from the given choices directly.", + "gt_response": "601270", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "610270", + "601200", + "601279", + "601270" + ], + "image_quantity_level": "Few", + "image": [ + "288.jpg", + "289.jpg", + "290.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 81, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the Estimated Budget for Pass-through Expenses?\nChoice list: \nA. $3,500.00\nB. $2,500.00\nC. $4,000.00\nD. $3,000.00*\nAnswer with the option's letter from the given choices directly.", + "gt_response": "$3000.00*", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "$3500.00", + "$2500.00", + "$4000.00", + "$3000.00*" + ], + "image_quantity_level": "Few", + "image": [ + "291.jpg", + "292.jpg", + "293.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 84, + "question": "Each group of pictures has a related question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: what is the name of marketing?\nChoice list: \nA. Targetbase\nB. Marketbase\nC. Brandbase\nD. Starbase\nAnswer with the option's letter from the given choices directly.", + "gt_response": "targetbase", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "targetbase", + "marketbase", + "brandbase", + "starbase" + ], + "image_quantity_level": "Few", + "image": [ + "301.jpg", + "302.jpg", + "303.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 86, + "question": "For each group of pictures, there is a question. Please give a short answer to it. You must choose your answer from the Choice List. \nQuestion: what is the area code mentioned in the given page?\nChoice list: \nA. 280\nB. 978\nC. 814\nD. 743\nAnswer with the option's letter from the given choices directly.", + "gt_response": "814", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "280", + "978", + "814", + "743" + ], + "image_quantity_level": "Few", + "image": [ + "309.jpg", + "310.jpg", + "311.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 87, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: To where the approved invoice sent to?\nChoice list: \nA. Sponsored Members\nB. Approving Sponsors\nC. Membership Sponsors\nD. Sponsoring Members\nAnswer with the option's letter from the given choices directly.", + "gt_response": "sponsoring members", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "sponsored members", + "approving sponsors", + "membership sponsors", + "sponsoring members" + ], + "image_quantity_level": "Few", + "image": [ + "312.jpg", + "313.jpg", + "314.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 96, + "question": "For each group of pictures, there is a question. Please give a short answer to it. You must choose your answer from the Choice List. \nQuestion: what is the estimated population in poverty in Macon county?\nChoice list: \nA. 3,363\nB. 3,366\nC. 3,360\nD. 3,362\nAnswer with the option's letter from the given choices directly.", + "gt_response": "3363", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "3363", + "3366", + "3360", + "3362" + ], + "image_quantity_level": "Few", + "image": [ + "347.jpg", + "348.jpg", + "349.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 109, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: what is number 2 priority as approved by the AHA board of directors?\nChoice list: \nA. Atherosclerosis\nB. Cardiovascular disease\nC. Coronary artery disease\nD. Hypertension\nAnswer with the option's letter from the given choices directly.", + "gt_response": "atherosclerosis", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "atherosclerosis", + "cardiovascular disease", + "coronary artery disease", + "hypertension" + ], + "image_quantity_level": "Few", + "image": [ + "390.jpg", + "391.jpg", + "392.jpg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 128, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the position title of William M. Grosvenor, Jr.?\nChoice list: \nA. Director of Research and Development\nB. Chief Financial Officer\nC. Vice President of Sales and Marketing\nD. Senior Project Manager\nAnswer with the option's letter from the given choices directly.", + "gt_response": "director of research and development", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "director of research and development", + "chief financial officer", + "vice president of sales and marketing", + "senior project manager" + ], + "image_quantity_level": "Few", + "image": [ + "467.jpg", + "468.jpg", + "469.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 141, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: Who is the marketing person responsible for the project?\nChoice list: \nA. Lucinda Sheer\nB. Lucinda Smith\nC. Lucinda Sheers\nD. Lucinda Sherman\nAnswer with the option's letter from the given choices directly.", + "gt_response": "lucinda sheer", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "lucinda sheer", + "lucinda smith", + "lucinda sheers", + "lucinda sherman" + ], + "image_quantity_level": "Few", + "image": [ + "515.jpg", + "516.jpg", + "517.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 152, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: What is the Payee's Account No.?\nChoice list: \nA. cg720t117pb\nB. cg721t117pb\nC. cg721t117qb\nD. cg721t116pb\nAnswer with the option's letter from the given choices directly.", + "gt_response": "cg721t117pb", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "cg720t117pb", + "cg721t117pb", + "cg721t117qb", + "cg721t116pb" + ], + "image_quantity_level": "Few", + "image": [ + "554.jpg", + "555.jpg", + "556.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 91, + "question": "Each group of pictures relates to a question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: What is the Mean of 2,4-D?\nChoice list: \nA. 0.08%\nB. 0.08\nC. 0.80%\nD. 0.80\nAnswer with the option's letter from the given choices directly.", + "gt_response": "0.08", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "0.08%", + "0.08", + "0.80%", + "0.80" + ], + "image_quantity_level": "Medium", + "image": [ + "327.jpg", + "328.jpg", + "329.jpg", + "330.jpg", + "331.jpg", + "332.jpg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 97, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the lounge booked for United States Brewers Association Medical Advisory Group?\nChoice list: \nA. Central Lounge\nB. Southwest Lounge\nC. South Lounge\nD. North Lounge\nAnswer with the option's letter from the given choices directly.", + "gt_response": "south lounge", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "central lounge", + "southwest lounge", + "south lounge", + "north lounge" + ], + "image_quantity_level": "Medium", + "image": [ + "350.jpg", + "351.jpg", + "352.jpg", + "353.jpg", + "354.jpg", + "355.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 103, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: What is the Daily Quantity for Turnip Greens?\nChoice list: \nA. 1/4 lb.\nB. 1/5 lb.\nC. 1/2 lb.\nD. 1/3 lb.\nAnswer with the option's letter from the given choices directly.", + "gt_response": "1 3 lb", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "1 4 lb", + "1 5 lb", + "1 2 lb", + "1 3 lb" + ], + "image_quantity_level": "Medium", + "image": [ + "369.jpg", + "370.jpg", + "371.jpg", + "372.jpg", + "373.jpg", + "374.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 115, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: Where is Mr.James Briggs working as an Editor?\nChoice list: \nA. Chicago Tribune\nB. Greeley Daily Tribune\nC. Los Angeles Times\nD. New York Times\nAnswer with the option's letter from the given choices directly.", + "gt_response": "greeley daily tribune", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "chicago tribune", + "greeley daily tribune", + "los angeles times", + "new york times" + ], + "image_quantity_level": "Medium", + "image": [ + "408.jpg", + "409.jpg", + "410.jpg", + "411.jpg", + "412.jpg", + "413.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 118, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: How much is the EFT $ Amount?\nChoice list: \nA. 3327.78\nB. $3327.79\nC. $3327.77\nD. $3327.80\nAnswer with the option's letter from the given choices directly.", + "gt_response": "3327.78", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "3327.78", + "$3327.79", + "$3327.77", + "$3327.80" + ], + "image_quantity_level": "Medium", + "image": [ + "418.jpg", + "419.jpg", + "420.jpg", + "421.jpg", + "422.jpg", + "423.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 125, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: who is the whole -time director( up to July 31,2009)?\nChoice list: \nA. Mr. V.S.Vhanbhag\nB. Ms. S.V.Shanbhag\nC. Mr. S.V.Shanbhag\nD. Mrs. S.V.Shanbhag\nAnswer with the option's letter from the given choices directly.", + "gt_response": "mr svshanbhag", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "mr vsvhanbhag", + "ms svshanbhag", + "mr svshanbhag", + "mrs svshanbhag" + ], + "image_quantity_level": "Medium", + "image": [ + "450.jpg", + "451.jpg", + "452.jpg", + "453.jpg", + "454.jpg", + "455.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 120, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: What is the full form of POV?\nChoice list: \nA. Public-Owned Vehicle\nB. Personal-Owned Vehicle\nC. Privately-Operated Vehicle\nD. Privately-Owned Vehicle\nAnswer with the option's letter from the given choices directly.", + "gt_response": "privately owned vehicle", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "public owned vehicle", + "personal owned vehicle", + "privately operated vehicle", + "privately owned vehicle" + ], + "image_quantity_level": "Few", + "image": [ + "428.jpg", + "429.jpg", + "430.jpg", + "431.jpg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 121, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the name of the university mentioned in the given form?\nChoice list: \nA. Washington university\nB. University of Washington\nC. George Washington University\nD. Washington State University\nAnswer with the option's letter from the given choices directly.", + "gt_response": "washington university", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "washington university", + "university of washington", + "george washington university", + "washington state university" + ], + "image_quantity_level": "Few", + "image": [ + "432.jpg", + "433.jpg", + "434.jpg", + "435.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 122, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the amount of Calories under Chemical Analyses?\nChoice list: \nA. 2,870 cal\nB. 2,872 kcal\nC. 2,872 cal\nD. 2,875 cal\nAnswer with the option's letter from the given choices directly.", + "gt_response": "2872 cal", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "2870 cal", + "2872 kcal", + "2872 cal", + "2875 cal" + ], + "image_quantity_level": "Few", + "image": [ + "436.jpg", + "437.jpg", + "438.jpg", + "439.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 131, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the name of the passenger mentioned in the given page?\nChoice list: \nA. Dr. W. J. Darby\nB. Dr. J. W. Darby\nC. W. J. Darby, MD\nD. Dr. William J. Darby\nAnswer with the option's letter from the given choices directly.", + "gt_response": "dr w j darby", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "dr w j darby", + "dr j w darby", + "w j darby md", + "dr william j darby" + ], + "image_quantity_level": "Few", + "image": [ + "474.jpg", + "475.jpg", + "476.jpg", + "477.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 134, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: Who is the Discussion chairman for application of the morning session on Wednesday, June 23?\nChoice list: \nA. John H. Brows\nB. John H. Brown\nC. John H. Browe\nD. John H. Browne\nAnswer with the option's letter from the given choices directly.", + "gt_response": "john h browe", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "john h brows", + "john h brown", + "john h browe", + "john h browne" + ], + "image_quantity_level": "Few", + "image": [ + "482.jpg", + "483.jpg", + "484.jpg", + "485.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 135, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the date mentioned in the given page?\nChoice list: \nA. december 6-8, 1958\nB. 12-6-1958\nC. Dec 6 to Dec 8, 1958\nD. December Sixth to Eighth, 1958\nAnswer with the option's letter from the given choices directly.", + "gt_response": "december 6 8 1958", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "december 6 8 1958", + "12 6 1958", + "dec 6 to dec 8 1958", + "december sixth to eighth 1958" + ], + "image_quantity_level": "Few", + "image": [ + "486.jpg", + "487.jpg", + "488.jpg", + "489.jpg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 145, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the Voucher Amount?\nChoice list: \nA. $210,391.01\nB. $210,391.10\nC. $210,390.00\nD. $210,391.00\nAnswer with the option's letter from the given choices directly.", + "gt_response": "$210391.00", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "$210391.01", + "$210391.10", + "$210390.00", + "$210391.00" + ], + "image_quantity_level": "Few", + "image": [ + "527.jpg", + "528.jpg", + "529.jpg", + "530.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 160, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: Who is the author of \" The Taming of the C.A.N.D.Y. Monster\"?\nChoice list: \nA. Lanski\nB. LANSKY\nC. Lanskey\nD. Langsky\nAnswer with the option's letter from the given choices directly.", + "gt_response": "lansky", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "lanski", + "lansky", + "lanskey", + "langsky" + ], + "image_quantity_level": "Few", + "image": [ + "577.jpg", + "578.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 164, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: Who is the Contact Person of agency?\nChoice list: \nA. John Smith\nB. David Moore\nC. David Johnson\nD. Michael Davis\nAnswer with the option's letter from the given choices directly.", + "gt_response": "david moore", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "john smith", + "david moore", + "david johnson", + "michael davis" + ], + "image_quantity_level": "Few", + "image": [ + "589.jpg", + "590.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 165, + "question": "Each group of pictures has a related question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: what is the origin?\nChoice list: \nA. LAX\nB. JFK\nC. CDG\nD. BEY\nAnswer with the option's letter from the given choices directly.", + "gt_response": "bey", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "lax", + "jfk", + "cdg", + "bey" + ], + "image_quantity_level": "Few", + "image": [ + "591.jpg", + "592.jpg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 166, + "question": "Each group of pictures has a related question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: What is the value of 'Deposits'?\nChoice list: \nA. $535.66\nB. \u00a5535.66\nC. \u20ac535.66\nD. $535.67\nAnswer with the option's letter from the given choices directly.", + "gt_response": "$535.66", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "$535.66", + "\u00a5535.66", + "\u20ac535.66", + "$535.67" + ], + "image_quantity_level": "Few", + "image": [ + "593.jpg", + "594.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 169, + "question": "There is a question for each group of pictures. Please answer it in a short way. You must choose your answer from the Choice List. \nQuestion: what is the name of the cashier mentioned?\nChoice list: \nA. Mr. Ralph T. Johnson\nB. Mr. John T. Smith and\nC. Ms. Rachel T. Smith\nD. Mr. Ralph T. Smith\nAnswer with the option's letter from the given choices directly.", + "gt_response": "mr ralph t smith", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "mr ralph t johnson", + "mr john t smith and", + "ms rachel t smith", + "mr ralph t smith" + ], + "image_quantity_level": "Few", + "image": [ + "602.jpg", + "603.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 172, + "question": "For each group of pictures, there is a question. Please give a short answer to it. You must choose your answer from the Choice List. \nQuestion: What is the heading of the REPORT?\nChoice list: \nA. progress report 11\nB. progress report 9\nC. progress report 10\nD. final report 10\nAnswer with the option's letter from the given choices directly.", + "gt_response": "progress report 10", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "progress report 11", + "progress report 9", + "progress report 10", + "final report 10" + ], + "image_quantity_level": "Few", + "image": [ + "611.jpg", + "612.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 174, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is a major source of sodium?\nChoice list: \nA. Salt is a major source of sodium.\nB. Water is a major source of sodium.\nC. Sugar is a major source of sodium.\nD. Potassium is a major source of sodium.\nAnswer with the option's letter from the given choices directly.", + "gt_response": "salt is a major source of sodium", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "salt is a major source of sodium", + "water is a major source of sodium", + "sugar is a major source of sodium", + "potassium is a major source of sodium" + ], + "image_quantity_level": "Few", + "image": [ + "619.jpg", + "620.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 176, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: What is the electronic signature code number?\nChoice list: \nA. 266-524-8\nB. 267-524-9\nC. 266-524-9\nD. 266-525-9\nAnswer with the option's letter from the given choices directly.", + "gt_response": "266 524 9", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "266 524 8", + "267 524 9", + "266 524 9", + "266 525 9" + ], + "image_quantity_level": "Few", + "image": [ + "626.jpg", + "627.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 178, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: Under which department, The Cleveland Clinic Foundation operates?\nChoice list: \nA. Department of Radiology\nB. Department of Orthopedic Surgery\nC. Department of Neuroscience\nD. Department of Cardiovascular Medicine.\nAnswer with the option's letter from the given choices directly.", + "gt_response": "department of cardiovascular medicine", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "department of radiology", + "department of orthopedic surgery", + "department of neuroscience", + "department of cardiovascular medicine" + ], + "image_quantity_level": "Few", + "image": [ + "631.jpg", + "632.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 179, + "question": "For each group of pictures, there is a question. Please give a short answer to it. You must choose your answer from the Choice List. \nQuestion: What is the product designation?\nChoice list: \nA. Product code\nB. Product name\nC. Sugarcane\nD. Sugar-cane\nAnswer with the option's letter from the given choices directly.", + "gt_response": "sugar cane", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "product code", + "product name", + "sugarcane", + "sugar cane" + ], + "image_quantity_level": "Few", + "image": [ + "633.jpg", + "634.jpg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 182, + "question": "Each group of pictures has a related question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: What is the cellular number of Gigi M. Chen?\nChoice list: \nA. 716.725.7102\nB. 716.725.7101\nC. 716.725.7103\nD. 716.725.7100\nAnswer with the option's letter from the given choices directly.", + "gt_response": "716.725.7100", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "716.725.7102", + "716.725.7101", + "716.725.7103", + "716.725.7100" + ], + "image_quantity_level": "Few", + "image": [ + "643.jpg", + "644.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 185, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: How many biographical data forms are attached?\nChoice list: \nA. 6 biographical data forms\nB. 5 biographical data forms\nC. 7 biographical data forms\nD. 4 biographical data forms\nAnswer with the option's letter from the given choices directly.", + "gt_response": "6 biographical data forms", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "6 biographical data forms", + "5 biographical data forms", + "7 biographical data forms", + "4 biographical data forms" + ], + "image_quantity_level": "Few", + "image": [ + "651.jpg", + "652.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 187, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: What is the proposed end date for Action # 11?\nChoice list: \nA. 3/24/03\nB. 12/24/03\nC. 3/24/05\nD. 3/24/04\nAnswer with the option's letter from the given choices directly.", + "gt_response": "3 24 03", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "3 24 03", + "12 24 03", + "3 24 05", + "3 24 04" + ], + "image_quantity_level": "Few", + "image": [ + "658.jpg", + "659.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 191, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: what is the given schedule at the time of 10:30 on monday, june 20?\nChoice list: \nA. meeting\nB. break\nC. lecture\nD. orientation\nAnswer with the option's letter from the given choices directly.", + "gt_response": "orientation", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "meeting", + "break", + "lecture", + "orientation" + ], + "image_quantity_level": "Few", + "image": [ + "674.jpg", + "675.jpg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 192, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: What is the Expiration Date?\nChoice list: \nA. 12/03/96\nB. 12/31/06\nC. 12/31/98\nD. 12/31/96\nAnswer with the option's letter from the given choices directly.", + "gt_response": "12 31 96", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "12 03 96", + "12 31 06", + "12 31 98", + "12 31 96" + ], + "image_quantity_level": "Few", + "image": [ + "676.jpg", + "677.jpg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 194, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is J. Brozek's topic from 9:30-10:00?\nChoice list: \nA. PHYSICAL CHARACTERISTICS OF MAN\nB. CHARACTERIZATION OF MAN'S PHYSIQUE\nC. CHARACTERIZATION OF PERSON'S PHYSIQUE\nD. CHARACTERIZATION OF WOMAN'S PHYSIQUE\nAnswer with the option's letter from the given choices directly.", + "gt_response": "characterization of man's physique", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "physical characteristics of man", + "characterization of man's physique", + "characterization of person's physique", + "characterization of woman's physique" + ], + "image_quantity_level": "Few", + "image": [ + "683.jpg", + "684.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 195, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: what is the quantity mentioned in the given page ?\nChoice list: \nA. 10\nB. 1\nC. 0\nD. 2\nAnswer with the option's letter from the given choices directly.", + "gt_response": "1", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "10", + "1", + "0", + "2" + ], + "image_quantity_level": "Few", + "image": [ + "685.jpg", + "686.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 147, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: What is the Total for Rent Paid for 2008?\nChoice list: \nA. 0.55\nB. $0.54\nC. $0.56\nD. 0.5502\nAnswer with the option's letter from the given choices directly.", + "gt_response": "0.55", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "0.55", + "$0.54", + "$0.56", + "0.5502" + ], + "image_quantity_level": "Few", + "image": [ + "536.jpg", + "537.jpg", + "538.jpg", + "539.jpg", + "540.jpg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 149, + "question": "For each group of pictures, there is a question. Please give a short answer to it. You must choose your answer from the Choice List. \nQuestion: what is the amount of potassium present in liver (R) as mentioned in the given page?\nChoice list: \nA. 300\nB. 295\nC. 296\nD. 297\nAnswer with the option's letter from the given choices directly.", + "gt_response": "296", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "300", + "295", + "296", + "297" + ], + "image_quantity_level": "Few", + "image": [ + "543.jpg", + "544.jpg", + "545.jpg", + "546.jpg", + "547.jpg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 153, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the amount of Potassium mg./100g. for Squash, Winter, seeds removed?\nChoice list: \nA. 407\nB. 409\nC. 406\nD. 408\nAnswer with the option's letter from the given choices directly.", + "gt_response": "407", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "407", + "409", + "406", + "408" + ], + "image_quantity_level": "Few", + "image": [ + "557.jpg", + "558.jpg", + "559.jpg", + "560.jpg", + "561.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 156, + "question": "For each group of pictures, there is a question. Please give a short answer to it. You must choose your answer from the Choice List. \nQuestion: Who is the author for \"Physiology of Vitamin A\"?\nChoice list: \nA. o. a. roels\nB. O. A. Roels\nC. Dr. O. A. Roels\nD. O. Roels\nAnswer with the option's letter from the given choices directly.", + "gt_response": "o a roels", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "o a roels", + "o a roels", + "dr o a roels", + "o roels" + ], + "image_quantity_level": "Few", + "image": [ + "566.jpg", + "567.jpg", + "568.jpg", + "569.jpg", + "570.jpg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 175, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is amount of composition for Printer number I?\nChoice list: \nA. \u20ac 3,436.98\nB. $ 3,436.98\nC. \u00a5 3,436.98\nD. $ 3,536.98\nAnswer with the option's letter from the given choices directly.", + "gt_response": "$ 3436.98", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "\u20ac 3436.98", + "$ 3436.98", + "\u00a5 3436.98", + "$ 3536.98" + ], + "image_quantity_level": "Few", + "image": [ + "621.jpg", + "622.jpg", + "623.jpg", + "624.jpg", + "625.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 180, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the amount of 2nd Quarter check mentioned in the form?\nChoice list: \nA. 150,204.69\nB. 150,204.79\nC. 150,204.39\nD. 150,207.79\nAnswer with the option's letter from the given choices directly.", + "gt_response": "150204.79", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "150204.69", + "150204.79", + "150204.39", + "150207.79" + ], + "image_quantity_level": "Few", + "image": [ + "635.jpg", + "636.jpg", + "637.jpg", + "638.jpg", + "639.jpg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 186, + "question": "There is a question for each group of pictures. Please answer it in a short way. You must choose your answer from the Choice List. \nQuestion: what is the issue date?\nChoice list: \nA. October 26, 1988\nB. October 16, 1998\nC. September 26, 1998\nD. October 26, 1998\nAnswer with the option's letter from the given choices directly.", + "gt_response": "october 26 1998", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "october 26 1988", + "october 16 1998", + "september 26 1998", + "october 26 1998" + ], + "image_quantity_level": "Few", + "image": [ + "653.jpg", + "654.jpg", + "655.jpg", + "656.jpg", + "657.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 193, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the location code?\nChoice list: \nA. 11-0010-000\nB. 11-0010-001\nC. 11-0010-010\nD. 11-0010-002\nAnswer with the option's letter from the given choices directly.", + "gt_response": "11 0010 001", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "11 0010 000", + "11 0010 001", + "11 0010 010", + "11 0010 002" + ], + "image_quantity_level": "Few", + "image": [ + "678.jpg", + "679.jpg", + "680.jpg", + "681.jpg", + "682.jpg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 198, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: To which department does Stephen Molnar belong to?\nChoice list: \nA. Ethnology\nB. Psychology\nC. Sociology\nD. Anthropology\nAnswer with the option's letter from the given choices directly.", + "gt_response": "anthropology", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "ethnology", + "psychology", + "sociology", + "anthropology" + ], + "image_quantity_level": "Few", + "image": [ + "694.jpg", + "695.jpg", + "696.jpg", + "697.jpg", + "698.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 161, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: Which company's name is mentioned?\nChoice list: \nA. Silver Tobacco Limited\nB. GOLDEN TOBACCO LIMITED\nC. Platinum Tobacco Limited\nD. Bronze Tobacco Limited\nAnswer with the option's letter from the given choices directly.", + "gt_response": "golden tobacco limited", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "silver tobacco limited", + "golden tobacco limited", + "platinum tobacco limited", + "bronze tobacco limited" + ], + "image_quantity_level": "Few", + "image": [ + "579.jpg", + "580.jpg", + "581.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 162, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: what are the expenses for meals (other than those at hotel )?\nChoice list: \nA. $27.50\nB. $25.50\nC. $26.00\nD. 26.50\nAnswer with the option's letter from the given choices directly.", + "gt_response": "26.50", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "$27.50", + "$25.50", + "$26.00", + "26.50" + ], + "image_quantity_level": "Few", + "image": [ + "582.jpg", + "583.jpg", + "584.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 167, + "question": "Each group of pictures has a related question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: What is the designation of Dr. William J. Darby?\nChoice list: \nA. CEO\nB. Chairman\nC. president\nD. Managing director\nAnswer with the option's letter from the given choices directly.", + "gt_response": "president", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "ceo", + "chairman", + "president", + "managing director" + ], + "image_quantity_level": "Few", + "image": [ + "595.jpg", + "596.jpg", + "597.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 171, + "question": "Each group of pictures relates to a question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: what should be able to be modified-expanded or contracted?\nChoice list: \nA. low priority designations\nB. incorrect priority designations\nC. high priority designations\nD. priority designations\nAnswer with the option's letter from the given choices directly.", + "gt_response": "low priority designations", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "low priority designations", + "incorrect priority designations", + "high priority designations", + "priority designations" + ], + "image_quantity_level": "Few", + "image": [ + "608.jpg", + "609.jpg", + "610.jpg" + ], + "extracted": "D", + "result": 0 + }, + { + "sample_id": 177, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: What is the issued date of this letter?\nChoice list: \nA. July 16, 1999\nB. July 15, 1999\nC. July 17, 1999\nD. June 16, 1999\nAnswer with the option's letter from the given choices directly.", + "gt_response": "july 16 1999", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "july 16 1999", + "july 15 1999", + "july 17 1999", + "june 16 1999" + ], + "image_quantity_level": "Few", + "image": [ + "628.jpg", + "629.jpg", + "630.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 181, + "question": "There is a related question for each group of pictures. Please answer it shortly. You must choose your answer from the Choice List. \nQuestion: Whats Lab name on the form?\nChoice list: \nA. ABBOT LABORATORIES\nB. ABOT LABS\nC. ABBOTT LABORATORIES\nD. ABBBOT LABS\nAnswer with the option's letter from the given choices directly.", + "gt_response": "abbott laboratories", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "abbot laboratories", + "abot labs", + "abbott laboratories", + "abbbot labs" + ], + "image_quantity_level": "Few", + "image": [ + "640.jpg", + "641.jpg", + "642.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 183, + "question": "Each group of pictures relates to a question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: What is the name of the project?\nChoice list: \nA. Project #7595\nB. #7595 Manuscript\nC. Project #7595 Manuscript\nD. Project Manuscript\nAnswer with the option's letter from the given choices directly.", + "gt_response": "#7595 manuscript", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "project #7595", + "#7595 manuscript", + "project #7595 manuscript", + "project manuscript" + ], + "image_quantity_level": "Few", + "image": [ + "645.jpg", + "646.jpg", + "647.jpg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 184, + "question": "For each group of pictures, there is a question. Please give a short answer to it. You must choose your answer from the Choice List. \nQuestion: What is the designation of Mrs. J. Heather Etheridge?\nChoice list: \nA. Secretary\nB. Office Manager\nC. Administrative Assistant\nD. Executive Assistant\nAnswer with the option's letter from the given choices directly.", + "gt_response": "administrative assistant", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "secretary", + "office manager", + "administrative assistant", + "executive assistant" + ], + "image_quantity_level": "Few", + "image": [ + "648.jpg", + "649.jpg", + "650.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 196, + "question": "Please answer briefly the question that corresponds to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the first line of the address mentioned at the top?\nChoice list: \nA. William Building\nB. Reynolds Building\nC. Johnson Building\nD. Smith Building\nAnswer with the option's letter from the given choices directly.", + "gt_response": "reynolds building", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "william building", + "reynolds building", + "johnson building", + "smith building" + ], + "image_quantity_level": "Few", + "image": [ + "687.jpg", + "688.jpg", + "689.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 199, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the estimated population in poverty in Putnam county?\nChoice list: \nA. 3,926,100\nB. 3,926,000\nC. 3,926\nD. 3,926,010\nAnswer with the option's letter from the given choices directly.", + "gt_response": "3926", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "3926100", + "3926000", + "3926", + "3926010" + ], + "image_quantity_level": "Few", + "image": [ + "699.jpg", + "700.jpg", + "701.jpg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 126, + "question": "Each group of pictures has a related question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: What is Written in the SPEC Status Field?\nChoice list: \nA. N/A\nB. CURRENT\nC. NOT CURRENT\nD. PASS\nAnswer with the option's letter from the given choices directly.", + "gt_response": "current", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "n a", + "current", + "not current", + "pass" + ], + "image_quantity_level": "Medium", + "image": [ + "456.jpg", + "457.jpg", + "458.jpg", + "459.jpg", + "460.jpg", + "461.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 136, + "question": "There is a question for each group of pictures. Please answer it in a short way. You must choose your answer from the Choice List. \nQuestion: What is the designation of the person undersigned?\nChoice list: \nA. General Manager\nB. Head Scientist\nC. Chief Chemist.\nD. Lead Researcher\nAnswer with the option's letter from the given choices directly.", + "gt_response": "chief chemist", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "general manager", + "head scientist", + "chief chemist", + "lead researcher" + ], + "image_quantity_level": "Medium", + "image": [ + "490.jpg", + "491.jpg", + "492.jpg", + "493.jpg", + "494.jpg", + "495.jpg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 138, + "question": "For each group of pictures, there is a question. Please give a short answer to it. You must choose your answer from the Choice List. \nQuestion: what are the number of professionals mentioned in the given form?\nChoice list: \nA. 10 Professionals\nB. 20 Professionals\nC. 15 Professionals\nD. 12 Professionals\nAnswer with the option's letter from the given choices directly.", + "gt_response": "12 professionals", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "10 professionals", + "20 professionals", + "15 professionals", + "12 professionals" + ], + "image_quantity_level": "Medium", + "image": [ + "501.jpg", + "502.jpg", + "503.jpg", + "504.jpg", + "505.jpg", + "506.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 139, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: What is the Home Phone?\nChoice list: \nA. 638-8723\nB. 744-5106\nC. 555-4982\nD. 454-5601\nAnswer with the option's letter from the given choices directly.", + "gt_response": "744 5106", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "638 8723", + "744 5106", + "555 4982", + "454 5601" + ], + "image_quantity_level": "Medium", + "image": [ + "507.jpg", + "508.jpg", + "509.jpg", + "510.jpg", + "511.jpg", + "512.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 173, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the title of position for charles L. Paxson?\nChoice list: \nA. detective\nB. researcher\nC. analyst\nD. investigator\nAnswer with the option's letter from the given choices directly.", + "gt_response": "investigator", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "detective", + "researcher", + "analyst", + "investigator" + ], + "image_quantity_level": "Medium", + "image": [ + "613.jpg", + "614.jpg", + "615.jpg", + "616.jpg", + "617.jpg", + "618.jpg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 188, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: What is th last Holding Date of Equity Owner mentioned for \"Dalmia Housing Finance Ltd\"?\nChoice list: \nA. 31-Dec-08\nB. 31-Dec-06\nC. 31-Dec-10\nD. 31-Dec-12\nAnswer with the option's letter from the given choices directly.", + "gt_response": "31 dec 08", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "31 dec 08", + "31 dec 06", + "31 dec 10", + "31 dec 12" + ], + "image_quantity_level": "Medium", + "image": [ + "660.jpg", + "661.jpg", + "662.jpg", + "663.jpg", + "664.jpg", + "665.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 150, + "question": "I will give you some pictures, and each group of pictures will correspond to a question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: what is the program endorsed by the committee?\nChoice list: \nA. history program\nB. home economics program\nC. mathematics program\nD. science program\nAnswer with the option's letter from the given choices directly.", + "gt_response": "home economics program", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "history program", + "home economics program", + "mathematics program", + "science program" + ], + "image_quantity_level": "Few", + "image": [ + "548.jpg", + "549.jpg", + "550.jpg", + "551.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 163, + "question": "Please give a short answer to the question that is related to each group of pictures. You must choose your answer from the Choice List. \nQuestion: When is the Memorandum dated on?\nChoice list: \nA. August 30, 1994\nB. January 30, 1994\nC. August 30, 1995\nD. August 3, 1994\nAnswer with the option's letter from the given choices directly.", + "gt_response": "august 30 1994", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "august 30 1994", + "january 30 1994", + "august 30 1995", + "august 3 1994" + ], + "image_quantity_level": "Few", + "image": [ + "585.jpg", + "586.jpg", + "587.jpg", + "588.jpg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 168, + "question": "There is a question for each group of pictures. Please answer it in a short way. You must choose your answer from the Choice List. \nQuestion: Who is Donald K. Tressler?\nChoice list: \nA. Donald K. Tressler is the CEO of the Institute\nB. Donald K. Tressler is a prominent scientist at the Institute\nC. Scientific Director of the Institute\nD. Donald K. Tressler is the Head of Research at the Institute\nAnswer with the option's letter from the given choices directly.", + "gt_response": "scientific director of the institute", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "donald k tressler is the ceo of the institute", + "donald k tressler is a prominent scientist at the institute", + "scientific director of the institute", + "donald k tressler is the head of research at the institute" + ], + "image_quantity_level": "Few", + "image": [ + "598.jpg", + "599.jpg", + "600.jpg", + "601.jpg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 170, + "question": "For each group of pictures, there is a question. Please give a short answer to it. You must choose your answer from the Choice List. \nQuestion: What is the minimal volume for GH RRA?\nChoice list: \nA. 2.5 cc\nB. 2.9 cc\nC. 3.1 cc\nD. 3.0 cc\nAnswer with the option's letter from the given choices directly.", + "gt_response": "3.0 cc", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "2.5 cc", + "2.9 cc", + "3.1 cc", + "3.0 cc" + ], + "image_quantity_level": "Few", + "image": [ + "604.jpg", + "605.jpg", + "606.jpg", + "607.jpg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 189, + "question": "Each group of pictures has a related question. Please give a brief answer to it. You must choose your answer from the Choice List. \nQuestion: what is the date of goods Rec?\nChoice list: \nA. 02/13/73\nB. 2/13/732\nC. 13/2/73\nD. 2/13/73\nAnswer with the option's letter from the given choices directly.", + "gt_response": "2 13 73", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "d", + "choice_list": [ + "02 13 73", + "2 13 732", + "13 2 73", + "2 13 73" + ], + "image_quantity_level": "Few", + "image": [ + "666.jpg", + "667.jpg", + "668.jpg", + "669.jpg" + ], + "extracted": "D", + "result": 1 + }, + { + "sample_id": 190, + "question": "For every group of pictures, there is a corresponding question. Please answer it briefly. You must choose your answer from the Choice List. \nQuestion: what is the name of the university mentioned?\nChoice list: \nA. Vanderbilt Institute\nB. Vanderbilt University\nC. Vanderbilt College\nD. Vandenburg University\nAnswer with the option's letter from the given choices directly.", + "gt_response": "vanderbilt university", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "vanderbilt institute", + "vanderbilt university", + "vanderbilt college", + "vandenburg university" + ], + "image_quantity_level": "Few", + "image": [ + "670.jpg", + "671.jpg", + "672.jpg", + "673.jpg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 197, + "question": "Please answer shortly the question that relates to each group of pictures. You must choose your answer from the Choice List. \nQuestion: what is the invoice number?\nChoice list: \nA. #154-0071\nB. #154-071\nC. #154-007\nD. #154-006\nAnswer with the option's letter from the given choices directly.", + "gt_response": "#154 0071", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "#154 0071", + "#154 071", + "#154 007", + "#154 006" + ], + "image_quantity_level": "Few", + "image": [ + "690.jpg", + "691.jpg", + "692.jpg", + "693.jpg" + ], + "extracted": "A", + "result": 1 + } +] \ No newline at end of file