diff --git "a/eval_milebench/CharacterOrder/pred_with_extracted.json" "b/eval_milebench/CharacterOrder/pred_with_extracted.json" new file mode 100644--- /dev/null +++ "b/eval_milebench/CharacterOrder/pred_with_extracted.json" @@ -0,0 +1,9701 @@ +[ + { + "sample_id": 0, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. l\nB. v\nC. e\nAnswer with the option's letter from the given choices directly.", + "gt_response": "l", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "l", + "v", + "e" + ], + "image_quantity_level": "Medium", + "image": [ + "video_1238/video_1238_0.jpeg", + "video_1238/video_1238_1.jpeg", + "video_1238/video_1238_2.jpeg", + "video_1238/video_1238_3.jpeg", + "video_1238/video_1238_4.jpeg", + "video_1238/video_1238_5.jpeg", + "video_1238/video_1238_6.jpeg", + "video_1238/video_1238_7.jpeg", + "video_1238/video_1238_8.jpeg", + "video_1238/video_1238_9.jpeg", + "video_1238/video_1238_10.jpeg", + "video_1238/video_1238_11.jpeg", + "video_1238/video_1238_12.jpeg", + "video_1238/video_1238_13.jpeg", + "video_1238/video_1238_14.jpeg", + "video_1238/video_1238_15.jpeg", + "video_1238/video_1238_16.jpeg", + "video_1238/video_1238_17.jpeg", + "video_1238/video_1238_18.jpeg", + "video_1238/video_1238_19.jpeg", + "video_1238/video_1238_20.jpeg", + "video_1238/video_1238_21.jpeg", + "video_1238/video_1238_22.jpeg", + "video_1238/video_1238_23.jpeg", + "video_1238/video_1238_24.jpeg", + "video_1238/video_1238_25.jpeg", + "video_1238/video_1238_26.jpeg", + "video_1238/video_1238_27.jpeg", + "video_1238/video_1238_28.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 4, + "question": "From the images supplied, answer the question relevant to character order. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. ceh\nB. hce\nC. hec\nAnswer with the option's letter from the given choices directly.", + "gt_response": "hec", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "ceh", + "hce", + "hec" + ], + "image_quantity_level": "Medium", + "image": [ + "video_11476/video_11476_0.jpeg", + "video_11476/video_11476_1.jpeg", + "video_11476/video_11476_2.jpeg", + "video_11476/video_11476_3.jpeg", + "video_11476/video_11476_4.jpeg", + "video_11476/video_11476_5.jpeg", + "video_11476/video_11476_6.jpeg", + "video_11476/video_11476_7.jpeg", + "video_11476/video_11476_8.jpeg", + "video_11476/video_11476_9.jpeg", + "video_11476/video_11476_10.jpeg", + "video_11476/video_11476_11.jpeg", + "video_11476/video_11476_12.jpeg", + "video_11476/video_11476_13.jpeg", + "video_11476/video_11476_14.jpeg", + "video_11476/video_11476_15.jpeg", + "video_11476/video_11476_16.jpeg", + "video_11476/video_11476_17.jpeg", + "video_11476/video_11476_18.jpeg", + "video_11476/video_11476_19.jpeg", + "video_11476/video_11476_20.jpeg", + "video_11476/video_11476_21.jpeg", + "video_11476/video_11476_22.jpeg", + "video_11476/video_11476_23.jpeg", + "video_11476/video_11476_24.jpeg", + "video_11476/video_11476_25.jpeg", + "video_11476/video_11476_26.jpeg", + "video_11476/video_11476_27.jpeg", + "video_11476/video_11476_28.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 1, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. mi\nB. xm\nC. mx\nAnswer with the option's letter from the given choices directly.", + "gt_response": "mx", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "mi", + "xm", + "mx" + ], + "image_quantity_level": "Medium", + "image": [ + "video_3157/video_3157_0.jpeg", + "video_3157/video_3157_1.jpeg", + "video_3157/video_3157_2.jpeg", + "video_3157/video_3157_3.jpeg", + "video_3157/video_3157_4.jpeg", + "video_3157/video_3157_5.jpeg", + "video_3157/video_3157_6.jpeg", + "video_3157/video_3157_7.jpeg", + "video_3157/video_3157_8.jpeg", + "video_3157/video_3157_9.jpeg", + "video_3157/video_3157_10.jpeg", + "video_3157/video_3157_11.jpeg", + "video_3157/video_3157_12.jpeg", + "video_3157/video_3157_13.jpeg", + "video_3157/video_3157_14.jpeg", + "video_3157/video_3157_15.jpeg", + "video_3157/video_3157_16.jpeg", + "video_3157/video_3157_17.jpeg", + "video_3157/video_3157_18.jpeg", + "video_3157/video_3157_19.jpeg", + "video_3157/video_3157_20.jpeg", + "video_3157/video_3157_21.jpeg", + "video_3157/video_3157_22.jpeg", + "video_3157/video_3157_23.jpeg", + "video_3157/video_3157_24.jpeg", + "video_3157/video_3157_25.jpeg", + "video_3157/video_3157_26.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 16, + "question": "From the images supplied, answer the question relevant to character order. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. wvc\nB. ovl\nC. lvo\nAnswer with the option's letter from the given choices directly.", + "gt_response": "lvo", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "wvc", + "ovl", + "lvo" + ], + "image_quantity_level": "Medium", + "image": [ + "video_9530/video_9530_0.jpeg", + "video_9530/video_9530_1.jpeg", + "video_9530/video_9530_2.jpeg", + "video_9530/video_9530_3.jpeg", + "video_9530/video_9530_4.jpeg", + "video_9530/video_9530_5.jpeg", + "video_9530/video_9530_6.jpeg", + "video_9530/video_9530_7.jpeg", + "video_9530/video_9530_8.jpeg", + "video_9530/video_9530_9.jpeg", + "video_9530/video_9530_10.jpeg", + "video_9530/video_9530_11.jpeg", + "video_9530/video_9530_12.jpeg", + "video_9530/video_9530_13.jpeg", + "video_9530/video_9530_14.jpeg", + "video_9530/video_9530_15.jpeg", + "video_9530/video_9530_16.jpeg", + "video_9530/video_9530_17.jpeg", + "video_9530/video_9530_18.jpeg", + "video_9530/video_9530_19.jpeg", + "video_9530/video_9530_20.jpeg", + "video_9530/video_9530_21.jpeg", + "video_9530/video_9530_22.jpeg", + "video_9530/video_9530_23.jpeg", + "video_9530/video_9530_24.jpeg", + "video_9530/video_9530_25.jpeg", + "video_9530/video_9530_26.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 2, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nWhat letters did the person type on the computer in order?\nChoice list: \nA. amxbnj\nB. rmtegh\nC. aqlbns\nAnswer with the option's letter from the given choices directly.", + "gt_response": "amxbnj", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "amxbnj", + "rmtegh", + "aqlbns" + ], + "image_quantity_level": "Medium", + "image": [ + "video_5766/video_5766_0.jpeg", + "video_5766/video_5766_1.jpeg", + "video_5766/video_5766_2.jpeg", + "video_5766/video_5766_3.jpeg", + "video_5766/video_5766_4.jpeg", + "video_5766/video_5766_5.jpeg", + "video_5766/video_5766_6.jpeg", + "video_5766/video_5766_7.jpeg", + "video_5766/video_5766_8.jpeg", + "video_5766/video_5766_9.jpeg", + "video_5766/video_5766_10.jpeg", + "video_5766/video_5766_11.jpeg", + "video_5766/video_5766_12.jpeg", + "video_5766/video_5766_13.jpeg", + "video_5766/video_5766_14.jpeg", + "video_5766/video_5766_15.jpeg", + "video_5766/video_5766_16.jpeg", + "video_5766/video_5766_17.jpeg", + "video_5766/video_5766_18.jpeg", + "video_5766/video_5766_19.jpeg", + "video_5766/video_5766_20.jpeg", + "video_5766/video_5766_21.jpeg", + "video_5766/video_5766_22.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 18, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. tel\nB. tun\nC. car\nAnswer with the option's letter from the given choices directly.", + "gt_response": "tel", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "tel", + "tun", + "car" + ], + "image_quantity_level": "Medium", + "image": [ + "video_9310/video_9310_0.jpeg", + "video_9310/video_9310_1.jpeg", + "video_9310/video_9310_2.jpeg", + "video_9310/video_9310_3.jpeg", + "video_9310/video_9310_4.jpeg", + "video_9310/video_9310_5.jpeg", + "video_9310/video_9310_6.jpeg", + "video_9310/video_9310_7.jpeg", + "video_9310/video_9310_8.jpeg", + "video_9310/video_9310_9.jpeg", + "video_9310/video_9310_10.jpeg", + "video_9310/video_9310_11.jpeg", + "video_9310/video_9310_12.jpeg", + "video_9310/video_9310_13.jpeg", + "video_9310/video_9310_14.jpeg", + "video_9310/video_9310_15.jpeg", + "video_9310/video_9310_16.jpeg", + "video_9310/video_9310_17.jpeg", + "video_9310/video_9310_18.jpeg", + "video_9310/video_9310_19.jpeg", + "video_9310/video_9310_20.jpeg", + "video_9310/video_9310_21.jpeg", + "video_9310/video_9310_22.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 30, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. evg\nB. efg\nC. esg\nAnswer with the option's letter from the given choices directly.", + "gt_response": "efg", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "evg", + "efg", + "esg" + ], + "image_quantity_level": "Medium", + "image": [ + "video_3976/video_3976_0.jpeg", + "video_3976/video_3976_1.jpeg", + "video_3976/video_3976_2.jpeg", + "video_3976/video_3976_3.jpeg", + "video_3976/video_3976_4.jpeg", + "video_3976/video_3976_5.jpeg", + "video_3976/video_3976_6.jpeg", + "video_3976/video_3976_7.jpeg", + "video_3976/video_3976_8.jpeg", + "video_3976/video_3976_9.jpeg", + "video_3976/video_3976_10.jpeg", + "video_3976/video_3976_11.jpeg", + "video_3976/video_3976_12.jpeg", + "video_3976/video_3976_13.jpeg", + "video_3976/video_3976_14.jpeg", + "video_3976/video_3976_15.jpeg", + "video_3976/video_3976_16.jpeg", + "video_3976/video_3976_17.jpeg", + "video_3976/video_3976_18.jpeg", + "video_3976/video_3976_19.jpeg", + "video_3976/video_3976_20.jpeg", + "video_3976/video_3976_21.jpeg", + "video_3976/video_3976_22.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 3, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nWhat would be the order of the written letters if the person had written them in reverse order?\nChoice list: \nA. rust\nB. tsur\nC. ruts\nAnswer with the option's letter from the given choices directly.", + "gt_response": "tsur", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "rust", + "tsur", + "ruts" + ], + "image_quantity_level": "Medium", + "image": [ + "video_4939/video_4939_0.jpeg", + "video_4939/video_4939_1.jpeg", + "video_4939/video_4939_2.jpeg", + "video_4939/video_4939_3.jpeg", + "video_4939/video_4939_4.jpeg", + "video_4939/video_4939_5.jpeg", + "video_4939/video_4939_6.jpeg", + "video_4939/video_4939_7.jpeg", + "video_4939/video_4939_8.jpeg", + "video_4939/video_4939_9.jpeg", + "video_4939/video_4939_10.jpeg", + "video_4939/video_4939_11.jpeg", + "video_4939/video_4939_12.jpeg", + "video_4939/video_4939_13.jpeg", + "video_4939/video_4939_14.jpeg", + "video_4939/video_4939_15.jpeg", + "video_4939/video_4939_16.jpeg", + "video_4939/video_4939_17.jpeg", + "video_4939/video_4939_18.jpeg", + "video_4939/video_4939_19.jpeg", + "video_4939/video_4939_20.jpeg", + "video_4939/video_4939_21.jpeg", + "video_4939/video_4939_22.jpeg", + "video_4939/video_4939_23.jpeg", + "video_4939/video_4939_24.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 7, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. luber\nB. luper\nC. lhpez\nAnswer with the option's letter from the given choices directly.", + "gt_response": "luper", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "luber", + "luper", + "lhpez" + ], + "image_quantity_level": "Medium", + "image": [ + "video_3041/video_3041_0.jpeg", + "video_3041/video_3041_1.jpeg", + "video_3041/video_3041_2.jpeg", + "video_3041/video_3041_3.jpeg", + "video_3041/video_3041_4.jpeg", + "video_3041/video_3041_5.jpeg", + "video_3041/video_3041_6.jpeg", + "video_3041/video_3041_7.jpeg", + "video_3041/video_3041_8.jpeg", + "video_3041/video_3041_9.jpeg", + "video_3041/video_3041_10.jpeg", + "video_3041/video_3041_11.jpeg", + "video_3041/video_3041_12.jpeg", + "video_3041/video_3041_13.jpeg", + "video_3041/video_3041_14.jpeg", + "video_3041/video_3041_15.jpeg", + "video_3041/video_3041_16.jpeg", + "video_3041/video_3041_17.jpeg", + "video_3041/video_3041_18.jpeg", + "video_3041/video_3041_19.jpeg", + "video_3041/video_3041_20.jpeg", + "video_3041/video_3041_21.jpeg", + "video_3041/video_3041_22.jpeg", + "video_3041/video_3041_23.jpeg", + "video_3041/video_3041_24.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 8, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat letter is likely to be shown next?\nChoice list: \nA. y\nB. h\nC. z\nAnswer with the option's letter from the given choices directly.", + "gt_response": "h", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "y", + "h", + "z" + ], + "image_quantity_level": "Medium", + "image": [ + "video_8069/video_8069_0.jpeg", + "video_8069/video_8069_1.jpeg", + "video_8069/video_8069_2.jpeg", + "video_8069/video_8069_3.jpeg", + "video_8069/video_8069_4.jpeg", + "video_8069/video_8069_5.jpeg", + "video_8069/video_8069_6.jpeg", + "video_8069/video_8069_7.jpeg", + "video_8069/video_8069_8.jpeg", + "video_8069/video_8069_9.jpeg", + "video_8069/video_8069_10.jpeg", + "video_8069/video_8069_11.jpeg", + "video_8069/video_8069_12.jpeg", + "video_8069/video_8069_13.jpeg", + "video_8069/video_8069_14.jpeg", + "video_8069/video_8069_15.jpeg", + "video_8069/video_8069_16.jpeg", + "video_8069/video_8069_17.jpeg", + "video_8069/video_8069_18.jpeg", + "video_8069/video_8069_19.jpeg", + "video_8069/video_8069_20.jpeg", + "video_8069/video_8069_21.jpeg", + "video_8069/video_8069_22.jpeg", + "video_8069/video_8069_23.jpeg", + "video_8069/video_8069_24.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 5, + "question": "Analyze the given visuals and answer the question about the order of character. You must choose your answer from the Choice List.\nWhat would be the order of the written letters if the person had written them in reverse order?\nChoice list: \nA. dog\nB. don\nC. god\nAnswer with the option's letter from the given choices directly.", + "gt_response": "dog", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "dog", + "don", + "god" + ], + "image_quantity_level": "Medium", + "image": [ + "video_3849/video_3849_0.jpeg", + "video_3849/video_3849_1.jpeg", + "video_3849/video_3849_2.jpeg", + "video_3849/video_3849_3.jpeg", + "video_3849/video_3849_4.jpeg", + "video_3849/video_3849_5.jpeg", + "video_3849/video_3849_6.jpeg", + "video_3849/video_3849_7.jpeg", + "video_3849/video_3849_8.jpeg", + "video_3849/video_3849_9.jpeg", + "video_3849/video_3849_10.jpeg", + "video_3849/video_3849_11.jpeg", + "video_3849/video_3849_12.jpeg", + "video_3849/video_3849_13.jpeg", + "video_3849/video_3849_14.jpeg", + "video_3849/video_3849_15.jpeg", + "video_3849/video_3849_16.jpeg", + "video_3849/video_3849_17.jpeg", + "video_3849/video_3849_18.jpeg", + "video_3849/video_3849_19.jpeg", + "video_3849/video_3849_20.jpeg", + "video_3849/video_3849_21.jpeg", + "video_3849/video_3849_22.jpeg", + "video_3849/video_3849_23.jpeg", + "video_3849/video_3849_24.jpeg", + "video_3849/video_3849_25.jpeg", + "video_3849/video_3849_26.jpeg", + "video_3849/video_3849_27.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 15, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. t\nB. a\nC. c\nAnswer with the option's letter from the given choices directly.", + "gt_response": "c", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "t", + "a", + "c" + ], + "image_quantity_level": "Medium", + "image": [ + "video_8823/video_8823_0.jpeg", + "video_8823/video_8823_1.jpeg", + "video_8823/video_8823_2.jpeg", + "video_8823/video_8823_3.jpeg", + "video_8823/video_8823_4.jpeg", + "video_8823/video_8823_5.jpeg", + "video_8823/video_8823_6.jpeg", + "video_8823/video_8823_7.jpeg", + "video_8823/video_8823_8.jpeg", + "video_8823/video_8823_9.jpeg", + "video_8823/video_8823_10.jpeg", + "video_8823/video_8823_11.jpeg", + "video_8823/video_8823_12.jpeg", + "video_8823/video_8823_13.jpeg", + "video_8823/video_8823_14.jpeg", + "video_8823/video_8823_15.jpeg", + "video_8823/video_8823_16.jpeg", + "video_8823/video_8823_17.jpeg", + "video_8823/video_8823_18.jpeg", + "video_8823/video_8823_19.jpeg", + "video_8823/video_8823_20.jpeg", + "video_8823/video_8823_21.jpeg", + "video_8823/video_8823_22.jpeg", + "video_8823/video_8823_23.jpeg", + "video_8823/video_8823_24.jpeg", + "video_8823/video_8823_25.jpeg", + "video_8823/video_8823_26.jpeg", + "video_8823/video_8823_27.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 43, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. def\nB. efd\nC. fed\nAnswer with the option's letter from the given choices directly.", + "gt_response": "def", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "def", + "efd", + "fed" + ], + "image_quantity_level": "Medium", + "image": [ + "video_11079/video_11079_0.jpeg", + "video_11079/video_11079_1.jpeg", + "video_11079/video_11079_2.jpeg", + "video_11079/video_11079_3.jpeg", + "video_11079/video_11079_4.jpeg", + "video_11079/video_11079_5.jpeg", + "video_11079/video_11079_6.jpeg", + "video_11079/video_11079_7.jpeg", + "video_11079/video_11079_8.jpeg", + "video_11079/video_11079_9.jpeg", + "video_11079/video_11079_10.jpeg", + "video_11079/video_11079_11.jpeg", + "video_11079/video_11079_12.jpeg", + "video_11079/video_11079_13.jpeg", + "video_11079/video_11079_14.jpeg", + "video_11079/video_11079_15.jpeg", + "video_11079/video_11079_16.jpeg", + "video_11079/video_11079_17.jpeg", + "video_11079/video_11079_18.jpeg", + "video_11079/video_11079_19.jpeg", + "video_11079/video_11079_20.jpeg", + "video_11079/video_11079_21.jpeg", + "video_11079/video_11079_22.jpeg", + "video_11079/video_11079_23.jpeg", + "video_11079/video_11079_24.jpeg", + "video_11079/video_11079_25.jpeg", + "video_11079/video_11079_26.jpeg", + "video_11079/video_11079_27.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 6, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nWhat letter is likely to be shown next?\nChoice list: \nA. g\nB. j\nC. s\nAnswer with the option's letter from the given choices directly.", + "gt_response": "s", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "g", + "j", + "s" + ], + "image_quantity_level": "Medium", + "image": [ + "video_190/video_190_0.jpeg", + "video_190/video_190_1.jpeg", + "video_190/video_190_2.jpeg", + "video_190/video_190_3.jpeg", + "video_190/video_190_4.jpeg", + "video_190/video_190_5.jpeg", + "video_190/video_190_6.jpeg", + "video_190/video_190_7.jpeg", + "video_190/video_190_8.jpeg", + "video_190/video_190_9.jpeg", + "video_190/video_190_10.jpeg", + "video_190/video_190_11.jpeg", + "video_190/video_190_12.jpeg", + "video_190/video_190_13.jpeg", + "video_190/video_190_14.jpeg", + "video_190/video_190_15.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 10, + "question": "From the images supplied, answer the question relevant to character order. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. efgh\nB. egfh\nC. empw\nAnswer with the option's letter from the given choices directly.", + "gt_response": "efgh", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "efgh", + "egfh", + "empw" + ], + "image_quantity_level": "Medium", + "image": [ + "video_8457/video_8457_0.jpeg", + "video_8457/video_8457_1.jpeg", + "video_8457/video_8457_2.jpeg", + "video_8457/video_8457_3.jpeg", + "video_8457/video_8457_4.jpeg", + "video_8457/video_8457_5.jpeg", + "video_8457/video_8457_6.jpeg", + "video_8457/video_8457_7.jpeg", + "video_8457/video_8457_8.jpeg", + "video_8457/video_8457_9.jpeg", + "video_8457/video_8457_10.jpeg", + "video_8457/video_8457_11.jpeg", + "video_8457/video_8457_12.jpeg", + "video_8457/video_8457_13.jpeg", + "video_8457/video_8457_14.jpeg", + "video_8457/video_8457_15.jpeg", + "video_8457/video_8457_16.jpeg", + "video_8457/video_8457_17.jpeg", + "video_8457/video_8457_18.jpeg", + "video_8457/video_8457_19.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 17, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. ad\nB. ma\nC. am\nAnswer with the option's letter from the given choices directly.", + "gt_response": "am", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "ad", + "ma", + "am" + ], + "image_quantity_level": "Medium", + "image": [ + "video_9770/video_9770_0.jpeg", + "video_9770/video_9770_1.jpeg", + "video_9770/video_9770_2.jpeg", + "video_9770/video_9770_3.jpeg", + "video_9770/video_9770_4.jpeg", + "video_9770/video_9770_5.jpeg", + "video_9770/video_9770_6.jpeg", + "video_9770/video_9770_7.jpeg", + "video_9770/video_9770_8.jpeg", + "video_9770/video_9770_9.jpeg", + "video_9770/video_9770_10.jpeg", + "video_9770/video_9770_11.jpeg", + "video_9770/video_9770_12.jpeg", + "video_9770/video_9770_13.jpeg", + "video_9770/video_9770_14.jpeg", + "video_9770/video_9770_15.jpeg", + "video_9770/video_9770_16.jpeg", + "video_9770/video_9770_17.jpeg", + "video_9770/video_9770_18.jpeg", + "video_9770/video_9770_19.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 12, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nWhat letter is likely to be shown next?\nChoice list: \nA. f\nB. u\nC. h\nAnswer with the option's letter from the given choices directly.", + "gt_response": "f", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "f", + "u", + "h" + ], + "image_quantity_level": "Medium", + "image": [ + "video_3025/video_3025_0.jpeg", + "video_3025/video_3025_1.jpeg", + "video_3025/video_3025_2.jpeg", + "video_3025/video_3025_3.jpeg", + "video_3025/video_3025_4.jpeg", + "video_3025/video_3025_5.jpeg", + "video_3025/video_3025_6.jpeg", + "video_3025/video_3025_7.jpeg", + "video_3025/video_3025_8.jpeg", + "video_3025/video_3025_9.jpeg", + "video_3025/video_3025_10.jpeg", + "video_3025/video_3025_11.jpeg", + "video_3025/video_3025_12.jpeg", + "video_3025/video_3025_13.jpeg", + "video_3025/video_3025_14.jpeg", + "video_3025/video_3025_15.jpeg", + "video_3025/video_3025_16.jpeg", + "video_3025/video_3025_17.jpeg", + "video_3025/video_3025_18.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 14, + "question": "Analyze the given visuals and answer the question about the order of character. You must choose your answer from the Choice List.\nWhat was the order of the letters on the table before shuffling?\nChoice list: \nA. calypso\nB. claoyps\nC. cyspola\nAnswer with the option's letter from the given choices directly.", + "gt_response": "calypso", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "calypso", + "claoyps", + "cyspola" + ], + "image_quantity_level": "Medium", + "image": [ + "video_5710/video_5710_0.jpeg", + "video_5710/video_5710_1.jpeg", + "video_5710/video_5710_2.jpeg", + "video_5710/video_5710_3.jpeg", + "video_5710/video_5710_4.jpeg", + "video_5710/video_5710_5.jpeg", + "video_5710/video_5710_6.jpeg", + "video_5710/video_5710_7.jpeg", + "video_5710/video_5710_8.jpeg", + "video_5710/video_5710_9.jpeg", + "video_5710/video_5710_10.jpeg", + "video_5710/video_5710_11.jpeg", + "video_5710/video_5710_12.jpeg", + "video_5710/video_5710_13.jpeg", + "video_5710/video_5710_14.jpeg", + "video_5710/video_5710_15.jpeg", + "video_5710/video_5710_16.jpeg", + "video_5710/video_5710_17.jpeg", + "video_5710/video_5710_18.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 13, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. hca\nB. ahc\nC. cha\nAnswer with the option's letter from the given choices directly.", + "gt_response": "ahc", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "hca", + "ahc", + "cha" + ], + "image_quantity_level": "Many", + "image": [ + "video_183/video_183_0.jpeg", + "video_183/video_183_1.jpeg", + "video_183/video_183_2.jpeg", + "video_183/video_183_3.jpeg", + "video_183/video_183_4.jpeg", + "video_183/video_183_5.jpeg", + "video_183/video_183_6.jpeg", + "video_183/video_183_7.jpeg", + "video_183/video_183_8.jpeg", + "video_183/video_183_9.jpeg", + "video_183/video_183_10.jpeg", + "video_183/video_183_11.jpeg", + "video_183/video_183_12.jpeg", + "video_183/video_183_13.jpeg", + "video_183/video_183_14.jpeg", + "video_183/video_183_15.jpeg", + "video_183/video_183_16.jpeg", + "video_183/video_183_17.jpeg", + "video_183/video_183_18.jpeg", + "video_183/video_183_19.jpeg", + "video_183/video_183_20.jpeg", + "video_183/video_183_21.jpeg", + "video_183/video_183_22.jpeg", + "video_183/video_183_23.jpeg", + "video_183/video_183_24.jpeg", + "video_183/video_183_25.jpeg", + "video_183/video_183_26.jpeg", + "video_183/video_183_27.jpeg", + "video_183/video_183_28.jpeg", + "video_183/video_183_29.jpeg", + "video_183/video_183_30.jpeg", + "video_183/video_183_31.jpeg", + "video_183/video_183_32.jpeg", + "video_183/video_183_33.jpeg", + "video_183/video_183_34.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 22, + "question": "Review the visuals provided and respond to the question concerning the character sequence. You must choose your answer from the Choice List.\nWhat would be the order of the written letters if the person had written them in reverse order?\nChoice list: \nA. dog\nB. zoy\nC. god\nAnswer with the option's letter from the given choices directly.", + "gt_response": "god", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "dog", + "zoy", + "god" + ], + "image_quantity_level": "Many", + "image": [ + "video_2635/video_2635_0.jpeg", + "video_2635/video_2635_1.jpeg", + "video_2635/video_2635_2.jpeg", + "video_2635/video_2635_3.jpeg", + "video_2635/video_2635_4.jpeg", + "video_2635/video_2635_5.jpeg", + "video_2635/video_2635_6.jpeg", + "video_2635/video_2635_7.jpeg", + "video_2635/video_2635_8.jpeg", + "video_2635/video_2635_9.jpeg", + "video_2635/video_2635_10.jpeg", + "video_2635/video_2635_11.jpeg", + "video_2635/video_2635_12.jpeg", + "video_2635/video_2635_13.jpeg", + "video_2635/video_2635_14.jpeg", + "video_2635/video_2635_15.jpeg", + "video_2635/video_2635_16.jpeg", + "video_2635/video_2635_17.jpeg", + "video_2635/video_2635_18.jpeg", + "video_2635/video_2635_19.jpeg", + "video_2635/video_2635_20.jpeg", + "video_2635/video_2635_21.jpeg", + "video_2635/video_2635_22.jpeg", + "video_2635/video_2635_23.jpeg", + "video_2635/video_2635_24.jpeg", + "video_2635/video_2635_25.jpeg", + "video_2635/video_2635_26.jpeg", + "video_2635/video_2635_27.jpeg", + "video_2635/video_2635_28.jpeg", + "video_2635/video_2635_29.jpeg", + "video_2635/video_2635_30.jpeg", + "video_2635/video_2635_31.jpeg", + "video_2635/video_2635_32.jpeg", + "video_2635/video_2635_33.jpeg", + "video_2635/video_2635_34.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 31, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. b\nB. a\nC. e\nAnswer with the option's letter from the given choices directly.", + "gt_response": "b", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "b", + "a", + "e" + ], + "image_quantity_level": "Many", + "image": [ + "video_10223/video_10223_0.jpeg", + "video_10223/video_10223_1.jpeg", + "video_10223/video_10223_2.jpeg", + "video_10223/video_10223_3.jpeg", + "video_10223/video_10223_4.jpeg", + "video_10223/video_10223_5.jpeg", + "video_10223/video_10223_6.jpeg", + "video_10223/video_10223_7.jpeg", + "video_10223/video_10223_8.jpeg", + "video_10223/video_10223_9.jpeg", + "video_10223/video_10223_10.jpeg", + "video_10223/video_10223_11.jpeg", + "video_10223/video_10223_12.jpeg", + "video_10223/video_10223_13.jpeg", + "video_10223/video_10223_14.jpeg", + "video_10223/video_10223_15.jpeg", + "video_10223/video_10223_16.jpeg", + "video_10223/video_10223_17.jpeg", + "video_10223/video_10223_18.jpeg", + "video_10223/video_10223_19.jpeg", + "video_10223/video_10223_20.jpeg", + "video_10223/video_10223_21.jpeg", + "video_10223/video_10223_22.jpeg", + "video_10223/video_10223_23.jpeg", + "video_10223/video_10223_24.jpeg", + "video_10223/video_10223_25.jpeg", + "video_10223/video_10223_26.jpeg", + "video_10223/video_10223_27.jpeg", + "video_10223/video_10223_28.jpeg", + "video_10223/video_10223_29.jpeg", + "video_10223/video_10223_30.jpeg", + "video_10223/video_10223_31.jpeg", + "video_10223/video_10223_32.jpeg", + "video_10223/video_10223_33.jpeg", + "video_10223/video_10223_34.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 33, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. abc\nB. obc\nC. wdc\nAnswer with the option's letter from the given choices directly.", + "gt_response": "abc", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "abc", + "obc", + "wdc" + ], + "image_quantity_level": "Many", + "image": [ + "video_9987/video_9987_0.jpeg", + "video_9987/video_9987_1.jpeg", + "video_9987/video_9987_2.jpeg", + "video_9987/video_9987_3.jpeg", + "video_9987/video_9987_4.jpeg", + "video_9987/video_9987_5.jpeg", + "video_9987/video_9987_6.jpeg", + "video_9987/video_9987_7.jpeg", + "video_9987/video_9987_8.jpeg", + "video_9987/video_9987_9.jpeg", + "video_9987/video_9987_10.jpeg", + "video_9987/video_9987_11.jpeg", + "video_9987/video_9987_12.jpeg", + "video_9987/video_9987_13.jpeg", + "video_9987/video_9987_14.jpeg", + "video_9987/video_9987_15.jpeg", + "video_9987/video_9987_16.jpeg", + "video_9987/video_9987_17.jpeg", + "video_9987/video_9987_18.jpeg", + "video_9987/video_9987_19.jpeg", + "video_9987/video_9987_20.jpeg", + "video_9987/video_9987_21.jpeg", + "video_9987/video_9987_22.jpeg", + "video_9987/video_9987_23.jpeg", + "video_9987/video_9987_24.jpeg", + "video_9987/video_9987_25.jpeg", + "video_9987/video_9987_26.jpeg", + "video_9987/video_9987_27.jpeg", + "video_9987/video_9987_28.jpeg", + "video_9987/video_9987_29.jpeg", + "video_9987/video_9987_30.jpeg", + "video_9987/video_9987_31.jpeg", + "video_9987/video_9987_32.jpeg", + "video_9987/video_9987_33.jpeg", + "video_9987/video_9987_34.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 46, + "question": "Analyze the given visuals and answer the question about the order of character. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. cha\nB. hac\nC. cah\nAnswer with the option's letter from the given choices directly.", + "gt_response": "cha", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "cha", + "hac", + "cah" + ], + "image_quantity_level": "Many", + "image": [ + "video_10461/video_10461_0.jpeg", + "video_10461/video_10461_1.jpeg", + "video_10461/video_10461_2.jpeg", + "video_10461/video_10461_3.jpeg", + "video_10461/video_10461_4.jpeg", + "video_10461/video_10461_5.jpeg", + "video_10461/video_10461_6.jpeg", + "video_10461/video_10461_7.jpeg", + "video_10461/video_10461_8.jpeg", + "video_10461/video_10461_9.jpeg", + "video_10461/video_10461_10.jpeg", + "video_10461/video_10461_11.jpeg", + "video_10461/video_10461_12.jpeg", + "video_10461/video_10461_13.jpeg", + "video_10461/video_10461_14.jpeg", + "video_10461/video_10461_15.jpeg", + "video_10461/video_10461_16.jpeg", + "video_10461/video_10461_17.jpeg", + "video_10461/video_10461_18.jpeg", + "video_10461/video_10461_19.jpeg", + "video_10461/video_10461_20.jpeg", + "video_10461/video_10461_21.jpeg", + "video_10461/video_10461_22.jpeg", + "video_10461/video_10461_23.jpeg", + "video_10461/video_10461_24.jpeg", + "video_10461/video_10461_25.jpeg", + "video_10461/video_10461_26.jpeg", + "video_10461/video_10461_27.jpeg", + "video_10461/video_10461_28.jpeg", + "video_10461/video_10461_29.jpeg", + "video_10461/video_10461_30.jpeg", + "video_10461/video_10461_31.jpeg", + "video_10461/video_10461_32.jpeg", + "video_10461/video_10461_33.jpeg", + "video_10461/video_10461_34.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 19, + "question": "Analyze the given visuals and answer the question about the order of character. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. ekq\nB. fqk\nC. bwk\nAnswer with the option's letter from the given choices directly.", + "gt_response": "ekq", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "ekq", + "fqk", + "bwk" + ], + "image_quantity_level": "Medium", + "image": [ + "video_9724/video_9724_0.jpeg", + "video_9724/video_9724_1.jpeg", + "video_9724/video_9724_2.jpeg", + "video_9724/video_9724_3.jpeg", + "video_9724/video_9724_4.jpeg", + "video_9724/video_9724_5.jpeg", + "video_9724/video_9724_6.jpeg", + "video_9724/video_9724_7.jpeg", + "video_9724/video_9724_8.jpeg", + "video_9724/video_9724_9.jpeg", + "video_9724/video_9724_10.jpeg", + "video_9724/video_9724_11.jpeg", + "video_9724/video_9724_12.jpeg", + "video_9724/video_9724_13.jpeg", + "video_9724/video_9724_14.jpeg", + "video_9724/video_9724_15.jpeg", + "video_9724/video_9724_16.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 21, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. umit\nB. iumt\nC. muti\nAnswer with the option's letter from the given choices directly.", + "gt_response": "umit", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "umit", + "iumt", + "muti" + ], + "image_quantity_level": "Medium", + "image": [ + "video_11140/video_11140_0.jpeg", + "video_11140/video_11140_1.jpeg", + "video_11140/video_11140_2.jpeg", + "video_11140/video_11140_3.jpeg", + "video_11140/video_11140_4.jpeg", + "video_11140/video_11140_5.jpeg", + "video_11140/video_11140_6.jpeg", + "video_11140/video_11140_7.jpeg", + "video_11140/video_11140_8.jpeg", + "video_11140/video_11140_9.jpeg", + "video_11140/video_11140_10.jpeg", + "video_11140/video_11140_11.jpeg", + "video_11140/video_11140_12.jpeg", + "video_11140/video_11140_13.jpeg", + "video_11140/video_11140_14.jpeg", + "video_11140/video_11140_15.jpeg", + "video_11140/video_11140_16.jpeg", + "video_11140/video_11140_17.jpeg", + "video_11140/video_11140_18.jpeg", + "video_11140/video_11140_19.jpeg", + "video_11140/video_11140_20.jpeg", + "video_11140/video_11140_21.jpeg", + "video_11140/video_11140_22.jpeg", + "video_11140/video_11140_23.jpeg", + "video_11140/video_11140_24.jpeg", + "video_11140/video_11140_25.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 51, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. t\nB. a\nC. c\nAnswer with the option's letter from the given choices directly.", + "gt_response": "a", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "t", + "a", + "c" + ], + "image_quantity_level": "Medium", + "image": [ + "video_5828/video_5828_0.jpeg", + "video_5828/video_5828_1.jpeg", + "video_5828/video_5828_2.jpeg", + "video_5828/video_5828_3.jpeg", + "video_5828/video_5828_4.jpeg", + "video_5828/video_5828_5.jpeg", + "video_5828/video_5828_6.jpeg", + "video_5828/video_5828_7.jpeg", + "video_5828/video_5828_8.jpeg", + "video_5828/video_5828_9.jpeg", + "video_5828/video_5828_10.jpeg", + "video_5828/video_5828_11.jpeg", + "video_5828/video_5828_12.jpeg", + "video_5828/video_5828_13.jpeg", + "video_5828/video_5828_14.jpeg", + "video_5828/video_5828_15.jpeg", + "video_5828/video_5828_16.jpeg", + "video_5828/video_5828_17.jpeg", + "video_5828/video_5828_18.jpeg", + "video_5828/video_5828_19.jpeg", + "video_5828/video_5828_20.jpeg", + "video_5828/video_5828_21.jpeg", + "video_5828/video_5828_22.jpeg", + "video_5828/video_5828_23.jpeg", + "video_5828/video_5828_24.jpeg", + "video_5828/video_5828_25.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 54, + "question": "Scrutinize the provided visuals and answer the question about the arrangement of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. ot\nB. te\nC. to\nAnswer with the option's letter from the given choices directly.", + "gt_response": "to", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "ot", + "te", + "to" + ], + "image_quantity_level": "Medium", + "image": [ + "video_1374/video_1374_0.jpeg", + "video_1374/video_1374_1.jpeg", + "video_1374/video_1374_2.jpeg", + "video_1374/video_1374_3.jpeg", + "video_1374/video_1374_4.jpeg", + "video_1374/video_1374_5.jpeg", + "video_1374/video_1374_6.jpeg", + "video_1374/video_1374_7.jpeg", + "video_1374/video_1374_8.jpeg", + "video_1374/video_1374_9.jpeg", + "video_1374/video_1374_10.jpeg", + "video_1374/video_1374_11.jpeg", + "video_1374/video_1374_12.jpeg", + "video_1374/video_1374_13.jpeg", + "video_1374/video_1374_14.jpeg", + "video_1374/video_1374_15.jpeg", + "video_1374/video_1374_16.jpeg", + "video_1374/video_1374_17.jpeg", + "video_1374/video_1374_18.jpeg", + "video_1374/video_1374_19.jpeg", + "video_1374/video_1374_20.jpeg", + "video_1374/video_1374_21.jpeg", + "video_1374/video_1374_22.jpeg", + "video_1374/video_1374_23.jpeg", + "video_1374/video_1374_24.jpeg", + "video_1374/video_1374_25.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 23, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. idm\nB. imd\nC. dim\nAnswer with the option's letter from the given choices directly.", + "gt_response": "dim", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "idm", + "imd", + "dim" + ], + "image_quantity_level": "Medium", + "image": [ + "video_7135/video_7135_0.jpeg", + "video_7135/video_7135_1.jpeg", + "video_7135/video_7135_2.jpeg", + "video_7135/video_7135_3.jpeg", + "video_7135/video_7135_4.jpeg", + "video_7135/video_7135_5.jpeg", + "video_7135/video_7135_6.jpeg", + "video_7135/video_7135_7.jpeg", + "video_7135/video_7135_8.jpeg", + "video_7135/video_7135_9.jpeg", + "video_7135/video_7135_10.jpeg", + "video_7135/video_7135_11.jpeg", + "video_7135/video_7135_12.jpeg", + "video_7135/video_7135_13.jpeg", + "video_7135/video_7135_14.jpeg", + "video_7135/video_7135_15.jpeg", + "video_7135/video_7135_16.jpeg", + "video_7135/video_7135_17.jpeg", + "video_7135/video_7135_18.jpeg", + "video_7135/video_7135_19.jpeg", + "video_7135/video_7135_20.jpeg", + "video_7135/video_7135_21.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 26, + "question": "Review the visuals provided and respond to the question concerning the character sequence. You must choose your answer from the Choice List.\nWhat letter is likely to be shown next?\nChoice list: \nA. u\nB. w\nC. g\nAnswer with the option's letter from the given choices directly.", + "gt_response": "g", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "u", + "w", + "g" + ], + "image_quantity_level": "Medium", + "image": [ + "video_905/video_905_0.jpeg", + "video_905/video_905_1.jpeg", + "video_905/video_905_2.jpeg", + "video_905/video_905_3.jpeg", + "video_905/video_905_4.jpeg", + "video_905/video_905_5.jpeg", + "video_905/video_905_6.jpeg", + "video_905/video_905_7.jpeg", + "video_905/video_905_8.jpeg", + "video_905/video_905_9.jpeg", + "video_905/video_905_10.jpeg", + "video_905/video_905_11.jpeg", + "video_905/video_905_12.jpeg", + "video_905/video_905_13.jpeg", + "video_905/video_905_14.jpeg", + "video_905/video_905_15.jpeg", + "video_905/video_905_16.jpeg", + "video_905/video_905_17.jpeg", + "video_905/video_905_18.jpeg", + "video_905/video_905_19.jpeg", + "video_905/video_905_20.jpeg", + "video_905/video_905_21.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 37, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. fbk\nB. fer\nC. ref\nAnswer with the option's letter from the given choices directly.", + "gt_response": "fer", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "fbk", + "fer", + "ref" + ], + "image_quantity_level": "Medium", + "image": [ + "video_7077/video_7077_0.jpeg", + "video_7077/video_7077_1.jpeg", + "video_7077/video_7077_2.jpeg", + "video_7077/video_7077_3.jpeg", + "video_7077/video_7077_4.jpeg", + "video_7077/video_7077_5.jpeg", + "video_7077/video_7077_6.jpeg", + "video_7077/video_7077_7.jpeg", + "video_7077/video_7077_8.jpeg", + "video_7077/video_7077_9.jpeg", + "video_7077/video_7077_10.jpeg", + "video_7077/video_7077_11.jpeg", + "video_7077/video_7077_12.jpeg", + "video_7077/video_7077_13.jpeg", + "video_7077/video_7077_14.jpeg", + "video_7077/video_7077_15.jpeg", + "video_7077/video_7077_16.jpeg", + "video_7077/video_7077_17.jpeg", + "video_7077/video_7077_18.jpeg", + "video_7077/video_7077_19.jpeg", + "video_7077/video_7077_20.jpeg", + "video_7077/video_7077_21.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 24, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. aojn\nB. qzcx\nC. alct\nAnswer with the option's letter from the given choices directly.", + "gt_response": "alct", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "aojn", + "qzcx", + "alct" + ], + "image_quantity_level": "Medium", + "image": [ + "video_683/video_683_0.jpeg", + "video_683/video_683_1.jpeg", + "video_683/video_683_2.jpeg", + "video_683/video_683_3.jpeg", + "video_683/video_683_4.jpeg", + "video_683/video_683_5.jpeg", + "video_683/video_683_6.jpeg", + "video_683/video_683_7.jpeg", + "video_683/video_683_8.jpeg", + "video_683/video_683_9.jpeg", + "video_683/video_683_10.jpeg", + "video_683/video_683_11.jpeg", + "video_683/video_683_12.jpeg", + "video_683/video_683_13.jpeg", + "video_683/video_683_14.jpeg", + "video_683/video_683_15.jpeg", + "video_683/video_683_16.jpeg", + "video_683/video_683_17.jpeg", + "video_683/video_683_18.jpeg", + "video_683/video_683_19.jpeg", + "video_683/video_683_20.jpeg", + "video_683/video_683_21.jpeg", + "video_683/video_683_22.jpeg", + "video_683/video_683_23.jpeg", + "video_683/video_683_24.jpeg", + "video_683/video_683_25.jpeg", + "video_683/video_683_26.jpeg", + "video_683/video_683_27.jpeg", + "video_683/video_683_28.jpeg", + "video_683/video_683_29.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 61, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. osp\nB. ops\nC. pos\nAnswer with the option's letter from the given choices directly.", + "gt_response": "osp", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "osp", + "ops", + "pos" + ], + "image_quantity_level": "Medium", + "image": [ + "video_5983/video_5983_0.jpeg", + "video_5983/video_5983_1.jpeg", + "video_5983/video_5983_2.jpeg", + "video_5983/video_5983_3.jpeg", + "video_5983/video_5983_4.jpeg", + "video_5983/video_5983_5.jpeg", + "video_5983/video_5983_6.jpeg", + "video_5983/video_5983_7.jpeg", + "video_5983/video_5983_8.jpeg", + "video_5983/video_5983_9.jpeg", + "video_5983/video_5983_10.jpeg", + "video_5983/video_5983_11.jpeg", + "video_5983/video_5983_12.jpeg", + "video_5983/video_5983_13.jpeg", + "video_5983/video_5983_14.jpeg", + "video_5983/video_5983_15.jpeg", + "video_5983/video_5983_16.jpeg", + "video_5983/video_5983_17.jpeg", + "video_5983/video_5983_18.jpeg", + "video_5983/video_5983_19.jpeg", + "video_5983/video_5983_20.jpeg", + "video_5983/video_5983_21.jpeg", + "video_5983/video_5983_22.jpeg", + "video_5983/video_5983_23.jpeg", + "video_5983/video_5983_24.jpeg", + "video_5983/video_5983_25.jpeg", + "video_5983/video_5983_26.jpeg", + "video_5983/video_5983_27.jpeg", + "video_5983/video_5983_28.jpeg", + "video_5983/video_5983_29.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 25, + "question": "From the images supplied, answer the question relevant to character order. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. pin\nB. ped\nC. pen\nAnswer with the option's letter from the given choices directly.", + "gt_response": "pen", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "pin", + "ped", + "pen" + ], + "image_quantity_level": "Medium", + "image": [ + "video_7909/video_7909_0.jpeg", + "video_7909/video_7909_1.jpeg", + "video_7909/video_7909_2.jpeg", + "video_7909/video_7909_3.jpeg", + "video_7909/video_7909_4.jpeg", + "video_7909/video_7909_5.jpeg", + "video_7909/video_7909_6.jpeg", + "video_7909/video_7909_7.jpeg", + "video_7909/video_7909_8.jpeg", + "video_7909/video_7909_9.jpeg", + "video_7909/video_7909_10.jpeg", + "video_7909/video_7909_11.jpeg", + "video_7909/video_7909_12.jpeg", + "video_7909/video_7909_13.jpeg", + "video_7909/video_7909_14.jpeg", + "video_7909/video_7909_15.jpeg", + "video_7909/video_7909_16.jpeg", + "video_7909/video_7909_17.jpeg", + "video_7909/video_7909_18.jpeg", + "video_7909/video_7909_19.jpeg", + "video_7909/video_7909_20.jpeg", + "video_7909/video_7909_21.jpeg", + "video_7909/video_7909_22.jpeg", + "video_7909/video_7909_23.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 35, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. bpx\nB. pxb\nC. bxp\nAnswer with the option's letter from the given choices directly.", + "gt_response": "bpx", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "bpx", + "pxb", + "bxp" + ], + "image_quantity_level": "Medium", + "image": [ + "video_11478/video_11478_0.jpeg", + "video_11478/video_11478_1.jpeg", + "video_11478/video_11478_2.jpeg", + "video_11478/video_11478_3.jpeg", + "video_11478/video_11478_4.jpeg", + "video_11478/video_11478_5.jpeg", + "video_11478/video_11478_6.jpeg", + "video_11478/video_11478_7.jpeg", + "video_11478/video_11478_8.jpeg", + "video_11478/video_11478_9.jpeg", + "video_11478/video_11478_10.jpeg", + "video_11478/video_11478_11.jpeg", + "video_11478/video_11478_12.jpeg", + "video_11478/video_11478_13.jpeg", + "video_11478/video_11478_14.jpeg", + "video_11478/video_11478_15.jpeg", + "video_11478/video_11478_16.jpeg", + "video_11478/video_11478_17.jpeg", + "video_11478/video_11478_18.jpeg", + "video_11478/video_11478_19.jpeg", + "video_11478/video_11478_20.jpeg", + "video_11478/video_11478_21.jpeg", + "video_11478/video_11478_22.jpeg", + "video_11478/video_11478_23.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 39, + "question": "Scrutinize the provided visuals and answer the question about the arrangement of characters. You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. a\nB. e\nC. t\nAnswer with the option's letter from the given choices directly.", + "gt_response": "e", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "a", + "e", + "t" + ], + "image_quantity_level": "Medium", + "image": [ + "video_8578/video_8578_0.jpeg", + "video_8578/video_8578_1.jpeg", + "video_8578/video_8578_2.jpeg", + "video_8578/video_8578_3.jpeg", + "video_8578/video_8578_4.jpeg", + "video_8578/video_8578_5.jpeg", + "video_8578/video_8578_6.jpeg", + "video_8578/video_8578_7.jpeg", + "video_8578/video_8578_8.jpeg", + "video_8578/video_8578_9.jpeg", + "video_8578/video_8578_10.jpeg", + "video_8578/video_8578_11.jpeg", + "video_8578/video_8578_12.jpeg", + "video_8578/video_8578_13.jpeg", + "video_8578/video_8578_14.jpeg", + "video_8578/video_8578_15.jpeg", + "video_8578/video_8578_16.jpeg", + "video_8578/video_8578_17.jpeg", + "video_8578/video_8578_18.jpeg", + "video_8578/video_8578_19.jpeg", + "video_8578/video_8578_20.jpeg", + "video_8578/video_8578_21.jpeg", + "video_8578/video_8578_22.jpeg", + "video_8578/video_8578_23.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 27, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. fdy\nB. cde\nC. cdf\nAnswer with the option's letter from the given choices directly.", + "gt_response": "cde", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "fdy", + "cde", + "cdf" + ], + "image_quantity_level": "Many", + "image": [ + "video_9961/video_9961_0.jpeg", + "video_9961/video_9961_1.jpeg", + "video_9961/video_9961_2.jpeg", + "video_9961/video_9961_3.jpeg", + "video_9961/video_9961_4.jpeg", + "video_9961/video_9961_5.jpeg", + "video_9961/video_9961_6.jpeg", + "video_9961/video_9961_7.jpeg", + "video_9961/video_9961_8.jpeg", + "video_9961/video_9961_9.jpeg", + "video_9961/video_9961_10.jpeg", + "video_9961/video_9961_11.jpeg", + "video_9961/video_9961_12.jpeg", + "video_9961/video_9961_13.jpeg", + "video_9961/video_9961_14.jpeg", + "video_9961/video_9961_15.jpeg", + "video_9961/video_9961_16.jpeg", + "video_9961/video_9961_17.jpeg", + "video_9961/video_9961_18.jpeg", + "video_9961/video_9961_19.jpeg", + "video_9961/video_9961_20.jpeg", + "video_9961/video_9961_21.jpeg", + "video_9961/video_9961_22.jpeg", + "video_9961/video_9961_23.jpeg", + "video_9961/video_9961_24.jpeg", + "video_9961/video_9961_25.jpeg", + "video_9961/video_9961_26.jpeg", + "video_9961/video_9961_27.jpeg", + "video_9961/video_9961_28.jpeg", + "video_9961/video_9961_29.jpeg", + "video_9961/video_9961_30.jpeg", + "video_9961/video_9961_31.jpeg", + "video_9961/video_9961_32.jpeg", + "video_9961/video_9961_33.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 29, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. c\nB. a\nC. t\nAnswer with the option's letter from the given choices directly.", + "gt_response": "c", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "c", + "a", + "t" + ], + "image_quantity_level": "Many", + "image": [ + "video_7451/video_7451_0.jpeg", + "video_7451/video_7451_1.jpeg", + "video_7451/video_7451_2.jpeg", + "video_7451/video_7451_3.jpeg", + "video_7451/video_7451_4.jpeg", + "video_7451/video_7451_5.jpeg", + "video_7451/video_7451_6.jpeg", + "video_7451/video_7451_7.jpeg", + "video_7451/video_7451_8.jpeg", + "video_7451/video_7451_9.jpeg", + "video_7451/video_7451_10.jpeg", + "video_7451/video_7451_11.jpeg", + "video_7451/video_7451_12.jpeg", + "video_7451/video_7451_13.jpeg", + "video_7451/video_7451_14.jpeg", + "video_7451/video_7451_15.jpeg", + "video_7451/video_7451_16.jpeg", + "video_7451/video_7451_17.jpeg", + "video_7451/video_7451_18.jpeg", + "video_7451/video_7451_19.jpeg", + "video_7451/video_7451_20.jpeg", + "video_7451/video_7451_21.jpeg", + "video_7451/video_7451_22.jpeg", + "video_7451/video_7451_23.jpeg", + "video_7451/video_7451_24.jpeg", + "video_7451/video_7451_25.jpeg", + "video_7451/video_7451_26.jpeg", + "video_7451/video_7451_27.jpeg", + "video_7451/video_7451_28.jpeg", + "video_7451/video_7451_29.jpeg", + "video_7451/video_7451_30.jpeg", + "video_7451/video_7451_31.jpeg", + "video_7451/video_7451_32.jpeg", + "video_7451/video_7451_33.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 32, + "question": "Analyze the given visuals and answer the question about the order of character. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. ated\nB. date\nC. taed\nAnswer with the option's letter from the given choices directly.", + "gt_response": "taed", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "ated", + "date", + "taed" + ], + "image_quantity_level": "Many", + "image": [ + "video_10782/video_10782_0.jpeg", + "video_10782/video_10782_1.jpeg", + "video_10782/video_10782_2.jpeg", + "video_10782/video_10782_3.jpeg", + "video_10782/video_10782_4.jpeg", + "video_10782/video_10782_5.jpeg", + "video_10782/video_10782_6.jpeg", + "video_10782/video_10782_7.jpeg", + "video_10782/video_10782_8.jpeg", + "video_10782/video_10782_9.jpeg", + "video_10782/video_10782_10.jpeg", + "video_10782/video_10782_11.jpeg", + "video_10782/video_10782_12.jpeg", + "video_10782/video_10782_13.jpeg", + "video_10782/video_10782_14.jpeg", + "video_10782/video_10782_15.jpeg", + "video_10782/video_10782_16.jpeg", + "video_10782/video_10782_17.jpeg", + "video_10782/video_10782_18.jpeg", + "video_10782/video_10782_19.jpeg", + "video_10782/video_10782_20.jpeg", + "video_10782/video_10782_21.jpeg", + "video_10782/video_10782_22.jpeg", + "video_10782/video_10782_23.jpeg", + "video_10782/video_10782_24.jpeg", + "video_10782/video_10782_25.jpeg", + "video_10782/video_10782_26.jpeg", + "video_10782/video_10782_27.jpeg", + "video_10782/video_10782_28.jpeg", + "video_10782/video_10782_29.jpeg", + "video_10782/video_10782_30.jpeg", + "video_10782/video_10782_31.jpeg", + "video_10782/video_10782_32.jpeg", + "video_10782/video_10782_33.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 49, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. e\nB. d\nC. r\nAnswer with the option's letter from the given choices directly.", + "gt_response": "e", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "e", + "d", + "r" + ], + "image_quantity_level": "Many", + "image": [ + "video_4380/video_4380_0.jpeg", + "video_4380/video_4380_1.jpeg", + "video_4380/video_4380_2.jpeg", + "video_4380/video_4380_3.jpeg", + "video_4380/video_4380_4.jpeg", + "video_4380/video_4380_5.jpeg", + "video_4380/video_4380_6.jpeg", + "video_4380/video_4380_7.jpeg", + "video_4380/video_4380_8.jpeg", + "video_4380/video_4380_9.jpeg", + "video_4380/video_4380_10.jpeg", + "video_4380/video_4380_11.jpeg", + "video_4380/video_4380_12.jpeg", + "video_4380/video_4380_13.jpeg", + "video_4380/video_4380_14.jpeg", + "video_4380/video_4380_15.jpeg", + "video_4380/video_4380_16.jpeg", + "video_4380/video_4380_17.jpeg", + "video_4380/video_4380_18.jpeg", + "video_4380/video_4380_19.jpeg", + "video_4380/video_4380_20.jpeg", + "video_4380/video_4380_21.jpeg", + "video_4380/video_4380_22.jpeg", + "video_4380/video_4380_23.jpeg", + "video_4380/video_4380_24.jpeg", + "video_4380/video_4380_25.jpeg", + "video_4380/video_4380_26.jpeg", + "video_4380/video_4380_27.jpeg", + "video_4380/video_4380_28.jpeg", + "video_4380/video_4380_29.jpeg", + "video_4380/video_4380_30.jpeg", + "video_4380/video_4380_31.jpeg", + "video_4380/video_4380_32.jpeg", + "video_4380/video_4380_33.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 65, + "question": "Analyze the given visuals and answer the question about the order of character. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. men\nB. mne\nC. emn\nAnswer with the option's letter from the given choices directly.", + "gt_response": "men", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "men", + "mne", + "emn" + ], + "image_quantity_level": "Many", + "image": [ + "video_10351/video_10351_0.jpeg", + "video_10351/video_10351_1.jpeg", + "video_10351/video_10351_2.jpeg", + "video_10351/video_10351_3.jpeg", + "video_10351/video_10351_4.jpeg", + "video_10351/video_10351_5.jpeg", + "video_10351/video_10351_6.jpeg", + "video_10351/video_10351_7.jpeg", + "video_10351/video_10351_8.jpeg", + "video_10351/video_10351_9.jpeg", + "video_10351/video_10351_10.jpeg", + "video_10351/video_10351_11.jpeg", + "video_10351/video_10351_12.jpeg", + "video_10351/video_10351_13.jpeg", + "video_10351/video_10351_14.jpeg", + "video_10351/video_10351_15.jpeg", + "video_10351/video_10351_16.jpeg", + "video_10351/video_10351_17.jpeg", + "video_10351/video_10351_18.jpeg", + "video_10351/video_10351_19.jpeg", + "video_10351/video_10351_20.jpeg", + "video_10351/video_10351_21.jpeg", + "video_10351/video_10351_22.jpeg", + "video_10351/video_10351_23.jpeg", + "video_10351/video_10351_24.jpeg", + "video_10351/video_10351_25.jpeg", + "video_10351/video_10351_26.jpeg", + "video_10351/video_10351_27.jpeg", + "video_10351/video_10351_28.jpeg", + "video_10351/video_10351_29.jpeg", + "video_10351/video_10351_30.jpeg", + "video_10351/video_10351_31.jpeg", + "video_10351/video_10351_32.jpeg", + "video_10351/video_10351_33.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 74, + "question": "Review the visuals provided and respond to the question concerning the character sequence. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. many\nB. jxne\nC. ywne\nAnswer with the option's letter from the given choices directly.", + "gt_response": "jxne", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "many", + "jxne", + "ywne" + ], + "image_quantity_level": "Many", + "image": [ + "video_9679/video_9679_0.jpeg", + "video_9679/video_9679_1.jpeg", + "video_9679/video_9679_2.jpeg", + "video_9679/video_9679_3.jpeg", + "video_9679/video_9679_4.jpeg", + "video_9679/video_9679_5.jpeg", + "video_9679/video_9679_6.jpeg", + "video_9679/video_9679_7.jpeg", + "video_9679/video_9679_8.jpeg", + "video_9679/video_9679_9.jpeg", + "video_9679/video_9679_10.jpeg", + "video_9679/video_9679_11.jpeg", + "video_9679/video_9679_12.jpeg", + "video_9679/video_9679_13.jpeg", + "video_9679/video_9679_14.jpeg", + "video_9679/video_9679_15.jpeg", + "video_9679/video_9679_16.jpeg", + "video_9679/video_9679_17.jpeg", + "video_9679/video_9679_18.jpeg", + "video_9679/video_9679_19.jpeg", + "video_9679/video_9679_20.jpeg", + "video_9679/video_9679_21.jpeg", + "video_9679/video_9679_22.jpeg", + "video_9679/video_9679_23.jpeg", + "video_9679/video_9679_24.jpeg", + "video_9679/video_9679_25.jpeg", + "video_9679/video_9679_26.jpeg", + "video_9679/video_9679_27.jpeg", + "video_9679/video_9679_28.jpeg", + "video_9679/video_9679_29.jpeg", + "video_9679/video_9679_30.jpeg", + "video_9679/video_9679_31.jpeg", + "video_9679/video_9679_32.jpeg", + "video_9679/video_9679_33.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 75, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. adb\nB. bad\nC. abd\nAnswer with the option's letter from the given choices directly.", + "gt_response": "abd", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "adb", + "bad", + "abd" + ], + "image_quantity_level": "Many", + "image": [ + "video_9019/video_9019_0.jpeg", + "video_9019/video_9019_1.jpeg", + "video_9019/video_9019_2.jpeg", + "video_9019/video_9019_3.jpeg", + "video_9019/video_9019_4.jpeg", + "video_9019/video_9019_5.jpeg", + "video_9019/video_9019_6.jpeg", + "video_9019/video_9019_7.jpeg", + "video_9019/video_9019_8.jpeg", + "video_9019/video_9019_9.jpeg", + "video_9019/video_9019_10.jpeg", + "video_9019/video_9019_11.jpeg", + "video_9019/video_9019_12.jpeg", + "video_9019/video_9019_13.jpeg", + "video_9019/video_9019_14.jpeg", + "video_9019/video_9019_15.jpeg", + "video_9019/video_9019_16.jpeg", + "video_9019/video_9019_17.jpeg", + "video_9019/video_9019_18.jpeg", + "video_9019/video_9019_19.jpeg", + "video_9019/video_9019_20.jpeg", + "video_9019/video_9019_21.jpeg", + "video_9019/video_9019_22.jpeg", + "video_9019/video_9019_23.jpeg", + "video_9019/video_9019_24.jpeg", + "video_9019/video_9019_25.jpeg", + "video_9019/video_9019_26.jpeg", + "video_9019/video_9019_27.jpeg", + "video_9019/video_9019_28.jpeg", + "video_9019/video_9019_29.jpeg", + "video_9019/video_9019_30.jpeg", + "video_9019/video_9019_31.jpeg", + "video_9019/video_9019_32.jpeg", + "video_9019/video_9019_33.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 28, + "question": "Scrutinize the provided visuals and answer the question about the arrangement of characters. You must choose your answer from the Choice List.\nWhat letters did the person type on the computer in order?\nChoice list: \nA. kag\nB. kav\nC. bag\nAnswer with the option's letter from the given choices directly.", + "gt_response": "bag", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "kag", + "kav", + "bag" + ], + "image_quantity_level": "Medium", + "image": [ + "video_5822/video_5822_0.jpeg", + "video_5822/video_5822_1.jpeg", + "video_5822/video_5822_2.jpeg", + "video_5822/video_5822_3.jpeg", + "video_5822/video_5822_4.jpeg", + "video_5822/video_5822_5.jpeg", + "video_5822/video_5822_6.jpeg", + "video_5822/video_5822_7.jpeg", + "video_5822/video_5822_8.jpeg", + "video_5822/video_5822_9.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 36, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. o\nB. c\nC. w\nAnswer with the option's letter from the given choices directly.", + "gt_response": "o", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "o", + "c", + "w" + ], + "image_quantity_level": "Medium", + "image": [ + "video_7864/video_7864_0.jpeg", + "video_7864/video_7864_1.jpeg", + "video_7864/video_7864_2.jpeg", + "video_7864/video_7864_3.jpeg", + "video_7864/video_7864_4.jpeg", + "video_7864/video_7864_5.jpeg", + "video_7864/video_7864_6.jpeg", + "video_7864/video_7864_7.jpeg", + "video_7864/video_7864_8.jpeg", + "video_7864/video_7864_9.jpeg", + "video_7864/video_7864_10.jpeg", + "video_7864/video_7864_11.jpeg", + "video_7864/video_7864_12.jpeg", + "video_7864/video_7864_13.jpeg", + "video_7864/video_7864_14.jpeg", + "video_7864/video_7864_15.jpeg", + "video_7864/video_7864_16.jpeg", + "video_7864/video_7864_17.jpeg", + "video_7864/video_7864_18.jpeg", + "video_7864/video_7864_19.jpeg", + "video_7864/video_7864_20.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 50, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nWhat would be the order of the written letters if the person had written them in reverse order?\nChoice list: \nA. ten\nB. teg\nC. bdg\nAnswer with the option's letter from the given choices directly.", + "gt_response": "teg", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "ten", + "teg", + "bdg" + ], + "image_quantity_level": "Medium", + "image": [ + "video_6288/video_6288_0.jpeg", + "video_6288/video_6288_1.jpeg", + "video_6288/video_6288_2.jpeg", + "video_6288/video_6288_3.jpeg", + "video_6288/video_6288_4.jpeg", + "video_6288/video_6288_5.jpeg", + "video_6288/video_6288_6.jpeg", + "video_6288/video_6288_7.jpeg", + "video_6288/video_6288_8.jpeg", + "video_6288/video_6288_9.jpeg", + "video_6288/video_6288_10.jpeg", + "video_6288/video_6288_11.jpeg", + "video_6288/video_6288_12.jpeg", + "video_6288/video_6288_13.jpeg", + "video_6288/video_6288_14.jpeg", + "video_6288/video_6288_15.jpeg", + "video_6288/video_6288_16.jpeg", + "video_6288/video_6288_17.jpeg", + "video_6288/video_6288_18.jpeg", + "video_6288/video_6288_19.jpeg", + "video_6288/video_6288_20.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 40, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nWhat was the order of the letters on the table before shuffling?\nChoice list: \nA. lkj\nB. kjl\nC. jkl\nAnswer with the option's letter from the given choices directly.", + "gt_response": "jkl", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "lkj", + "kjl", + "jkl" + ], + "image_quantity_level": "Medium", + "image": [ + "video_3707/video_3707_0.jpeg", + "video_3707/video_3707_1.jpeg", + "video_3707/video_3707_2.jpeg", + "video_3707/video_3707_3.jpeg", + "video_3707/video_3707_4.jpeg", + "video_3707/video_3707_5.jpeg", + "video_3707/video_3707_6.jpeg", + "video_3707/video_3707_7.jpeg", + "video_3707/video_3707_8.jpeg", + "video_3707/video_3707_9.jpeg", + "video_3707/video_3707_10.jpeg", + "video_3707/video_3707_11.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 44, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nWhat letter is likely to be shown next?\nChoice list: \nA. o\nB. l\nC. e\nAnswer with the option's letter from the given choices directly.", + "gt_response": "o", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "o", + "l", + "e" + ], + "image_quantity_level": "Many", + "image": [ + "video_9824/video_9824_0.jpeg", + "video_9824/video_9824_1.jpeg", + "video_9824/video_9824_2.jpeg", + "video_9824/video_9824_3.jpeg", + "video_9824/video_9824_4.jpeg", + "video_9824/video_9824_5.jpeg", + "video_9824/video_9824_6.jpeg", + "video_9824/video_9824_7.jpeg", + "video_9824/video_9824_8.jpeg", + "video_9824/video_9824_9.jpeg", + "video_9824/video_9824_10.jpeg", + "video_9824/video_9824_11.jpeg", + "video_9824/video_9824_12.jpeg", + "video_9824/video_9824_13.jpeg", + "video_9824/video_9824_14.jpeg", + "video_9824/video_9824_15.jpeg", + "video_9824/video_9824_16.jpeg", + "video_9824/video_9824_17.jpeg", + "video_9824/video_9824_18.jpeg", + "video_9824/video_9824_19.jpeg", + "video_9824/video_9824_20.jpeg", + "video_9824/video_9824_21.jpeg", + "video_9824/video_9824_22.jpeg", + "video_9824/video_9824_23.jpeg", + "video_9824/video_9824_24.jpeg", + "video_9824/video_9824_25.jpeg", + "video_9824/video_9824_26.jpeg", + "video_9824/video_9824_27.jpeg", + "video_9824/video_9824_28.jpeg", + "video_9824/video_9824_29.jpeg", + "video_9824/video_9824_30.jpeg", + "video_9824/video_9824_31.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 58, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. dog\nB. dgo\nC. god\nAnswer with the option's letter from the given choices directly.", + "gt_response": "dog", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "dog", + "dgo", + "god" + ], + "image_quantity_level": "Many", + "image": [ + "video_8882/video_8882_0.jpeg", + "video_8882/video_8882_1.jpeg", + "video_8882/video_8882_2.jpeg", + "video_8882/video_8882_3.jpeg", + "video_8882/video_8882_4.jpeg", + "video_8882/video_8882_5.jpeg", + "video_8882/video_8882_6.jpeg", + "video_8882/video_8882_7.jpeg", + "video_8882/video_8882_8.jpeg", + "video_8882/video_8882_9.jpeg", + "video_8882/video_8882_10.jpeg", + "video_8882/video_8882_11.jpeg", + "video_8882/video_8882_12.jpeg", + "video_8882/video_8882_13.jpeg", + "video_8882/video_8882_14.jpeg", + "video_8882/video_8882_15.jpeg", + "video_8882/video_8882_16.jpeg", + "video_8882/video_8882_17.jpeg", + "video_8882/video_8882_18.jpeg", + "video_8882/video_8882_19.jpeg", + "video_8882/video_8882_20.jpeg", + "video_8882/video_8882_21.jpeg", + "video_8882/video_8882_22.jpeg", + "video_8882/video_8882_23.jpeg", + "video_8882/video_8882_24.jpeg", + "video_8882/video_8882_25.jpeg", + "video_8882/video_8882_26.jpeg", + "video_8882/video_8882_27.jpeg", + "video_8882/video_8882_28.jpeg", + "video_8882/video_8882_29.jpeg", + "video_8882/video_8882_30.jpeg", + "video_8882/video_8882_31.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 69, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. niw\nB. win\nC. ain\nAnswer with the option's letter from the given choices directly.", + "gt_response": "win", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "niw", + "win", + "ain" + ], + "image_quantity_level": "Many", + "image": [ + "video_2384/video_2384_0.jpeg", + "video_2384/video_2384_1.jpeg", + "video_2384/video_2384_2.jpeg", + "video_2384/video_2384_3.jpeg", + "video_2384/video_2384_4.jpeg", + "video_2384/video_2384_5.jpeg", + "video_2384/video_2384_6.jpeg", + "video_2384/video_2384_7.jpeg", + "video_2384/video_2384_8.jpeg", + "video_2384/video_2384_9.jpeg", + "video_2384/video_2384_10.jpeg", + "video_2384/video_2384_11.jpeg", + "video_2384/video_2384_12.jpeg", + "video_2384/video_2384_13.jpeg", + "video_2384/video_2384_14.jpeg", + "video_2384/video_2384_15.jpeg", + "video_2384/video_2384_16.jpeg", + "video_2384/video_2384_17.jpeg", + "video_2384/video_2384_18.jpeg", + "video_2384/video_2384_19.jpeg", + "video_2384/video_2384_20.jpeg", + "video_2384/video_2384_21.jpeg", + "video_2384/video_2384_22.jpeg", + "video_2384/video_2384_23.jpeg", + "video_2384/video_2384_24.jpeg", + "video_2384/video_2384_25.jpeg", + "video_2384/video_2384_26.jpeg", + "video_2384/video_2384_27.jpeg", + "video_2384/video_2384_28.jpeg", + "video_2384/video_2384_29.jpeg", + "video_2384/video_2384_30.jpeg", + "video_2384/video_2384_31.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 77, + "question": "Review the visuals provided and respond to the question concerning the character sequence. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. rst\nB. ost\nC. tsr\nAnswer with the option's letter from the given choices directly.", + "gt_response": "rst", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "rst", + "ost", + "tsr" + ], + "image_quantity_level": "Many", + "image": [ + "video_5530/video_5530_0.jpeg", + "video_5530/video_5530_1.jpeg", + "video_5530/video_5530_2.jpeg", + "video_5530/video_5530_3.jpeg", + "video_5530/video_5530_4.jpeg", + "video_5530/video_5530_5.jpeg", + "video_5530/video_5530_6.jpeg", + "video_5530/video_5530_7.jpeg", + "video_5530/video_5530_8.jpeg", + "video_5530/video_5530_9.jpeg", + "video_5530/video_5530_10.jpeg", + "video_5530/video_5530_11.jpeg", + "video_5530/video_5530_12.jpeg", + "video_5530/video_5530_13.jpeg", + "video_5530/video_5530_14.jpeg", + "video_5530/video_5530_15.jpeg", + "video_5530/video_5530_16.jpeg", + "video_5530/video_5530_17.jpeg", + "video_5530/video_5530_18.jpeg", + "video_5530/video_5530_19.jpeg", + "video_5530/video_5530_20.jpeg", + "video_5530/video_5530_21.jpeg", + "video_5530/video_5530_22.jpeg", + "video_5530/video_5530_23.jpeg", + "video_5530/video_5530_24.jpeg", + "video_5530/video_5530_25.jpeg", + "video_5530/video_5530_26.jpeg", + "video_5530/video_5530_27.jpeg", + "video_5530/video_5530_28.jpeg", + "video_5530/video_5530_29.jpeg", + "video_5530/video_5530_30.jpeg", + "video_5530/video_5530_31.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 52, + "question": "Analyze the given visuals and answer the question about the order of character. You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. g\nB. d\nC. i\nAnswer with the option's letter from the given choices directly.", + "gt_response": "i", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "g", + "d", + "i" + ], + "image_quantity_level": "Many", + "image": [ + "video_10287/video_10287_0.jpeg", + "video_10287/video_10287_1.jpeg", + "video_10287/video_10287_2.jpeg", + "video_10287/video_10287_3.jpeg", + "video_10287/video_10287_4.jpeg", + "video_10287/video_10287_5.jpeg", + "video_10287/video_10287_6.jpeg", + "video_10287/video_10287_7.jpeg", + "video_10287/video_10287_8.jpeg", + "video_10287/video_10287_9.jpeg", + "video_10287/video_10287_10.jpeg", + "video_10287/video_10287_11.jpeg", + "video_10287/video_10287_12.jpeg", + "video_10287/video_10287_13.jpeg", + "video_10287/video_10287_14.jpeg", + "video_10287/video_10287_15.jpeg", + "video_10287/video_10287_16.jpeg", + "video_10287/video_10287_17.jpeg", + "video_10287/video_10287_18.jpeg", + "video_10287/video_10287_19.jpeg", + "video_10287/video_10287_20.jpeg", + "video_10287/video_10287_21.jpeg", + "video_10287/video_10287_22.jpeg", + "video_10287/video_10287_23.jpeg", + "video_10287/video_10287_24.jpeg", + "video_10287/video_10287_25.jpeg", + "video_10287/video_10287_26.jpeg", + "video_10287/video_10287_27.jpeg", + "video_10287/video_10287_28.jpeg", + "video_10287/video_10287_29.jpeg", + "video_10287/video_10287_30.jpeg", + "video_10287/video_10287_31.jpeg", + "video_10287/video_10287_32.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 53, + "question": "From the images supplied, answer the question relevant to character order. You must choose your answer from the Choice List.\nWhat would be the order of the written letters if the person had written them in reverse order?\nChoice list: \nA. cow\nB. woc\nC. qob\nAnswer with the option's letter from the given choices directly.", + "gt_response": "woc", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "cow", + "woc", + "qob" + ], + "image_quantity_level": "Medium", + "image": [ + "video_6535/video_6535_0.jpeg", + "video_6535/video_6535_1.jpeg", + "video_6535/video_6535_2.jpeg", + "video_6535/video_6535_3.jpeg", + "video_6535/video_6535_4.jpeg", + "video_6535/video_6535_5.jpeg", + "video_6535/video_6535_6.jpeg", + "video_6535/video_6535_7.jpeg", + "video_6535/video_6535_8.jpeg", + "video_6535/video_6535_9.jpeg", + "video_6535/video_6535_10.jpeg", + "video_6535/video_6535_11.jpeg", + "video_6535/video_6535_12.jpeg", + "video_6535/video_6535_13.jpeg", + "video_6535/video_6535_14.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 66, + "question": "From the images supplied, answer the question relevant to character order. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. liex\nB. jnyv\nC. liev\nAnswer with the option's letter from the given choices directly.", + "gt_response": "liev", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "liex", + "jnyv", + "liev" + ], + "image_quantity_level": "Medium", + "image": [ + "video_6837/video_6837_0.jpeg", + "video_6837/video_6837_1.jpeg", + "video_6837/video_6837_2.jpeg", + "video_6837/video_6837_3.jpeg", + "video_6837/video_6837_4.jpeg", + "video_6837/video_6837_5.jpeg", + "video_6837/video_6837_6.jpeg", + "video_6837/video_6837_7.jpeg", + "video_6837/video_6837_8.jpeg", + "video_6837/video_6837_9.jpeg", + "video_6837/video_6837_10.jpeg", + "video_6837/video_6837_11.jpeg", + "video_6837/video_6837_12.jpeg", + "video_6837/video_6837_13.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 72, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters on the table at the end?\nChoice list: \nA. boy\nB. oby\nC. byo\nAnswer with the option's letter from the given choices directly.", + "gt_response": "boy", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "boy", + "oby", + "byo" + ], + "image_quantity_level": "Medium", + "image": [ + "video_6814/video_6814_0.jpeg", + "video_6814/video_6814_1.jpeg", + "video_6814/video_6814_2.jpeg", + "video_6814/video_6814_3.jpeg", + "video_6814/video_6814_4.jpeg", + "video_6814/video_6814_5.jpeg", + "video_6814/video_6814_6.jpeg", + "video_6814/video_6814_7.jpeg", + "video_6814/video_6814_8.jpeg", + "video_6814/video_6814_9.jpeg", + "video_6814/video_6814_10.jpeg", + "video_6814/video_6814_11.jpeg", + "video_6814/video_6814_12.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 84, + "question": "Analyze the given visuals and answer the question about the order of character. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. eta\nB. eat\nC. aet\nAnswer with the option's letter from the given choices directly.", + "gt_response": "eat", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "eta", + "eat", + "aet" + ], + "image_quantity_level": "Medium", + "image": [ + "video_2418/video_2418_0.jpeg", + "video_2418/video_2418_1.jpeg", + "video_2418/video_2418_2.jpeg", + "video_2418/video_2418_3.jpeg", + "video_2418/video_2418_4.jpeg", + "video_2418/video_2418_5.jpeg", + "video_2418/video_2418_6.jpeg", + "video_2418/video_2418_7.jpeg", + "video_2418/video_2418_8.jpeg", + "video_2418/video_2418_9.jpeg", + "video_2418/video_2418_10.jpeg", + "video_2418/video_2418_11.jpeg", + "video_2418/video_2418_12.jpeg", + "video_2418/video_2418_13.jpeg", + "video_2418/video_2418_14.jpeg", + "video_2418/video_2418_15.jpeg", + "video_2418/video_2418_16.jpeg", + "video_2418/video_2418_17.jpeg", + "video_2418/video_2418_18.jpeg", + "video_2418/video_2418_19.jpeg", + "video_2418/video_2418_20.jpeg", + "video_2418/video_2418_21.jpeg", + "video_2418/video_2418_22.jpeg", + "video_2418/video_2418_23.jpeg", + "video_2418/video_2418_24.jpeg", + "video_2418/video_2418_25.jpeg", + "video_2418/video_2418_26.jpeg", + "video_2418/video_2418_27.jpeg", + "video_2418/video_2418_28.jpeg", + "video_2418/video_2418_29.jpeg", + "video_2418/video_2418_30.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 106, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. jxoa\nB. jxne\nC. jgnq\nAnswer with the option's letter from the given choices directly.", + "gt_response": "jxne", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "jxoa", + "jxne", + "jgnq" + ], + "image_quantity_level": "Medium", + "image": [ + "video_9683/video_9683_0.jpeg", + "video_9683/video_9683_1.jpeg", + "video_9683/video_9683_2.jpeg", + "video_9683/video_9683_3.jpeg", + "video_9683/video_9683_4.jpeg", + "video_9683/video_9683_5.jpeg", + "video_9683/video_9683_6.jpeg", + "video_9683/video_9683_7.jpeg", + "video_9683/video_9683_8.jpeg", + "video_9683/video_9683_9.jpeg", + "video_9683/video_9683_10.jpeg", + "video_9683/video_9683_11.jpeg", + "video_9683/video_9683_12.jpeg", + "video_9683/video_9683_13.jpeg", + "video_9683/video_9683_14.jpeg", + "video_9683/video_9683_15.jpeg", + "video_9683/video_9683_16.jpeg", + "video_9683/video_9683_17.jpeg", + "video_9683/video_9683_18.jpeg", + "video_9683/video_9683_19.jpeg", + "video_9683/video_9683_20.jpeg", + "video_9683/video_9683_21.jpeg", + "video_9683/video_9683_22.jpeg", + "video_9683/video_9683_23.jpeg", + "video_9683/video_9683_24.jpeg", + "video_9683/video_9683_25.jpeg", + "video_9683/video_9683_26.jpeg", + "video_9683/video_9683_27.jpeg", + "video_9683/video_9683_28.jpeg", + "video_9683/video_9683_29.jpeg", + "video_9683/video_9683_30.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 103, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. str\nB. rst\nC. rsn\nAnswer with the option's letter from the given choices directly.", + "gt_response": "rst", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "str", + "rst", + "rsn" + ], + "image_quantity_level": "Medium", + "image": [ + "video_8373/video_8373_0.jpeg", + "video_8373/video_8373_1.jpeg", + "video_8373/video_8373_2.jpeg", + "video_8373/video_8373_3.jpeg", + "video_8373/video_8373_4.jpeg", + "video_8373/video_8373_5.jpeg", + "video_8373/video_8373_6.jpeg", + "video_8373/video_8373_7.jpeg", + "video_8373/video_8373_8.jpeg", + "video_8373/video_8373_9.jpeg", + "video_8373/video_8373_10.jpeg", + "video_8373/video_8373_11.jpeg", + "video_8373/video_8373_12.jpeg", + "video_8373/video_8373_13.jpeg", + "video_8373/video_8373_14.jpeg", + "video_8373/video_8373_15.jpeg", + "video_8373/video_8373_16.jpeg", + "video_8373/video_8373_17.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 107, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters on the table at the end?\nChoice list: \nA. xqko\nB. xkoq\nC. qxok\nAnswer with the option's letter from the given choices directly.", + "gt_response": "qxok", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "xqko", + "xkoq", + "qxok" + ], + "image_quantity_level": "Medium", + "image": [ + "video_9686/video_9686_0.jpeg", + "video_9686/video_9686_1.jpeg", + "video_9686/video_9686_2.jpeg", + "video_9686/video_9686_3.jpeg", + "video_9686/video_9686_4.jpeg", + "video_9686/video_9686_5.jpeg", + "video_9686/video_9686_6.jpeg", + "video_9686/video_9686_7.jpeg", + "video_9686/video_9686_8.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 20, + "question": "From the images supplied, answer the question relevant to character order. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. etum\nB. mute\nC. suhe\nAnswer with the option's letter from the given choices directly.", + "gt_response": "mute", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "etum", + "mute", + "suhe" + ], + "image_quantity_level": "Medium", + "image": [ + "video_8956/video_8956_0.jpeg", + "video_8956/video_8956_1.jpeg", + "video_8956/video_8956_2.jpeg", + "video_8956/video_8956_3.jpeg", + "video_8956/video_8956_4.jpeg", + "video_8956/video_8956_5.jpeg", + "video_8956/video_8956_6.jpeg", + "video_8956/video_8956_7.jpeg", + "video_8956/video_8956_8.jpeg", + "video_8956/video_8956_9.jpeg", + "video_8956/video_8956_10.jpeg", + "video_8956/video_8956_11.jpeg", + "video_8956/video_8956_12.jpeg", + "video_8956/video_8956_13.jpeg", + "video_8956/video_8956_14.jpeg", + "video_8956/video_8956_15.jpeg", + "video_8956/video_8956_16.jpeg", + "video_8956/video_8956_17.jpeg", + "video_8956/video_8956_18.jpeg", + "video_8956/video_8956_19.jpeg", + "video_8956/video_8956_20.jpeg", + "video_8956/video_8956_21.jpeg", + "video_8956/video_8956_22.jpeg", + "video_8956/video_8956_23.jpeg", + "video_8956/video_8956_24.jpeg", + "video_8956/video_8956_25.jpeg", + "video_8956/video_8956_26.jpeg", + "video_8956/video_8956_27.jpeg", + "video_8956/video_8956_28.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 45, + "question": "Analyze the given visuals and answer the question about the order of character. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. hkxnam\nB. knamxh\nC. xhakmn\nAnswer with the option's letter from the given choices directly.", + "gt_response": "xhakmn", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "hkxnam", + "knamxh", + "xhakmn" + ], + "image_quantity_level": "Medium", + "image": [ + "video_8170/video_8170_0.jpeg", + "video_8170/video_8170_1.jpeg", + "video_8170/video_8170_2.jpeg", + "video_8170/video_8170_3.jpeg", + "video_8170/video_8170_4.jpeg", + "video_8170/video_8170_5.jpeg", + "video_8170/video_8170_6.jpeg", + "video_8170/video_8170_7.jpeg", + "video_8170/video_8170_8.jpeg", + "video_8170/video_8170_9.jpeg", + "video_8170/video_8170_10.jpeg", + "video_8170/video_8170_11.jpeg", + "video_8170/video_8170_12.jpeg", + "video_8170/video_8170_13.jpeg", + "video_8170/video_8170_14.jpeg", + "video_8170/video_8170_15.jpeg", + "video_8170/video_8170_16.jpeg", + "video_8170/video_8170_17.jpeg", + "video_8170/video_8170_18.jpeg", + "video_8170/video_8170_19.jpeg", + "video_8170/video_8170_20.jpeg", + "video_8170/video_8170_21.jpeg", + "video_8170/video_8170_22.jpeg", + "video_8170/video_8170_23.jpeg", + "video_8170/video_8170_24.jpeg", + "video_8170/video_8170_25.jpeg", + "video_8170/video_8170_26.jpeg", + "video_8170/video_8170_27.jpeg", + "video_8170/video_8170_28.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 79, + "question": "Scrutinize the provided visuals and answer the question about the arrangement of characters. You must choose your answer from the Choice List.\nWhat would be the order of the written letters if the person had written them in reverse order?\nChoice list: \nA. ael\nB. aet\nC. tae\nAnswer with the option's letter from the given choices directly.", + "gt_response": "aet", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "ael", + "aet", + "tae" + ], + "image_quantity_level": "Medium", + "image": [ + "video_6250/video_6250_0.jpeg", + "video_6250/video_6250_1.jpeg", + "video_6250/video_6250_2.jpeg", + "video_6250/video_6250_3.jpeg", + "video_6250/video_6250_4.jpeg", + "video_6250/video_6250_5.jpeg", + "video_6250/video_6250_6.jpeg", + "video_6250/video_6250_7.jpeg", + "video_6250/video_6250_8.jpeg", + "video_6250/video_6250_9.jpeg", + "video_6250/video_6250_10.jpeg", + "video_6250/video_6250_11.jpeg", + "video_6250/video_6250_12.jpeg", + "video_6250/video_6250_13.jpeg", + "video_6250/video_6250_14.jpeg", + "video_6250/video_6250_15.jpeg", + "video_6250/video_6250_16.jpeg", + "video_6250/video_6250_17.jpeg", + "video_6250/video_6250_18.jpeg", + "video_6250/video_6250_19.jpeg", + "video_6250/video_6250_20.jpeg", + "video_6250/video_6250_21.jpeg", + "video_6250/video_6250_22.jpeg", + "video_6250/video_6250_23.jpeg", + "video_6250/video_6250_24.jpeg", + "video_6250/video_6250_25.jpeg", + "video_6250/video_6250_26.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 42, + "question": "Analyze the given visuals and answer the question about the order of character. You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. w\nB. o\nC. c\nAnswer with the option's letter from the given choices directly.", + "gt_response": "c", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "w", + "o", + "c" + ], + "image_quantity_level": "Medium", + "image": [ + "video_1726/video_1726_0.jpeg", + "video_1726/video_1726_1.jpeg", + "video_1726/video_1726_2.jpeg", + "video_1726/video_1726_3.jpeg", + "video_1726/video_1726_4.jpeg", + "video_1726/video_1726_5.jpeg", + "video_1726/video_1726_6.jpeg", + "video_1726/video_1726_7.jpeg", + "video_1726/video_1726_8.jpeg", + "video_1726/video_1726_9.jpeg", + "video_1726/video_1726_10.jpeg", + "video_1726/video_1726_11.jpeg", + "video_1726/video_1726_12.jpeg", + "video_1726/video_1726_13.jpeg", + "video_1726/video_1726_14.jpeg", + "video_1726/video_1726_15.jpeg", + "video_1726/video_1726_16.jpeg", + "video_1726/video_1726_17.jpeg", + "video_1726/video_1726_18.jpeg", + "video_1726/video_1726_19.jpeg", + "video_1726/video_1726_20.jpeg", + "video_1726/video_1726_21.jpeg", + "video_1726/video_1726_22.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 138, + "question": "Scrutinize the provided visuals and answer the question about the arrangement of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. mid\nB. dim\nC. idm\nAnswer with the option's letter from the given choices directly.", + "gt_response": "mid", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "mid", + "dim", + "idm" + ], + "image_quantity_level": "Medium", + "image": [ + "video_1469/video_1469_0.jpeg", + "video_1469/video_1469_1.jpeg", + "video_1469/video_1469_2.jpeg", + "video_1469/video_1469_3.jpeg", + "video_1469/video_1469_4.jpeg", + "video_1469/video_1469_5.jpeg", + "video_1469/video_1469_6.jpeg", + "video_1469/video_1469_7.jpeg", + "video_1469/video_1469_8.jpeg", + "video_1469/video_1469_9.jpeg", + "video_1469/video_1469_10.jpeg", + "video_1469/video_1469_11.jpeg", + "video_1469/video_1469_12.jpeg", + "video_1469/video_1469_13.jpeg", + "video_1469/video_1469_14.jpeg", + "video_1469/video_1469_15.jpeg", + "video_1469/video_1469_16.jpeg", + "video_1469/video_1469_17.jpeg", + "video_1469/video_1469_18.jpeg", + "video_1469/video_1469_19.jpeg", + "video_1469/video_1469_20.jpeg", + "video_1469/video_1469_21.jpeg", + "video_1469/video_1469_22.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 9, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nWhat would be the order of the written letters if the person had written them in reverse order?\nChoice list: \nA. bce\nB. zlb\nC. ecb\nAnswer with the option's letter from the given choices directly.", + "gt_response": "ecb", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "bce", + "zlb", + "ecb" + ], + "image_quantity_level": "Medium", + "image": [ + "video_9537/video_9537_0.jpeg", + "video_9537/video_9537_1.jpeg", + "video_9537/video_9537_2.jpeg", + "video_9537/video_9537_3.jpeg", + "video_9537/video_9537_4.jpeg", + "video_9537/video_9537_5.jpeg", + "video_9537/video_9537_6.jpeg", + "video_9537/video_9537_7.jpeg", + "video_9537/video_9537_8.jpeg", + "video_9537/video_9537_9.jpeg", + "video_9537/video_9537_10.jpeg", + "video_9537/video_9537_11.jpeg", + "video_9537/video_9537_12.jpeg", + "video_9537/video_9537_13.jpeg", + "video_9537/video_9537_14.jpeg", + "video_9537/video_9537_15.jpeg", + "video_9537/video_9537_16.jpeg", + "video_9537/video_9537_17.jpeg", + "video_9537/video_9537_18.jpeg", + "video_9537/video_9537_19.jpeg", + "video_9537/video_9537_20.jpeg", + "video_9537/video_9537_21.jpeg", + "video_9537/video_9537_22.jpeg", + "video_9537/video_9537_23.jpeg", + "video_9537/video_9537_24.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 11, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. mf\nB. ma\nC. am\nAnswer with the option's letter from the given choices directly.", + "gt_response": "ma", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "mf", + "ma", + "am" + ], + "image_quantity_level": "Medium", + "image": [ + "video_6912/video_6912_0.jpeg", + "video_6912/video_6912_1.jpeg", + "video_6912/video_6912_2.jpeg", + "video_6912/video_6912_3.jpeg", + "video_6912/video_6912_4.jpeg", + "video_6912/video_6912_5.jpeg", + "video_6912/video_6912_6.jpeg", + "video_6912/video_6912_7.jpeg", + "video_6912/video_6912_8.jpeg", + "video_6912/video_6912_9.jpeg", + "video_6912/video_6912_10.jpeg", + "video_6912/video_6912_11.jpeg", + "video_6912/video_6912_12.jpeg", + "video_6912/video_6912_13.jpeg", + "video_6912/video_6912_14.jpeg", + "video_6912/video_6912_15.jpeg", + "video_6912/video_6912_16.jpeg", + "video_6912/video_6912_17.jpeg", + "video_6912/video_6912_18.jpeg", + "video_6912/video_6912_19.jpeg", + "video_6912/video_6912_20.jpeg", + "video_6912/video_6912_21.jpeg", + "video_6912/video_6912_22.jpeg", + "video_6912/video_6912_23.jpeg", + "video_6912/video_6912_24.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 123, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. dsa\nB. sad\nC. asd\nAnswer with the option's letter from the given choices directly.", + "gt_response": "sad", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "dsa", + "sad", + "asd" + ], + "image_quantity_level": "Medium", + "image": [ + "video_6098/video_6098_0.jpeg", + "video_6098/video_6098_1.jpeg", + "video_6098/video_6098_2.jpeg", + "video_6098/video_6098_3.jpeg", + "video_6098/video_6098_4.jpeg", + "video_6098/video_6098_5.jpeg", + "video_6098/video_6098_6.jpeg", + "video_6098/video_6098_7.jpeg", + "video_6098/video_6098_8.jpeg", + "video_6098/video_6098_9.jpeg", + "video_6098/video_6098_10.jpeg", + "video_6098/video_6098_11.jpeg", + "video_6098/video_6098_12.jpeg", + "video_6098/video_6098_13.jpeg", + "video_6098/video_6098_14.jpeg", + "video_6098/video_6098_15.jpeg", + "video_6098/video_6098_16.jpeg", + "video_6098/video_6098_17.jpeg", + "video_6098/video_6098_18.jpeg", + "video_6098/video_6098_19.jpeg", + "video_6098/video_6098_20.jpeg", + "video_6098/video_6098_21.jpeg", + "video_6098/video_6098_22.jpeg", + "video_6098/video_6098_23.jpeg", + "video_6098/video_6098_24.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 64, + "question": "Scrutinize the provided visuals and answer the question about the arrangement of characters. You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. i\nB. u\nC. t\nAnswer with the option's letter from the given choices directly.", + "gt_response": "t", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "i", + "u", + "t" + ], + "image_quantity_level": "Medium", + "image": [ + "video_5924/video_5924_0.jpeg", + "video_5924/video_5924_1.jpeg", + "video_5924/video_5924_2.jpeg", + "video_5924/video_5924_3.jpeg", + "video_5924/video_5924_4.jpeg", + "video_5924/video_5924_5.jpeg", + "video_5924/video_5924_6.jpeg", + "video_5924/video_5924_7.jpeg", + "video_5924/video_5924_8.jpeg", + "video_5924/video_5924_9.jpeg", + "video_5924/video_5924_10.jpeg", + "video_5924/video_5924_11.jpeg", + "video_5924/video_5924_12.jpeg", + "video_5924/video_5924_13.jpeg", + "video_5924/video_5924_14.jpeg", + "video_5924/video_5924_15.jpeg", + "video_5924/video_5924_16.jpeg", + "video_5924/video_5924_17.jpeg", + "video_5924/video_5924_18.jpeg", + "video_5924/video_5924_19.jpeg", + "video_5924/video_5924_20.jpeg", + "video_5924/video_5924_21.jpeg", + "video_5924/video_5924_22.jpeg", + "video_5924/video_5924_23.jpeg", + "video_5924/video_5924_24.jpeg", + "video_5924/video_5924_25.jpeg", + "video_5924/video_5924_26.jpeg", + "video_5924/video_5924_27.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 82, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. a\nB. t\nC. c\nAnswer with the option's letter from the given choices directly.", + "gt_response": "a", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "a", + "t", + "c" + ], + "image_quantity_level": "Medium", + "image": [ + "video_9427/video_9427_0.jpeg", + "video_9427/video_9427_1.jpeg", + "video_9427/video_9427_2.jpeg", + "video_9427/video_9427_3.jpeg", + "video_9427/video_9427_4.jpeg", + "video_9427/video_9427_5.jpeg", + "video_9427/video_9427_6.jpeg", + "video_9427/video_9427_7.jpeg", + "video_9427/video_9427_8.jpeg", + "video_9427/video_9427_9.jpeg", + "video_9427/video_9427_10.jpeg", + "video_9427/video_9427_11.jpeg", + "video_9427/video_9427_12.jpeg", + "video_9427/video_9427_13.jpeg", + "video_9427/video_9427_14.jpeg", + "video_9427/video_9427_15.jpeg", + "video_9427/video_9427_16.jpeg", + "video_9427/video_9427_17.jpeg", + "video_9427/video_9427_18.jpeg", + "video_9427/video_9427_19.jpeg", + "video_9427/video_9427_20.jpeg", + "video_9427/video_9427_21.jpeg", + "video_9427/video_9427_22.jpeg", + "video_9427/video_9427_23.jpeg", + "video_9427/video_9427_24.jpeg", + "video_9427/video_9427_25.jpeg", + "video_9427/video_9427_26.jpeg", + "video_9427/video_9427_27.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 105, + "question": "Scrutinize the provided visuals and answer the question about the arrangement of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. sad\nB. ads\nC. asd\nAnswer with the option's letter from the given choices directly.", + "gt_response": "sad", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "sad", + "ads", + "asd" + ], + "image_quantity_level": "Medium", + "image": [ + "video_6578/video_6578_0.jpeg", + "video_6578/video_6578_1.jpeg", + "video_6578/video_6578_2.jpeg", + "video_6578/video_6578_3.jpeg", + "video_6578/video_6578_4.jpeg", + "video_6578/video_6578_5.jpeg", + "video_6578/video_6578_6.jpeg", + "video_6578/video_6578_7.jpeg", + "video_6578/video_6578_8.jpeg", + "video_6578/video_6578_9.jpeg", + "video_6578/video_6578_10.jpeg", + "video_6578/video_6578_11.jpeg", + "video_6578/video_6578_12.jpeg", + "video_6578/video_6578_13.jpeg", + "video_6578/video_6578_14.jpeg", + "video_6578/video_6578_15.jpeg", + "video_6578/video_6578_16.jpeg", + "video_6578/video_6578_17.jpeg", + "video_6578/video_6578_18.jpeg", + "video_6578/video_6578_19.jpeg", + "video_6578/video_6578_20.jpeg", + "video_6578/video_6578_21.jpeg", + "video_6578/video_6578_22.jpeg", + "video_6578/video_6578_23.jpeg", + "video_6578/video_6578_24.jpeg", + "video_6578/video_6578_25.jpeg", + "video_6578/video_6578_26.jpeg", + "video_6578/video_6578_27.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 94, + "question": "Scrutinize the provided visuals and answer the question about the arrangement of characters. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. cta\nB. tca\nC. cat\nAnswer with the option's letter from the given choices directly.", + "gt_response": "cat", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "cta", + "tca", + "cat" + ], + "image_quantity_level": "Medium", + "image": [ + "video_726/video_726_0.jpeg", + "video_726/video_726_1.jpeg", + "video_726/video_726_2.jpeg", + "video_726/video_726_3.jpeg", + "video_726/video_726_4.jpeg", + "video_726/video_726_5.jpeg", + "video_726/video_726_6.jpeg", + "video_726/video_726_7.jpeg", + "video_726/video_726_8.jpeg", + "video_726/video_726_9.jpeg", + "video_726/video_726_10.jpeg", + "video_726/video_726_11.jpeg", + "video_726/video_726_12.jpeg", + "video_726/video_726_13.jpeg", + "video_726/video_726_14.jpeg", + "video_726/video_726_15.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 34, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. xae\nB. axe\nC. hme\nAnswer with the option's letter from the given choices directly.", + "gt_response": "axe", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "xae", + "axe", + "hme" + ], + "image_quantity_level": "Medium", + "image": [ + "video_6422/video_6422_0.jpeg", + "video_6422/video_6422_1.jpeg", + "video_6422/video_6422_2.jpeg", + "video_6422/video_6422_3.jpeg", + "video_6422/video_6422_4.jpeg", + "video_6422/video_6422_5.jpeg", + "video_6422/video_6422_6.jpeg", + "video_6422/video_6422_7.jpeg", + "video_6422/video_6422_8.jpeg", + "video_6422/video_6422_9.jpeg", + "video_6422/video_6422_10.jpeg", + "video_6422/video_6422_11.jpeg", + "video_6422/video_6422_12.jpeg", + "video_6422/video_6422_13.jpeg", + "video_6422/video_6422_14.jpeg", + "video_6422/video_6422_15.jpeg", + "video_6422/video_6422_16.jpeg", + "video_6422/video_6422_17.jpeg", + "video_6422/video_6422_18.jpeg", + "video_6422/video_6422_19.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 48, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. a\nB. m\nC. g\nAnswer with the option's letter from the given choices directly.", + "gt_response": "a", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "a", + "m", + "g" + ], + "image_quantity_level": "Medium", + "image": [ + "video_4432/video_4432_0.jpeg", + "video_4432/video_4432_1.jpeg", + "video_4432/video_4432_2.jpeg", + "video_4432/video_4432_3.jpeg", + "video_4432/video_4432_4.jpeg", + "video_4432/video_4432_5.jpeg", + "video_4432/video_4432_6.jpeg", + "video_4432/video_4432_7.jpeg", + "video_4432/video_4432_8.jpeg", + "video_4432/video_4432_9.jpeg", + "video_4432/video_4432_10.jpeg", + "video_4432/video_4432_11.jpeg", + "video_4432/video_4432_12.jpeg", + "video_4432/video_4432_13.jpeg", + "video_4432/video_4432_14.jpeg", + "video_4432/video_4432_15.jpeg", + "video_4432/video_4432_16.jpeg", + "video_4432/video_4432_17.jpeg", + "video_4432/video_4432_18.jpeg", + "video_4432/video_4432_19.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 38, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. f\nB. g\nC. o\nAnswer with the option's letter from the given choices directly.", + "gt_response": "o", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "f", + "g", + "o" + ], + "image_quantity_level": "Medium", + "image": [ + "video_6796/video_6796_0.jpeg", + "video_6796/video_6796_1.jpeg", + "video_6796/video_6796_2.jpeg", + "video_6796/video_6796_3.jpeg", + "video_6796/video_6796_4.jpeg", + "video_6796/video_6796_5.jpeg", + "video_6796/video_6796_6.jpeg", + "video_6796/video_6796_7.jpeg", + "video_6796/video_6796_8.jpeg", + "video_6796/video_6796_9.jpeg", + "video_6796/video_6796_10.jpeg", + "video_6796/video_6796_11.jpeg", + "video_6796/video_6796_12.jpeg", + "video_6796/video_6796_13.jpeg", + "video_6796/video_6796_14.jpeg", + "video_6796/video_6796_15.jpeg", + "video_6796/video_6796_16.jpeg", + "video_6796/video_6796_17.jpeg", + "video_6796/video_6796_18.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 70, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. efg\nB. ofg\nC. ecg\nAnswer with the option's letter from the given choices directly.", + "gt_response": "efg", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "efg", + "ofg", + "ecg" + ], + "image_quantity_level": "Medium", + "image": [ + "video_9583/video_9583_0.jpeg", + "video_9583/video_9583_1.jpeg", + "video_9583/video_9583_2.jpeg", + "video_9583/video_9583_3.jpeg", + "video_9583/video_9583_4.jpeg", + "video_9583/video_9583_5.jpeg", + "video_9583/video_9583_6.jpeg", + "video_9583/video_9583_7.jpeg", + "video_9583/video_9583_8.jpeg", + "video_9583/video_9583_9.jpeg", + "video_9583/video_9583_10.jpeg", + "video_9583/video_9583_11.jpeg", + "video_9583/video_9583_12.jpeg", + "video_9583/video_9583_13.jpeg", + "video_9583/video_9583_14.jpeg", + "video_9583/video_9583_15.jpeg", + "video_9583/video_9583_16.jpeg", + "video_9583/video_9583_17.jpeg", + "video_9583/video_9583_18.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 47, + "question": "Review the visuals provided and respond to the question concerning the character sequence. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. men\nB. enm\nC. emn\nAnswer with the option's letter from the given choices directly.", + "gt_response": "men", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "men", + "enm", + "emn" + ], + "image_quantity_level": "Many", + "image": [ + "video_10857/video_10857_0.jpeg", + "video_10857/video_10857_1.jpeg", + "video_10857/video_10857_2.jpeg", + "video_10857/video_10857_3.jpeg", + "video_10857/video_10857_4.jpeg", + "video_10857/video_10857_5.jpeg", + "video_10857/video_10857_6.jpeg", + "video_10857/video_10857_7.jpeg", + "video_10857/video_10857_8.jpeg", + "video_10857/video_10857_9.jpeg", + "video_10857/video_10857_10.jpeg", + "video_10857/video_10857_11.jpeg", + "video_10857/video_10857_12.jpeg", + "video_10857/video_10857_13.jpeg", + "video_10857/video_10857_14.jpeg", + "video_10857/video_10857_15.jpeg", + "video_10857/video_10857_16.jpeg", + "video_10857/video_10857_17.jpeg", + "video_10857/video_10857_18.jpeg", + "video_10857/video_10857_19.jpeg", + "video_10857/video_10857_20.jpeg", + "video_10857/video_10857_21.jpeg", + "video_10857/video_10857_22.jpeg", + "video_10857/video_10857_23.jpeg", + "video_10857/video_10857_24.jpeg", + "video_10857/video_10857_25.jpeg", + "video_10857/video_10857_26.jpeg", + "video_10857/video_10857_27.jpeg", + "video_10857/video_10857_28.jpeg", + "video_10857/video_10857_29.jpeg", + "video_10857/video_10857_30.jpeg", + "video_10857/video_10857_31.jpeg", + "video_10857/video_10857_32.jpeg", + "video_10857/video_10857_33.jpeg", + "video_10857/video_10857_34.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 60, + "question": "Scrutinize the provided visuals and answer the question about the arrangement of characters. You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. o\nB. d\nC. g\nAnswer with the option's letter from the given choices directly.", + "gt_response": "o", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "o", + "d", + "g" + ], + "image_quantity_level": "Many", + "image": [ + "video_432/video_432_0.jpeg", + "video_432/video_432_1.jpeg", + "video_432/video_432_2.jpeg", + "video_432/video_432_3.jpeg", + "video_432/video_432_4.jpeg", + "video_432/video_432_5.jpeg", + "video_432/video_432_6.jpeg", + "video_432/video_432_7.jpeg", + "video_432/video_432_8.jpeg", + "video_432/video_432_9.jpeg", + "video_432/video_432_10.jpeg", + "video_432/video_432_11.jpeg", + "video_432/video_432_12.jpeg", + "video_432/video_432_13.jpeg", + "video_432/video_432_14.jpeg", + "video_432/video_432_15.jpeg", + "video_432/video_432_16.jpeg", + "video_432/video_432_17.jpeg", + "video_432/video_432_18.jpeg", + "video_432/video_432_19.jpeg", + "video_432/video_432_20.jpeg", + "video_432/video_432_21.jpeg", + "video_432/video_432_22.jpeg", + "video_432/video_432_23.jpeg", + "video_432/video_432_24.jpeg", + "video_432/video_432_25.jpeg", + "video_432/video_432_26.jpeg", + "video_432/video_432_27.jpeg", + "video_432/video_432_28.jpeg", + "video_432/video_432_29.jpeg", + "video_432/video_432_30.jpeg", + "video_432/video_432_31.jpeg", + "video_432/video_432_32.jpeg", + "video_432/video_432_33.jpeg", + "video_432/video_432_34.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 63, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. ual\nB. lua\nC. alu\nAnswer with the option's letter from the given choices directly.", + "gt_response": "lua", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "ual", + "lua", + "alu" + ], + "image_quantity_level": "Many", + "image": [ + "video_7228/video_7228_0.jpeg", + "video_7228/video_7228_1.jpeg", + "video_7228/video_7228_2.jpeg", + "video_7228/video_7228_3.jpeg", + "video_7228/video_7228_4.jpeg", + "video_7228/video_7228_5.jpeg", + "video_7228/video_7228_6.jpeg", + "video_7228/video_7228_7.jpeg", + "video_7228/video_7228_8.jpeg", + "video_7228/video_7228_9.jpeg", + "video_7228/video_7228_10.jpeg", + "video_7228/video_7228_11.jpeg", + "video_7228/video_7228_12.jpeg", + "video_7228/video_7228_13.jpeg", + "video_7228/video_7228_14.jpeg", + "video_7228/video_7228_15.jpeg", + "video_7228/video_7228_16.jpeg", + "video_7228/video_7228_17.jpeg", + "video_7228/video_7228_18.jpeg", + "video_7228/video_7228_19.jpeg", + "video_7228/video_7228_20.jpeg", + "video_7228/video_7228_21.jpeg", + "video_7228/video_7228_22.jpeg", + "video_7228/video_7228_23.jpeg", + "video_7228/video_7228_24.jpeg", + "video_7228/video_7228_25.jpeg", + "video_7228/video_7228_26.jpeg", + "video_7228/video_7228_27.jpeg", + "video_7228/video_7228_28.jpeg", + "video_7228/video_7228_29.jpeg", + "video_7228/video_7228_30.jpeg", + "video_7228/video_7228_31.jpeg", + "video_7228/video_7228_32.jpeg", + "video_7228/video_7228_33.jpeg", + "video_7228/video_7228_34.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 67, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. xbdm\nB. mxbd\nC. bxmd\nAnswer with the option's letter from the given choices directly.", + "gt_response": "xbdm", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "xbdm", + "mxbd", + "bxmd" + ], + "image_quantity_level": "Many", + "image": [ + "video_9985/video_9985_0.jpeg", + "video_9985/video_9985_1.jpeg", + "video_9985/video_9985_2.jpeg", + "video_9985/video_9985_3.jpeg", + "video_9985/video_9985_4.jpeg", + "video_9985/video_9985_5.jpeg", + "video_9985/video_9985_6.jpeg", + "video_9985/video_9985_7.jpeg", + "video_9985/video_9985_8.jpeg", + "video_9985/video_9985_9.jpeg", + "video_9985/video_9985_10.jpeg", + "video_9985/video_9985_11.jpeg", + "video_9985/video_9985_12.jpeg", + "video_9985/video_9985_13.jpeg", + "video_9985/video_9985_14.jpeg", + "video_9985/video_9985_15.jpeg", + "video_9985/video_9985_16.jpeg", + "video_9985/video_9985_17.jpeg", + "video_9985/video_9985_18.jpeg", + "video_9985/video_9985_19.jpeg", + "video_9985/video_9985_20.jpeg", + "video_9985/video_9985_21.jpeg", + "video_9985/video_9985_22.jpeg", + "video_9985/video_9985_23.jpeg", + "video_9985/video_9985_24.jpeg", + "video_9985/video_9985_25.jpeg", + "video_9985/video_9985_26.jpeg", + "video_9985/video_9985_27.jpeg", + "video_9985/video_9985_28.jpeg", + "video_9985/video_9985_29.jpeg", + "video_9985/video_9985_30.jpeg", + "video_9985/video_9985_31.jpeg", + "video_9985/video_9985_32.jpeg", + "video_9985/video_9985_33.jpeg", + "video_9985/video_9985_34.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 76, + "question": "From the images supplied, answer the question relevant to character order. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. bxmd\nB. mbxd\nC. bdxm\nAnswer with the option's letter from the given choices directly.", + "gt_response": "mbxd", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "bxmd", + "mbxd", + "bdxm" + ], + "image_quantity_level": "Many", + "image": [ + "video_5724/video_5724_0.jpeg", + "video_5724/video_5724_1.jpeg", + "video_5724/video_5724_2.jpeg", + "video_5724/video_5724_3.jpeg", + "video_5724/video_5724_4.jpeg", + "video_5724/video_5724_5.jpeg", + "video_5724/video_5724_6.jpeg", + "video_5724/video_5724_7.jpeg", + "video_5724/video_5724_8.jpeg", + "video_5724/video_5724_9.jpeg", + "video_5724/video_5724_10.jpeg", + "video_5724/video_5724_11.jpeg", + "video_5724/video_5724_12.jpeg", + "video_5724/video_5724_13.jpeg", + "video_5724/video_5724_14.jpeg", + "video_5724/video_5724_15.jpeg", + "video_5724/video_5724_16.jpeg", + "video_5724/video_5724_17.jpeg", + "video_5724/video_5724_18.jpeg", + "video_5724/video_5724_19.jpeg", + "video_5724/video_5724_20.jpeg", + "video_5724/video_5724_21.jpeg", + "video_5724/video_5724_22.jpeg", + "video_5724/video_5724_23.jpeg", + "video_5724/video_5724_24.jpeg", + "video_5724/video_5724_25.jpeg", + "video_5724/video_5724_26.jpeg", + "video_5724/video_5724_27.jpeg", + "video_5724/video_5724_28.jpeg", + "video_5724/video_5724_29.jpeg", + "video_5724/video_5724_30.jpeg", + "video_5724/video_5724_31.jpeg", + "video_5724/video_5724_32.jpeg", + "video_5724/video_5724_33.jpeg", + "video_5724/video_5724_34.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 68, + "question": "Analyze the given visuals and answer the question about the order of character. You must choose your answer from the Choice List.\nWhat would be the order of the written letters if the person had written them in reverse order?\nChoice list: \nA. lhb\nB. gab\nC. bag\nAnswer with the option's letter from the given choices directly.", + "gt_response": "gab", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "lhb", + "gab", + "bag" + ], + "image_quantity_level": "Medium", + "image": [ + "video_977/video_977_0.jpeg", + "video_977/video_977_1.jpeg", + "video_977/video_977_2.jpeg", + "video_977/video_977_3.jpeg", + "video_977/video_977_4.jpeg", + "video_977/video_977_5.jpeg", + "video_977/video_977_6.jpeg", + "video_977/video_977_7.jpeg", + "video_977/video_977_8.jpeg", + "video_977/video_977_9.jpeg", + "video_977/video_977_10.jpeg", + "video_977/video_977_11.jpeg", + "video_977/video_977_12.jpeg", + "video_977/video_977_13.jpeg", + "video_977/video_977_14.jpeg", + "video_977/video_977_15.jpeg", + "video_977/video_977_16.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 71, + "question": "Review the visuals provided and respond to the question concerning the character sequence. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. qm\nB. tq\nC. qt\nAnswer with the option's letter from the given choices directly.", + "gt_response": "qt", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "qm", + "tq", + "qt" + ], + "image_quantity_level": "Medium", + "image": [ + "video_558/video_558_0.jpeg", + "video_558/video_558_1.jpeg", + "video_558/video_558_2.jpeg", + "video_558/video_558_3.jpeg", + "video_558/video_558_4.jpeg", + "video_558/video_558_5.jpeg", + "video_558/video_558_6.jpeg", + "video_558/video_558_7.jpeg", + "video_558/video_558_8.jpeg", + "video_558/video_558_9.jpeg", + "video_558/video_558_10.jpeg", + "video_558/video_558_11.jpeg", + "video_558/video_558_12.jpeg", + "video_558/video_558_13.jpeg", + "video_558/video_558_14.jpeg", + "video_558/video_558_15.jpeg", + "video_558/video_558_16.jpeg", + "video_558/video_558_17.jpeg", + "video_558/video_558_18.jpeg", + "video_558/video_558_19.jpeg", + "video_558/video_558_20.jpeg", + "video_558/video_558_21.jpeg", + "video_558/video_558_22.jpeg", + "video_558/video_558_23.jpeg", + "video_558/video_558_24.jpeg", + "video_558/video_558_25.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 73, + "question": "Review the visuals provided and respond to the question concerning the character sequence. You must choose your answer from the Choice List.\nWhat was the order of the letters on the table before shuffling?\nChoice list: \nA. exs\nB. esx\nC. sex\nAnswer with the option's letter from the given choices directly.", + "gt_response": "sex", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "exs", + "esx", + "sex" + ], + "image_quantity_level": "Medium", + "image": [ + "video_6437/video_6437_0.jpeg", + "video_6437/video_6437_1.jpeg", + "video_6437/video_6437_2.jpeg", + "video_6437/video_6437_3.jpeg", + "video_6437/video_6437_4.jpeg", + "video_6437/video_6437_5.jpeg", + "video_6437/video_6437_6.jpeg", + "video_6437/video_6437_7.jpeg", + "video_6437/video_6437_8.jpeg", + "video_6437/video_6437_9.jpeg", + "video_6437/video_6437_10.jpeg", + "video_6437/video_6437_11.jpeg", + "video_6437/video_6437_12.jpeg", + "video_6437/video_6437_13.jpeg", + "video_6437/video_6437_14.jpeg", + "video_6437/video_6437_15.jpeg", + "video_6437/video_6437_16.jpeg", + "video_6437/video_6437_17.jpeg", + "video_6437/video_6437_18.jpeg", + "video_6437/video_6437_19.jpeg", + "video_6437/video_6437_20.jpeg", + "video_6437/video_6437_21.jpeg", + "video_6437/video_6437_22.jpeg", + "video_6437/video_6437_23.jpeg", + "video_6437/video_6437_24.jpeg", + "video_6437/video_6437_25.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 80, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. e\nB. a\nC. x\nAnswer with the option's letter from the given choices directly.", + "gt_response": "x", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "e", + "a", + "x" + ], + "image_quantity_level": "Medium", + "image": [ + "video_3597/video_3597_0.jpeg", + "video_3597/video_3597_1.jpeg", + "video_3597/video_3597_2.jpeg", + "video_3597/video_3597_3.jpeg", + "video_3597/video_3597_4.jpeg", + "video_3597/video_3597_5.jpeg", + "video_3597/video_3597_6.jpeg", + "video_3597/video_3597_7.jpeg", + "video_3597/video_3597_8.jpeg", + "video_3597/video_3597_9.jpeg", + "video_3597/video_3597_10.jpeg", + "video_3597/video_3597_11.jpeg", + "video_3597/video_3597_12.jpeg", + "video_3597/video_3597_13.jpeg", + "video_3597/video_3597_14.jpeg", + "video_3597/video_3597_15.jpeg", + "video_3597/video_3597_16.jpeg", + "video_3597/video_3597_17.jpeg", + "video_3597/video_3597_18.jpeg", + "video_3597/video_3597_19.jpeg", + "video_3597/video_3597_20.jpeg", + "video_3597/video_3597_21.jpeg", + "video_3597/video_3597_22.jpeg", + "video_3597/video_3597_23.jpeg", + "video_3597/video_3597_24.jpeg", + "video_3597/video_3597_25.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 41, + "question": "Review the visuals provided and respond to the question concerning the character sequence. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. edf\nB. obf\nC. def\nAnswer with the option's letter from the given choices directly.", + "gt_response": "def", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "edf", + "obf", + "def" + ], + "image_quantity_level": "Medium", + "image": [ + "video_9397/video_9397_0.jpeg", + "video_9397/video_9397_1.jpeg", + "video_9397/video_9397_2.jpeg", + "video_9397/video_9397_3.jpeg", + "video_9397/video_9397_4.jpeg", + "video_9397/video_9397_5.jpeg", + "video_9397/video_9397_6.jpeg", + "video_9397/video_9397_7.jpeg", + "video_9397/video_9397_8.jpeg", + "video_9397/video_9397_9.jpeg", + "video_9397/video_9397_10.jpeg", + "video_9397/video_9397_11.jpeg", + "video_9397/video_9397_12.jpeg", + "video_9397/video_9397_13.jpeg", + "video_9397/video_9397_14.jpeg", + "video_9397/video_9397_15.jpeg", + "video_9397/video_9397_16.jpeg", + "video_9397/video_9397_17.jpeg", + "video_9397/video_9397_18.jpeg", + "video_9397/video_9397_19.jpeg", + "video_9397/video_9397_20.jpeg", + "video_9397/video_9397_21.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 59, + "question": "Analyze the given visuals and answer the question about the order of character. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. efg\nB. eag\nC. efd\nAnswer with the option's letter from the given choices directly.", + "gt_response": "efg", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "efg", + "eag", + "efd" + ], + "image_quantity_level": "Medium", + "image": [ + "video_1175/video_1175_0.jpeg", + "video_1175/video_1175_1.jpeg", + "video_1175/video_1175_2.jpeg", + "video_1175/video_1175_3.jpeg", + "video_1175/video_1175_4.jpeg", + "video_1175/video_1175_5.jpeg", + "video_1175/video_1175_6.jpeg", + "video_1175/video_1175_7.jpeg", + "video_1175/video_1175_8.jpeg", + "video_1175/video_1175_9.jpeg", + "video_1175/video_1175_10.jpeg", + "video_1175/video_1175_11.jpeg", + "video_1175/video_1175_12.jpeg", + "video_1175/video_1175_13.jpeg", + "video_1175/video_1175_14.jpeg", + "video_1175/video_1175_15.jpeg", + "video_1175/video_1175_16.jpeg", + "video_1175/video_1175_17.jpeg", + "video_1175/video_1175_18.jpeg", + "video_1175/video_1175_19.jpeg", + "video_1175/video_1175_20.jpeg", + "video_1175/video_1175_21.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 124, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. csgz\nB. rsvp\nC. osvd\nAnswer with the option's letter from the given choices directly.", + "gt_response": "rsvp", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "csgz", + "rsvp", + "osvd" + ], + "image_quantity_level": "Medium", + "image": [ + "video_722/video_722_0.jpeg", + "video_722/video_722_1.jpeg", + "video_722/video_722_2.jpeg", + "video_722/video_722_3.jpeg", + "video_722/video_722_4.jpeg", + "video_722/video_722_5.jpeg", + "video_722/video_722_6.jpeg", + "video_722/video_722_7.jpeg", + "video_722/video_722_8.jpeg", + "video_722/video_722_9.jpeg", + "video_722/video_722_10.jpeg", + "video_722/video_722_11.jpeg", + "video_722/video_722_12.jpeg", + "video_722/video_722_13.jpeg", + "video_722/video_722_14.jpeg", + "video_722/video_722_15.jpeg", + "video_722/video_722_16.jpeg", + "video_722/video_722_17.jpeg", + "video_722/video_722_18.jpeg", + "video_722/video_722_19.jpeg", + "video_722/video_722_20.jpeg", + "video_722/video_722_21.jpeg", + "video_722/video_722_22.jpeg", + "video_722/video_722_23.jpeg", + "video_722/video_722_24.jpeg", + "video_722/video_722_25.jpeg", + "video_722/video_722_26.jpeg", + "video_722/video_722_27.jpeg", + "video_722/video_722_28.jpeg", + "video_722/video_722_29.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 55, + "question": "Scrutinize the provided visuals and answer the question about the arrangement of characters. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. iwt\nB. wit\nC. wyt\nAnswer with the option's letter from the given choices directly.", + "gt_response": "wit", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "iwt", + "wit", + "wyt" + ], + "image_quantity_level": "Medium", + "image": [ + "video_4861/video_4861_0.jpeg", + "video_4861/video_4861_1.jpeg", + "video_4861/video_4861_2.jpeg", + "video_4861/video_4861_3.jpeg", + "video_4861/video_4861_4.jpeg", + "video_4861/video_4861_5.jpeg", + "video_4861/video_4861_6.jpeg", + "video_4861/video_4861_7.jpeg", + "video_4861/video_4861_8.jpeg", + "video_4861/video_4861_9.jpeg", + "video_4861/video_4861_10.jpeg", + "video_4861/video_4861_11.jpeg", + "video_4861/video_4861_12.jpeg", + "video_4861/video_4861_13.jpeg", + "video_4861/video_4861_14.jpeg", + "video_4861/video_4861_15.jpeg", + "video_4861/video_4861_16.jpeg", + "video_4861/video_4861_17.jpeg", + "video_4861/video_4861_18.jpeg", + "video_4861/video_4861_19.jpeg", + "video_4861/video_4861_20.jpeg", + "video_4861/video_4861_21.jpeg", + "video_4861/video_4861_22.jpeg", + "video_4861/video_4861_23.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 57, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. pmg\nB. prk\nC. pmt\nAnswer with the option's letter from the given choices directly.", + "gt_response": "pmg", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "pmg", + "prk", + "pmt" + ], + "image_quantity_level": "Medium", + "image": [ + "video_6844/video_6844_0.jpeg", + "video_6844/video_6844_1.jpeg", + "video_6844/video_6844_2.jpeg", + "video_6844/video_6844_3.jpeg", + "video_6844/video_6844_4.jpeg", + "video_6844/video_6844_5.jpeg", + "video_6844/video_6844_6.jpeg", + "video_6844/video_6844_7.jpeg", + "video_6844/video_6844_8.jpeg", + "video_6844/video_6844_9.jpeg", + "video_6844/video_6844_10.jpeg", + "video_6844/video_6844_11.jpeg", + "video_6844/video_6844_12.jpeg", + "video_6844/video_6844_13.jpeg", + "video_6844/video_6844_14.jpeg", + "video_6844/video_6844_15.jpeg", + "video_6844/video_6844_16.jpeg", + "video_6844/video_6844_17.jpeg", + "video_6844/video_6844_18.jpeg", + "video_6844/video_6844_19.jpeg", + "video_6844/video_6844_20.jpeg", + "video_6844/video_6844_21.jpeg", + "video_6844/video_6844_22.jpeg", + "video_6844/video_6844_23.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 83, + "question": "Analyze the given visuals and answer the question about the order of character. You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. o\nB. v\nC. l\nAnswer with the option's letter from the given choices directly.", + "gt_response": "l", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "o", + "v", + "l" + ], + "image_quantity_level": "Medium", + "image": [ + "video_2248/video_2248_0.jpeg", + "video_2248/video_2248_1.jpeg", + "video_2248/video_2248_2.jpeg", + "video_2248/video_2248_3.jpeg", + "video_2248/video_2248_4.jpeg", + "video_2248/video_2248_5.jpeg", + "video_2248/video_2248_6.jpeg", + "video_2248/video_2248_7.jpeg", + "video_2248/video_2248_8.jpeg", + "video_2248/video_2248_9.jpeg", + "video_2248/video_2248_10.jpeg", + "video_2248/video_2248_11.jpeg", + "video_2248/video_2248_12.jpeg", + "video_2248/video_2248_13.jpeg", + "video_2248/video_2248_14.jpeg", + "video_2248/video_2248_15.jpeg", + "video_2248/video_2248_16.jpeg", + "video_2248/video_2248_17.jpeg", + "video_2248/video_2248_18.jpeg", + "video_2248/video_2248_19.jpeg", + "video_2248/video_2248_20.jpeg", + "video_2248/video_2248_21.jpeg", + "video_2248/video_2248_22.jpeg", + "video_2248/video_2248_23.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 85, + "question": "Scrutinize the provided visuals and answer the question about the arrangement of characters. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. bcdef\nB. krdvl\nC. bsdhf\nAnswer with the option's letter from the given choices directly.", + "gt_response": "bcdef", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "bcdef", + "krdvl", + "bsdhf" + ], + "image_quantity_level": "Many", + "image": [ + "video_5578/video_5578_0.jpeg", + "video_5578/video_5578_1.jpeg", + "video_5578/video_5578_2.jpeg", + "video_5578/video_5578_3.jpeg", + "video_5578/video_5578_4.jpeg", + "video_5578/video_5578_5.jpeg", + "video_5578/video_5578_6.jpeg", + "video_5578/video_5578_7.jpeg", + "video_5578/video_5578_8.jpeg", + "video_5578/video_5578_9.jpeg", + "video_5578/video_5578_10.jpeg", + "video_5578/video_5578_11.jpeg", + "video_5578/video_5578_12.jpeg", + "video_5578/video_5578_13.jpeg", + "video_5578/video_5578_14.jpeg", + "video_5578/video_5578_15.jpeg", + "video_5578/video_5578_16.jpeg", + "video_5578/video_5578_17.jpeg", + "video_5578/video_5578_18.jpeg", + "video_5578/video_5578_19.jpeg", + "video_5578/video_5578_20.jpeg", + "video_5578/video_5578_21.jpeg", + "video_5578/video_5578_22.jpeg", + "video_5578/video_5578_23.jpeg", + "video_5578/video_5578_24.jpeg", + "video_5578/video_5578_25.jpeg", + "video_5578/video_5578_26.jpeg", + "video_5578/video_5578_27.jpeg", + "video_5578/video_5578_28.jpeg", + "video_5578/video_5578_29.jpeg", + "video_5578/video_5578_30.jpeg", + "video_5578/video_5578_31.jpeg", + "video_5578/video_5578_32.jpeg", + "video_5578/video_5578_33.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 86, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. mer\nB. emr\nC. rme\nAnswer with the option's letter from the given choices directly.", + "gt_response": "emr", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "mer", + "emr", + "rme" + ], + "image_quantity_level": "Many", + "image": [ + "video_567/video_567_0.jpeg", + "video_567/video_567_1.jpeg", + "video_567/video_567_2.jpeg", + "video_567/video_567_3.jpeg", + "video_567/video_567_4.jpeg", + "video_567/video_567_5.jpeg", + "video_567/video_567_6.jpeg", + "video_567/video_567_7.jpeg", + "video_567/video_567_8.jpeg", + "video_567/video_567_9.jpeg", + "video_567/video_567_10.jpeg", + "video_567/video_567_11.jpeg", + "video_567/video_567_12.jpeg", + "video_567/video_567_13.jpeg", + "video_567/video_567_14.jpeg", + "video_567/video_567_15.jpeg", + "video_567/video_567_16.jpeg", + "video_567/video_567_17.jpeg", + "video_567/video_567_18.jpeg", + "video_567/video_567_19.jpeg", + "video_567/video_567_20.jpeg", + "video_567/video_567_21.jpeg", + "video_567/video_567_22.jpeg", + "video_567/video_567_23.jpeg", + "video_567/video_567_24.jpeg", + "video_567/video_567_25.jpeg", + "video_567/video_567_26.jpeg", + "video_567/video_567_27.jpeg", + "video_567/video_567_28.jpeg", + "video_567/video_567_29.jpeg", + "video_567/video_567_30.jpeg", + "video_567/video_567_31.jpeg", + "video_567/video_567_32.jpeg", + "video_567/video_567_33.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 92, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nWhat letter is likely to be shown next?\nChoice list: \nA. r\nB. f\nC. k\nAnswer with the option's letter from the given choices directly.", + "gt_response": "f", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "r", + "f", + "k" + ], + "image_quantity_level": "Many", + "image": [ + "video_3161/video_3161_0.jpeg", + "video_3161/video_3161_1.jpeg", + "video_3161/video_3161_2.jpeg", + "video_3161/video_3161_3.jpeg", + "video_3161/video_3161_4.jpeg", + "video_3161/video_3161_5.jpeg", + "video_3161/video_3161_6.jpeg", + "video_3161/video_3161_7.jpeg", + "video_3161/video_3161_8.jpeg", + "video_3161/video_3161_9.jpeg", + "video_3161/video_3161_10.jpeg", + "video_3161/video_3161_11.jpeg", + "video_3161/video_3161_12.jpeg", + "video_3161/video_3161_13.jpeg", + "video_3161/video_3161_14.jpeg", + "video_3161/video_3161_15.jpeg", + "video_3161/video_3161_16.jpeg", + "video_3161/video_3161_17.jpeg", + "video_3161/video_3161_18.jpeg", + "video_3161/video_3161_19.jpeg", + "video_3161/video_3161_20.jpeg", + "video_3161/video_3161_21.jpeg", + "video_3161/video_3161_22.jpeg", + "video_3161/video_3161_23.jpeg", + "video_3161/video_3161_24.jpeg", + "video_3161/video_3161_25.jpeg", + "video_3161/video_3161_26.jpeg", + "video_3161/video_3161_27.jpeg", + "video_3161/video_3161_28.jpeg", + "video_3161/video_3161_29.jpeg", + "video_3161/video_3161_30.jpeg", + "video_3161/video_3161_31.jpeg", + "video_3161/video_3161_32.jpeg", + "video_3161/video_3161_33.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 101, + "question": "Analyze the given visuals and answer the question about the order of character. You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. e\nB. b\nC. d\nAnswer with the option's letter from the given choices directly.", + "gt_response": "e", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "e", + "b", + "d" + ], + "image_quantity_level": "Many", + "image": [ + "video_4559/video_4559_0.jpeg", + "video_4559/video_4559_1.jpeg", + "video_4559/video_4559_2.jpeg", + "video_4559/video_4559_3.jpeg", + "video_4559/video_4559_4.jpeg", + "video_4559/video_4559_5.jpeg", + "video_4559/video_4559_6.jpeg", + "video_4559/video_4559_7.jpeg", + "video_4559/video_4559_8.jpeg", + "video_4559/video_4559_9.jpeg", + "video_4559/video_4559_10.jpeg", + "video_4559/video_4559_11.jpeg", + "video_4559/video_4559_12.jpeg", + "video_4559/video_4559_13.jpeg", + "video_4559/video_4559_14.jpeg", + "video_4559/video_4559_15.jpeg", + "video_4559/video_4559_16.jpeg", + "video_4559/video_4559_17.jpeg", + "video_4559/video_4559_18.jpeg", + "video_4559/video_4559_19.jpeg", + "video_4559/video_4559_20.jpeg", + "video_4559/video_4559_21.jpeg", + "video_4559/video_4559_22.jpeg", + "video_4559/video_4559_23.jpeg", + "video_4559/video_4559_24.jpeg", + "video_4559/video_4559_25.jpeg", + "video_4559/video_4559_26.jpeg", + "video_4559/video_4559_27.jpeg", + "video_4559/video_4559_28.jpeg", + "video_4559/video_4559_29.jpeg", + "video_4559/video_4559_30.jpeg", + "video_4559/video_4559_31.jpeg", + "video_4559/video_4559_32.jpeg", + "video_4559/video_4559_33.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 114, + "question": "Analyze the given visuals and answer the question about the order of character. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. tea\nB. ate\nC. eat\nAnswer with the option's letter from the given choices directly.", + "gt_response": "eat", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "tea", + "ate", + "eat" + ], + "image_quantity_level": "Many", + "image": [ + "video_8754/video_8754_0.jpeg", + "video_8754/video_8754_1.jpeg", + "video_8754/video_8754_2.jpeg", + "video_8754/video_8754_3.jpeg", + "video_8754/video_8754_4.jpeg", + "video_8754/video_8754_5.jpeg", + "video_8754/video_8754_6.jpeg", + "video_8754/video_8754_7.jpeg", + "video_8754/video_8754_8.jpeg", + "video_8754/video_8754_9.jpeg", + "video_8754/video_8754_10.jpeg", + "video_8754/video_8754_11.jpeg", + "video_8754/video_8754_12.jpeg", + "video_8754/video_8754_13.jpeg", + "video_8754/video_8754_14.jpeg", + "video_8754/video_8754_15.jpeg", + "video_8754/video_8754_16.jpeg", + "video_8754/video_8754_17.jpeg", + "video_8754/video_8754_18.jpeg", + "video_8754/video_8754_19.jpeg", + "video_8754/video_8754_20.jpeg", + "video_8754/video_8754_21.jpeg", + "video_8754/video_8754_22.jpeg", + "video_8754/video_8754_23.jpeg", + "video_8754/video_8754_24.jpeg", + "video_8754/video_8754_25.jpeg", + "video_8754/video_8754_26.jpeg", + "video_8754/video_8754_27.jpeg", + "video_8754/video_8754_28.jpeg", + "video_8754/video_8754_29.jpeg", + "video_8754/video_8754_30.jpeg", + "video_8754/video_8754_31.jpeg", + "video_8754/video_8754_32.jpeg", + "video_8754/video_8754_33.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 120, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. hoyc\nB. hyoc\nC. cyoh\nAnswer with the option's letter from the given choices directly.", + "gt_response": "hoyc", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "hoyc", + "hyoc", + "cyoh" + ], + "image_quantity_level": "Many", + "image": [ + "video_203/video_203_0.jpeg", + "video_203/video_203_1.jpeg", + "video_203/video_203_2.jpeg", + "video_203/video_203_3.jpeg", + "video_203/video_203_4.jpeg", + "video_203/video_203_5.jpeg", + "video_203/video_203_6.jpeg", + "video_203/video_203_7.jpeg", + "video_203/video_203_8.jpeg", + "video_203/video_203_9.jpeg", + "video_203/video_203_10.jpeg", + "video_203/video_203_11.jpeg", + "video_203/video_203_12.jpeg", + "video_203/video_203_13.jpeg", + "video_203/video_203_14.jpeg", + "video_203/video_203_15.jpeg", + "video_203/video_203_16.jpeg", + "video_203/video_203_17.jpeg", + "video_203/video_203_18.jpeg", + "video_203/video_203_19.jpeg", + "video_203/video_203_20.jpeg", + "video_203/video_203_21.jpeg", + "video_203/video_203_22.jpeg", + "video_203/video_203_23.jpeg", + "video_203/video_203_24.jpeg", + "video_203/video_203_25.jpeg", + "video_203/video_203_26.jpeg", + "video_203/video_203_27.jpeg", + "video_203/video_203_28.jpeg", + "video_203/video_203_29.jpeg", + "video_203/video_203_30.jpeg", + "video_203/video_203_31.jpeg", + "video_203/video_203_32.jpeg", + "video_203/video_203_33.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 125, + "question": "Analyze the given visuals and answer the question about the order of character. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. daobgc\nB. dwrbgc\nC. dqobgc\nAnswer with the option's letter from the given choices directly.", + "gt_response": "daobgc", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "daobgc", + "dwrbgc", + "dqobgc" + ], + "image_quantity_level": "Many", + "image": [ + "video_2142/video_2142_0.jpeg", + "video_2142/video_2142_1.jpeg", + "video_2142/video_2142_2.jpeg", + "video_2142/video_2142_3.jpeg", + "video_2142/video_2142_4.jpeg", + "video_2142/video_2142_5.jpeg", + "video_2142/video_2142_6.jpeg", + "video_2142/video_2142_7.jpeg", + "video_2142/video_2142_8.jpeg", + "video_2142/video_2142_9.jpeg", + "video_2142/video_2142_10.jpeg", + "video_2142/video_2142_11.jpeg", + "video_2142/video_2142_12.jpeg", + "video_2142/video_2142_13.jpeg", + "video_2142/video_2142_14.jpeg", + "video_2142/video_2142_15.jpeg", + "video_2142/video_2142_16.jpeg", + "video_2142/video_2142_17.jpeg", + "video_2142/video_2142_18.jpeg", + "video_2142/video_2142_19.jpeg", + "video_2142/video_2142_20.jpeg", + "video_2142/video_2142_21.jpeg", + "video_2142/video_2142_22.jpeg", + "video_2142/video_2142_23.jpeg", + "video_2142/video_2142_24.jpeg", + "video_2142/video_2142_25.jpeg", + "video_2142/video_2142_26.jpeg", + "video_2142/video_2142_27.jpeg", + "video_2142/video_2142_28.jpeg", + "video_2142/video_2142_29.jpeg", + "video_2142/video_2142_30.jpeg", + "video_2142/video_2142_31.jpeg", + "video_2142/video_2142_32.jpeg", + "video_2142/video_2142_33.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 144, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nWhat was the order of the letters on the table before shuffling?\nChoice list: \nA. hanhdm\nB. hdhman\nC. hhdamn\nAnswer with the option's letter from the given choices directly.", + "gt_response": "hhdamn", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "hanhdm", + "hdhman", + "hhdamn" + ], + "image_quantity_level": "Medium", + "image": [ + "video_8155/video_8155_0.jpeg", + "video_8155/video_8155_1.jpeg", + "video_8155/video_8155_2.jpeg", + "video_8155/video_8155_3.jpeg", + "video_8155/video_8155_4.jpeg", + "video_8155/video_8155_5.jpeg", + "video_8155/video_8155_6.jpeg", + "video_8155/video_8155_7.jpeg", + "video_8155/video_8155_8.jpeg", + "video_8155/video_8155_9.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 90, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. tism\nB. xkqm\nC. thsm\nAnswer with the option's letter from the given choices directly.", + "gt_response": "tism", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "tism", + "xkqm", + "thsm" + ], + "image_quantity_level": "Medium", + "image": [ + "video_1012/video_1012_0.jpeg", + "video_1012/video_1012_1.jpeg", + "video_1012/video_1012_2.jpeg", + "video_1012/video_1012_3.jpeg", + "video_1012/video_1012_4.jpeg", + "video_1012/video_1012_5.jpeg", + "video_1012/video_1012_6.jpeg", + "video_1012/video_1012_7.jpeg", + "video_1012/video_1012_8.jpeg", + "video_1012/video_1012_9.jpeg", + "video_1012/video_1012_10.jpeg", + "video_1012/video_1012_11.jpeg", + "video_1012/video_1012_12.jpeg", + "video_1012/video_1012_13.jpeg", + "video_1012/video_1012_14.jpeg", + "video_1012/video_1012_15.jpeg", + "video_1012/video_1012_16.jpeg", + "video_1012/video_1012_17.jpeg", + "video_1012/video_1012_18.jpeg", + "video_1012/video_1012_19.jpeg", + "video_1012/video_1012_20.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 126, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat would be the order of the written letters if the person had written them in reverse order?\nChoice list: \nA. nbw\nB. niw\nC. win\nAnswer with the option's letter from the given choices directly.", + "gt_response": "niw", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "nbw", + "niw", + "win" + ], + "image_quantity_level": "Medium", + "image": [ + "video_2277/video_2277_0.jpeg", + "video_2277/video_2277_1.jpeg", + "video_2277/video_2277_2.jpeg", + "video_2277/video_2277_3.jpeg", + "video_2277/video_2277_4.jpeg", + "video_2277/video_2277_5.jpeg", + "video_2277/video_2277_6.jpeg", + "video_2277/video_2277_7.jpeg", + "video_2277/video_2277_8.jpeg", + "video_2277/video_2277_9.jpeg", + "video_2277/video_2277_10.jpeg", + "video_2277/video_2277_11.jpeg", + "video_2277/video_2277_12.jpeg", + "video_2277/video_2277_13.jpeg", + "video_2277/video_2277_14.jpeg", + "video_2277/video_2277_15.jpeg", + "video_2277/video_2277_16.jpeg", + "video_2277/video_2277_17.jpeg", + "video_2277/video_2277_18.jpeg", + "video_2277/video_2277_19.jpeg", + "video_2277/video_2277_20.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 111, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. o\nB. v\nC. l\nAnswer with the option's letter from the given choices directly.", + "gt_response": "o", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "o", + "v", + "l" + ], + "image_quantity_level": "Medium", + "image": [ + "video_11021/video_11021_0.jpeg", + "video_11021/video_11021_1.jpeg", + "video_11021/video_11021_2.jpeg", + "video_11021/video_11021_3.jpeg", + "video_11021/video_11021_4.jpeg", + "video_11021/video_11021_5.jpeg", + "video_11021/video_11021_6.jpeg", + "video_11021/video_11021_7.jpeg", + "video_11021/video_11021_8.jpeg", + "video_11021/video_11021_9.jpeg", + "video_11021/video_11021_10.jpeg", + "video_11021/video_11021_11.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 78, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. e\nB. m\nC. o\nAnswer with the option's letter from the given choices directly.", + "gt_response": "o", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "e", + "m", + "o" + ], + "image_quantity_level": "Many", + "image": [ + "video_6696/video_6696_0.jpeg", + "video_6696/video_6696_1.jpeg", + "video_6696/video_6696_2.jpeg", + "video_6696/video_6696_3.jpeg", + "video_6696/video_6696_4.jpeg", + "video_6696/video_6696_5.jpeg", + "video_6696/video_6696_6.jpeg", + "video_6696/video_6696_7.jpeg", + "video_6696/video_6696_8.jpeg", + "video_6696/video_6696_9.jpeg", + "video_6696/video_6696_10.jpeg", + "video_6696/video_6696_11.jpeg", + "video_6696/video_6696_12.jpeg", + "video_6696/video_6696_13.jpeg", + "video_6696/video_6696_14.jpeg", + "video_6696/video_6696_15.jpeg", + "video_6696/video_6696_16.jpeg", + "video_6696/video_6696_17.jpeg", + "video_6696/video_6696_18.jpeg", + "video_6696/video_6696_19.jpeg", + "video_6696/video_6696_20.jpeg", + "video_6696/video_6696_21.jpeg", + "video_6696/video_6696_22.jpeg", + "video_6696/video_6696_23.jpeg", + "video_6696/video_6696_24.jpeg", + "video_6696/video_6696_25.jpeg", + "video_6696/video_6696_26.jpeg", + "video_6696/video_6696_27.jpeg", + "video_6696/video_6696_28.jpeg", + "video_6696/video_6696_29.jpeg", + "video_6696/video_6696_30.jpeg", + "video_6696/video_6696_31.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 88, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. gra\nB. arg\nC. rag\nAnswer with the option's letter from the given choices directly.", + "gt_response": "gra", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "gra", + "arg", + "rag" + ], + "image_quantity_level": "Many", + "image": [ + "video_7982/video_7982_0.jpeg", + "video_7982/video_7982_1.jpeg", + "video_7982/video_7982_2.jpeg", + "video_7982/video_7982_3.jpeg", + "video_7982/video_7982_4.jpeg", + "video_7982/video_7982_5.jpeg", + "video_7982/video_7982_6.jpeg", + "video_7982/video_7982_7.jpeg", + "video_7982/video_7982_8.jpeg", + "video_7982/video_7982_9.jpeg", + "video_7982/video_7982_10.jpeg", + "video_7982/video_7982_11.jpeg", + "video_7982/video_7982_12.jpeg", + "video_7982/video_7982_13.jpeg", + "video_7982/video_7982_14.jpeg", + "video_7982/video_7982_15.jpeg", + "video_7982/video_7982_16.jpeg", + "video_7982/video_7982_17.jpeg", + "video_7982/video_7982_18.jpeg", + "video_7982/video_7982_19.jpeg", + "video_7982/video_7982_20.jpeg", + "video_7982/video_7982_21.jpeg", + "video_7982/video_7982_22.jpeg", + "video_7982/video_7982_23.jpeg", + "video_7982/video_7982_24.jpeg", + "video_7982/video_7982_25.jpeg", + "video_7982/video_7982_26.jpeg", + "video_7982/video_7982_27.jpeg", + "video_7982/video_7982_28.jpeg", + "video_7982/video_7982_29.jpeg", + "video_7982/video_7982_30.jpeg", + "video_7982/video_7982_31.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 98, + "question": "From the images supplied, answer the question relevant to character order. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. abchij\nB. lpczqj\nC. angyrz\nAnswer with the option's letter from the given choices directly.", + "gt_response": "abchij", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "abchij", + "lpczqj", + "angyrz" + ], + "image_quantity_level": "Many", + "image": [ + "video_3863/video_3863_0.jpeg", + "video_3863/video_3863_1.jpeg", + "video_3863/video_3863_2.jpeg", + "video_3863/video_3863_3.jpeg", + "video_3863/video_3863_4.jpeg", + "video_3863/video_3863_5.jpeg", + "video_3863/video_3863_6.jpeg", + "video_3863/video_3863_7.jpeg", + "video_3863/video_3863_8.jpeg", + "video_3863/video_3863_9.jpeg", + "video_3863/video_3863_10.jpeg", + "video_3863/video_3863_11.jpeg", + "video_3863/video_3863_12.jpeg", + "video_3863/video_3863_13.jpeg", + "video_3863/video_3863_14.jpeg", + "video_3863/video_3863_15.jpeg", + "video_3863/video_3863_16.jpeg", + "video_3863/video_3863_17.jpeg", + "video_3863/video_3863_18.jpeg", + "video_3863/video_3863_19.jpeg", + "video_3863/video_3863_20.jpeg", + "video_3863/video_3863_21.jpeg", + "video_3863/video_3863_22.jpeg", + "video_3863/video_3863_23.jpeg", + "video_3863/video_3863_24.jpeg", + "video_3863/video_3863_25.jpeg", + "video_3863/video_3863_26.jpeg", + "video_3863/video_3863_27.jpeg", + "video_3863/video_3863_28.jpeg", + "video_3863/video_3863_29.jpeg", + "video_3863/video_3863_30.jpeg", + "video_3863/video_3863_31.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 116, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. hec\nB. che\nC. hce\nAnswer with the option's letter from the given choices directly.", + "gt_response": "hec", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "hec", + "che", + "hce" + ], + "image_quantity_level": "Many", + "image": [ + "video_477/video_477_0.jpeg", + "video_477/video_477_1.jpeg", + "video_477/video_477_2.jpeg", + "video_477/video_477_3.jpeg", + "video_477/video_477_4.jpeg", + "video_477/video_477_5.jpeg", + "video_477/video_477_6.jpeg", + "video_477/video_477_7.jpeg", + "video_477/video_477_8.jpeg", + "video_477/video_477_9.jpeg", + "video_477/video_477_10.jpeg", + "video_477/video_477_11.jpeg", + "video_477/video_477_12.jpeg", + "video_477/video_477_13.jpeg", + "video_477/video_477_14.jpeg", + "video_477/video_477_15.jpeg", + "video_477/video_477_16.jpeg", + "video_477/video_477_17.jpeg", + "video_477/video_477_18.jpeg", + "video_477/video_477_19.jpeg", + "video_477/video_477_20.jpeg", + "video_477/video_477_21.jpeg", + "video_477/video_477_22.jpeg", + "video_477/video_477_23.jpeg", + "video_477/video_477_24.jpeg", + "video_477/video_477_25.jpeg", + "video_477/video_477_26.jpeg", + "video_477/video_477_27.jpeg", + "video_477/video_477_28.jpeg", + "video_477/video_477_29.jpeg", + "video_477/video_477_30.jpeg", + "video_477/video_477_31.jpeg", + "video_477/video_477_32.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 95, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. hji\nB. hij\nC. doj\nAnswer with the option's letter from the given choices directly.", + "gt_response": "hij", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "hji", + "hij", + "doj" + ], + "image_quantity_level": "Medium", + "image": [ + "video_10565/video_10565_0.jpeg", + "video_10565/video_10565_1.jpeg", + "video_10565/video_10565_2.jpeg", + "video_10565/video_10565_3.jpeg", + "video_10565/video_10565_4.jpeg", + "video_10565/video_10565_5.jpeg", + "video_10565/video_10565_6.jpeg", + "video_10565/video_10565_7.jpeg", + "video_10565/video_10565_8.jpeg", + "video_10565/video_10565_9.jpeg", + "video_10565/video_10565_10.jpeg", + "video_10565/video_10565_11.jpeg", + "video_10565/video_10565_12.jpeg", + "video_10565/video_10565_13.jpeg", + "video_10565/video_10565_14.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 113, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nWhat letters did the person type on the computer in order?\nChoice list: \nA. lmae\nB. lame\nC. kame\nAnswer with the option's letter from the given choices directly.", + "gt_response": "lame", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "lmae", + "lame", + "kame" + ], + "image_quantity_level": "Medium", + "image": [ + "video_9617/video_9617_0.jpeg", + "video_9617/video_9617_1.jpeg", + "video_9617/video_9617_2.jpeg", + "video_9617/video_9617_3.jpeg", + "video_9617/video_9617_4.jpeg", + "video_9617/video_9617_5.jpeg", + "video_9617/video_9617_6.jpeg", + "video_9617/video_9617_7.jpeg", + "video_9617/video_9617_8.jpeg", + "video_9617/video_9617_9.jpeg", + "video_9617/video_9617_10.jpeg", + "video_9617/video_9617_11.jpeg", + "video_9617/video_9617_12.jpeg", + "video_9617/video_9617_13.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 152, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. xvc\nB. tec\nC. xuc\nAnswer with the option's letter from the given choices directly.", + "gt_response": "xvc", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "xvc", + "tec", + "xuc" + ], + "image_quantity_level": "Medium", + "image": [ + "video_11563/video_11563_0.jpeg", + "video_11563/video_11563_1.jpeg", + "video_11563/video_11563_2.jpeg", + "video_11563/video_11563_3.jpeg", + "video_11563/video_11563_4.jpeg", + "video_11563/video_11563_5.jpeg", + "video_11563/video_11563_6.jpeg", + "video_11563/video_11563_7.jpeg", + "video_11563/video_11563_8.jpeg", + "video_11563/video_11563_9.jpeg", + "video_11563/video_11563_10.jpeg", + "video_11563/video_11563_11.jpeg", + "video_11563/video_11563_12.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 108, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. t\nB. r\nC. o\nAnswer with the option's letter from the given choices directly.", + "gt_response": "r", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "t", + "r", + "o" + ], + "image_quantity_level": "Medium", + "image": [ + "video_1114/video_1114_0.jpeg", + "video_1114/video_1114_1.jpeg", + "video_1114/video_1114_2.jpeg", + "video_1114/video_1114_3.jpeg", + "video_1114/video_1114_4.jpeg", + "video_1114/video_1114_5.jpeg", + "video_1114/video_1114_6.jpeg", + "video_1114/video_1114_7.jpeg", + "video_1114/video_1114_8.jpeg", + "video_1114/video_1114_9.jpeg", + "video_1114/video_1114_10.jpeg", + "video_1114/video_1114_11.jpeg", + "video_1114/video_1114_12.jpeg", + "video_1114/video_1114_13.jpeg", + "video_1114/video_1114_14.jpeg", + "video_1114/video_1114_15.jpeg", + "video_1114/video_1114_16.jpeg", + "video_1114/video_1114_17.jpeg", + "video_1114/video_1114_18.jpeg", + "video_1114/video_1114_19.jpeg", + "video_1114/video_1114_20.jpeg", + "video_1114/video_1114_21.jpeg", + "video_1114/video_1114_22.jpeg", + "video_1114/video_1114_23.jpeg", + "video_1114/video_1114_24.jpeg", + "video_1114/video_1114_25.jpeg", + "video_1114/video_1114_26.jpeg", + "video_1114/video_1114_27.jpeg", + "video_1114/video_1114_28.jpeg", + "video_1114/video_1114_29.jpeg", + "video_1114/video_1114_30.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 115, + "question": "Review the visuals provided and respond to the question concerning the character sequence. You must choose your answer from the Choice List.\nWhat was the order of the letters on the table before shuffling?\nChoice list: \nA. bfcade\nB. afcdbe\nC. abcdef\nAnswer with the option's letter from the given choices directly.", + "gt_response": "abcdef", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "bfcade", + "afcdbe", + "abcdef" + ], + "image_quantity_level": "Medium", + "image": [ + "video_3557/video_3557_0.jpeg", + "video_3557/video_3557_1.jpeg", + "video_3557/video_3557_2.jpeg", + "video_3557/video_3557_3.jpeg", + "video_3557/video_3557_4.jpeg", + "video_3557/video_3557_5.jpeg", + "video_3557/video_3557_6.jpeg", + "video_3557/video_3557_7.jpeg", + "video_3557/video_3557_8.jpeg", + "video_3557/video_3557_9.jpeg", + "video_3557/video_3557_10.jpeg", + "video_3557/video_3557_11.jpeg", + "video_3557/video_3557_12.jpeg", + "video_3557/video_3557_13.jpeg", + "video_3557/video_3557_14.jpeg", + "video_3557/video_3557_15.jpeg", + "video_3557/video_3557_16.jpeg", + "video_3557/video_3557_17.jpeg", + "video_3557/video_3557_18.jpeg", + "video_3557/video_3557_19.jpeg", + "video_3557/video_3557_20.jpeg", + "video_3557/video_3557_21.jpeg", + "video_3557/video_3557_22.jpeg", + "video_3557/video_3557_23.jpeg", + "video_3557/video_3557_24.jpeg", + "video_3557/video_3557_25.jpeg", + "video_3557/video_3557_26.jpeg", + "video_3557/video_3557_27.jpeg", + "video_3557/video_3557_28.jpeg", + "video_3557/video_3557_29.jpeg", + "video_3557/video_3557_30.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 154, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nWhat would be the order of the written letters if the person had written them in reverse order?\nChoice list: \nA. jfau\nB. ecab\nC. bace\nAnswer with the option's letter from the given choices directly.", + "gt_response": "ecab", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "jfau", + "ecab", + "bace" + ], + "image_quantity_level": "Medium", + "image": [ + "video_6338/video_6338_0.jpeg", + "video_6338/video_6338_1.jpeg", + "video_6338/video_6338_2.jpeg", + "video_6338/video_6338_3.jpeg", + "video_6338/video_6338_4.jpeg", + "video_6338/video_6338_5.jpeg", + "video_6338/video_6338_6.jpeg", + "video_6338/video_6338_7.jpeg", + "video_6338/video_6338_8.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 56, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. muse\nB. mute\nC. etum\nAnswer with the option's letter from the given choices directly.", + "gt_response": "mute", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "muse", + "mute", + "etum" + ], + "image_quantity_level": "Medium", + "image": [ + "video_7210/video_7210_0.jpeg", + "video_7210/video_7210_1.jpeg", + "video_7210/video_7210_2.jpeg", + "video_7210/video_7210_3.jpeg", + "video_7210/video_7210_4.jpeg", + "video_7210/video_7210_5.jpeg", + "video_7210/video_7210_6.jpeg", + "video_7210/video_7210_7.jpeg", + "video_7210/video_7210_8.jpeg", + "video_7210/video_7210_9.jpeg", + "video_7210/video_7210_10.jpeg", + "video_7210/video_7210_11.jpeg", + "video_7210/video_7210_12.jpeg", + "video_7210/video_7210_13.jpeg", + "video_7210/video_7210_14.jpeg", + "video_7210/video_7210_15.jpeg", + "video_7210/video_7210_16.jpeg", + "video_7210/video_7210_17.jpeg", + "video_7210/video_7210_18.jpeg", + "video_7210/video_7210_19.jpeg", + "video_7210/video_7210_20.jpeg", + "video_7210/video_7210_21.jpeg", + "video_7210/video_7210_22.jpeg", + "video_7210/video_7210_23.jpeg", + "video_7210/video_7210_24.jpeg", + "video_7210/video_7210_25.jpeg", + "video_7210/video_7210_26.jpeg", + "video_7210/video_7210_27.jpeg", + "video_7210/video_7210_28.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 110, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. o\nB. u\nC. y\nAnswer with the option's letter from the given choices directly.", + "gt_response": "o", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "o", + "u", + "y" + ], + "image_quantity_level": "Medium", + "image": [ + "video_4112/video_4112_0.jpeg", + "video_4112/video_4112_1.jpeg", + "video_4112/video_4112_2.jpeg", + "video_4112/video_4112_3.jpeg", + "video_4112/video_4112_4.jpeg", + "video_4112/video_4112_5.jpeg", + "video_4112/video_4112_6.jpeg", + "video_4112/video_4112_7.jpeg", + "video_4112/video_4112_8.jpeg", + "video_4112/video_4112_9.jpeg", + "video_4112/video_4112_10.jpeg", + "video_4112/video_4112_11.jpeg", + "video_4112/video_4112_12.jpeg", + "video_4112/video_4112_13.jpeg", + "video_4112/video_4112_14.jpeg", + "video_4112/video_4112_15.jpeg", + "video_4112/video_4112_16.jpeg", + "video_4112/video_4112_17.jpeg", + "video_4112/video_4112_18.jpeg", + "video_4112/video_4112_19.jpeg", + "video_4112/video_4112_20.jpeg", + "video_4112/video_4112_21.jpeg", + "video_4112/video_4112_22.jpeg", + "video_4112/video_4112_23.jpeg", + "video_4112/video_4112_24.jpeg", + "video_4112/video_4112_25.jpeg", + "video_4112/video_4112_26.jpeg", + "video_4112/video_4112_27.jpeg", + "video_4112/video_4112_28.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 81, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. emn\nB. nme\nC. noe\nAnswer with the option's letter from the given choices directly.", + "gt_response": "nme", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "emn", + "nme", + "noe" + ], + "image_quantity_level": "Medium", + "image": [ + "video_7294/video_7294_0.jpeg", + "video_7294/video_7294_1.jpeg", + "video_7294/video_7294_2.jpeg", + "video_7294/video_7294_3.jpeg", + "video_7294/video_7294_4.jpeg", + "video_7294/video_7294_5.jpeg", + "video_7294/video_7294_6.jpeg", + "video_7294/video_7294_7.jpeg", + "video_7294/video_7294_8.jpeg", + "video_7294/video_7294_9.jpeg", + "video_7294/video_7294_10.jpeg", + "video_7294/video_7294_11.jpeg", + "video_7294/video_7294_12.jpeg", + "video_7294/video_7294_13.jpeg", + "video_7294/video_7294_14.jpeg", + "video_7294/video_7294_15.jpeg", + "video_7294/video_7294_16.jpeg", + "video_7294/video_7294_17.jpeg", + "video_7294/video_7294_18.jpeg", + "video_7294/video_7294_19.jpeg", + "video_7294/video_7294_20.jpeg", + "video_7294/video_7294_21.jpeg", + "video_7294/video_7294_22.jpeg", + "video_7294/video_7294_23.jpeg", + "video_7294/video_7294_24.jpeg", + "video_7294/video_7294_25.jpeg", + "video_7294/video_7294_26.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 141, + "question": "Review the visuals provided and respond to the question concerning the character sequence. You must choose your answer from the Choice List.\nWhat letters did the person type on the computer in order?\nChoice list: \nA. teh\nB. hte\nC. tqh\nAnswer with the option's letter from the given choices directly.", + "gt_response": "teh", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "teh", + "hte", + "tqh" + ], + "image_quantity_level": "Medium", + "image": [ + "video_5492/video_5492_0.jpeg", + "video_5492/video_5492_1.jpeg", + "video_5492/video_5492_2.jpeg", + "video_5492/video_5492_3.jpeg", + "video_5492/video_5492_4.jpeg", + "video_5492/video_5492_5.jpeg", + "video_5492/video_5492_6.jpeg", + "video_5492/video_5492_7.jpeg", + "video_5492/video_5492_8.jpeg", + "video_5492/video_5492_9.jpeg", + "video_5492/video_5492_10.jpeg", + "video_5492/video_5492_11.jpeg", + "video_5492/video_5492_12.jpeg", + "video_5492/video_5492_13.jpeg", + "video_5492/video_5492_14.jpeg", + "video_5492/video_5492_15.jpeg", + "video_5492/video_5492_16.jpeg", + "video_5492/video_5492_17.jpeg", + "video_5492/video_5492_18.jpeg", + "video_5492/video_5492_19.jpeg", + "video_5492/video_5492_20.jpeg", + "video_5492/video_5492_21.jpeg", + "video_5492/video_5492_22.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 146, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. t\nB. c\nC. a\nAnswer with the option's letter from the given choices directly.", + "gt_response": "c", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "t", + "c", + "a" + ], + "image_quantity_level": "Medium", + "image": [ + "video_5325/video_5325_0.jpeg", + "video_5325/video_5325_1.jpeg", + "video_5325/video_5325_2.jpeg", + "video_5325/video_5325_3.jpeg", + "video_5325/video_5325_4.jpeg", + "video_5325/video_5325_5.jpeg", + "video_5325/video_5325_6.jpeg", + "video_5325/video_5325_7.jpeg", + "video_5325/video_5325_8.jpeg", + "video_5325/video_5325_9.jpeg", + "video_5325/video_5325_10.jpeg", + "video_5325/video_5325_11.jpeg", + "video_5325/video_5325_12.jpeg", + "video_5325/video_5325_13.jpeg", + "video_5325/video_5325_14.jpeg", + "video_5325/video_5325_15.jpeg", + "video_5325/video_5325_16.jpeg", + "video_5325/video_5325_17.jpeg", + "video_5325/video_5325_18.jpeg", + "video_5325/video_5325_19.jpeg", + "video_5325/video_5325_20.jpeg", + "video_5325/video_5325_21.jpeg", + "video_5325/video_5325_22.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 142, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. pgoq\nB. pmoz\nC. umow\nAnswer with the option's letter from the given choices directly.", + "gt_response": "pmoz", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "pgoq", + "pmoz", + "umow" + ], + "image_quantity_level": "Medium", + "image": [ + "video_7528/video_7528_0.jpeg", + "video_7528/video_7528_1.jpeg", + "video_7528/video_7528_2.jpeg", + "video_7528/video_7528_3.jpeg", + "video_7528/video_7528_4.jpeg", + "video_7528/video_7528_5.jpeg", + "video_7528/video_7528_6.jpeg", + "video_7528/video_7528_7.jpeg", + "video_7528/video_7528_8.jpeg", + "video_7528/video_7528_9.jpeg", + "video_7528/video_7528_10.jpeg", + "video_7528/video_7528_11.jpeg", + "video_7528/video_7528_12.jpeg", + "video_7528/video_7528_13.jpeg", + "video_7528/video_7528_14.jpeg", + "video_7528/video_7528_15.jpeg", + "video_7528/video_7528_16.jpeg", + "video_7528/video_7528_17.jpeg", + "video_7528/video_7528_18.jpeg", + "video_7528/video_7528_19.jpeg", + "video_7528/video_7528_20.jpeg", + "video_7528/video_7528_21.jpeg", + "video_7528/video_7528_22.jpeg", + "video_7528/video_7528_23.jpeg", + "video_7528/video_7528_24.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 161, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nWhat letter is likely to be shown next?\nChoice list: \nA. y\nB. f\nC. o\nAnswer with the option's letter from the given choices directly.", + "gt_response": "o", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "y", + "f", + "o" + ], + "image_quantity_level": "Medium", + "image": [ + "video_11339/video_11339_0.jpeg", + "video_11339/video_11339_1.jpeg", + "video_11339/video_11339_2.jpeg", + "video_11339/video_11339_3.jpeg", + "video_11339/video_11339_4.jpeg", + "video_11339/video_11339_5.jpeg", + "video_11339/video_11339_6.jpeg", + "video_11339/video_11339_7.jpeg", + "video_11339/video_11339_8.jpeg", + "video_11339/video_11339_9.jpeg", + "video_11339/video_11339_10.jpeg", + "video_11339/video_11339_11.jpeg", + "video_11339/video_11339_12.jpeg", + "video_11339/video_11339_13.jpeg", + "video_11339/video_11339_14.jpeg", + "video_11339/video_11339_15.jpeg", + "video_11339/video_11339_16.jpeg", + "video_11339/video_11339_17.jpeg", + "video_11339/video_11339_18.jpeg", + "video_11339/video_11339_19.jpeg", + "video_11339/video_11339_20.jpeg", + "video_11339/video_11339_21.jpeg", + "video_11339/video_11339_22.jpeg", + "video_11339/video_11339_23.jpeg", + "video_11339/video_11339_24.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 177, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nWhat was the order of the letters on the table before shuffling?\nChoice list: \nA. bcaogd\nB. abcdgo\nC. aodgbc\nAnswer with the option's letter from the given choices directly.", + "gt_response": "bcaogd", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "bcaogd", + "abcdgo", + "aodgbc" + ], + "image_quantity_level": "Medium", + "image": [ + "video_3898/video_3898_0.jpeg", + "video_3898/video_3898_1.jpeg", + "video_3898/video_3898_2.jpeg", + "video_3898/video_3898_3.jpeg", + "video_3898/video_3898_4.jpeg", + "video_3898/video_3898_5.jpeg", + "video_3898/video_3898_6.jpeg", + "video_3898/video_3898_7.jpeg", + "video_3898/video_3898_8.jpeg", + "video_3898/video_3898_9.jpeg", + "video_3898/video_3898_10.jpeg", + "video_3898/video_3898_11.jpeg", + "video_3898/video_3898_12.jpeg", + "video_3898/video_3898_13.jpeg", + "video_3898/video_3898_14.jpeg", + "video_3898/video_3898_15.jpeg", + "video_3898/video_3898_16.jpeg", + "video_3898/video_3898_17.jpeg", + "video_3898/video_3898_18.jpeg", + "video_3898/video_3898_19.jpeg", + "video_3898/video_3898_20.jpeg", + "video_3898/video_3898_21.jpeg", + "video_3898/video_3898_22.jpeg", + "video_3898/video_3898_23.jpeg", + "video_3898/video_3898_24.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 132, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. cdzw\nB. cehi\nC. seht\nAnswer with the option's letter from the given choices directly.", + "gt_response": "cehi", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "cdzw", + "cehi", + "seht" + ], + "image_quantity_level": "Medium", + "image": [ + "video_10049/video_10049_0.jpeg", + "video_10049/video_10049_1.jpeg", + "video_10049/video_10049_2.jpeg", + "video_10049/video_10049_3.jpeg", + "video_10049/video_10049_4.jpeg", + "video_10049/video_10049_5.jpeg", + "video_10049/video_10049_6.jpeg", + "video_10049/video_10049_7.jpeg", + "video_10049/video_10049_8.jpeg", + "video_10049/video_10049_9.jpeg", + "video_10049/video_10049_10.jpeg", + "video_10049/video_10049_11.jpeg", + "video_10049/video_10049_12.jpeg", + "video_10049/video_10049_13.jpeg", + "video_10049/video_10049_14.jpeg", + "video_10049/video_10049_15.jpeg", + "video_10049/video_10049_16.jpeg", + "video_10049/video_10049_17.jpeg", + "video_10049/video_10049_18.jpeg", + "video_10049/video_10049_19.jpeg", + "video_10049/video_10049_20.jpeg", + "video_10049/video_10049_21.jpeg", + "video_10049/video_10049_22.jpeg", + "video_10049/video_10049_23.jpeg", + "video_10049/video_10049_24.jpeg", + "video_10049/video_10049_25.jpeg", + "video_10049/video_10049_26.jpeg", + "video_10049/video_10049_27.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 134, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. w\nB. o\nC. t\nAnswer with the option's letter from the given choices directly.", + "gt_response": "w", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "w", + "o", + "t" + ], + "image_quantity_level": "Medium", + "image": [ + "video_4029/video_4029_0.jpeg", + "video_4029/video_4029_1.jpeg", + "video_4029/video_4029_2.jpeg", + "video_4029/video_4029_3.jpeg", + "video_4029/video_4029_4.jpeg", + "video_4029/video_4029_5.jpeg", + "video_4029/video_4029_6.jpeg", + "video_4029/video_4029_7.jpeg", + "video_4029/video_4029_8.jpeg", + "video_4029/video_4029_9.jpeg", + "video_4029/video_4029_10.jpeg", + "video_4029/video_4029_11.jpeg", + "video_4029/video_4029_12.jpeg", + "video_4029/video_4029_13.jpeg", + "video_4029/video_4029_14.jpeg", + "video_4029/video_4029_15.jpeg", + "video_4029/video_4029_16.jpeg", + "video_4029/video_4029_17.jpeg", + "video_4029/video_4029_18.jpeg", + "video_4029/video_4029_19.jpeg", + "video_4029/video_4029_20.jpeg", + "video_4029/video_4029_21.jpeg", + "video_4029/video_4029_22.jpeg", + "video_4029/video_4029_23.jpeg", + "video_4029/video_4029_24.jpeg", + "video_4029/video_4029_25.jpeg", + "video_4029/video_4029_26.jpeg", + "video_4029/video_4029_27.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 145, + "question": "Scrutinize the provided visuals and answer the question about the arrangement of characters. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. lypi\nB. qyve\nC. love\nAnswer with the option's letter from the given choices directly.", + "gt_response": "love", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "lypi", + "qyve", + "love" + ], + "image_quantity_level": "Medium", + "image": [ + "video_2476/video_2476_0.jpeg", + "video_2476/video_2476_1.jpeg", + "video_2476/video_2476_2.jpeg", + "video_2476/video_2476_3.jpeg", + "video_2476/video_2476_4.jpeg", + "video_2476/video_2476_5.jpeg", + "video_2476/video_2476_6.jpeg", + "video_2476/video_2476_7.jpeg", + "video_2476/video_2476_8.jpeg", + "video_2476/video_2476_9.jpeg", + "video_2476/video_2476_10.jpeg", + "video_2476/video_2476_11.jpeg", + "video_2476/video_2476_12.jpeg", + "video_2476/video_2476_13.jpeg", + "video_2476/video_2476_14.jpeg", + "video_2476/video_2476_15.jpeg", + "video_2476/video_2476_16.jpeg", + "video_2476/video_2476_17.jpeg", + "video_2476/video_2476_18.jpeg", + "video_2476/video_2476_19.jpeg", + "video_2476/video_2476_20.jpeg", + "video_2476/video_2476_21.jpeg", + "video_2476/video_2476_22.jpeg", + "video_2476/video_2476_23.jpeg", + "video_2476/video_2476_24.jpeg", + "video_2476/video_2476_25.jpeg", + "video_2476/video_2476_26.jpeg", + "video_2476/video_2476_27.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 182, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. d\nB. g\nC. o\nAnswer with the option's letter from the given choices directly.", + "gt_response": "g", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "d", + "g", + "o" + ], + "image_quantity_level": "Medium", + "image": [ + "video_9035/video_9035_0.jpeg", + "video_9035/video_9035_1.jpeg", + "video_9035/video_9035_2.jpeg", + "video_9035/video_9035_3.jpeg", + "video_9035/video_9035_4.jpeg", + "video_9035/video_9035_5.jpeg", + "video_9035/video_9035_6.jpeg", + "video_9035/video_9035_7.jpeg", + "video_9035/video_9035_8.jpeg", + "video_9035/video_9035_9.jpeg", + "video_9035/video_9035_10.jpeg", + "video_9035/video_9035_11.jpeg", + "video_9035/video_9035_12.jpeg", + "video_9035/video_9035_13.jpeg", + "video_9035/video_9035_14.jpeg", + "video_9035/video_9035_15.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 62, + "question": "Review the visuals provided and respond to the question concerning the character sequence. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. yxs\nB. uxz\nC. ybw\nAnswer with the option's letter from the given choices directly.", + "gt_response": "yxs", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "yxs", + "uxz", + "ybw" + ], + "image_quantity_level": "Medium", + "image": [ + "video_10383/video_10383_0.jpeg", + "video_10383/video_10383_1.jpeg", + "video_10383/video_10383_2.jpeg", + "video_10383/video_10383_3.jpeg", + "video_10383/video_10383_4.jpeg", + "video_10383/video_10383_5.jpeg", + "video_10383/video_10383_6.jpeg", + "video_10383/video_10383_7.jpeg", + "video_10383/video_10383_8.jpeg", + "video_10383/video_10383_9.jpeg", + "video_10383/video_10383_10.jpeg", + "video_10383/video_10383_11.jpeg", + "video_10383/video_10383_12.jpeg", + "video_10383/video_10383_13.jpeg", + "video_10383/video_10383_14.jpeg", + "video_10383/video_10383_15.jpeg", + "video_10383/video_10383_16.jpeg", + "video_10383/video_10383_17.jpeg", + "video_10383/video_10383_18.jpeg", + "video_10383/video_10383_19.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 117, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. as\nB. ao\nC. sa\nAnswer with the option's letter from the given choices directly.", + "gt_response": "as", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "as", + "ao", + "sa" + ], + "image_quantity_level": "Medium", + "image": [ + "video_4405/video_4405_0.jpeg", + "video_4405/video_4405_1.jpeg", + "video_4405/video_4405_2.jpeg", + "video_4405/video_4405_3.jpeg", + "video_4405/video_4405_4.jpeg", + "video_4405/video_4405_5.jpeg", + "video_4405/video_4405_6.jpeg", + "video_4405/video_4405_7.jpeg", + "video_4405/video_4405_8.jpeg", + "video_4405/video_4405_9.jpeg", + "video_4405/video_4405_10.jpeg", + "video_4405/video_4405_11.jpeg", + "video_4405/video_4405_12.jpeg", + "video_4405/video_4405_13.jpeg", + "video_4405/video_4405_14.jpeg", + "video_4405/video_4405_15.jpeg", + "video_4405/video_4405_16.jpeg", + "video_4405/video_4405_17.jpeg", + "video_4405/video_4405_18.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 151, + "question": "Scrutinize the provided visuals and answer the question about the arrangement of characters. You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. l\nB. a\nC. p\nAnswer with the option's letter from the given choices directly.", + "gt_response": "l", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "l", + "a", + "p" + ], + "image_quantity_level": "Medium", + "image": [ + "video_285/video_285_0.jpeg", + "video_285/video_285_1.jpeg", + "video_285/video_285_2.jpeg", + "video_285/video_285_3.jpeg", + "video_285/video_285_4.jpeg", + "video_285/video_285_5.jpeg", + "video_285/video_285_6.jpeg", + "video_285/video_285_7.jpeg", + "video_285/video_285_8.jpeg", + "video_285/video_285_9.jpeg", + "video_285/video_285_10.jpeg", + "video_285/video_285_11.jpeg", + "video_285/video_285_12.jpeg", + "video_285/video_285_13.jpeg", + "video_285/video_285_14.jpeg", + "video_285/video_285_15.jpeg", + "video_285/video_285_16.jpeg", + "video_285/video_285_17.jpeg", + "video_285/video_285_18.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 87, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nWhat letter is likely to be shown next?\nChoice list: \nA. g\nB. a\nC. j\nAnswer with the option's letter from the given choices directly.", + "gt_response": "g", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "g", + "a", + "j" + ], + "image_quantity_level": "Many", + "image": [ + "video_6885/video_6885_0.jpeg", + "video_6885/video_6885_1.jpeg", + "video_6885/video_6885_2.jpeg", + "video_6885/video_6885_3.jpeg", + "video_6885/video_6885_4.jpeg", + "video_6885/video_6885_5.jpeg", + "video_6885/video_6885_6.jpeg", + "video_6885/video_6885_7.jpeg", + "video_6885/video_6885_8.jpeg", + "video_6885/video_6885_9.jpeg", + "video_6885/video_6885_10.jpeg", + "video_6885/video_6885_11.jpeg", + "video_6885/video_6885_12.jpeg", + "video_6885/video_6885_13.jpeg", + "video_6885/video_6885_14.jpeg", + "video_6885/video_6885_15.jpeg", + "video_6885/video_6885_16.jpeg", + "video_6885/video_6885_17.jpeg", + "video_6885/video_6885_18.jpeg", + "video_6885/video_6885_19.jpeg", + "video_6885/video_6885_20.jpeg", + "video_6885/video_6885_21.jpeg", + "video_6885/video_6885_22.jpeg", + "video_6885/video_6885_23.jpeg", + "video_6885/video_6885_24.jpeg", + "video_6885/video_6885_25.jpeg", + "video_6885/video_6885_26.jpeg", + "video_6885/video_6885_27.jpeg", + "video_6885/video_6885_28.jpeg", + "video_6885/video_6885_29.jpeg", + "video_6885/video_6885_30.jpeg", + "video_6885/video_6885_31.jpeg", + "video_6885/video_6885_32.jpeg", + "video_6885/video_6885_33.jpeg", + "video_6885/video_6885_34.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 97, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. rwe\nB. wre\nC. ewr\nAnswer with the option's letter from the given choices directly.", + "gt_response": "ewr", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "rwe", + "wre", + "ewr" + ], + "image_quantity_level": "Many", + "image": [ + "video_1715/video_1715_0.jpeg", + "video_1715/video_1715_1.jpeg", + "video_1715/video_1715_2.jpeg", + "video_1715/video_1715_3.jpeg", + "video_1715/video_1715_4.jpeg", + "video_1715/video_1715_5.jpeg", + "video_1715/video_1715_6.jpeg", + "video_1715/video_1715_7.jpeg", + "video_1715/video_1715_8.jpeg", + "video_1715/video_1715_9.jpeg", + "video_1715/video_1715_10.jpeg", + "video_1715/video_1715_11.jpeg", + "video_1715/video_1715_12.jpeg", + "video_1715/video_1715_13.jpeg", + "video_1715/video_1715_14.jpeg", + "video_1715/video_1715_15.jpeg", + "video_1715/video_1715_16.jpeg", + "video_1715/video_1715_17.jpeg", + "video_1715/video_1715_18.jpeg", + "video_1715/video_1715_19.jpeg", + "video_1715/video_1715_20.jpeg", + "video_1715/video_1715_21.jpeg", + "video_1715/video_1715_22.jpeg", + "video_1715/video_1715_23.jpeg", + "video_1715/video_1715_24.jpeg", + "video_1715/video_1715_25.jpeg", + "video_1715/video_1715_26.jpeg", + "video_1715/video_1715_27.jpeg", + "video_1715/video_1715_28.jpeg", + "video_1715/video_1715_29.jpeg", + "video_1715/video_1715_30.jpeg", + "video_1715/video_1715_31.jpeg", + "video_1715/video_1715_32.jpeg", + "video_1715/video_1715_33.jpeg", + "video_1715/video_1715_34.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 100, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. vol\nB. lvo\nC. ovl\nAnswer with the option's letter from the given choices directly.", + "gt_response": "vol", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "vol", + "lvo", + "ovl" + ], + "image_quantity_level": "Many", + "image": [ + "video_1517/video_1517_0.jpeg", + "video_1517/video_1517_1.jpeg", + "video_1517/video_1517_2.jpeg", + "video_1517/video_1517_3.jpeg", + "video_1517/video_1517_4.jpeg", + "video_1517/video_1517_5.jpeg", + "video_1517/video_1517_6.jpeg", + "video_1517/video_1517_7.jpeg", + "video_1517/video_1517_8.jpeg", + "video_1517/video_1517_9.jpeg", + "video_1517/video_1517_10.jpeg", + "video_1517/video_1517_11.jpeg", + "video_1517/video_1517_12.jpeg", + "video_1517/video_1517_13.jpeg", + "video_1517/video_1517_14.jpeg", + "video_1517/video_1517_15.jpeg", + "video_1517/video_1517_16.jpeg", + "video_1517/video_1517_17.jpeg", + "video_1517/video_1517_18.jpeg", + "video_1517/video_1517_19.jpeg", + "video_1517/video_1517_20.jpeg", + "video_1517/video_1517_21.jpeg", + "video_1517/video_1517_22.jpeg", + "video_1517/video_1517_23.jpeg", + "video_1517/video_1517_24.jpeg", + "video_1517/video_1517_25.jpeg", + "video_1517/video_1517_26.jpeg", + "video_1517/video_1517_27.jpeg", + "video_1517/video_1517_28.jpeg", + "video_1517/video_1517_29.jpeg", + "video_1517/video_1517_30.jpeg", + "video_1517/video_1517_31.jpeg", + "video_1517/video_1517_32.jpeg", + "video_1517/video_1517_33.jpeg", + "video_1517/video_1517_34.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 127, + "question": "Analyze the given visuals and answer the question about the order of character. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. sel\nB. les\nC. esl\nAnswer with the option's letter from the given choices directly.", + "gt_response": "sel", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "sel", + "les", + "esl" + ], + "image_quantity_level": "Many", + "image": [ + "video_11183/video_11183_0.jpeg", + "video_11183/video_11183_1.jpeg", + "video_11183/video_11183_2.jpeg", + "video_11183/video_11183_3.jpeg", + "video_11183/video_11183_4.jpeg", + "video_11183/video_11183_5.jpeg", + "video_11183/video_11183_6.jpeg", + "video_11183/video_11183_7.jpeg", + "video_11183/video_11183_8.jpeg", + "video_11183/video_11183_9.jpeg", + "video_11183/video_11183_10.jpeg", + "video_11183/video_11183_11.jpeg", + "video_11183/video_11183_12.jpeg", + "video_11183/video_11183_13.jpeg", + "video_11183/video_11183_14.jpeg", + "video_11183/video_11183_15.jpeg", + "video_11183/video_11183_16.jpeg", + "video_11183/video_11183_17.jpeg", + "video_11183/video_11183_18.jpeg", + "video_11183/video_11183_19.jpeg", + "video_11183/video_11183_20.jpeg", + "video_11183/video_11183_21.jpeg", + "video_11183/video_11183_22.jpeg", + "video_11183/video_11183_23.jpeg", + "video_11183/video_11183_24.jpeg", + "video_11183/video_11183_25.jpeg", + "video_11183/video_11183_26.jpeg", + "video_11183/video_11183_27.jpeg", + "video_11183/video_11183_28.jpeg", + "video_11183/video_11183_29.jpeg", + "video_11183/video_11183_30.jpeg", + "video_11183/video_11183_31.jpeg", + "video_11183/video_11183_32.jpeg", + "video_11183/video_11183_33.jpeg", + "video_11183/video_11183_34.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 131, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nWhat letter is likely to be shown next?\nChoice list: \nA. h\nB. o\nC. j\nAnswer with the option's letter from the given choices directly.", + "gt_response": "o", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "h", + "o", + "j" + ], + "image_quantity_level": "Many", + "image": [ + "video_1916/video_1916_0.jpeg", + "video_1916/video_1916_1.jpeg", + "video_1916/video_1916_2.jpeg", + "video_1916/video_1916_3.jpeg", + "video_1916/video_1916_4.jpeg", + "video_1916/video_1916_5.jpeg", + "video_1916/video_1916_6.jpeg", + "video_1916/video_1916_7.jpeg", + "video_1916/video_1916_8.jpeg", + "video_1916/video_1916_9.jpeg", + "video_1916/video_1916_10.jpeg", + "video_1916/video_1916_11.jpeg", + "video_1916/video_1916_12.jpeg", + "video_1916/video_1916_13.jpeg", + "video_1916/video_1916_14.jpeg", + "video_1916/video_1916_15.jpeg", + "video_1916/video_1916_16.jpeg", + "video_1916/video_1916_17.jpeg", + "video_1916/video_1916_18.jpeg", + "video_1916/video_1916_19.jpeg", + "video_1916/video_1916_20.jpeg", + "video_1916/video_1916_21.jpeg", + "video_1916/video_1916_22.jpeg", + "video_1916/video_1916_23.jpeg", + "video_1916/video_1916_24.jpeg", + "video_1916/video_1916_25.jpeg", + "video_1916/video_1916_26.jpeg", + "video_1916/video_1916_27.jpeg", + "video_1916/video_1916_28.jpeg", + "video_1916/video_1916_29.jpeg", + "video_1916/video_1916_30.jpeg", + "video_1916/video_1916_31.jpeg", + "video_1916/video_1916_32.jpeg", + "video_1916/video_1916_33.jpeg", + "video_1916/video_1916_34.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 91, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat would be the order of the written letters if the person had written them in reverse order?\nChoice list: \nA. gab\nB. gjb\nC. bag\nAnswer with the option's letter from the given choices directly.", + "gt_response": "gab", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "gab", + "gjb", + "bag" + ], + "image_quantity_level": "Medium", + "image": [ + "video_9948/video_9948_0.jpeg", + "video_9948/video_9948_1.jpeg", + "video_9948/video_9948_2.jpeg", + "video_9948/video_9948_3.jpeg", + "video_9948/video_9948_4.jpeg", + "video_9948/video_9948_5.jpeg", + "video_9948/video_9948_6.jpeg", + "video_9948/video_9948_7.jpeg", + "video_9948/video_9948_8.jpeg", + "video_9948/video_9948_9.jpeg", + "video_9948/video_9948_10.jpeg", + "video_9948/video_9948_11.jpeg", + "video_9948/video_9948_12.jpeg", + "video_9948/video_9948_13.jpeg", + "video_9948/video_9948_14.jpeg", + "video_9948/video_9948_15.jpeg", + "video_9948/video_9948_16.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 96, + "question": "Review the visuals provided and respond to the question concerning the character sequence. You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. c\nB. r\nC. a\nAnswer with the option's letter from the given choices directly.", + "gt_response": "a", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "c", + "r", + "a" + ], + "image_quantity_level": "Medium", + "image": [ + "video_1289/video_1289_0.jpeg", + "video_1289/video_1289_1.jpeg", + "video_1289/video_1289_2.jpeg", + "video_1289/video_1289_3.jpeg", + "video_1289/video_1289_4.jpeg", + "video_1289/video_1289_5.jpeg", + "video_1289/video_1289_6.jpeg", + "video_1289/video_1289_7.jpeg", + "video_1289/video_1289_8.jpeg", + "video_1289/video_1289_9.jpeg", + "video_1289/video_1289_10.jpeg", + "video_1289/video_1289_11.jpeg", + "video_1289/video_1289_12.jpeg", + "video_1289/video_1289_13.jpeg", + "video_1289/video_1289_14.jpeg", + "video_1289/video_1289_15.jpeg", + "video_1289/video_1289_16.jpeg", + "video_1289/video_1289_17.jpeg", + "video_1289/video_1289_18.jpeg", + "video_1289/video_1289_19.jpeg", + "video_1289/video_1289_20.jpeg", + "video_1289/video_1289_21.jpeg", + "video_1289/video_1289_22.jpeg", + "video_1289/video_1289_23.jpeg", + "video_1289/video_1289_24.jpeg", + "video_1289/video_1289_25.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 102, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. aimr\nB. airm\nC. mira\nAnswer with the option's letter from the given choices directly.", + "gt_response": "airm", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "aimr", + "airm", + "mira" + ], + "image_quantity_level": "Medium", + "image": [ + "video_3115/video_3115_0.jpeg", + "video_3115/video_3115_1.jpeg", + "video_3115/video_3115_2.jpeg", + "video_3115/video_3115_3.jpeg", + "video_3115/video_3115_4.jpeg", + "video_3115/video_3115_5.jpeg", + "video_3115/video_3115_6.jpeg", + "video_3115/video_3115_7.jpeg", + "video_3115/video_3115_8.jpeg", + "video_3115/video_3115_9.jpeg", + "video_3115/video_3115_10.jpeg", + "video_3115/video_3115_11.jpeg", + "video_3115/video_3115_12.jpeg", + "video_3115/video_3115_13.jpeg", + "video_3115/video_3115_14.jpeg", + "video_3115/video_3115_15.jpeg", + "video_3115/video_3115_16.jpeg", + "video_3115/video_3115_17.jpeg", + "video_3115/video_3115_18.jpeg", + "video_3115/video_3115_19.jpeg", + "video_3115/video_3115_20.jpeg", + "video_3115/video_3115_21.jpeg", + "video_3115/video_3115_22.jpeg", + "video_3115/video_3115_23.jpeg", + "video_3115/video_3115_24.jpeg", + "video_3115/video_3115_25.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 121, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. t\nB. a\nC. c\nAnswer with the option's letter from the given choices directly.", + "gt_response": "c", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "t", + "a", + "c" + ], + "image_quantity_level": "Medium", + "image": [ + "video_6195/video_6195_0.jpeg", + "video_6195/video_6195_1.jpeg", + "video_6195/video_6195_2.jpeg", + "video_6195/video_6195_3.jpeg", + "video_6195/video_6195_4.jpeg", + "video_6195/video_6195_5.jpeg", + "video_6195/video_6195_6.jpeg", + "video_6195/video_6195_7.jpeg", + "video_6195/video_6195_8.jpeg", + "video_6195/video_6195_9.jpeg", + "video_6195/video_6195_10.jpeg", + "video_6195/video_6195_11.jpeg", + "video_6195/video_6195_12.jpeg", + "video_6195/video_6195_13.jpeg", + "video_6195/video_6195_14.jpeg", + "video_6195/video_6195_15.jpeg", + "video_6195/video_6195_16.jpeg", + "video_6195/video_6195_17.jpeg", + "video_6195/video_6195_18.jpeg", + "video_6195/video_6195_19.jpeg", + "video_6195/video_6195_20.jpeg", + "video_6195/video_6195_21.jpeg", + "video_6195/video_6195_22.jpeg", + "video_6195/video_6195_23.jpeg", + "video_6195/video_6195_24.jpeg", + "video_6195/video_6195_25.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 104, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. ma\nB. aj\nC. am\nAnswer with the option's letter from the given choices directly.", + "gt_response": "am", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "ma", + "aj", + "am" + ], + "image_quantity_level": "Medium", + "image": [ + "video_8884/video_8884_0.jpeg", + "video_8884/video_8884_1.jpeg", + "video_8884/video_8884_2.jpeg", + "video_8884/video_8884_3.jpeg", + "video_8884/video_8884_4.jpeg", + "video_8884/video_8884_5.jpeg", + "video_8884/video_8884_6.jpeg", + "video_8884/video_8884_7.jpeg", + "video_8884/video_8884_8.jpeg", + "video_8884/video_8884_9.jpeg", + "video_8884/video_8884_10.jpeg", + "video_8884/video_8884_11.jpeg", + "video_8884/video_8884_12.jpeg", + "video_8884/video_8884_13.jpeg", + "video_8884/video_8884_14.jpeg", + "video_8884/video_8884_15.jpeg", + "video_8884/video_8884_16.jpeg", + "video_8884/video_8884_17.jpeg", + "video_8884/video_8884_18.jpeg", + "video_8884/video_8884_19.jpeg", + "video_8884/video_8884_20.jpeg", + "video_8884/video_8884_21.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 156, + "question": "From the images supplied, answer the question relevant to character order. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. mf\nB. am\nC. ma\nAnswer with the option's letter from the given choices directly.", + "gt_response": "ma", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "mf", + "am", + "ma" + ], + "image_quantity_level": "Medium", + "image": [ + "video_9409/video_9409_0.jpeg", + "video_9409/video_9409_1.jpeg", + "video_9409/video_9409_2.jpeg", + "video_9409/video_9409_3.jpeg", + "video_9409/video_9409_4.jpeg", + "video_9409/video_9409_5.jpeg", + "video_9409/video_9409_6.jpeg", + "video_9409/video_9409_7.jpeg", + "video_9409/video_9409_8.jpeg", + "video_9409/video_9409_9.jpeg", + "video_9409/video_9409_10.jpeg", + "video_9409/video_9409_11.jpeg", + "video_9409/video_9409_12.jpeg", + "video_9409/video_9409_13.jpeg", + "video_9409/video_9409_14.jpeg", + "video_9409/video_9409_15.jpeg", + "video_9409/video_9409_16.jpeg", + "video_9409/video_9409_17.jpeg", + "video_9409/video_9409_18.jpeg", + "video_9409/video_9409_19.jpeg", + "video_9409/video_9409_20.jpeg", + "video_9409/video_9409_21.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 140, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. supev\nB. super\nC. supzr\nAnswer with the option's letter from the given choices directly.", + "gt_response": "super", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "supev", + "super", + "supzr" + ], + "image_quantity_level": "Medium", + "image": [ + "video_9456/video_9456_0.jpeg", + "video_9456/video_9456_1.jpeg", + "video_9456/video_9456_2.jpeg", + "video_9456/video_9456_3.jpeg", + "video_9456/video_9456_4.jpeg", + "video_9456/video_9456_5.jpeg", + "video_9456/video_9456_6.jpeg", + "video_9456/video_9456_7.jpeg", + "video_9456/video_9456_8.jpeg", + "video_9456/video_9456_9.jpeg", + "video_9456/video_9456_10.jpeg", + "video_9456/video_9456_11.jpeg", + "video_9456/video_9456_12.jpeg", + "video_9456/video_9456_13.jpeg", + "video_9456/video_9456_14.jpeg", + "video_9456/video_9456_15.jpeg", + "video_9456/video_9456_16.jpeg", + "video_9456/video_9456_17.jpeg", + "video_9456/video_9456_18.jpeg", + "video_9456/video_9456_19.jpeg", + "video_9456/video_9456_20.jpeg", + "video_9456/video_9456_21.jpeg", + "video_9456/video_9456_22.jpeg", + "video_9456/video_9456_23.jpeg", + "video_9456/video_9456_24.jpeg", + "video_9456/video_9456_25.jpeg", + "video_9456/video_9456_26.jpeg", + "video_9456/video_9456_27.jpeg", + "video_9456/video_9456_28.jpeg", + "video_9456/video_9456_29.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 93, + "question": "Review the visuals provided and respond to the question concerning the character sequence. You must choose your answer from the Choice List.\nWhat was the order of the letters on the table before shuffling?\nChoice list: \nA. npor\nB. opnr\nC. porn\nAnswer with the option's letter from the given choices directly.", + "gt_response": "opnr", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "npor", + "opnr", + "porn" + ], + "image_quantity_level": "Medium", + "image": [ + "video_3168/video_3168_0.jpeg", + "video_3168/video_3168_1.jpeg", + "video_3168/video_3168_2.jpeg", + "video_3168/video_3168_3.jpeg", + "video_3168/video_3168_4.jpeg", + "video_3168/video_3168_5.jpeg", + "video_3168/video_3168_6.jpeg", + "video_3168/video_3168_7.jpeg", + "video_3168/video_3168_8.jpeg", + "video_3168/video_3168_9.jpeg", + "video_3168/video_3168_10.jpeg", + "video_3168/video_3168_11.jpeg", + "video_3168/video_3168_12.jpeg", + "video_3168/video_3168_13.jpeg", + "video_3168/video_3168_14.jpeg", + "video_3168/video_3168_15.jpeg", + "video_3168/video_3168_16.jpeg", + "video_3168/video_3168_17.jpeg", + "video_3168/video_3168_18.jpeg", + "video_3168/video_3168_19.jpeg", + "video_3168/video_3168_20.jpeg", + "video_3168/video_3168_21.jpeg", + "video_3168/video_3168_22.jpeg", + "video_3168/video_3168_23.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 109, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. deu\nB. dvg\nC. eud\nAnswer with the option's letter from the given choices directly.", + "gt_response": "deu", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "deu", + "dvg", + "eud" + ], + "image_quantity_level": "Medium", + "image": [ + "video_32/video_32_0.jpeg", + "video_32/video_32_1.jpeg", + "video_32/video_32_2.jpeg", + "video_32/video_32_3.jpeg", + "video_32/video_32_4.jpeg", + "video_32/video_32_5.jpeg", + "video_32/video_32_6.jpeg", + "video_32/video_32_7.jpeg", + "video_32/video_32_8.jpeg", + "video_32/video_32_9.jpeg", + "video_32/video_32_10.jpeg", + "video_32/video_32_11.jpeg", + "video_32/video_32_12.jpeg", + "video_32/video_32_13.jpeg", + "video_32/video_32_14.jpeg", + "video_32/video_32_15.jpeg", + "video_32/video_32_16.jpeg", + "video_32/video_32_17.jpeg", + "video_32/video_32_18.jpeg", + "video_32/video_32_19.jpeg", + "video_32/video_32_20.jpeg", + "video_32/video_32_21.jpeg", + "video_32/video_32_22.jpeg", + "video_32/video_32_23.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 162, + "question": "Analyze the given visuals and answer the question about the order of character. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. vahb\nB. abhv\nC. bavh\nAnswer with the option's letter from the given choices directly.", + "gt_response": "bavh", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "vahb", + "abhv", + "bavh" + ], + "image_quantity_level": "Medium", + "image": [ + "video_6470/video_6470_0.jpeg", + "video_6470/video_6470_1.jpeg", + "video_6470/video_6470_2.jpeg", + "video_6470/video_6470_3.jpeg", + "video_6470/video_6470_4.jpeg", + "video_6470/video_6470_5.jpeg", + "video_6470/video_6470_6.jpeg", + "video_6470/video_6470_7.jpeg", + "video_6470/video_6470_8.jpeg", + "video_6470/video_6470_9.jpeg", + "video_6470/video_6470_10.jpeg", + "video_6470/video_6470_11.jpeg", + "video_6470/video_6470_12.jpeg", + "video_6470/video_6470_13.jpeg", + "video_6470/video_6470_14.jpeg", + "video_6470/video_6470_15.jpeg", + "video_6470/video_6470_16.jpeg", + "video_6470/video_6470_17.jpeg", + "video_6470/video_6470_18.jpeg", + "video_6470/video_6470_19.jpeg", + "video_6470/video_6470_20.jpeg", + "video_6470/video_6470_21.jpeg", + "video_6470/video_6470_22.jpeg", + "video_6470/video_6470_23.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 128, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. zxy\nB. xyr\nC. xyz\nAnswer with the option's letter from the given choices directly.", + "gt_response": "xyz", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "zxy", + "xyr", + "xyz" + ], + "image_quantity_level": "Many", + "image": [ + "video_3929/video_3929_0.jpeg", + "video_3929/video_3929_1.jpeg", + "video_3929/video_3929_2.jpeg", + "video_3929/video_3929_3.jpeg", + "video_3929/video_3929_4.jpeg", + "video_3929/video_3929_5.jpeg", + "video_3929/video_3929_6.jpeg", + "video_3929/video_3929_7.jpeg", + "video_3929/video_3929_8.jpeg", + "video_3929/video_3929_9.jpeg", + "video_3929/video_3929_10.jpeg", + "video_3929/video_3929_11.jpeg", + "video_3929/video_3929_12.jpeg", + "video_3929/video_3929_13.jpeg", + "video_3929/video_3929_14.jpeg", + "video_3929/video_3929_15.jpeg", + "video_3929/video_3929_16.jpeg", + "video_3929/video_3929_17.jpeg", + "video_3929/video_3929_18.jpeg", + "video_3929/video_3929_19.jpeg", + "video_3929/video_3929_20.jpeg", + "video_3929/video_3929_21.jpeg", + "video_3929/video_3929_22.jpeg", + "video_3929/video_3929_23.jpeg", + "video_3929/video_3929_24.jpeg", + "video_3929/video_3929_25.jpeg", + "video_3929/video_3929_26.jpeg", + "video_3929/video_3929_27.jpeg", + "video_3929/video_3929_28.jpeg", + "video_3929/video_3929_29.jpeg", + "video_3929/video_3929_30.jpeg", + "video_3929/video_3929_31.jpeg", + "video_3929/video_3929_32.jpeg", + "video_3929/video_3929_33.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 130, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. ochy\nB. hoyc\nC. cyoh\nAnswer with the option's letter from the given choices directly.", + "gt_response": "cyoh", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "ochy", + "hoyc", + "cyoh" + ], + "image_quantity_level": "Many", + "image": [ + "video_8772/video_8772_0.jpeg", + "video_8772/video_8772_1.jpeg", + "video_8772/video_8772_2.jpeg", + "video_8772/video_8772_3.jpeg", + "video_8772/video_8772_4.jpeg", + "video_8772/video_8772_5.jpeg", + "video_8772/video_8772_6.jpeg", + "video_8772/video_8772_7.jpeg", + "video_8772/video_8772_8.jpeg", + "video_8772/video_8772_9.jpeg", + "video_8772/video_8772_10.jpeg", + "video_8772/video_8772_11.jpeg", + "video_8772/video_8772_12.jpeg", + "video_8772/video_8772_13.jpeg", + "video_8772/video_8772_14.jpeg", + "video_8772/video_8772_15.jpeg", + "video_8772/video_8772_16.jpeg", + "video_8772/video_8772_17.jpeg", + "video_8772/video_8772_18.jpeg", + "video_8772/video_8772_19.jpeg", + "video_8772/video_8772_20.jpeg", + "video_8772/video_8772_21.jpeg", + "video_8772/video_8772_22.jpeg", + "video_8772/video_8772_23.jpeg", + "video_8772/video_8772_24.jpeg", + "video_8772/video_8772_25.jpeg", + "video_8772/video_8772_26.jpeg", + "video_8772/video_8772_27.jpeg", + "video_8772/video_8772_28.jpeg", + "video_8772/video_8772_29.jpeg", + "video_8772/video_8772_30.jpeg", + "video_8772/video_8772_31.jpeg", + "video_8772/video_8772_32.jpeg", + "video_8772/video_8772_33.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 137, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. dc\nB. zd\nC. cd\nAnswer with the option's letter from the given choices directly.", + "gt_response": "cd", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "dc", + "zd", + "cd" + ], + "image_quantity_level": "Many", + "image": [ + "video_2565/video_2565_0.jpeg", + "video_2565/video_2565_1.jpeg", + "video_2565/video_2565_2.jpeg", + "video_2565/video_2565_3.jpeg", + "video_2565/video_2565_4.jpeg", + "video_2565/video_2565_5.jpeg", + "video_2565/video_2565_6.jpeg", + "video_2565/video_2565_7.jpeg", + "video_2565/video_2565_8.jpeg", + "video_2565/video_2565_9.jpeg", + "video_2565/video_2565_10.jpeg", + "video_2565/video_2565_11.jpeg", + "video_2565/video_2565_12.jpeg", + "video_2565/video_2565_13.jpeg", + "video_2565/video_2565_14.jpeg", + "video_2565/video_2565_15.jpeg", + "video_2565/video_2565_16.jpeg", + "video_2565/video_2565_17.jpeg", + "video_2565/video_2565_18.jpeg", + "video_2565/video_2565_19.jpeg", + "video_2565/video_2565_20.jpeg", + "video_2565/video_2565_21.jpeg", + "video_2565/video_2565_22.jpeg", + "video_2565/video_2565_23.jpeg", + "video_2565/video_2565_24.jpeg", + "video_2565/video_2565_25.jpeg", + "video_2565/video_2565_26.jpeg", + "video_2565/video_2565_27.jpeg", + "video_2565/video_2565_28.jpeg", + "video_2565/video_2565_29.jpeg", + "video_2565/video_2565_30.jpeg", + "video_2565/video_2565_31.jpeg", + "video_2565/video_2565_32.jpeg", + "video_2565/video_2565_33.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 139, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. alic\nB. laic\nC. ical\nAnswer with the option's letter from the given choices directly.", + "gt_response": "alic", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "alic", + "laic", + "ical" + ], + "image_quantity_level": "Many", + "image": [ + "video_4370/video_4370_0.jpeg", + "video_4370/video_4370_1.jpeg", + "video_4370/video_4370_2.jpeg", + "video_4370/video_4370_3.jpeg", + "video_4370/video_4370_4.jpeg", + "video_4370/video_4370_5.jpeg", + "video_4370/video_4370_6.jpeg", + "video_4370/video_4370_7.jpeg", + "video_4370/video_4370_8.jpeg", + "video_4370/video_4370_9.jpeg", + "video_4370/video_4370_10.jpeg", + "video_4370/video_4370_11.jpeg", + "video_4370/video_4370_12.jpeg", + "video_4370/video_4370_13.jpeg", + "video_4370/video_4370_14.jpeg", + "video_4370/video_4370_15.jpeg", + "video_4370/video_4370_16.jpeg", + "video_4370/video_4370_17.jpeg", + "video_4370/video_4370_18.jpeg", + "video_4370/video_4370_19.jpeg", + "video_4370/video_4370_20.jpeg", + "video_4370/video_4370_21.jpeg", + "video_4370/video_4370_22.jpeg", + "video_4370/video_4370_23.jpeg", + "video_4370/video_4370_24.jpeg", + "video_4370/video_4370_25.jpeg", + "video_4370/video_4370_26.jpeg", + "video_4370/video_4370_27.jpeg", + "video_4370/video_4370_28.jpeg", + "video_4370/video_4370_29.jpeg", + "video_4370/video_4370_30.jpeg", + "video_4370/video_4370_31.jpeg", + "video_4370/video_4370_32.jpeg", + "video_4370/video_4370_33.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 148, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. box\nB. lom\nC. xob\nAnswer with the option's letter from the given choices directly.", + "gt_response": "box", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "box", + "lom", + "xob" + ], + "image_quantity_level": "Many", + "image": [ + "video_11060/video_11060_0.jpeg", + "video_11060/video_11060_1.jpeg", + "video_11060/video_11060_2.jpeg", + "video_11060/video_11060_3.jpeg", + "video_11060/video_11060_4.jpeg", + "video_11060/video_11060_5.jpeg", + "video_11060/video_11060_6.jpeg", + "video_11060/video_11060_7.jpeg", + "video_11060/video_11060_8.jpeg", + "video_11060/video_11060_9.jpeg", + "video_11060/video_11060_10.jpeg", + "video_11060/video_11060_11.jpeg", + "video_11060/video_11060_12.jpeg", + "video_11060/video_11060_13.jpeg", + "video_11060/video_11060_14.jpeg", + "video_11060/video_11060_15.jpeg", + "video_11060/video_11060_16.jpeg", + "video_11060/video_11060_17.jpeg", + "video_11060/video_11060_18.jpeg", + "video_11060/video_11060_19.jpeg", + "video_11060/video_11060_20.jpeg", + "video_11060/video_11060_21.jpeg", + "video_11060/video_11060_22.jpeg", + "video_11060/video_11060_23.jpeg", + "video_11060/video_11060_24.jpeg", + "video_11060/video_11060_25.jpeg", + "video_11060/video_11060_26.jpeg", + "video_11060/video_11060_27.jpeg", + "video_11060/video_11060_28.jpeg", + "video_11060/video_11060_29.jpeg", + "video_11060/video_11060_30.jpeg", + "video_11060/video_11060_31.jpeg", + "video_11060/video_11060_32.jpeg", + "video_11060/video_11060_33.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 150, + "question": "Scrutinize the provided visuals and answer the question about the arrangement of characters. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. hrdae\nB. darhe\nC. ehrad\nAnswer with the option's letter from the given choices directly.", + "gt_response": "darhe", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "hrdae", + "darhe", + "ehrad" + ], + "image_quantity_level": "Many", + "image": [ + "video_10826/video_10826_0.jpeg", + "video_10826/video_10826_1.jpeg", + "video_10826/video_10826_2.jpeg", + "video_10826/video_10826_3.jpeg", + "video_10826/video_10826_4.jpeg", + "video_10826/video_10826_5.jpeg", + "video_10826/video_10826_6.jpeg", + "video_10826/video_10826_7.jpeg", + "video_10826/video_10826_8.jpeg", + "video_10826/video_10826_9.jpeg", + "video_10826/video_10826_10.jpeg", + "video_10826/video_10826_11.jpeg", + "video_10826/video_10826_12.jpeg", + "video_10826/video_10826_13.jpeg", + "video_10826/video_10826_14.jpeg", + "video_10826/video_10826_15.jpeg", + "video_10826/video_10826_16.jpeg", + "video_10826/video_10826_17.jpeg", + "video_10826/video_10826_18.jpeg", + "video_10826/video_10826_19.jpeg", + "video_10826/video_10826_20.jpeg", + "video_10826/video_10826_21.jpeg", + "video_10826/video_10826_22.jpeg", + "video_10826/video_10826_23.jpeg", + "video_10826/video_10826_24.jpeg", + "video_10826/video_10826_25.jpeg", + "video_10826/video_10826_26.jpeg", + "video_10826/video_10826_27.jpeg", + "video_10826/video_10826_28.jpeg", + "video_10826/video_10826_29.jpeg", + "video_10826/video_10826_30.jpeg", + "video_10826/video_10826_31.jpeg", + "video_10826/video_10826_32.jpeg", + "video_10826/video_10826_33.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 159, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. m\nB. l\nC. o\nAnswer with the option's letter from the given choices directly.", + "gt_response": "o", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "m", + "l", + "o" + ], + "image_quantity_level": "Many", + "image": [ + "video_10484/video_10484_0.jpeg", + "video_10484/video_10484_1.jpeg", + "video_10484/video_10484_2.jpeg", + "video_10484/video_10484_3.jpeg", + "video_10484/video_10484_4.jpeg", + "video_10484/video_10484_5.jpeg", + "video_10484/video_10484_6.jpeg", + "video_10484/video_10484_7.jpeg", + "video_10484/video_10484_8.jpeg", + "video_10484/video_10484_9.jpeg", + "video_10484/video_10484_10.jpeg", + "video_10484/video_10484_11.jpeg", + "video_10484/video_10484_12.jpeg", + "video_10484/video_10484_13.jpeg", + "video_10484/video_10484_14.jpeg", + "video_10484/video_10484_15.jpeg", + "video_10484/video_10484_16.jpeg", + "video_10484/video_10484_17.jpeg", + "video_10484/video_10484_18.jpeg", + "video_10484/video_10484_19.jpeg", + "video_10484/video_10484_20.jpeg", + "video_10484/video_10484_21.jpeg", + "video_10484/video_10484_22.jpeg", + "video_10484/video_10484_23.jpeg", + "video_10484/video_10484_24.jpeg", + "video_10484/video_10484_25.jpeg", + "video_10484/video_10484_26.jpeg", + "video_10484/video_10484_27.jpeg", + "video_10484/video_10484_28.jpeg", + "video_10484/video_10484_29.jpeg", + "video_10484/video_10484_30.jpeg", + "video_10484/video_10484_31.jpeg", + "video_10484/video_10484_32.jpeg", + "video_10484/video_10484_33.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 171, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters on the table at the end?\nChoice list: \nA. abn\nB. nab\nC. bna\nAnswer with the option's letter from the given choices directly.", + "gt_response": "bna", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "abn", + "nab", + "bna" + ], + "image_quantity_level": "Medium", + "image": [ + "video_4451/video_4451_0.jpeg", + "video_4451/video_4451_1.jpeg", + "video_4451/video_4451_2.jpeg", + "video_4451/video_4451_3.jpeg", + "video_4451/video_4451_4.jpeg", + "video_4451/video_4451_5.jpeg", + "video_4451/video_4451_6.jpeg", + "video_4451/video_4451_7.jpeg", + "video_4451/video_4451_8.jpeg", + "video_4451/video_4451_9.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 143, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. z\nB. o\nC. n\nAnswer with the option's letter from the given choices directly.", + "gt_response": "z", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "z", + "o", + "n" + ], + "image_quantity_level": "Medium", + "image": [ + "video_4910/video_4910_0.jpeg", + "video_4910/video_4910_1.jpeg", + "video_4910/video_4910_2.jpeg", + "video_4910/video_4910_3.jpeg", + "video_4910/video_4910_4.jpeg", + "video_4910/video_4910_5.jpeg", + "video_4910/video_4910_6.jpeg", + "video_4910/video_4910_7.jpeg", + "video_4910/video_4910_8.jpeg", + "video_4910/video_4910_9.jpeg", + "video_4910/video_4910_10.jpeg", + "video_4910/video_4910_11.jpeg", + "video_4910/video_4910_12.jpeg", + "video_4910/video_4910_13.jpeg", + "video_4910/video_4910_14.jpeg", + "video_4910/video_4910_15.jpeg", + "video_4910/video_4910_16.jpeg", + "video_4910/video_4910_17.jpeg", + "video_4910/video_4910_18.jpeg", + "video_4910/video_4910_19.jpeg", + "video_4910/video_4910_20.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 153, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nWhat letter is likely to be shown next?\nChoice list: \nA. a\nB. e\nC. m\nAnswer with the option's letter from the given choices directly.", + "gt_response": "e", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "a", + "e", + "m" + ], + "image_quantity_level": "Medium", + "image": [ + "video_7145/video_7145_0.jpeg", + "video_7145/video_7145_1.jpeg", + "video_7145/video_7145_2.jpeg", + "video_7145/video_7145_3.jpeg", + "video_7145/video_7145_4.jpeg", + "video_7145/video_7145_5.jpeg", + "video_7145/video_7145_6.jpeg", + "video_7145/video_7145_7.jpeg", + "video_7145/video_7145_8.jpeg", + "video_7145/video_7145_9.jpeg", + "video_7145/video_7145_10.jpeg", + "video_7145/video_7145_11.jpeg", + "video_7145/video_7145_12.jpeg", + "video_7145/video_7145_13.jpeg", + "video_7145/video_7145_14.jpeg", + "video_7145/video_7145_15.jpeg", + "video_7145/video_7145_16.jpeg", + "video_7145/video_7145_17.jpeg", + "video_7145/video_7145_18.jpeg", + "video_7145/video_7145_19.jpeg", + "video_7145/video_7145_20.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 160, + "question": "Scrutinize the provided visuals and answer the question about the arrangement of characters. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. haik\nB. habo\nC. aobh\nAnswer with the option's letter from the given choices directly.", + "gt_response": "aobh", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "haik", + "habo", + "aobh" + ], + "image_quantity_level": "Medium", + "image": [ + "video_8323/video_8323_0.jpeg", + "video_8323/video_8323_1.jpeg", + "video_8323/video_8323_2.jpeg", + "video_8323/video_8323_3.jpeg", + "video_8323/video_8323_4.jpeg", + "video_8323/video_8323_5.jpeg", + "video_8323/video_8323_6.jpeg", + "video_8323/video_8323_7.jpeg", + "video_8323/video_8323_8.jpeg", + "video_8323/video_8323_9.jpeg", + "video_8323/video_8323_10.jpeg", + "video_8323/video_8323_11.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 99, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. ech\nB. hec\nC. che\nAnswer with the option's letter from the given choices directly.", + "gt_response": "hec", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "ech", + "hec", + "che" + ], + "image_quantity_level": "Many", + "image": [ + "video_10765/video_10765_0.jpeg", + "video_10765/video_10765_1.jpeg", + "video_10765/video_10765_2.jpeg", + "video_10765/video_10765_3.jpeg", + "video_10765/video_10765_4.jpeg", + "video_10765/video_10765_5.jpeg", + "video_10765/video_10765_6.jpeg", + "video_10765/video_10765_7.jpeg", + "video_10765/video_10765_8.jpeg", + "video_10765/video_10765_9.jpeg", + "video_10765/video_10765_10.jpeg", + "video_10765/video_10765_11.jpeg", + "video_10765/video_10765_12.jpeg", + "video_10765/video_10765_13.jpeg", + "video_10765/video_10765_14.jpeg", + "video_10765/video_10765_15.jpeg", + "video_10765/video_10765_16.jpeg", + "video_10765/video_10765_17.jpeg", + "video_10765/video_10765_18.jpeg", + "video_10765/video_10765_19.jpeg", + "video_10765/video_10765_20.jpeg", + "video_10765/video_10765_21.jpeg", + "video_10765/video_10765_22.jpeg", + "video_10765/video_10765_23.jpeg", + "video_10765/video_10765_24.jpeg", + "video_10765/video_10765_25.jpeg", + "video_10765/video_10765_26.jpeg", + "video_10765/video_10765_27.jpeg", + "video_10765/video_10765_28.jpeg", + "video_10765/video_10765_29.jpeg", + "video_10765/video_10765_30.jpeg", + "video_10765/video_10765_31.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 118, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. red\nB. rde\nC. erd\nAnswer with the option's letter from the given choices directly.", + "gt_response": "red", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "red", + "rde", + "erd" + ], + "image_quantity_level": "Many", + "image": [ + "video_9769/video_9769_0.jpeg", + "video_9769/video_9769_1.jpeg", + "video_9769/video_9769_2.jpeg", + "video_9769/video_9769_3.jpeg", + "video_9769/video_9769_4.jpeg", + "video_9769/video_9769_5.jpeg", + "video_9769/video_9769_6.jpeg", + "video_9769/video_9769_7.jpeg", + "video_9769/video_9769_8.jpeg", + "video_9769/video_9769_9.jpeg", + "video_9769/video_9769_10.jpeg", + "video_9769/video_9769_11.jpeg", + "video_9769/video_9769_12.jpeg", + "video_9769/video_9769_13.jpeg", + "video_9769/video_9769_14.jpeg", + "video_9769/video_9769_15.jpeg", + "video_9769/video_9769_16.jpeg", + "video_9769/video_9769_17.jpeg", + "video_9769/video_9769_18.jpeg", + "video_9769/video_9769_19.jpeg", + "video_9769/video_9769_20.jpeg", + "video_9769/video_9769_21.jpeg", + "video_9769/video_9769_22.jpeg", + "video_9769/video_9769_23.jpeg", + "video_9769/video_9769_24.jpeg", + "video_9769/video_9769_25.jpeg", + "video_9769/video_9769_26.jpeg", + "video_9769/video_9769_27.jpeg", + "video_9769/video_9769_28.jpeg", + "video_9769/video_9769_29.jpeg", + "video_9769/video_9769_30.jpeg", + "video_9769/video_9769_31.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 133, + "question": "From the images supplied, answer the question relevant to character order. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. av\nB. va\nC. ya\nAnswer with the option's letter from the given choices directly.", + "gt_response": "va", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "av", + "va", + "ya" + ], + "image_quantity_level": "Many", + "image": [ + "video_9001/video_9001_0.jpeg", + "video_9001/video_9001_1.jpeg", + "video_9001/video_9001_2.jpeg", + "video_9001/video_9001_3.jpeg", + "video_9001/video_9001_4.jpeg", + "video_9001/video_9001_5.jpeg", + "video_9001/video_9001_6.jpeg", + "video_9001/video_9001_7.jpeg", + "video_9001/video_9001_8.jpeg", + "video_9001/video_9001_9.jpeg", + "video_9001/video_9001_10.jpeg", + "video_9001/video_9001_11.jpeg", + "video_9001/video_9001_12.jpeg", + "video_9001/video_9001_13.jpeg", + "video_9001/video_9001_14.jpeg", + "video_9001/video_9001_15.jpeg", + "video_9001/video_9001_16.jpeg", + "video_9001/video_9001_17.jpeg", + "video_9001/video_9001_18.jpeg", + "video_9001/video_9001_19.jpeg", + "video_9001/video_9001_20.jpeg", + "video_9001/video_9001_21.jpeg", + "video_9001/video_9001_22.jpeg", + "video_9001/video_9001_23.jpeg", + "video_9001/video_9001_24.jpeg", + "video_9001/video_9001_25.jpeg", + "video_9001/video_9001_26.jpeg", + "video_9001/video_9001_27.jpeg", + "video_9001/video_9001_28.jpeg", + "video_9001/video_9001_29.jpeg", + "video_9001/video_9001_30.jpeg", + "video_9001/video_9001_31.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 122, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. eat\nB. ate\nC. tae\nAnswer with the option's letter from the given choices directly.", + "gt_response": "eat", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "eat", + "ate", + "tae" + ], + "image_quantity_level": "Many", + "image": [ + "video_3496/video_3496_0.jpeg", + "video_3496/video_3496_1.jpeg", + "video_3496/video_3496_2.jpeg", + "video_3496/video_3496_3.jpeg", + "video_3496/video_3496_4.jpeg", + "video_3496/video_3496_5.jpeg", + "video_3496/video_3496_6.jpeg", + "video_3496/video_3496_7.jpeg", + "video_3496/video_3496_8.jpeg", + "video_3496/video_3496_9.jpeg", + "video_3496/video_3496_10.jpeg", + "video_3496/video_3496_11.jpeg", + "video_3496/video_3496_12.jpeg", + "video_3496/video_3496_13.jpeg", + "video_3496/video_3496_14.jpeg", + "video_3496/video_3496_15.jpeg", + "video_3496/video_3496_16.jpeg", + "video_3496/video_3496_17.jpeg", + "video_3496/video_3496_18.jpeg", + "video_3496/video_3496_19.jpeg", + "video_3496/video_3496_20.jpeg", + "video_3496/video_3496_21.jpeg", + "video_3496/video_3496_22.jpeg", + "video_3496/video_3496_23.jpeg", + "video_3496/video_3496_24.jpeg", + "video_3496/video_3496_25.jpeg", + "video_3496/video_3496_26.jpeg", + "video_3496/video_3496_27.jpeg", + "video_3496/video_3496_28.jpeg", + "video_3496/video_3496_29.jpeg", + "video_3496/video_3496_30.jpeg", + "video_3496/video_3496_31.jpeg", + "video_3496/video_3496_32.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 164, + "question": "From the images supplied, answer the question relevant to character order. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. hcj\nB. hij\nC. uzj\nAnswer with the option's letter from the given choices directly.", + "gt_response": "hij", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "hcj", + "hij", + "uzj" + ], + "image_quantity_level": "Medium", + "image": [ + "video_11167/video_11167_0.jpeg", + "video_11167/video_11167_1.jpeg", + "video_11167/video_11167_2.jpeg", + "video_11167/video_11167_3.jpeg", + "video_11167/video_11167_4.jpeg", + "video_11167/video_11167_5.jpeg", + "video_11167/video_11167_6.jpeg", + "video_11167/video_11167_7.jpeg", + "video_11167/video_11167_8.jpeg", + "video_11167/video_11167_9.jpeg", + "video_11167/video_11167_10.jpeg", + "video_11167/video_11167_11.jpeg", + "video_11167/video_11167_12.jpeg", + "video_11167/video_11167_13.jpeg", + "video_11167/video_11167_14.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 170, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nWhat was the order of the letters on the table before shuffling?\nChoice list: \nA. fdacbe\nB. abcdef\nC. dacbef\nAnswer with the option's letter from the given choices directly.", + "gt_response": "abcdef", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "fdacbe", + "abcdef", + "dacbef" + ], + "image_quantity_level": "Medium", + "image": [ + "video_4232/video_4232_0.jpeg", + "video_4232/video_4232_1.jpeg", + "video_4232/video_4232_2.jpeg", + "video_4232/video_4232_3.jpeg", + "video_4232/video_4232_4.jpeg", + "video_4232/video_4232_5.jpeg", + "video_4232/video_4232_6.jpeg", + "video_4232/video_4232_7.jpeg", + "video_4232/video_4232_8.jpeg", + "video_4232/video_4232_9.jpeg", + "video_4232/video_4232_10.jpeg", + "video_4232/video_4232_11.jpeg", + "video_4232/video_4232_12.jpeg", + "video_4232/video_4232_13.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 136, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. feba\nB. feby\nC. fdsa\nAnswer with the option's letter from the given choices directly.", + "gt_response": "feba", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "feba", + "feby", + "fdsa" + ], + "image_quantity_level": "Medium", + "image": [ + "video_10031/video_10031_0.jpeg", + "video_10031/video_10031_1.jpeg", + "video_10031/video_10031_2.jpeg", + "video_10031/video_10031_3.jpeg", + "video_10031/video_10031_4.jpeg", + "video_10031/video_10031_5.jpeg", + "video_10031/video_10031_6.jpeg", + "video_10031/video_10031_7.jpeg", + "video_10031/video_10031_8.jpeg", + "video_10031/video_10031_9.jpeg", + "video_10031/video_10031_10.jpeg", + "video_10031/video_10031_11.jpeg", + "video_10031/video_10031_12.jpeg", + "video_10031/video_10031_13.jpeg", + "video_10031/video_10031_14.jpeg", + "video_10031/video_10031_15.jpeg", + "video_10031/video_10031_16.jpeg", + "video_10031/video_10031_17.jpeg", + "video_10031/video_10031_18.jpeg", + "video_10031/video_10031_19.jpeg", + "video_10031/video_10031_20.jpeg", + "video_10031/video_10031_21.jpeg", + "video_10031/video_10031_22.jpeg", + "video_10031/video_10031_23.jpeg", + "video_10031/video_10031_24.jpeg", + "video_10031/video_10031_25.jpeg", + "video_10031/video_10031_26.jpeg", + "video_10031/video_10031_27.jpeg", + "video_10031/video_10031_28.jpeg", + "video_10031/video_10031_29.jpeg", + "video_10031/video_10031_30.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 157, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. gair\nB. ragi\nC. gari\nAnswer with the option's letter from the given choices directly.", + "gt_response": "ragi", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "gair", + "ragi", + "gari" + ], + "image_quantity_level": "Medium", + "image": [ + "video_1168/video_1168_0.jpeg", + "video_1168/video_1168_1.jpeg", + "video_1168/video_1168_2.jpeg", + "video_1168/video_1168_3.jpeg", + "video_1168/video_1168_4.jpeg", + "video_1168/video_1168_5.jpeg", + "video_1168/video_1168_6.jpeg", + "video_1168/video_1168_7.jpeg", + "video_1168/video_1168_8.jpeg", + "video_1168/video_1168_9.jpeg", + "video_1168/video_1168_10.jpeg", + "video_1168/video_1168_11.jpeg", + "video_1168/video_1168_12.jpeg", + "video_1168/video_1168_13.jpeg", + "video_1168/video_1168_14.jpeg", + "video_1168/video_1168_15.jpeg", + "video_1168/video_1168_16.jpeg", + "video_1168/video_1168_17.jpeg", + "video_1168/video_1168_18.jpeg", + "video_1168/video_1168_19.jpeg", + "video_1168/video_1168_20.jpeg", + "video_1168/video_1168_21.jpeg", + "video_1168/video_1168_22.jpeg", + "video_1168/video_1168_23.jpeg", + "video_1168/video_1168_24.jpeg", + "video_1168/video_1168_25.jpeg", + "video_1168/video_1168_26.jpeg", + "video_1168/video_1168_27.jpeg", + "video_1168/video_1168_28.jpeg", + "video_1168/video_1168_29.jpeg", + "video_1168/video_1168_30.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 112, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. o\nB. n\nC. e\nAnswer with the option's letter from the given choices directly.", + "gt_response": "n", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "o", + "n", + "e" + ], + "image_quantity_level": "Medium", + "image": [ + "video_2125/video_2125_0.jpeg", + "video_2125/video_2125_1.jpeg", + "video_2125/video_2125_2.jpeg", + "video_2125/video_2125_3.jpeg", + "video_2125/video_2125_4.jpeg", + "video_2125/video_2125_5.jpeg", + "video_2125/video_2125_6.jpeg", + "video_2125/video_2125_7.jpeg", + "video_2125/video_2125_8.jpeg", + "video_2125/video_2125_9.jpeg", + "video_2125/video_2125_10.jpeg", + "video_2125/video_2125_11.jpeg", + "video_2125/video_2125_12.jpeg", + "video_2125/video_2125_13.jpeg", + "video_2125/video_2125_14.jpeg", + "video_2125/video_2125_15.jpeg", + "video_2125/video_2125_16.jpeg", + "video_2125/video_2125_17.jpeg", + "video_2125/video_2125_18.jpeg", + "video_2125/video_2125_19.jpeg", + "video_2125/video_2125_20.jpeg", + "video_2125/video_2125_21.jpeg", + "video_2125/video_2125_22.jpeg", + "video_2125/video_2125_23.jpeg", + "video_2125/video_2125_24.jpeg", + "video_2125/video_2125_25.jpeg", + "video_2125/video_2125_26.jpeg", + "video_2125/video_2125_27.jpeg", + "video_2125/video_2125_28.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 119, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters on the table at the end?\nChoice list: \nA. ghijkl\nB. kglhji\nC. igljkh\nAnswer with the option's letter from the given choices directly.", + "gt_response": "ghijkl", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "ghijkl", + "kglhji", + "igljkh" + ], + "image_quantity_level": "Medium", + "image": [ + "video_4871/video_4871_0.jpeg", + "video_4871/video_4871_1.jpeg", + "video_4871/video_4871_2.jpeg", + "video_4871/video_4871_3.jpeg", + "video_4871/video_4871_4.jpeg", + "video_4871/video_4871_5.jpeg", + "video_4871/video_4871_6.jpeg", + "video_4871/video_4871_7.jpeg", + "video_4871/video_4871_8.jpeg", + "video_4871/video_4871_9.jpeg", + "video_4871/video_4871_10.jpeg", + "video_4871/video_4871_11.jpeg", + "video_4871/video_4871_12.jpeg", + "video_4871/video_4871_13.jpeg", + "video_4871/video_4871_14.jpeg", + "video_4871/video_4871_15.jpeg", + "video_4871/video_4871_16.jpeg", + "video_4871/video_4871_17.jpeg", + "video_4871/video_4871_18.jpeg", + "video_4871/video_4871_19.jpeg", + "video_4871/video_4871_20.jpeg", + "video_4871/video_4871_21.jpeg", + "video_4871/video_4871_22.jpeg", + "video_4871/video_4871_23.jpeg", + "video_4871/video_4871_24.jpeg", + "video_4871/video_4871_25.jpeg", + "video_4871/video_4871_26.jpeg", + "video_4871/video_4871_27.jpeg", + "video_4871/video_4871_28.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 89, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nWhat was the order of the letters on the table before shuffling?\nChoice list: \nA. eupckac\nB. cupcake\nC. pkcaeuc\nAnswer with the option's letter from the given choices directly.", + "gt_response": "eupckac", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "eupckac", + "cupcake", + "pkcaeuc" + ], + "image_quantity_level": "Medium", + "image": [ + "video_3195/video_3195_0.jpeg", + "video_3195/video_3195_1.jpeg", + "video_3195/video_3195_2.jpeg", + "video_3195/video_3195_3.jpeg", + "video_3195/video_3195_4.jpeg", + "video_3195/video_3195_5.jpeg", + "video_3195/video_3195_6.jpeg", + "video_3195/video_3195_7.jpeg", + "video_3195/video_3195_8.jpeg", + "video_3195/video_3195_9.jpeg", + "video_3195/video_3195_10.jpeg", + "video_3195/video_3195_11.jpeg", + "video_3195/video_3195_12.jpeg", + "video_3195/video_3195_13.jpeg", + "video_3195/video_3195_14.jpeg", + "video_3195/video_3195_15.jpeg", + "video_3195/video_3195_16.jpeg", + "video_3195/video_3195_17.jpeg", + "video_3195/video_3195_18.jpeg", + "video_3195/video_3195_19.jpeg", + "video_3195/video_3195_20.jpeg", + "video_3195/video_3195_21.jpeg", + "video_3195/video_3195_22.jpeg", + "video_3195/video_3195_23.jpeg", + "video_3195/video_3195_24.jpeg", + "video_3195/video_3195_25.jpeg", + "video_3195/video_3195_26.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 163, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. jklmn\nB. jxigc\nC. jsrho\nAnswer with the option's letter from the given choices directly.", + "gt_response": "jklmn", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "jklmn", + "jxigc", + "jsrho" + ], + "image_quantity_level": "Medium", + "image": [ + "video_355/video_355_0.jpeg", + "video_355/video_355_1.jpeg", + "video_355/video_355_2.jpeg", + "video_355/video_355_3.jpeg", + "video_355/video_355_4.jpeg", + "video_355/video_355_5.jpeg", + "video_355/video_355_6.jpeg", + "video_355/video_355_7.jpeg", + "video_355/video_355_8.jpeg", + "video_355/video_355_9.jpeg", + "video_355/video_355_10.jpeg", + "video_355/video_355_11.jpeg", + "video_355/video_355_12.jpeg", + "video_355/video_355_13.jpeg", + "video_355/video_355_14.jpeg", + "video_355/video_355_15.jpeg", + "video_355/video_355_16.jpeg", + "video_355/video_355_17.jpeg", + "video_355/video_355_18.jpeg", + "video_355/video_355_19.jpeg", + "video_355/video_355_20.jpeg", + "video_355/video_355_21.jpeg", + "video_355/video_355_22.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 198, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. a\nB. e\nC. r\nAnswer with the option's letter from the given choices directly.", + "gt_response": "r", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "a", + "e", + "r" + ], + "image_quantity_level": "Medium", + "image": [ + "video_9723/video_9723_0.jpeg", + "video_9723/video_9723_1.jpeg", + "video_9723/video_9723_2.jpeg", + "video_9723/video_9723_3.jpeg", + "video_9723/video_9723_4.jpeg", + "video_9723/video_9723_5.jpeg", + "video_9723/video_9723_6.jpeg", + "video_9723/video_9723_7.jpeg", + "video_9723/video_9723_8.jpeg", + "video_9723/video_9723_9.jpeg", + "video_9723/video_9723_10.jpeg", + "video_9723/video_9723_11.jpeg", + "video_9723/video_9723_12.jpeg", + "video_9723/video_9723_13.jpeg", + "video_9723/video_9723_14.jpeg", + "video_9723/video_9723_15.jpeg", + "video_9723/video_9723_16.jpeg", + "video_9723/video_9723_17.jpeg", + "video_9723/video_9723_18.jpeg", + "video_9723/video_9723_19.jpeg", + "video_9723/video_9723_20.jpeg", + "video_9723/video_9723_21.jpeg", + "video_9723/video_9723_22.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 188, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. bavh\nB. abhv\nC. vhba\nAnswer with the option's letter from the given choices directly.", + "gt_response": "bavh", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "bavh", + "abhv", + "vhba" + ], + "image_quantity_level": "Medium", + "image": [ + "video_7119/video_7119_0.jpeg", + "video_7119/video_7119_1.jpeg", + "video_7119/video_7119_2.jpeg", + "video_7119/video_7119_3.jpeg", + "video_7119/video_7119_4.jpeg", + "video_7119/video_7119_5.jpeg", + "video_7119/video_7119_6.jpeg", + "video_7119/video_7119_7.jpeg", + "video_7119/video_7119_8.jpeg", + "video_7119/video_7119_9.jpeg", + "video_7119/video_7119_10.jpeg", + "video_7119/video_7119_11.jpeg", + "video_7119/video_7119_12.jpeg", + "video_7119/video_7119_13.jpeg", + "video_7119/video_7119_14.jpeg", + "video_7119/video_7119_15.jpeg", + "video_7119/video_7119_16.jpeg", + "video_7119/video_7119_17.jpeg", + "video_7119/video_7119_18.jpeg", + "video_7119/video_7119_19.jpeg", + "video_7119/video_7119_20.jpeg", + "video_7119/video_7119_21.jpeg", + "video_7119/video_7119_22.jpeg", + "video_7119/video_7119_23.jpeg", + "video_7119/video_7119_24.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 194, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. o\nB. n\nC. h\nAnswer with the option's letter from the given choices directly.", + "gt_response": "o", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "o", + "n", + "h" + ], + "image_quantity_level": "Medium", + "image": [ + "video_6373/video_6373_0.jpeg", + "video_6373/video_6373_1.jpeg", + "video_6373/video_6373_2.jpeg", + "video_6373/video_6373_3.jpeg", + "video_6373/video_6373_4.jpeg", + "video_6373/video_6373_5.jpeg", + "video_6373/video_6373_6.jpeg", + "video_6373/video_6373_7.jpeg", + "video_6373/video_6373_8.jpeg", + "video_6373/video_6373_9.jpeg", + "video_6373/video_6373_10.jpeg", + "video_6373/video_6373_11.jpeg", + "video_6373/video_6373_12.jpeg", + "video_6373/video_6373_13.jpeg", + "video_6373/video_6373_14.jpeg", + "video_6373/video_6373_15.jpeg", + "video_6373/video_6373_16.jpeg", + "video_6373/video_6373_17.jpeg", + "video_6373/video_6373_18.jpeg", + "video_6373/video_6373_19.jpeg", + "video_6373/video_6373_20.jpeg", + "video_6373/video_6373_21.jpeg", + "video_6373/video_6373_22.jpeg", + "video_6373/video_6373_23.jpeg", + "video_6373/video_6373_24.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 196, + "question": "Review the visuals provided and respond to the question concerning the character sequence. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. sml\nB. fle\nC. efl\nAnswer with the option's letter from the given choices directly.", + "gt_response": "efl", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "sml", + "fle", + "efl" + ], + "image_quantity_level": "Medium", + "image": [ + "video_9023/video_9023_0.jpeg", + "video_9023/video_9023_1.jpeg", + "video_9023/video_9023_2.jpeg", + "video_9023/video_9023_3.jpeg", + "video_9023/video_9023_4.jpeg", + "video_9023/video_9023_5.jpeg", + "video_9023/video_9023_6.jpeg", + "video_9023/video_9023_7.jpeg", + "video_9023/video_9023_8.jpeg", + "video_9023/video_9023_9.jpeg", + "video_9023/video_9023_10.jpeg", + "video_9023/video_9023_11.jpeg", + "video_9023/video_9023_12.jpeg", + "video_9023/video_9023_13.jpeg", + "video_9023/video_9023_14.jpeg", + "video_9023/video_9023_15.jpeg", + "video_9023/video_9023_16.jpeg", + "video_9023/video_9023_17.jpeg", + "video_9023/video_9023_18.jpeg", + "video_9023/video_9023_19.jpeg", + "video_9023/video_9023_20.jpeg", + "video_9023/video_9023_21.jpeg", + "video_9023/video_9023_22.jpeg", + "video_9023/video_9023_23.jpeg", + "video_9023/video_9023_24.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 147, + "question": "From the images supplied, answer the question relevant to character order. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. va\nB. vj\nC. av\nAnswer with the option's letter from the given choices directly.", + "gt_response": "va", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "va", + "vj", + "av" + ], + "image_quantity_level": "Medium", + "image": [ + "video_7661/video_7661_0.jpeg", + "video_7661/video_7661_1.jpeg", + "video_7661/video_7661_2.jpeg", + "video_7661/video_7661_3.jpeg", + "video_7661/video_7661_4.jpeg", + "video_7661/video_7661_5.jpeg", + "video_7661/video_7661_6.jpeg", + "video_7661/video_7661_7.jpeg", + "video_7661/video_7661_8.jpeg", + "video_7661/video_7661_9.jpeg", + "video_7661/video_7661_10.jpeg", + "video_7661/video_7661_11.jpeg", + "video_7661/video_7661_12.jpeg", + "video_7661/video_7661_13.jpeg", + "video_7661/video_7661_14.jpeg", + "video_7661/video_7661_15.jpeg", + "video_7661/video_7661_16.jpeg", + "video_7661/video_7661_17.jpeg", + "video_7661/video_7661_18.jpeg", + "video_7661/video_7661_19.jpeg", + "video_7661/video_7661_20.jpeg", + "video_7661/video_7661_21.jpeg", + "video_7661/video_7661_22.jpeg", + "video_7661/video_7661_23.jpeg", + "video_7661/video_7661_24.jpeg", + "video_7661/video_7661_25.jpeg", + "video_7661/video_7661_26.jpeg", + "video_7661/video_7661_27.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 189, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. d\nB. r\nC. e\nAnswer with the option's letter from the given choices directly.", + "gt_response": "r", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "d", + "r", + "e" + ], + "image_quantity_level": "Medium", + "image": [ + "video_3518/video_3518_0.jpeg", + "video_3518/video_3518_1.jpeg", + "video_3518/video_3518_2.jpeg", + "video_3518/video_3518_3.jpeg", + "video_3518/video_3518_4.jpeg", + "video_3518/video_3518_5.jpeg", + "video_3518/video_3518_6.jpeg", + "video_3518/video_3518_7.jpeg", + "video_3518/video_3518_8.jpeg", + "video_3518/video_3518_9.jpeg", + "video_3518/video_3518_10.jpeg", + "video_3518/video_3518_11.jpeg", + "video_3518/video_3518_12.jpeg", + "video_3518/video_3518_13.jpeg", + "video_3518/video_3518_14.jpeg", + "video_3518/video_3518_15.jpeg", + "video_3518/video_3518_16.jpeg", + "video_3518/video_3518_17.jpeg", + "video_3518/video_3518_18.jpeg", + "video_3518/video_3518_19.jpeg", + "video_3518/video_3518_20.jpeg", + "video_3518/video_3518_21.jpeg", + "video_3518/video_3518_22.jpeg", + "video_3518/video_3518_23.jpeg", + "video_3518/video_3518_24.jpeg", + "video_3518/video_3518_25.jpeg", + "video_3518/video_3518_26.jpeg", + "video_3518/video_3518_27.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 193, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. rna\nB. nav\nC. nar\nAnswer with the option's letter from the given choices directly.", + "gt_response": "nar", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "rna", + "nav", + "nar" + ], + "image_quantity_level": "Medium", + "image": [ + "video_10960/video_10960_0.jpeg", + "video_10960/video_10960_1.jpeg", + "video_10960/video_10960_2.jpeg", + "video_10960/video_10960_3.jpeg", + "video_10960/video_10960_4.jpeg", + "video_10960/video_10960_5.jpeg", + "video_10960/video_10960_6.jpeg", + "video_10960/video_10960_7.jpeg", + "video_10960/video_10960_8.jpeg", + "video_10960/video_10960_9.jpeg", + "video_10960/video_10960_10.jpeg", + "video_10960/video_10960_11.jpeg", + "video_10960/video_10960_12.jpeg", + "video_10960/video_10960_13.jpeg", + "video_10960/video_10960_14.jpeg", + "video_10960/video_10960_15.jpeg", + "video_10960/video_10960_16.jpeg", + "video_10960/video_10960_17.jpeg", + "video_10960/video_10960_18.jpeg", + "video_10960/video_10960_19.jpeg", + "video_10960/video_10960_20.jpeg", + "video_10960/video_10960_21.jpeg", + "video_10960/video_10960_22.jpeg", + "video_10960/video_10960_23.jpeg", + "video_10960/video_10960_24.jpeg", + "video_10960/video_10960_25.jpeg", + "video_10960/video_10960_26.jpeg", + "video_10960/video_10960_27.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 181, + "question": "Review the visuals provided and respond to the question concerning the character sequence. You must choose your answer from the Choice List.\nWhat letter is likely to be shown next?\nChoice list: \nA. b\nB. i\nC. e\nAnswer with the option's letter from the given choices directly.", + "gt_response": "e", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "b", + "i", + "e" + ], + "image_quantity_level": "Medium", + "image": [ + "video_4591/video_4591_0.jpeg", + "video_4591/video_4591_1.jpeg", + "video_4591/video_4591_2.jpeg", + "video_4591/video_4591_3.jpeg", + "video_4591/video_4591_4.jpeg", + "video_4591/video_4591_5.jpeg", + "video_4591/video_4591_6.jpeg", + "video_4591/video_4591_7.jpeg", + "video_4591/video_4591_8.jpeg", + "video_4591/video_4591_9.jpeg", + "video_4591/video_4591_10.jpeg", + "video_4591/video_4591_11.jpeg", + "video_4591/video_4591_12.jpeg", + "video_4591/video_4591_13.jpeg", + "video_4591/video_4591_14.jpeg", + "video_4591/video_4591_15.jpeg", + "video_4591/video_4591_16.jpeg", + "video_4591/video_4591_17.jpeg", + "video_4591/video_4591_18.jpeg", + "video_4591/video_4591_19.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 191, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. ymc\nB. abc\nC. hbc\nAnswer with the option's letter from the given choices directly.", + "gt_response": "abc", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "ymc", + "abc", + "hbc" + ], + "image_quantity_level": "Medium", + "image": [ + "video_11469/video_11469_0.jpeg", + "video_11469/video_11469_1.jpeg", + "video_11469/video_11469_2.jpeg", + "video_11469/video_11469_3.jpeg", + "video_11469/video_11469_4.jpeg", + "video_11469/video_11469_5.jpeg", + "video_11469/video_11469_6.jpeg", + "video_11469/video_11469_7.jpeg", + "video_11469/video_11469_8.jpeg", + "video_11469/video_11469_9.jpeg", + "video_11469/video_11469_10.jpeg", + "video_11469/video_11469_11.jpeg", + "video_11469/video_11469_12.jpeg", + "video_11469/video_11469_13.jpeg", + "video_11469/video_11469_14.jpeg", + "video_11469/video_11469_15.jpeg", + "video_11469/video_11469_16.jpeg", + "video_11469/video_11469_17.jpeg", + "video_11469/video_11469_18.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 135, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nWhat would be the order of the written letters if the person had written them in reverse order?\nChoice list: \nA. car\nB. cer\nC. rac\nAnswer with the option's letter from the given choices directly.", + "gt_response": "rac", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "car", + "cer", + "rac" + ], + "image_quantity_level": "Many", + "image": [ + "video_1860/video_1860_0.jpeg", + "video_1860/video_1860_1.jpeg", + "video_1860/video_1860_2.jpeg", + "video_1860/video_1860_3.jpeg", + "video_1860/video_1860_4.jpeg", + "video_1860/video_1860_5.jpeg", + "video_1860/video_1860_6.jpeg", + "video_1860/video_1860_7.jpeg", + "video_1860/video_1860_8.jpeg", + "video_1860/video_1860_9.jpeg", + "video_1860/video_1860_10.jpeg", + "video_1860/video_1860_11.jpeg", + "video_1860/video_1860_12.jpeg", + "video_1860/video_1860_13.jpeg", + "video_1860/video_1860_14.jpeg", + "video_1860/video_1860_15.jpeg", + "video_1860/video_1860_16.jpeg", + "video_1860/video_1860_17.jpeg", + "video_1860/video_1860_18.jpeg", + "video_1860/video_1860_19.jpeg", + "video_1860/video_1860_20.jpeg", + "video_1860/video_1860_21.jpeg", + "video_1860/video_1860_22.jpeg", + "video_1860/video_1860_23.jpeg", + "video_1860/video_1860_24.jpeg", + "video_1860/video_1860_25.jpeg", + "video_1860/video_1860_26.jpeg", + "video_1860/video_1860_27.jpeg", + "video_1860/video_1860_28.jpeg", + "video_1860/video_1860_29.jpeg", + "video_1860/video_1860_30.jpeg", + "video_1860/video_1860_31.jpeg", + "video_1860/video_1860_32.jpeg", + "video_1860/video_1860_33.jpeg", + "video_1860/video_1860_34.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 168, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. se\nB. ye\nC. es\nAnswer with the option's letter from the given choices directly.", + "gt_response": "se", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "se", + "ye", + "es" + ], + "image_quantity_level": "Many", + "image": [ + "video_6232/video_6232_0.jpeg", + "video_6232/video_6232_1.jpeg", + "video_6232/video_6232_2.jpeg", + "video_6232/video_6232_3.jpeg", + "video_6232/video_6232_4.jpeg", + "video_6232/video_6232_5.jpeg", + "video_6232/video_6232_6.jpeg", + "video_6232/video_6232_7.jpeg", + "video_6232/video_6232_8.jpeg", + "video_6232/video_6232_9.jpeg", + "video_6232/video_6232_10.jpeg", + "video_6232/video_6232_11.jpeg", + "video_6232/video_6232_12.jpeg", + "video_6232/video_6232_13.jpeg", + "video_6232/video_6232_14.jpeg", + "video_6232/video_6232_15.jpeg", + "video_6232/video_6232_16.jpeg", + "video_6232/video_6232_17.jpeg", + "video_6232/video_6232_18.jpeg", + "video_6232/video_6232_19.jpeg", + "video_6232/video_6232_20.jpeg", + "video_6232/video_6232_21.jpeg", + "video_6232/video_6232_22.jpeg", + "video_6232/video_6232_23.jpeg", + "video_6232/video_6232_24.jpeg", + "video_6232/video_6232_25.jpeg", + "video_6232/video_6232_26.jpeg", + "video_6232/video_6232_27.jpeg", + "video_6232/video_6232_28.jpeg", + "video_6232/video_6232_29.jpeg", + "video_6232/video_6232_30.jpeg", + "video_6232/video_6232_31.jpeg", + "video_6232/video_6232_32.jpeg", + "video_6232/video_6232_33.jpeg", + "video_6232/video_6232_34.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 178, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat would be the order of the written letters if the person had written them in reverse order?\nChoice list: \nA. gjicf\nB. yppah\nC. happy\nAnswer with the option's letter from the given choices directly.", + "gt_response": "yppah", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "gjicf", + "yppah", + "happy" + ], + "image_quantity_level": "Many", + "image": [ + "video_2492/video_2492_0.jpeg", + "video_2492/video_2492_1.jpeg", + "video_2492/video_2492_2.jpeg", + "video_2492/video_2492_3.jpeg", + "video_2492/video_2492_4.jpeg", + "video_2492/video_2492_5.jpeg", + "video_2492/video_2492_6.jpeg", + "video_2492/video_2492_7.jpeg", + "video_2492/video_2492_8.jpeg", + "video_2492/video_2492_9.jpeg", + "video_2492/video_2492_10.jpeg", + "video_2492/video_2492_11.jpeg", + "video_2492/video_2492_12.jpeg", + "video_2492/video_2492_13.jpeg", + "video_2492/video_2492_14.jpeg", + "video_2492/video_2492_15.jpeg", + "video_2492/video_2492_16.jpeg", + "video_2492/video_2492_17.jpeg", + "video_2492/video_2492_18.jpeg", + "video_2492/video_2492_19.jpeg", + "video_2492/video_2492_20.jpeg", + "video_2492/video_2492_21.jpeg", + "video_2492/video_2492_22.jpeg", + "video_2492/video_2492_23.jpeg", + "video_2492/video_2492_24.jpeg", + "video_2492/video_2492_25.jpeg", + "video_2492/video_2492_26.jpeg", + "video_2492/video_2492_27.jpeg", + "video_2492/video_2492_28.jpeg", + "video_2492/video_2492_29.jpeg", + "video_2492/video_2492_30.jpeg", + "video_2492/video_2492_31.jpeg", + "video_2492/video_2492_32.jpeg", + "video_2492/video_2492_33.jpeg", + "video_2492/video_2492_34.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 184, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. i\nB. k\nC. b\nAnswer with the option's letter from the given choices directly.", + "gt_response": "b", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "i", + "k", + "b" + ], + "image_quantity_level": "Many", + "image": [ + "video_861/video_861_0.jpeg", + "video_861/video_861_1.jpeg", + "video_861/video_861_2.jpeg", + "video_861/video_861_3.jpeg", + "video_861/video_861_4.jpeg", + "video_861/video_861_5.jpeg", + "video_861/video_861_6.jpeg", + "video_861/video_861_7.jpeg", + "video_861/video_861_8.jpeg", + "video_861/video_861_9.jpeg", + "video_861/video_861_10.jpeg", + "video_861/video_861_11.jpeg", + "video_861/video_861_12.jpeg", + "video_861/video_861_13.jpeg", + "video_861/video_861_14.jpeg", + "video_861/video_861_15.jpeg", + "video_861/video_861_16.jpeg", + "video_861/video_861_17.jpeg", + "video_861/video_861_18.jpeg", + "video_861/video_861_19.jpeg", + "video_861/video_861_20.jpeg", + "video_861/video_861_21.jpeg", + "video_861/video_861_22.jpeg", + "video_861/video_861_23.jpeg", + "video_861/video_861_24.jpeg", + "video_861/video_861_25.jpeg", + "video_861/video_861_26.jpeg", + "video_861/video_861_27.jpeg", + "video_861/video_861_28.jpeg", + "video_861/video_861_29.jpeg", + "video_861/video_861_30.jpeg", + "video_861/video_861_31.jpeg", + "video_861/video_861_32.jpeg", + "video_861/video_861_33.jpeg", + "video_861/video_861_34.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 192, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. erm\nB. emr\nC. rem\nAnswer with the option's letter from the given choices directly.", + "gt_response": "emr", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "erm", + "emr", + "rem" + ], + "image_quantity_level": "Many", + "image": [ + "video_1021/video_1021_0.jpeg", + "video_1021/video_1021_1.jpeg", + "video_1021/video_1021_2.jpeg", + "video_1021/video_1021_3.jpeg", + "video_1021/video_1021_4.jpeg", + "video_1021/video_1021_5.jpeg", + "video_1021/video_1021_6.jpeg", + "video_1021/video_1021_7.jpeg", + "video_1021/video_1021_8.jpeg", + "video_1021/video_1021_9.jpeg", + "video_1021/video_1021_10.jpeg", + "video_1021/video_1021_11.jpeg", + "video_1021/video_1021_12.jpeg", + "video_1021/video_1021_13.jpeg", + "video_1021/video_1021_14.jpeg", + "video_1021/video_1021_15.jpeg", + "video_1021/video_1021_16.jpeg", + "video_1021/video_1021_17.jpeg", + "video_1021/video_1021_18.jpeg", + "video_1021/video_1021_19.jpeg", + "video_1021/video_1021_20.jpeg", + "video_1021/video_1021_21.jpeg", + "video_1021/video_1021_22.jpeg", + "video_1021/video_1021_23.jpeg", + "video_1021/video_1021_24.jpeg", + "video_1021/video_1021_25.jpeg", + "video_1021/video_1021_26.jpeg", + "video_1021/video_1021_27.jpeg", + "video_1021/video_1021_28.jpeg", + "video_1021/video_1021_29.jpeg", + "video_1021/video_1021_30.jpeg", + "video_1021/video_1021_31.jpeg", + "video_1021/video_1021_32.jpeg", + "video_1021/video_1021_33.jpeg", + "video_1021/video_1021_34.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 167, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. hgf\nB. fgh\nC. hfg\nAnswer with the option's letter from the given choices directly.", + "gt_response": "fgh", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "hgf", + "fgh", + "hfg" + ], + "image_quantity_level": "Medium", + "image": [ + "video_1458/video_1458_0.jpeg", + "video_1458/video_1458_1.jpeg", + "video_1458/video_1458_2.jpeg", + "video_1458/video_1458_3.jpeg", + "video_1458/video_1458_4.jpeg", + "video_1458/video_1458_5.jpeg", + "video_1458/video_1458_6.jpeg", + "video_1458/video_1458_7.jpeg", + "video_1458/video_1458_8.jpeg", + "video_1458/video_1458_9.jpeg", + "video_1458/video_1458_10.jpeg", + "video_1458/video_1458_11.jpeg", + "video_1458/video_1458_12.jpeg", + "video_1458/video_1458_13.jpeg", + "video_1458/video_1458_14.jpeg", + "video_1458/video_1458_15.jpeg", + "video_1458/video_1458_16.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 129, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nWhat is the order of the letters on the table at the end?\nChoice list: \nA. sbeac\nB. bsace\nC. seacb\nAnswer with the option's letter from the given choices directly.", + "gt_response": "sbeac", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "sbeac", + "bsace", + "seacb" + ], + "image_quantity_level": "Medium", + "image": [ + "video_178/video_178_0.jpeg", + "video_178/video_178_1.jpeg", + "video_178/video_178_2.jpeg", + "video_178/video_178_3.jpeg", + "video_178/video_178_4.jpeg", + "video_178/video_178_5.jpeg", + "video_178/video_178_6.jpeg", + "video_178/video_178_7.jpeg", + "video_178/video_178_8.jpeg", + "video_178/video_178_9.jpeg", + "video_178/video_178_10.jpeg", + "video_178/video_178_11.jpeg", + "video_178/video_178_12.jpeg", + "video_178/video_178_13.jpeg", + "video_178/video_178_14.jpeg", + "video_178/video_178_15.jpeg", + "video_178/video_178_16.jpeg", + "video_178/video_178_17.jpeg", + "video_178/video_178_18.jpeg", + "video_178/video_178_19.jpeg", + "video_178/video_178_20.jpeg", + "video_178/video_178_21.jpeg", + "video_178/video_178_22.jpeg", + "video_178/video_178_23.jpeg", + "video_178/video_178_24.jpeg", + "video_178/video_178_25.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 169, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nWhat was the order of the letters on the table before shuffling?\nChoice list: \nA. money\nB. nmeoy\nC. emnoy\nAnswer with the option's letter from the given choices directly.", + "gt_response": "nmeoy", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "money", + "nmeoy", + "emnoy" + ], + "image_quantity_level": "Medium", + "image": [ + "video_6536/video_6536_0.jpeg", + "video_6536/video_6536_1.jpeg", + "video_6536/video_6536_2.jpeg", + "video_6536/video_6536_3.jpeg", + "video_6536/video_6536_4.jpeg", + "video_6536/video_6536_5.jpeg", + "video_6536/video_6536_6.jpeg", + "video_6536/video_6536_7.jpeg", + "video_6536/video_6536_8.jpeg", + "video_6536/video_6536_9.jpeg", + "video_6536/video_6536_10.jpeg", + "video_6536/video_6536_11.jpeg", + "video_6536/video_6536_12.jpeg", + "video_6536/video_6536_13.jpeg", + "video_6536/video_6536_14.jpeg", + "video_6536/video_6536_15.jpeg", + "video_6536/video_6536_16.jpeg", + "video_6536/video_6536_17.jpeg", + "video_6536/video_6536_18.jpeg", + "video_6536/video_6536_19.jpeg", + "video_6536/video_6536_20.jpeg", + "video_6536/video_6536_21.jpeg", + "video_6536/video_6536_22.jpeg", + "video_6536/video_6536_23.jpeg", + "video_6536/video_6536_24.jpeg", + "video_6536/video_6536_25.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 165, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nWhat letter is likely to be shown next?\nChoice list: \nA. m\nB. f\nC. q\nAnswer with the option's letter from the given choices directly.", + "gt_response": "q", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "m", + "f", + "q" + ], + "image_quantity_level": "Medium", + "image": [ + "video_701/video_701_0.jpeg", + "video_701/video_701_1.jpeg", + "video_701/video_701_2.jpeg", + "video_701/video_701_3.jpeg", + "video_701/video_701_4.jpeg", + "video_701/video_701_5.jpeg", + "video_701/video_701_6.jpeg", + "video_701/video_701_7.jpeg", + "video_701/video_701_8.jpeg", + "video_701/video_701_9.jpeg", + "video_701/video_701_10.jpeg", + "video_701/video_701_11.jpeg", + "video_701/video_701_12.jpeg", + "video_701/video_701_13.jpeg", + "video_701/video_701_14.jpeg", + "video_701/video_701_15.jpeg", + "video_701/video_701_16.jpeg", + "video_701/video_701_17.jpeg", + "video_701/video_701_18.jpeg", + "video_701/video_701_19.jpeg", + "video_701/video_701_20.jpeg", + "video_701/video_701_21.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 187, + "question": "Review the visuals provided and respond to the question concerning the character sequence. You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. g\nB. d\nC. o\nAnswer with the option's letter from the given choices directly.", + "gt_response": "o", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "g", + "d", + "o" + ], + "image_quantity_level": "Medium", + "image": [ + "video_8470/video_8470_0.jpeg", + "video_8470/video_8470_1.jpeg", + "video_8470/video_8470_2.jpeg", + "video_8470/video_8470_3.jpeg", + "video_8470/video_8470_4.jpeg", + "video_8470/video_8470_5.jpeg", + "video_8470/video_8470_6.jpeg", + "video_8470/video_8470_7.jpeg", + "video_8470/video_8470_8.jpeg", + "video_8470/video_8470_9.jpeg", + "video_8470/video_8470_10.jpeg", + "video_8470/video_8470_11.jpeg", + "video_8470/video_8470_12.jpeg", + "video_8470/video_8470_13.jpeg", + "video_8470/video_8470_14.jpeg", + "video_8470/video_8470_15.jpeg", + "video_8470/video_8470_16.jpeg", + "video_8470/video_8470_17.jpeg", + "video_8470/video_8470_18.jpeg", + "video_8470/video_8470_19.jpeg", + "video_8470/video_8470_20.jpeg", + "video_8470/video_8470_21.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 197, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. t\nB. i\nC. z\nAnswer with the option's letter from the given choices directly.", + "gt_response": "i", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "t", + "i", + "z" + ], + "image_quantity_level": "Medium", + "image": [ + "video_3553/video_3553_0.jpeg", + "video_3553/video_3553_1.jpeg", + "video_3553/video_3553_2.jpeg", + "video_3553/video_3553_3.jpeg", + "video_3553/video_3553_4.jpeg", + "video_3553/video_3553_5.jpeg", + "video_3553/video_3553_6.jpeg", + "video_3553/video_3553_7.jpeg", + "video_3553/video_3553_8.jpeg", + "video_3553/video_3553_9.jpeg", + "video_3553/video_3553_10.jpeg", + "video_3553/video_3553_11.jpeg", + "video_3553/video_3553_12.jpeg", + "video_3553/video_3553_13.jpeg", + "video_3553/video_3553_14.jpeg", + "video_3553/video_3553_15.jpeg", + "video_3553/video_3553_16.jpeg", + "video_3553/video_3553_17.jpeg", + "video_3553/video_3553_18.jpeg", + "video_3553/video_3553_19.jpeg", + "video_3553/video_3553_20.jpeg", + "video_3553/video_3553_21.jpeg", + "video_3553/video_3553_22.jpeg", + "video_3553/video_3553_23.jpeg", + "video_3553/video_3553_24.jpeg", + "video_3553/video_3553_25.jpeg", + "video_3553/video_3553_26.jpeg", + "video_3553/video_3553_27.jpeg", + "video_3553/video_3553_28.jpeg", + "video_3553/video_3553_29.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 172, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. gxvd\nB. love\nC. lyne\nAnswer with the option's letter from the given choices directly.", + "gt_response": "love", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "gxvd", + "love", + "lyne" + ], + "image_quantity_level": "Medium", + "image": [ + "video_6040/video_6040_0.jpeg", + "video_6040/video_6040_1.jpeg", + "video_6040/video_6040_2.jpeg", + "video_6040/video_6040_3.jpeg", + "video_6040/video_6040_4.jpeg", + "video_6040/video_6040_5.jpeg", + "video_6040/video_6040_6.jpeg", + "video_6040/video_6040_7.jpeg", + "video_6040/video_6040_8.jpeg", + "video_6040/video_6040_9.jpeg", + "video_6040/video_6040_10.jpeg", + "video_6040/video_6040_11.jpeg", + "video_6040/video_6040_12.jpeg", + "video_6040/video_6040_13.jpeg", + "video_6040/video_6040_14.jpeg", + "video_6040/video_6040_15.jpeg", + "video_6040/video_6040_16.jpeg", + "video_6040/video_6040_17.jpeg", + "video_6040/video_6040_18.jpeg", + "video_6040/video_6040_19.jpeg", + "video_6040/video_6040_20.jpeg", + "video_6040/video_6040_21.jpeg", + "video_6040/video_6040_22.jpeg", + "video_6040/video_6040_23.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 174, + "question": "Scrutinize the provided visuals and answer the question about the arrangement of characters. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. luper\nB. rpleu\nC. lupbi\nAnswer with the option's letter from the given choices directly.", + "gt_response": "luper", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "a", + "choice_list": [ + "luper", + "rpleu", + "lupbi" + ], + "image_quantity_level": "Medium", + "image": [ + "video_9366/video_9366_0.jpeg", + "video_9366/video_9366_1.jpeg", + "video_9366/video_9366_2.jpeg", + "video_9366/video_9366_3.jpeg", + "video_9366/video_9366_4.jpeg", + "video_9366/video_9366_5.jpeg", + "video_9366/video_9366_6.jpeg", + "video_9366/video_9366_7.jpeg", + "video_9366/video_9366_8.jpeg", + "video_9366/video_9366_9.jpeg", + "video_9366/video_9366_10.jpeg", + "video_9366/video_9366_11.jpeg", + "video_9366/video_9366_12.jpeg", + "video_9366/video_9366_13.jpeg", + "video_9366/video_9366_14.jpeg", + "video_9366/video_9366_15.jpeg", + "video_9366/video_9366_16.jpeg", + "video_9366/video_9366_17.jpeg", + "video_9366/video_9366_18.jpeg", + "video_9366/video_9366_19.jpeg", + "video_9366/video_9366_20.jpeg", + "video_9366/video_9366_21.jpeg", + "video_9366/video_9366_22.jpeg", + "video_9366/video_9366_23.jpeg" + ], + "extracted": "A", + "result": 1 + }, + { + "sample_id": 199, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. emoz\nB. pmqr\nC. pmoz\nAnswer with the option's letter from the given choices directly.", + "gt_response": "pmoz", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "emoz", + "pmqr", + "pmoz" + ], + "image_quantity_level": "Medium", + "image": [ + "video_10634/video_10634_0.jpeg", + "video_10634/video_10634_1.jpeg", + "video_10634/video_10634_2.jpeg", + "video_10634/video_10634_3.jpeg", + "video_10634/video_10634_4.jpeg", + "video_10634/video_10634_5.jpeg", + "video_10634/video_10634_6.jpeg", + "video_10634/video_10634_7.jpeg", + "video_10634/video_10634_8.jpeg", + "video_10634/video_10634_9.jpeg", + "video_10634/video_10634_10.jpeg", + "video_10634/video_10634_11.jpeg", + "video_10634/video_10634_12.jpeg", + "video_10634/video_10634_13.jpeg", + "video_10634/video_10634_14.jpeg", + "video_10634/video_10634_15.jpeg", + "video_10634/video_10634_16.jpeg", + "video_10634/video_10634_17.jpeg", + "video_10634/video_10634_18.jpeg", + "video_10634/video_10634_19.jpeg", + "video_10634/video_10634_20.jpeg", + "video_10634/video_10634_21.jpeg", + "video_10634/video_10634_22.jpeg", + "video_10634/video_10634_23.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 166, + "question": "Analyze the given visuals and answer the question about the order of character. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. ilac\nB. laci\nC. alic\nAnswer with the option's letter from the given choices directly.", + "gt_response": "alic", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "ilac", + "laci", + "alic" + ], + "image_quantity_level": "Many", + "image": [ + "video_4471/video_4471_0.jpeg", + "video_4471/video_4471_1.jpeg", + "video_4471/video_4471_2.jpeg", + "video_4471/video_4471_3.jpeg", + "video_4471/video_4471_4.jpeg", + "video_4471/video_4471_5.jpeg", + "video_4471/video_4471_6.jpeg", + "video_4471/video_4471_7.jpeg", + "video_4471/video_4471_8.jpeg", + "video_4471/video_4471_9.jpeg", + "video_4471/video_4471_10.jpeg", + "video_4471/video_4471_11.jpeg", + "video_4471/video_4471_12.jpeg", + "video_4471/video_4471_13.jpeg", + "video_4471/video_4471_14.jpeg", + "video_4471/video_4471_15.jpeg", + "video_4471/video_4471_16.jpeg", + "video_4471/video_4471_17.jpeg", + "video_4471/video_4471_18.jpeg", + "video_4471/video_4471_19.jpeg", + "video_4471/video_4471_20.jpeg", + "video_4471/video_4471_21.jpeg", + "video_4471/video_4471_22.jpeg", + "video_4471/video_4471_23.jpeg", + "video_4471/video_4471_24.jpeg", + "video_4471/video_4471_25.jpeg", + "video_4471/video_4471_26.jpeg", + "video_4471/video_4471_27.jpeg", + "video_4471/video_4471_28.jpeg", + "video_4471/video_4471_29.jpeg", + "video_4471/video_4471_30.jpeg", + "video_4471/video_4471_31.jpeg", + "video_4471/video_4471_32.jpeg", + "video_4471/video_4471_33.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 173, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. malin\nB. nialm\nC. inlam\nAnswer with the option's letter from the given choices directly.", + "gt_response": "inlam", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "malin", + "nialm", + "inlam" + ], + "image_quantity_level": "Many", + "image": [ + "video_11119/video_11119_0.jpeg", + "video_11119/video_11119_1.jpeg", + "video_11119/video_11119_2.jpeg", + "video_11119/video_11119_3.jpeg", + "video_11119/video_11119_4.jpeg", + "video_11119/video_11119_5.jpeg", + "video_11119/video_11119_6.jpeg", + "video_11119/video_11119_7.jpeg", + "video_11119/video_11119_8.jpeg", + "video_11119/video_11119_9.jpeg", + "video_11119/video_11119_10.jpeg", + "video_11119/video_11119_11.jpeg", + "video_11119/video_11119_12.jpeg", + "video_11119/video_11119_13.jpeg", + "video_11119/video_11119_14.jpeg", + "video_11119/video_11119_15.jpeg", + "video_11119/video_11119_16.jpeg", + "video_11119/video_11119_17.jpeg", + "video_11119/video_11119_18.jpeg", + "video_11119/video_11119_19.jpeg", + "video_11119/video_11119_20.jpeg", + "video_11119/video_11119_21.jpeg", + "video_11119/video_11119_22.jpeg", + "video_11119/video_11119_23.jpeg", + "video_11119/video_11119_24.jpeg", + "video_11119/video_11119_25.jpeg", + "video_11119/video_11119_26.jpeg", + "video_11119/video_11119_27.jpeg", + "video_11119/video_11119_28.jpeg", + "video_11119/video_11119_29.jpeg", + "video_11119/video_11119_30.jpeg", + "video_11119/video_11119_31.jpeg", + "video_11119/video_11119_32.jpeg", + "video_11119/video_11119_33.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 175, + "question": "Review the visuals provided and respond to the question concerning the character sequence. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. alic\nB. lcai\nC. ical\nAnswer with the option's letter from the given choices directly.", + "gt_response": "ical", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "alic", + "lcai", + "ical" + ], + "image_quantity_level": "Many", + "image": [ + "video_8117/video_8117_0.jpeg", + "video_8117/video_8117_1.jpeg", + "video_8117/video_8117_2.jpeg", + "video_8117/video_8117_3.jpeg", + "video_8117/video_8117_4.jpeg", + "video_8117/video_8117_5.jpeg", + "video_8117/video_8117_6.jpeg", + "video_8117/video_8117_7.jpeg", + "video_8117/video_8117_8.jpeg", + "video_8117/video_8117_9.jpeg", + "video_8117/video_8117_10.jpeg", + "video_8117/video_8117_11.jpeg", + "video_8117/video_8117_12.jpeg", + "video_8117/video_8117_13.jpeg", + "video_8117/video_8117_14.jpeg", + "video_8117/video_8117_15.jpeg", + "video_8117/video_8117_16.jpeg", + "video_8117/video_8117_17.jpeg", + "video_8117/video_8117_18.jpeg", + "video_8117/video_8117_19.jpeg", + "video_8117/video_8117_20.jpeg", + "video_8117/video_8117_21.jpeg", + "video_8117/video_8117_22.jpeg", + "video_8117/video_8117_23.jpeg", + "video_8117/video_8117_24.jpeg", + "video_8117/video_8117_25.jpeg", + "video_8117/video_8117_26.jpeg", + "video_8117/video_8117_27.jpeg", + "video_8117/video_8117_28.jpeg", + "video_8117/video_8117_29.jpeg", + "video_8117/video_8117_30.jpeg", + "video_8117/video_8117_31.jpeg", + "video_8117/video_8117_32.jpeg", + "video_8117/video_8117_33.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 176, + "question": "From the images supplied, answer the question relevant to character order. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. bfd\nB. bid\nC. bkd\nAnswer with the option's letter from the given choices directly.", + "gt_response": "bid", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "bfd", + "bid", + "bkd" + ], + "image_quantity_level": "Many", + "image": [ + "video_5943/video_5943_0.jpeg", + "video_5943/video_5943_1.jpeg", + "video_5943/video_5943_2.jpeg", + "video_5943/video_5943_3.jpeg", + "video_5943/video_5943_4.jpeg", + "video_5943/video_5943_5.jpeg", + "video_5943/video_5943_6.jpeg", + "video_5943/video_5943_7.jpeg", + "video_5943/video_5943_8.jpeg", + "video_5943/video_5943_9.jpeg", + "video_5943/video_5943_10.jpeg", + "video_5943/video_5943_11.jpeg", + "video_5943/video_5943_12.jpeg", + "video_5943/video_5943_13.jpeg", + "video_5943/video_5943_14.jpeg", + "video_5943/video_5943_15.jpeg", + "video_5943/video_5943_16.jpeg", + "video_5943/video_5943_17.jpeg", + "video_5943/video_5943_18.jpeg", + "video_5943/video_5943_19.jpeg", + "video_5943/video_5943_20.jpeg", + "video_5943/video_5943_21.jpeg", + "video_5943/video_5943_22.jpeg", + "video_5943/video_5943_23.jpeg", + "video_5943/video_5943_24.jpeg", + "video_5943/video_5943_25.jpeg", + "video_5943/video_5943_26.jpeg", + "video_5943/video_5943_27.jpeg", + "video_5943/video_5943_28.jpeg", + "video_5943/video_5943_29.jpeg", + "video_5943/video_5943_30.jpeg", + "video_5943/video_5943_31.jpeg", + "video_5943/video_5943_32.jpeg", + "video_5943/video_5943_33.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 180, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nWhat was the second letter written on the paper?\nChoice list: \nA. i\nB. p\nC. g\nAnswer with the option's letter from the given choices directly.", + "gt_response": "i", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "i", + "p", + "g" + ], + "image_quantity_level": "Many", + "image": [ + "video_341/video_341_0.jpeg", + "video_341/video_341_1.jpeg", + "video_341/video_341_2.jpeg", + "video_341/video_341_3.jpeg", + "video_341/video_341_4.jpeg", + "video_341/video_341_5.jpeg", + "video_341/video_341_6.jpeg", + "video_341/video_341_7.jpeg", + "video_341/video_341_8.jpeg", + "video_341/video_341_9.jpeg", + "video_341/video_341_10.jpeg", + "video_341/video_341_11.jpeg", + "video_341/video_341_12.jpeg", + "video_341/video_341_13.jpeg", + "video_341/video_341_14.jpeg", + "video_341/video_341_15.jpeg", + "video_341/video_341_16.jpeg", + "video_341/video_341_17.jpeg", + "video_341/video_341_18.jpeg", + "video_341/video_341_19.jpeg", + "video_341/video_341_20.jpeg", + "video_341/video_341_21.jpeg", + "video_341/video_341_22.jpeg", + "video_341/video_341_23.jpeg", + "video_341/video_341_24.jpeg", + "video_341/video_341_25.jpeg", + "video_341/video_341_26.jpeg", + "video_341/video_341_27.jpeg", + "video_341/video_341_28.jpeg", + "video_341/video_341_29.jpeg", + "video_341/video_341_30.jpeg", + "video_341/video_341_31.jpeg", + "video_341/video_341_32.jpeg", + "video_341/video_341_33.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 183, + "question": "Assess the given images and answer the question related to the order of characters. You must choose your answer from the Choice List.\nWhat letter is likely to be shown next?\nChoice list: \nA. c\nB. o\nC. g\nAnswer with the option's letter from the given choices directly.", + "gt_response": "g", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "c", + "o", + "g" + ], + "image_quantity_level": "Many", + "image": [ + "video_1583/video_1583_0.jpeg", + "video_1583/video_1583_1.jpeg", + "video_1583/video_1583_2.jpeg", + "video_1583/video_1583_3.jpeg", + "video_1583/video_1583_4.jpeg", + "video_1583/video_1583_5.jpeg", + "video_1583/video_1583_6.jpeg", + "video_1583/video_1583_7.jpeg", + "video_1583/video_1583_8.jpeg", + "video_1583/video_1583_9.jpeg", + "video_1583/video_1583_10.jpeg", + "video_1583/video_1583_11.jpeg", + "video_1583/video_1583_12.jpeg", + "video_1583/video_1583_13.jpeg", + "video_1583/video_1583_14.jpeg", + "video_1583/video_1583_15.jpeg", + "video_1583/video_1583_16.jpeg", + "video_1583/video_1583_17.jpeg", + "video_1583/video_1583_18.jpeg", + "video_1583/video_1583_19.jpeg", + "video_1583/video_1583_20.jpeg", + "video_1583/video_1583_21.jpeg", + "video_1583/video_1583_22.jpeg", + "video_1583/video_1583_23.jpeg", + "video_1583/video_1583_24.jpeg", + "video_1583/video_1583_25.jpeg", + "video_1583/video_1583_26.jpeg", + "video_1583/video_1583_27.jpeg", + "video_1583/video_1583_28.jpeg", + "video_1583/video_1583_29.jpeg", + "video_1583/video_1583_30.jpeg", + "video_1583/video_1583_31.jpeg", + "video_1583/video_1583_32.jpeg", + "video_1583/video_1583_33.jpeg" + ], + "extracted": "A", + "result": 0 + }, + { + "sample_id": 190, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nWhat letters did the person show in order?\nChoice list: \nA. mcdev\nB. bcdef\nC. edbfc\nAnswer with the option's letter from the given choices directly.", + "gt_response": "bcdef", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "mcdev", + "bcdef", + "edbfc" + ], + "image_quantity_level": "Many", + "image": [ + "video_3718/video_3718_0.jpeg", + "video_3718/video_3718_1.jpeg", + "video_3718/video_3718_2.jpeg", + "video_3718/video_3718_3.jpeg", + "video_3718/video_3718_4.jpeg", + "video_3718/video_3718_5.jpeg", + "video_3718/video_3718_6.jpeg", + "video_3718/video_3718_7.jpeg", + "video_3718/video_3718_8.jpeg", + "video_3718/video_3718_9.jpeg", + "video_3718/video_3718_10.jpeg", + "video_3718/video_3718_11.jpeg", + "video_3718/video_3718_12.jpeg", + "video_3718/video_3718_13.jpeg", + "video_3718/video_3718_14.jpeg", + "video_3718/video_3718_15.jpeg", + "video_3718/video_3718_16.jpeg", + "video_3718/video_3718_17.jpeg", + "video_3718/video_3718_18.jpeg", + "video_3718/video_3718_19.jpeg", + "video_3718/video_3718_20.jpeg", + "video_3718/video_3718_21.jpeg", + "video_3718/video_3718_22.jpeg", + "video_3718/video_3718_23.jpeg", + "video_3718/video_3718_24.jpeg", + "video_3718/video_3718_25.jpeg", + "video_3718/video_3718_26.jpeg", + "video_3718/video_3718_27.jpeg", + "video_3718/video_3718_28.jpeg", + "video_3718/video_3718_29.jpeg", + "video_3718/video_3718_30.jpeg", + "video_3718/video_3718_31.jpeg", + "video_3718/video_3718_32.jpeg", + "video_3718/video_3718_33.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 185, + "question": "From the images supplied, answer the question relevant to character order. You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. n\nB. f\nC. u\nAnswer with the option's letter from the given choices directly.", + "gt_response": "f", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "n", + "f", + "u" + ], + "image_quantity_level": "Medium", + "image": [ + "video_8747/video_8747_0.jpeg", + "video_8747/video_8747_1.jpeg", + "video_8747/video_8747_2.jpeg", + "video_8747/video_8747_3.jpeg", + "video_8747/video_8747_4.jpeg", + "video_8747/video_8747_5.jpeg", + "video_8747/video_8747_6.jpeg", + "video_8747/video_8747_7.jpeg", + "video_8747/video_8747_8.jpeg", + "video_8747/video_8747_9.jpeg", + "video_8747/video_8747_10.jpeg", + "video_8747/video_8747_11.jpeg", + "video_8747/video_8747_12.jpeg", + "video_8747/video_8747_13.jpeg", + "video_8747/video_8747_14.jpeg", + "video_8747/video_8747_15.jpeg", + "video_8747/video_8747_16.jpeg", + "video_8747/video_8747_17.jpeg", + "video_8747/video_8747_18.jpeg", + "video_8747/video_8747_19.jpeg", + "video_8747/video_8747_20.jpeg" + ], + "extracted": "B", + "result": 1 + }, + { + "sample_id": 186, + "question": "Using the supplied images, answer the question concerning the order of characters. You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. fal\nB. lfa\nC. afl\nAnswer with the option's letter from the given choices directly.", + "gt_response": "fal", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "fal", + "lfa", + "afl" + ], + "image_quantity_level": "Medium", + "image": [ + "video_4956/video_4956_0.jpeg", + "video_4956/video_4956_1.jpeg", + "video_4956/video_4956_2.jpeg", + "video_4956/video_4956_3.jpeg", + "video_4956/video_4956_4.jpeg", + "video_4956/video_4956_5.jpeg", + "video_4956/video_4956_6.jpeg", + "video_4956/video_4956_7.jpeg", + "video_4956/video_4956_8.jpeg", + "video_4956/video_4956_9.jpeg", + "video_4956/video_4956_10.jpeg", + "video_4956/video_4956_11.jpeg", + "video_4956/video_4956_12.jpeg", + "video_4956/video_4956_13.jpeg", + "video_4956/video_4956_14.jpeg", + "video_4956/video_4956_15.jpeg", + "video_4956/video_4956_16.jpeg", + "video_4956/video_4956_17.jpeg", + "video_4956/video_4956_18.jpeg", + "video_4956/video_4956_19.jpeg", + "video_4956/video_4956_20.jpeg" + ], + "extracted": "C", + "result": 0 + }, + { + "sample_id": 149, + "question": "Examine the visuals provided and respond to the query about the sequence of characters. You must choose your answer from the Choice List.\nWhat was the order of the letters at the beginning?\nChoice list: \nA. levo\nB. vole\nC. love\nAnswer with the option's letter from the given choices directly.", + "gt_response": "love", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "levo", + "vole", + "love" + ], + "image_quantity_level": "Many", + "image": [ + "video_4335/video_4335_0.jpeg", + "video_4335/video_4335_1.jpeg", + "video_4335/video_4335_2.jpeg", + "video_4335/video_4335_3.jpeg", + "video_4335/video_4335_4.jpeg", + "video_4335/video_4335_5.jpeg", + "video_4335/video_4335_6.jpeg", + "video_4335/video_4335_7.jpeg", + "video_4335/video_4335_8.jpeg", + "video_4335/video_4335_9.jpeg", + "video_4335/video_4335_10.jpeg", + "video_4335/video_4335_11.jpeg", + "video_4335/video_4335_12.jpeg", + "video_4335/video_4335_13.jpeg", + "video_4335/video_4335_14.jpeg", + "video_4335/video_4335_15.jpeg", + "video_4335/video_4335_16.jpeg", + "video_4335/video_4335_17.jpeg", + "video_4335/video_4335_18.jpeg", + "video_4335/video_4335_19.jpeg", + "video_4335/video_4335_20.jpeg", + "video_4335/video_4335_21.jpeg", + "video_4335/video_4335_22.jpeg", + "video_4335/video_4335_23.jpeg", + "video_4335/video_4335_24.jpeg", + "video_4335/video_4335_25.jpeg", + "video_4335/video_4335_26.jpeg", + "video_4335/video_4335_27.jpeg", + "video_4335/video_4335_28.jpeg", + "video_4335/video_4335_29.jpeg", + "video_4335/video_4335_30.jpeg", + "video_4335/video_4335_31.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 155, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nWhat letter did the person write first on the paper?\nChoice list: \nA. f\nB. a\nC. b\nAnswer with the option's letter from the given choices directly.", + "gt_response": "b", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "f", + "a", + "b" + ], + "image_quantity_level": "Many", + "image": [ + "video_10762/video_10762_0.jpeg", + "video_10762/video_10762_1.jpeg", + "video_10762/video_10762_2.jpeg", + "video_10762/video_10762_3.jpeg", + "video_10762/video_10762_4.jpeg", + "video_10762/video_10762_5.jpeg", + "video_10762/video_10762_6.jpeg", + "video_10762/video_10762_7.jpeg", + "video_10762/video_10762_8.jpeg", + "video_10762/video_10762_9.jpeg", + "video_10762/video_10762_10.jpeg", + "video_10762/video_10762_11.jpeg", + "video_10762/video_10762_12.jpeg", + "video_10762/video_10762_13.jpeg", + "video_10762/video_10762_14.jpeg", + "video_10762/video_10762_15.jpeg", + "video_10762/video_10762_16.jpeg", + "video_10762/video_10762_17.jpeg", + "video_10762/video_10762_18.jpeg", + "video_10762/video_10762_19.jpeg", + "video_10762/video_10762_20.jpeg", + "video_10762/video_10762_21.jpeg", + "video_10762/video_10762_22.jpeg", + "video_10762/video_10762_23.jpeg", + "video_10762/video_10762_24.jpeg", + "video_10762/video_10762_25.jpeg", + "video_10762/video_10762_26.jpeg", + "video_10762/video_10762_27.jpeg", + "video_10762/video_10762_28.jpeg", + "video_10762/video_10762_29.jpeg", + "video_10762/video_10762_30.jpeg", + "video_10762/video_10762_31.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 195, + "question": "Based on the provided images, answer the question related to the character order You must choose your answer from the Choice List.\nWhat is the order of the letters at the end?\nChoice list: \nA. odco\nB. oocd\nC. dooc\nAnswer with the option's letter from the given choices directly.", + "gt_response": "dooc", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "c", + "choice_list": [ + "odco", + "oocd", + "dooc" + ], + "image_quantity_level": "Many", + "image": [ + "video_3010/video_3010_0.jpeg", + "video_3010/video_3010_1.jpeg", + "video_3010/video_3010_2.jpeg", + "video_3010/video_3010_3.jpeg", + "video_3010/video_3010_4.jpeg", + "video_3010/video_3010_5.jpeg", + "video_3010/video_3010_6.jpeg", + "video_3010/video_3010_7.jpeg", + "video_3010/video_3010_8.jpeg", + "video_3010/video_3010_9.jpeg", + "video_3010/video_3010_10.jpeg", + "video_3010/video_3010_11.jpeg", + "video_3010/video_3010_12.jpeg", + "video_3010/video_3010_13.jpeg", + "video_3010/video_3010_14.jpeg", + "video_3010/video_3010_15.jpeg", + "video_3010/video_3010_16.jpeg", + "video_3010/video_3010_17.jpeg", + "video_3010/video_3010_18.jpeg", + "video_3010/video_3010_19.jpeg", + "video_3010/video_3010_20.jpeg", + "video_3010/video_3010_21.jpeg", + "video_3010/video_3010_22.jpeg", + "video_3010/video_3010_23.jpeg", + "video_3010/video_3010_24.jpeg", + "video_3010/video_3010_25.jpeg", + "video_3010/video_3010_26.jpeg", + "video_3010/video_3010_27.jpeg", + "video_3010/video_3010_28.jpeg", + "video_3010/video_3010_29.jpeg", + "video_3010/video_3010_30.jpeg", + "video_3010/video_3010_31.jpeg" + ], + "extracted": "C", + "result": 1 + }, + { + "sample_id": 158, + "question": "Interpret the images given and provide an answer to the question regarding the character order. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. ant\nB. act\nC. hnv\nAnswer with the option's letter from the given choices directly.", + "gt_response": "ant", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "ant", + "act", + "hnv" + ], + "image_quantity_level": "Medium", + "image": [ + "video_8466/video_8466_0.jpeg", + "video_8466/video_8466_1.jpeg", + "video_8466/video_8466_2.jpeg", + "video_8466/video_8466_3.jpeg", + "video_8466/video_8466_4.jpeg", + "video_8466/video_8466_5.jpeg", + "video_8466/video_8466_6.jpeg", + "video_8466/video_8466_7.jpeg", + "video_8466/video_8466_8.jpeg", + "video_8466/video_8466_9.jpeg", + "video_8466/video_8466_10.jpeg", + "video_8466/video_8466_11.jpeg", + "video_8466/video_8466_12.jpeg", + "video_8466/video_8466_13.jpeg", + "video_8466/video_8466_14.jpeg", + "video_8466/video_8466_15.jpeg", + "video_8466/video_8466_16.jpeg", + "video_8466/video_8466_17.jpeg", + "video_8466/video_8466_18.jpeg", + "video_8466/video_8466_19.jpeg", + "video_8466/video_8466_20.jpeg", + "video_8466/video_8466_21.jpeg", + "video_8466/video_8466_22.jpeg", + "video_8466/video_8466_23.jpeg", + "video_8466/video_8466_24.jpeg", + "video_8466/video_8466_25.jpeg", + "video_8466/video_8466_26.jpeg", + "video_8466/video_8466_27.jpeg", + "video_8466/video_8466_28.jpeg", + "video_8466/video_8466_29.jpeg", + "video_8466/video_8466_30.jpeg" + ], + "extracted": "B", + "result": 0 + }, + { + "sample_id": 179, + "question": "Utilize the images provided to answer the given question about the character order. You must choose your answer from the Choice List.\nIn what order did the person put the letters on the table?\nChoice list: \nA. fab\nB. cat\nC. fhb\nAnswer with the option's letter from the given choices directly.", + "gt_response": "fab", + "gen_kwargs": { + "do_sample": false, + "num_beams": 1, + "max_new_tokens": 32, + "eos_token_id": 92542 + }, + "pred_response": "b", + "choice_list": [ + "fab", + "cat", + "fhb" + ], + "image_quantity_level": "Medium", + "image": [ + "video_11481/video_11481_0.jpeg", + "video_11481/video_11481_1.jpeg", + "video_11481/video_11481_2.jpeg", + "video_11481/video_11481_3.jpeg", + "video_11481/video_11481_4.jpeg", + "video_11481/video_11481_5.jpeg", + "video_11481/video_11481_6.jpeg", + "video_11481/video_11481_7.jpeg", + "video_11481/video_11481_8.jpeg", + "video_11481/video_11481_9.jpeg", + "video_11481/video_11481_10.jpeg", + "video_11481/video_11481_11.jpeg", + "video_11481/video_11481_12.jpeg", + "video_11481/video_11481_13.jpeg", + "video_11481/video_11481_14.jpeg", + "video_11481/video_11481_15.jpeg", + "video_11481/video_11481_16.jpeg", + "video_11481/video_11481_17.jpeg", + "video_11481/video_11481_18.jpeg", + "video_11481/video_11481_19.jpeg", + "video_11481/video_11481_20.jpeg", + "video_11481/video_11481_21.jpeg", + "video_11481/video_11481_22.jpeg", + "video_11481/video_11481_23.jpeg", + "video_11481/video_11481_24.jpeg", + "video_11481/video_11481_25.jpeg", + "video_11481/video_11481_26.jpeg", + "video_11481/video_11481_27.jpeg", + "video_11481/video_11481_28.jpeg", + "video_11481/video_11481_29.jpeg", + "video_11481/video_11481_30.jpeg" + ], + "extracted": "B", + "result": 0 + } +] \ No newline at end of file