| { | |
| "edges": [ | |
| { | |
| "arrows": "to", | |
| "from": "agbenchmark/generate_test.py::TestWriteFile::test_method[challenge_data0]", | |
| "id": "agbenchmark/generate_test.py::TestWriteFile::test_method[challenge_data0]_to_agbenchmark/generate_test.py::TestReadFile::test_method[challenge_data0]", | |
| "to": "agbenchmark/generate_test.py::TestReadFile::test_method[challenge_data0]" | |
| }, | |
| { | |
| "arrows": "to", | |
| "from": "agbenchmark/generate_test.py::TestSearch::test_method[challenge_data0]", | |
| "id": "agbenchmark/generate_test.py::TestSearch::test_method[challenge_data0]_to_agbenchmark/generate_test.py::TestBasicRetrieval::test_method[challenge_data0]", | |
| "to": "agbenchmark/generate_test.py::TestBasicRetrieval::test_method[challenge_data0]" | |
| }, | |
| { | |
| "arrows": "to", | |
| "from": "agbenchmark/generate_test.py::TestWriteFile::test_method[challenge_data0]", | |
| "id": "agbenchmark/generate_test.py::TestWriteFile::test_method[challenge_data0]_to_agbenchmark/generate_test.py::TestSearch::test_method[challenge_data0]", | |
| "to": "agbenchmark/generate_test.py::TestSearch::test_method[challenge_data0]" | |
| }, | |
| { | |
| "arrows": "to", | |
| "from": "agbenchmark/generate_test.py::TestRevenueRetrieval2::test_method[challenge_data0]", | |
| "id": "agbenchmark/generate_test.py::TestRevenueRetrieval2::test_method[challenge_data0]_to_agbenchmark/generate_test.py::TestTestGetInformation::test_method[challenge_data0]", | |
| "to": "agbenchmark/generate_test.py::TestTestGetInformation::test_method[challenge_data0]" | |
| }, | |
| { | |
| "arrows": "to", | |
| "from": "agbenchmark/generate_test.py::TestRevenueRetrieval::test_method[challenge_data0]", | |
| "id": "agbenchmark/generate_test.py::TestRevenueRetrieval::test_method[challenge_data0]_to_agbenchmark/generate_test.py::TestRevenueRetrieval2::test_method[challenge_data0]", | |
| "to": "agbenchmark/generate_test.py::TestRevenueRetrieval2::test_method[challenge_data0]" | |
| }, | |
| { | |
| "arrows": "to", | |
| "from": "agbenchmark/generate_test.py::TestBasicRetrieval::test_method[challenge_data0]", | |
| "id": "agbenchmark/generate_test.py::TestBasicRetrieval::test_method[challenge_data0]_to_agbenchmark/generate_test.py::TestRevenueRetrieval::test_method[challenge_data0]", | |
| "to": "agbenchmark/generate_test.py::TestRevenueRetrieval::test_method[challenge_data0]" | |
| }, | |
| { | |
| "arrows": "to", | |
| "from": "agbenchmark/generate_test.py::TestReadFile::test_method[challenge_data0]", | |
| "id": "agbenchmark/generate_test.py::TestReadFile::test_method[challenge_data0]_to_agbenchmark/generate_test.py::TestSynthesizeInfo::test_method[challenge_data0]", | |
| "to": "agbenchmark/generate_test.py::TestSynthesizeInfo::test_method[challenge_data0]" | |
| } | |
| ], | |
| "nodes": [ | |
| { | |
| "color": "grey", | |
| "data": { | |
| "category": [ | |
| "general", | |
| "coding", | |
| "scrape_synthesize", | |
| "data" | |
| ], | |
| "cutoff": 60, | |
| "dependencies": [ | |
| "TestWriteFile" | |
| ], | |
| "eval_id": "f219f3d3-a41b-45a9-a3d0-389832086ee8", | |
| "ground": { | |
| "answer": "The content of output.txt should be 'Hello World!'", | |
| "eval": { | |
| "type": "file" | |
| }, | |
| "files": [ | |
| "output.txt" | |
| ], | |
| "should_contain": [ | |
| "Hello World!" | |
| ] | |
| }, | |
| "info": { | |
| "description": "Tests if the agent can read a file.", | |
| "difficulty": "interface", | |
| "side_effects": [ | |
| "" | |
| ] | |
| }, | |
| "name": "TestReadFile", | |
| "task": "Read the file called file_to_read.txt and write its content to a file called output.txt" | |
| }, | |
| "id": "agbenchmark/generate_test.py::TestReadFile::test_method[challenge_data0]", | |
| "label": "ReadFile", | |
| "shape": "dot" | |
| }, | |
| { | |
| "color": "grey", | |
| "data": { | |
| "category": [ | |
| "general", | |
| "coding", | |
| "scrape_synthesize", | |
| "data" | |
| ], | |
| "cutoff": 60, | |
| "dependencies": [], | |
| "eval_id": "021c695a-6cc4-46c2-b93a-f3a9b0f4d123", | |
| "ground": { | |
| "answer": "The word 'Washington', printed to a .txt file named anything", | |
| "eval": { | |
| "type": "file" | |
| }, | |
| "files": [ | |
| ".txt" | |
| ], | |
| "should_contain": [ | |
| "Washington" | |
| ], | |
| "should_not_contain": [] | |
| }, | |
| "info": { | |
| "description": "Tests if the agent can write a file", | |
| "difficulty": "interface", | |
| "side_effects": [ | |
| "" | |
| ] | |
| }, | |
| "name": "TestWriteFile", | |
| "task": "Write the word 'Washington' to a .txt file" | |
| }, | |
| "id": "agbenchmark/generate_test.py::TestWriteFile::test_method[challenge_data0]", | |
| "label": "WriteFile", | |
| "shape": "dot" | |
| }, | |
| { | |
| "color": "grey", | |
| "data": { | |
| "category": [ | |
| "scrape_synthesize", | |
| "general" | |
| ], | |
| "cutoff": 60, | |
| "dependencies": [ | |
| "TestSearch" | |
| ], | |
| "eval_id": "cd96e6b2-779d-4a4a-8367-d520023e27ae", | |
| "ground": { | |
| "answer": "\u00a325.89", | |
| "eval": { | |
| "type": "file" | |
| }, | |
| "files": [ | |
| ".txt" | |
| ], | |
| "should_contain": [ | |
| "25.89" | |
| ], | |
| "should_not_contain": [] | |
| }, | |
| "info": { | |
| "description": "Tests if the agent can retrieve a specific information from a website.", | |
| "difficulty": "basic", | |
| "side_effects": [] | |
| }, | |
| "name": "TestBasicRetrieval", | |
| "task": "Write the price of the book in this url 'http://books.toscrape.com/catalogue/meditations_33/index.html' to a .txt file." | |
| }, | |
| "id": "agbenchmark/generate_test.py::TestBasicRetrieval::test_method[challenge_data0]", | |
| "label": "BasicRetrieval", | |
| "shape": "dot" | |
| }, | |
| { | |
| "color": "grey", | |
| "data": { | |
| "category": [ | |
| "general", | |
| "scrape_synthesize" | |
| ], | |
| "cutoff": 120, | |
| "dependencies": [ | |
| "TestWriteFile" | |
| ], | |
| "eval_id": "0bb23182-b434-402b-a73e-9c226469b959", | |
| "ground": { | |
| "answer": "This is a Heading\nThis is a paragraph.", | |
| "eval": { | |
| "type": "file" | |
| }, | |
| "files": [ | |
| ".txt" | |
| ], | |
| "should_contain": [ | |
| "Heading", | |
| "paragraph" | |
| ], | |
| "should_not_contain": [ | |
| "The", | |
| "the" | |
| ] | |
| }, | |
| "info": { | |
| "description": "Tests if the agent can search.", | |
| "difficulty": "interface", | |
| "side_effects": [ | |
| "" | |
| ] | |
| }, | |
| "name": "TestSearch", | |
| "task": "Open 'https://silennaihin.com/random/plain.html' and paste all of the text on the page in a .txt file" | |
| }, | |
| "id": "agbenchmark/generate_test.py::TestSearch::test_method[challenge_data0]", | |
| "label": "Search", | |
| "shape": "dot" | |
| }, | |
| { | |
| "color": "grey", | |
| "data": { | |
| "category": [ | |
| "scrape_synthesize", | |
| "general" | |
| ], | |
| "cutoff": 60, | |
| "dependencies": [ | |
| "TestRevenueRetrieval2" | |
| ], | |
| "eval_id": "1758058c-f726-484f-96fa-f05e278e5ff5", | |
| "ground": { | |
| "answer": "The twitter handles of the two hosts of Latent Space.", | |
| "eval": { | |
| "type": "file" | |
| }, | |
| "files": [ | |
| "output.txt" | |
| ], | |
| "should_contain": [ | |
| "swyx", | |
| "FanaHOVA" | |
| ], | |
| "should_not_contain": [] | |
| }, | |
| "info": { | |
| "description": "Tests if the agent can retrieve twitter handles given a vague description.", | |
| "difficulty": "intermediate", | |
| "side_effects": [ | |
| "" | |
| ] | |
| }, | |
| "name": "TestTestGetInformation", | |
| "task": "Write the twitter handle of the two hosts of Latent Space to a file called output.txt" | |
| }, | |
| "id": "agbenchmark/generate_test.py::TestTestGetInformation::test_method[challenge_data0]", | |
| "label": "TestGetInformation", | |
| "shape": "dot" | |
| }, | |
| { | |
| "color": "grey", | |
| "data": { | |
| "category": [ | |
| "scrape_synthesize" | |
| ], | |
| "cutoff": 60, | |
| "dependencies": [ | |
| "TestRevenueRetrieval" | |
| ], | |
| "eval_id": "552bdf23-db40-4bd1-b123-4ed820886cc1", | |
| "ground": { | |
| "answer": "15 Millions\n112 Millions\n117 Millions\n204 Millions\n413 Millions\n2,014 Millions\n3,198 Millions\n4,046 Millions\n7,000 Millions\n11,759 Millions\n21,461 Millions\n24,578 Millions\n31,536 Millions\n53,823 Millions\n81,462 Millions", | |
| "eval": { | |
| "type": "file" | |
| }, | |
| "files": [ | |
| ".txt" | |
| ], | |
| "should_contain": [ | |
| "15", | |
| "112", | |
| "117", | |
| "204", | |
| "413", | |
| "2,014", | |
| "3,198", | |
| "4,046", | |
| "7,000", | |
| "11,759", | |
| "21,461", | |
| "24,578", | |
| "31,536", | |
| "53,823", | |
| "81,462" | |
| ], | |
| "should_not_contain": [] | |
| }, | |
| "info": { | |
| "description": "Tests if the agent can retrieve all the revenues of Tesla since its creation.", | |
| "difficulty": "intermediate", | |
| "side_effects": [ | |
| "tests if there is in fact an LLM attached" | |
| ] | |
| }, | |
| "name": "TestRevenueRetrieval2", | |
| "task": "Write tesla's revenue every year since its creation into a .txt file. Use the US notation, with a precision rounded to the nearest million dollars (for instance, $31,578 million)." | |
| }, | |
| "id": "agbenchmark/generate_test.py::TestRevenueRetrieval2::test_method[challenge_data0]", | |
| "label": "RevenueRetrieval2", | |
| "shape": "dot" | |
| }, | |
| { | |
| "color": "grey", | |
| "data": { | |
| "category": [ | |
| "scrape_synthesize", | |
| "general" | |
| ], | |
| "cutoff": 60, | |
| "dependencies": [ | |
| "TestBasicRetrieval" | |
| ], | |
| "eval_id": "dc2114d7-1597-4c9b-bed0-a97937ad977f", | |
| "ground": { | |
| "answer": "It was $81.462 billion in 2022. In millions the answer is 81,462.", | |
| "eval": { | |
| "type": "file" | |
| }, | |
| "files": [ | |
| ".txt" | |
| ], | |
| "should_contain": [ | |
| "81,462" | |
| ], | |
| "should_not_contain": [] | |
| }, | |
| "info": { | |
| "description": "Tests if the agent can retrieve Tesla's revenue in 2022.", | |
| "difficulty": "intermediate", | |
| "side_effects": [] | |
| }, | |
| "name": "TestRevenueRetrieval", | |
| "task": "Write tesla's exact revenue in 2022 into a .txt file. Use the US notation, with a precision rounded to the nearest million dollars (for instance, $31,578 million)." | |
| }, | |
| "id": "agbenchmark/generate_test.py::TestRevenueRetrieval::test_method[challenge_data0]", | |
| "label": "RevenueRetrieval", | |
| "shape": "dot" | |
| }, | |
| { | |
| "color": "grey", | |
| "data": { | |
| "category": [ | |
| "scrape_synthesize", | |
| "general" | |
| ], | |
| "cutoff": 240, | |
| "dependencies": [ | |
| "TestReadFile" | |
| ], | |
| "eval_id": "895ae28a-4513-44ea-a872-0164771d1597", | |
| "ground": { | |
| "answer": "A report highlighting elements from the 2 files.", | |
| "eval": { | |
| "scoring": "binary", | |
| "template": "question", | |
| "type": "llm" | |
| }, | |
| "files": [ | |
| "output.txt" | |
| ], | |
| "should_contain": [ | |
| "Is the company mentioned in the output actively addressing or capitalizing on the challenges or trends listed?" | |
| ], | |
| "should_not_contain": [] | |
| }, | |
| "info": { | |
| "description": "Tests if the agent can generate content based on the content of 2 files.", | |
| "difficulty": "basic", | |
| "side_effects": [] | |
| }, | |
| "name": "TestSynthesizeInfo", | |
| "task": "Create a brief report or summary highlighting how one or more companies from companies.txt are addressing or capitalizing on challenges or trends from challenges.txt. Write a file called output.txt." | |
| }, | |
| "id": "agbenchmark/generate_test.py::TestSynthesizeInfo::test_method[challenge_data0]", | |
| "label": "SynthesizeInfo", | |
| "shape": "dot" | |
| } | |
| ] | |
| } | |