{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":290909192,"defaultBranch":"main","name":"lm-evaluation-harness","ownerLogin":"EleutherAI","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2020-08-28T00:09:15.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/68924597?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1726817747.0","currentOid":""},"activityList":{"items":[{"before":null,"after":"623727a4c257545b7f076927069ce2839956bec1","ref":"refs/heads/openai","pushedAt":"2024-09-20T07:35:47.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"better error message; fix greedy matching","shortMessageHtmlLink":"better error message; fix greedy matching"}},{"before":"8a78ea24c4ebe39fdba80f642917f9797f338897","after":"f691592460d7f2d381426033fd58d2d22cd2c610","ref":"refs/heads/ifeval_rank","pushedAt":"2024-09-19T19:08:56.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"nit","shortMessageHtmlLink":"nit"}},{"before":"5e8998273c5a67d27ad32489bc819f4946420c15","after":"8a78ea24c4ebe39fdba80f642917f9797f338897","ref":"refs/heads/ifeval_rank","pushedAt":"2024-09-19T19:02:44.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"remove `time`","shortMessageHtmlLink":"remove <code>time</code>"}},{"before":"9d5ea23b32fc44c68efee5ad558ff74c955fd7ce","after":"a84c102b3cf2564fd035941beb4f71f3755d5cf4","ref":"refs/heads/mathvista","pushedAt":"2024-09-19T07:02:58.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"add ai2d doc_to_image","shortMessageHtmlLink":"add ai2d doc_to_image"}},{"before":"1c2424855aaeacddad6f3df97f9359f0c23e0c8d","after":"9d5ea23b32fc44c68efee5ad558ff74c955fd7ce","ref":"refs/heads/mathvista","pushedAt":"2024-09-18T21:57:18.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"fix regex","shortMessageHtmlLink":"fix regex"}},{"before":"1a28d18f1b978883a293e6a537136ee51d0f8ddf","after":"1c2424855aaeacddad6f3df97f9359f0c23e0c8d","ref":"refs/heads/mathvista","pushedAt":"2024-09-18T21:40:52.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"pad images if exception","shortMessageHtmlLink":"pad images if exception"}},{"before":"88ea85b4e54d0554e6051da71e30bf955a614954","after":"9a092f374bdc6d6032ae2b878b7a49b97801ab69","ref":"refs/heads/main","pushedAt":"2024-09-18T20:15:57.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"Update neuron backend (#2314)\n\n* feat(neuron): align with latest optimum-neuron\r\n\r\n* feat(neuron): support pre-exported neuron models\r\n\r\n* fix(neuron): correctly use max_length\r\n\r\n* fix(neuron): adapt loglikelihood\r\n\r\nThe evaluation of log likelihood was not working for neuron models\r\nusing continuous batching, such as all cached neuron LLama models.\r\n\r\n* refactor(neuron): remove dead code","shortMessageHtmlLink":"Update neuron backend (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2531339700\" data-permission-text=\"Title is private\" data-url=\"https://github.com/EleutherAI/lm-evaluation-harness/issues/2314\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/EleutherAI/lm-evaluation-harness/pull/2314/hovercard\" href=\"https://github.com/EleutherAI/lm-evaluation-harness/pull/2314\">#2314</a>)"}},{"before":"cc723f2b737fabb3df72bb4d79795168784931f4","after":"f2a9b4c4ec79cbec6d99518401084a2968e2ad79","ref":"refs/heads/parentheses","pushedAt":"2024-09-18T17:25:38.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"StellaAthena","name":"Stella Biderman","path":"/StellaAthena","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15899312?s=80&v=4"},"commit":{"message":"Update _default_template_yaml","shortMessageHtmlLink":"Update _default_template_yaml"}},{"before":"88ea85b4e54d0554e6051da71e30bf955a614954","after":"cc723f2b737fabb3df72bb4d79795168784931f4","ref":"refs/heads/parentheses","pushedAt":"2024-09-18T17:23:54.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"StellaAthena","name":"Stella Biderman","path":"/StellaAthena","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15899312?s=80&v=4"},"commit":{"message":"Update _default_template_yaml","shortMessageHtmlLink":"Update _default_template_yaml"}},{"before":null,"after":"88ea85b4e54d0554e6051da71e30bf955a614954","ref":"refs/heads/parentheses","pushedAt":"2024-09-18T17:22:43.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"StellaAthena","name":"Stella Biderman","path":"/StellaAthena","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/15899312?s=80&v=4"},"commit":{"message":"repr bug (#2315)","shortMessageHtmlLink":"repr bug (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2531600438\" data-permission-text=\"Title is private\" data-url=\"https://github.com/EleutherAI/lm-evaluation-harness/issues/2315\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/EleutherAI/lm-evaluation-harness/pull/2315/hovercard\" href=\"https://github.com/EleutherAI/lm-evaluation-harness/pull/2315\">#2315</a>)"}},{"before":null,"after":"7bc960c4f57eb0f762f45ffebe23b552458c270b","ref":"refs/heads/tag_g","pushedAt":"2024-09-18T16:31:18.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"change group to tags in task `eus_exams` task configs","shortMessageHtmlLink":"change group to tags in task <code>eus_exams</code> task configs"}},{"before":"c6259cf7016058fa5106b51e6b236f4fcef5f8d4","after":"1a28d18f1b978883a293e6a537136ee51d0f8ddf","ref":"refs/heads/mathvista","pushedAt":"2024-09-18T09:41:17.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"add custom regex","shortMessageHtmlLink":"add custom regex"}},{"before":"93dbe5c7d2a51d255b6435e598efc48421151cb9","after":null,"ref":"refs/heads/nit_task","pushedAt":"2024-09-17T21:35:29.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"lintangsutawika","name":"Lintang Sutawika","path":"/lintangsutawika","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/5774558?s=80&v=4"}},{"before":"a5e0adcb52915788262ac346a023a01eb25b6339","after":"88ea85b4e54d0554e6051da71e30bf955a614954","ref":"refs/heads/main","pushedAt":"2024-09-17T21:35:28.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"lintangsutawika","name":"Lintang Sutawika","path":"/lintangsutawika","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/5774558?s=80&v=4"},"commit":{"message":"repr bug (#2315)","shortMessageHtmlLink":"repr bug (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2531600438\" data-permission-text=\"Title is private\" data-url=\"https://github.com/EleutherAI/lm-evaluation-harness/issues/2315\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/EleutherAI/lm-evaluation-harness/pull/2315/hovercard\" href=\"https://github.com/EleutherAI/lm-evaluation-harness/pull/2315\">#2315</a>)"}},{"before":"b27bc18efd29ac859123849b4bd843d8623f5d5a","after":"c6259cf7016058fa5106b51e6b236f4fcef5f8d4","ref":"refs/heads/mathvista","pushedAt":"2024-09-17T21:22:16.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"doc_to_image is list[img]","shortMessageHtmlLink":"doc_to_image is list[img]"}},{"before":"fb963f0f0a5b28b69763590bb59676072cf43a01","after":"a5e0adcb52915788262ac346a023a01eb25b6339","ref":"refs/heads/main","pushedAt":"2024-09-17T20:17:40.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"Update README.md (#2297)\n\n* Update README.md\r\n\r\nI encounter some Git buffer size limits when trying to download all commits history of the repository, such as:\r\n```error: RPC failed; curl 18 transfer closed with outstanding read data remaining\r\nerror: 5815 bytes of body are still expected\r\nfetch-pack: unexpected disconnect while reading sideband packet\r\nfatal: early EOF```\r\n\r\ntherefore the installation is faster and there are not errors when I download only the last version of the repository\r\n\r\n* Fix linting issue","shortMessageHtmlLink":"Update README.md (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2526441226\" data-permission-text=\"Title is private\" data-url=\"https://github.com/EleutherAI/lm-evaluation-harness/issues/2297\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/EleutherAI/lm-evaluation-harness/pull/2297/hovercard\" href=\"https://github.com/EleutherAI/lm-evaluation-harness/pull/2297\">#2297</a>)"}},{"before":"0d187eda23f55ebce7a01e9df301e44f17e8b048","after":"b27bc18efd29ac859123849b4bd843d8623f5d5a","ref":"refs/heads/mathvista","pushedAt":"2024-09-17T20:02:40.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"add processing code","shortMessageHtmlLink":"add processing code"}},{"before":null,"after":"93dbe5c7d2a51d255b6435e598efc48421151cb9","ref":"refs/heads/nit_task","pushedAt":"2024-09-17T16:33:51.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"repr bug","shortMessageHtmlLink":"repr bug"}},{"before":"6508efaacb33d2d41aacd5f064b432cc92c0aa8c","after":"569f9e6e2ec4992fc04e81890640926a1910d0c1","ref":"refs/heads/b_limit","pushedAt":"2024-09-17T13:15:22.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"nit","shortMessageHtmlLink":"nit"}},{"before":"8335e43a668ac01d34da2921f57ab3c0ca2fc53a","after":"6508efaacb33d2d41aacd5f064b432cc92c0aa8c","ref":"refs/heads/b_limit","pushedAt":"2024-09-17T13:13:52.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"nit","shortMessageHtmlLink":"nit"}},{"before":null,"after":"8335e43a668ac01d34da2921f57ab3c0ca2fc53a","ref":"refs/heads/b_limit","pushedAt":"2024-09-17T13:09:00.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"add batch_size to `get_sample_size`","shortMessageHtmlLink":"add batch_size to <code>get_sample_size</code>"}},{"before":null,"after":"0d187eda23f55ebce7a01e9df301e44f17e8b048","ref":"refs/heads/mathvista","pushedAt":"2024-09-16T19:46:37.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"add mathvista","shortMessageHtmlLink":"add mathvista"}},{"before":"9b03a3c54426fd55845200ac4ead796f36b237da","after":"d9d45c9edbc3483ab90194bcd998270a79dc0a87","ref":"refs/heads/mmlu_llama","pushedAt":"2024-09-16T14:45:38.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"add llama 3.1 mmlu","shortMessageHtmlLink":"add llama 3.1 mmlu"}},{"before":null,"after":"9b03a3c54426fd55845200ac4ead796f36b237da","ref":"refs/heads/mmlu_llama","pushedAt":"2024-09-16T12:58:48.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"test","shortMessageHtmlLink":"test"}},{"before":"dac8b534a26f7e580ba1b63008b0ff4b072f3134","after":"4eecbabb78c96941d011aecc44adcddc8a672736","ref":"refs/heads/prefill","pushedAt":"2024-09-16T12:47:54.000Z","pushType":"push","commitsCount":7,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"},"commit":{"message":"Merge branch 'main' into prefill","shortMessageHtmlLink":"Merge branch 'main' into prefill"}},{"before":"9789f83e635742aa63f6da6a59ea1c10f96e268c","after":null,"ref":"refs/heads/bmmodal","pushedAt":"2024-09-14T11:55:46.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"baberabb","name":"Baber Abbasi","path":"/baberabb","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/92168766?s=80&v=4"}},{"before":"d85c3b654780a76fde6cbbd06360c362d3e4f5c2","after":null,"ref":"refs/heads/multimodal-prototyping","pushedAt":"2024-09-13T17:47:03.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"haileyschoelkopf","name":"Hailey Schoelkopf","path":"/haileyschoelkopf","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/65563625?s=80&v=4"}},{"before":"decc533d02222f3b866d9a89263277fe0cc2fcb2","after":"fb963f0f0a5b28b69763590bb59676072cf43a01","ref":"refs/heads/main","pushedAt":"2024-09-13T17:47:03.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"haileyschoelkopf","name":"Hailey Schoelkopf","path":"/haileyschoelkopf","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/65563625?s=80&v=4"},"commit":{"message":"Multimodal prototyping (#2243)\n\n* add WIP hf vlm class\r\n\r\n* add doc_to_image\r\n\r\n* add mmmu tasks\r\n\r\n* fix merge conflicts\r\n\r\n* add lintang's changes to hf_vlms.py\r\n\r\n* fix doc_to_image\r\n\r\n* added yaml_path for config-loading\r\n\r\n* revert\r\n\r\n* add line to process str type v\r\n\r\n* update\r\n\r\n* modeling cleanup\r\n\r\n* add aggregation for mmmu\r\n\r\n* rewrite MMMU processing code based on only MMMU authors' repo (doc_to_image still WIP)\r\n\r\n* implemented doc_to_image\r\n\r\n* update doc_to_image to accept list of features\r\n\r\n* update functions\r\n\r\n* readd image processed\r\n\r\n* update args process\r\n\r\n* bugfix for repeated images fed to model\r\n\r\n* push WIP loglikelihood code\r\n\r\n* commit most recent code (generative ; qwen2-vl testing)\r\n\r\n* preliminary image_token_id handling\r\n\r\n* small mmmu update: some qs have >4 mcqa options\r\n\r\n* push updated modeling code\r\n\r\n* use processor.apply_chat_template\r\n\r\n* add mathvista draft\r\n\r\n* nit\r\n\r\n* nit\r\n\r\n* ensure no footguns in text<>multimodal LM<>task incompatibility\r\n\r\n* add notification to readme regarding launch of prototype!\r\n\r\n* fix compatibility check\r\n\r\n* reorganize mmmu configs\r\n\r\n* chat_template=None\r\n\r\n* add interleave chat_template\r\n\r\n* add condition\r\n\r\n* add max_images; interleave=true\r\n\r\n* nit\r\n\r\n* testmini_mcq\r\n\r\n* nit\r\n\r\n* pass image string; convert img\r\n\r\n* add vllm\r\n\r\n* add init\r\n\r\n* vlm add multi attr\r\n\r\n* fixup\r\n\r\n* pass max images to vllm model init\r\n\r\n* nit\r\n\r\n* encoding to device\r\n\r\n* fix HFMultimodalLM.chat_template ?\r\n\r\n* add mmmu readme\r\n\r\n* remove erroneous prints\r\n\r\n* use HFMultimodalLM.chat_template ; restore tasks/__init__.py\r\n\r\n* add docstring for replace_placeholders in utils\r\n\r\n* fix `replace_placeholders`; set image_string=None\r\n\r\n* fix typo\r\n\r\n* cleanup + fix merge conflicts\r\n\r\n* update MMMU readme\r\n\r\n* del mathvista\r\n\r\n* add some sample scores\r\n\r\n* Update README.md\r\n\r\n* add log msg for image_string value\r\n\r\n---------\r\n\r\nCo-authored-by: haileyschoelkopf <hailey@eleuther.ai>\r\nCo-authored-by: Baber Abbasi <baber@eleuther.ai>\r\nCo-authored-by: Baber <baber@hey.com>\r\nCo-authored-by: Hailey Schoelkopf <65563625+haileyschoelkopf@users.noreply.github.com>","shortMessageHtmlLink":"Multimodal prototyping (<a class=\"issue-link js-issue-link\" data-error-text=\"Failed to load title\" data-id=\"2481914176\" data-permission-text=\"Title is private\" data-url=\"https://github.com/EleutherAI/lm-evaluation-harness/issues/2243\" data-hovercard-type=\"pull_request\" data-hovercard-url=\"/EleutherAI/lm-evaluation-harness/pull/2243/hovercard\" href=\"https://github.com/EleutherAI/lm-evaluation-harness/pull/2243\">#2243</a>)"}},{"before":"5f76efd2ae467898b191675fd6d5a8f5fdfd8392","after":"d85c3b654780a76fde6cbbd06360c362d3e4f5c2","ref":"refs/heads/multimodal-prototyping","pushedAt":"2024-09-13T17:45:02.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"haileyschoelkopf","name":"Hailey Schoelkopf","path":"/haileyschoelkopf","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/65563625?s=80&v=4"},"commit":{"message":"Merge branch 'multimodal-prototyping' of https://github.com/EleutherAI/lm-evaluation-harness into multimodal-prototyping","shortMessageHtmlLink":"Merge branch 'multimodal-prototyping' of <a href=\"https://github.com/EleutherAI/lm-evaluation-harness\">https://github.com/EleutherA…</a>"}},{"before":"a3bb2f15006ac113655ff52d13c6e6d3ac051c77","after":"5f76efd2ae467898b191675fd6d5a8f5fdfd8392","ref":"refs/heads/multimodal-prototyping","pushedAt":"2024-09-13T17:38:04.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"haileyschoelkopf","name":"Hailey Schoelkopf","path":"/haileyschoelkopf","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/65563625?s=80&v=4"},"commit":{"message":"Update README.md","shortMessageHtmlLink":"Update README.md"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"Y3Vyc29yOnYyOpK7MjAyNC0wOS0yMFQwNzozNTo0Ny4wMDAwMDBazwAAAAS7rFpj","startCursor":"Y3Vyc29yOnYyOpK7MjAyNC0wOS0yMFQwNzozNTo0Ny4wMDAwMDBazwAAAAS7rFpj","endCursor":"Y3Vyc29yOnYyOpK7MjAyNC0wOS0xM1QxNzozODowNC4wMDAwMDBazwAAAAS1vjue"}},"title":"Activity · EleutherAI/lm-evaluation-harness"}