From 26611fc4e6dc44db63221f80cef29ffb0833572f Mon Sep 17 00:00:00 2001 From: capjamesg Date: Thu, 24 Oct 2024 01:28:24 +0000 Subject: [PATCH] Update results --- index.html | 196 +++++++++++++++++++--------------------- results/2024-10-24.json | 106 ++++++++++++++++++++++ 2 files changed, 198 insertions(+), 104 deletions(-) create mode 100644 results/2024-10-24.json diff --git a/index.html b/index.html index 0f05381..2ab865f 100644 --- a/index.html +++ b/index.html @@ -40,7 +40,7 @@

How's GPT-4o Doing?

You can contribute your own tests, too! See the GitHub README for contributing instructions.

-

Tests are run every day at 1am PT. Last updated October 23, 2024.

+

Tests are run every day at 1am PT. Last updated October 24, 2024.

Made with ❤️ by the team at Roboflow.

@@ -58,12 +58,12 @@

How's GPT-4o Doing?

Response Time

-

Today, the average response time to receive results from our tests was 3.95 seconds per request.

+

Today, the average response time to receive results from our tests was 3.94 seconds per request.

This number only accounts for requests made by this application.

-

3.95 s

+

3.94 s

@@ -122,17 +122,17 @@

Prompt

Image

Image of the input into GPT-4

Result

- 
7
+ 
8

Test submitted by Roboflow

- +
-

Handwriting OCR

-

Can GPT-4V read handwriting?

+

Object Detection

+

Can GPT-4V detect objects in an image?

@@ -145,48 +145,48 @@

Handwriting OCR

Last 7-Day Performance
-
+
-
+
-
+
-
+
-
+
-
+
-

Of the last 7 tests, conducted daily, this test has passed 86.0% of the time.

+

Of the last 7 tests, conducted daily, this test has passed 0% of the time.

Today's request cost $0.009

Method

- 
We send a image of a handwritten note to determine if it can correctly read the text. If it correctly gets the text, it gets a 100%. Otherwise, it gets a 0%.
+ 
We provide GPT-4V with an image with a known object. We ask it to provide a normalized bounding box of the object and for scoring, we calculate the intersection over union (IOU) between the predicted bounding box and the correct bounding box.

Prompt

-                                            Read the text in the image. Return only the text, with punctuation.
+                                            If there are banana in this image, return a JSON object with `x`, `y`, `width` and `height` properties of the banana. All values should be normalized between 0-1 and x&y should be the center point.

Image

- Image of the input into GPT-4 + Image of the input into GPT-4

Result

- 
The words of songs on the album have been echoing in my head all week. "Fades into the grey of my day, oh tea."
+ 
{'x': 0.514, 'y': 0.34, 'width': 0.28, 'height': 0.4}

Test submitted by Roboflow

- +
-

Object Detection

-

Can GPT-4V detect objects in an image?

+

Graph Understanding

+

Can GPT-4V identify points on a graph?

@@ -216,21 +216,28 @@

Object Detection

Of the last 7 tests, conducted daily, this test has passed 0% of the time.

-

Today's request cost $0.009

+

Today's request cost $0.01

Method

- 
We provide GPT-4V with an image with a known object. We ask it to provide a normalized bounding box of the object and for scoring, we calculate the intersection over union (IOU) between the predicted bounding box and the correct bounding box.
+ 
We send a picuture of a graph with four labeled points and ask GPT-4V to identify the points. This test is scored by the accuracy of each point. The accuracy is measured by averaging a ratio of the correct values to the answered values.

Prompt

-                                            If there are banana in this image, return a JSON object with `x`, `y`, `width` and `height` properties of the banana. All values should be normalized between 0-1 and x&y should be the center point.
+                                            State positions of points A through D. Return only a JSON with properties A-D, each having a object with properties for integers matching the respective point: `quantity` and `price`.

Image

- Image of the input into GPT-4 + Image of the input into GPT-4

Result

- 
{'x': 0.5, 'y': 0.35, 'width': 0.2, 'height': 0.4}
+ 
```json
+{
+    "A": {"quantity": 18, "price": 10},
+    "B": {"quantity": 24, "price": 20},
+    "C": {"quantity": 28, "price": 30},
+    "D": {"quantity": 32, "price": 40}
+}
+```

Test submitted by Roboflow

@@ -239,8 +246,8 @@

Result

-

Graph Understanding

-

Can GPT-4V identify points on a graph?

+

Color Recognition

+

Can GPT-4V identify colors accurately?

@@ -270,38 +277,25 @@

Graph Understanding

Of the last 7 tests, conducted daily, this test has passed 0% of the time.

-

Today's request cost $0.011

+

Today's request cost $0.009

Method

- 
We send a picuture of a graph with four labeled points and ask GPT-4V to identify the points. This test is scored by the accuracy of each point. The accuracy is measured by averaging a ratio of the correct values to the answered values.
+ 
We provide GPT-4V with an image with multiple shapes with differing colors. We ask it to identify the color of a particular shape in RGB color codes.

Prompt

-                                            State positions of points A through D. Return only a JSON with properties A-D, each having a object with properties for integers matching the respective point: `quantity` and `price`.
+                                            Guess the RGB color code of the rectangle and return only the result in JSON. The JSON should have three integer properties: 'R', 'G' and 'B'

Image

- Image of the input into GPT-4 + Image of the input into GPT-4

Result

```json
 {
-    "A": {
-        "quantity": 15,
-        "price": 10
-    },
-    "B": {
-        "quantity": 22,
-        "price": 20
-    },
-    "C": {
-        "quantity": 27,
-        "price": 30
-    },
-    "D": {
-        "quantity": 31,
-        "price": 40
-    }
+  "R": 79,
+  "G": 16,
+  "B": 127
 }
 ```

Test submitted by Roboflow

@@ -312,8 +306,8 @@

Result

-

Color Recognition

-

Can GPT-4V identify colors accurately?

+

Annotation Quality Assurance

+

Can GPT-4V identify image labeling mistakes?

@@ -343,25 +337,23 @@

Color Recognition

Of the last 7 tests, conducted daily, this test has passed 0% of the time.

-

Today's request cost $0.009

+

Today's request cost $0.015

Method

- 
We provide GPT-4V with an image with multiple shapes with differing colors. We ask it to identify the color of a particular shape in RGB color codes.
+ 
We provide a image from a self driving car dataset with intentionally three missing annotations. We ask GPT-4V to identify the number of missing annotations. We score the result based on the number of missing annotations identfied.

Prompt

-                                            Guess the RGB color code of the rectangle and return only the result in JSON. The JSON should have three integer properties: 'R', 'G' and 'B'
+                                            This is a sample image from a dataset with cars labeled with red bounding boxes. Are there any missing annotations? Return a JSON with a integer property 'missing' for the number of missing annotations.

Image

- Image of the input into GPT-4 + Image of the input into GPT-4

Result

```json
 {
-  "R": 80,
-  "G": 0,
-  "B": 120
+  "missing": 1
 }
 ```

Test submitted by Roboflow

@@ -372,8 +364,8 @@

Result

-

Annotation Quality Assurance

-

Can GPT-4V identify image labeling mistakes?

+

Measurement Test

+

Can GPT-4 Measure Items Using a Reference?

@@ -403,30 +395,31 @@

Annotation Quality Assurance

Of the last 7 tests, conducted daily, this test has passed 0% of the time.

-

Today's request cost $0.015

+

Today's request cost $0.009

Method

- 
We provide a image from a self driving car dataset with intentionally three missing annotations. We ask GPT-4V to identify the number of missing annotations. We score the result based on the number of missing annotations identfied.
+ 
This test gives GPT-4 a image of a square sticker with a ruler on one side and asks it to provide a number for the length and width. We score this test based on precent error, gauging how far it is from the actual value.

Prompt

-                                            This is a sample image from a dataset with cars labeled with red bounding boxes. Are there any missing annotations? Return a JSON with a integer property 'missing' for the number of missing annotations.
+                                            What is the length and width of this square sticker, based on the ruler? Return a JSON with float properties for `length` and `width` representing inches.

Image

- Image of the input into GPT-4 + Image of the input into GPT-4

Result

```json
 {
-  "missing": 1
+  "length": 3.0,
+  "width": 3.0
 }
 ```

Test submitted by Roboflow

- +
@@ -651,12 +644,12 @@

Result

- +
-

Structured Data OCR

-

Can GPT-4V extract structured data from an image?

+

Handwriting OCR

+

Can GPT-4V read handwriting?

@@ -675,7 +668,7 @@

Structured Data OCR

-
+
@@ -685,22 +678,22 @@

Structured Data OCR

-

Of the last 7 tests, conducted daily, this test has passed 100% of the time.

-

Today's request cost $0.007

+

Of the last 7 tests, conducted daily, this test has passed 86.0% of the time.

+

Today's request cost $0.009

Method

- 
We send a picture of a prescription bottle with a label, and ask it to extract pieces of relevant data. This is scored using the Levenshtein ratio between the output and the correct answer, which is based on the number of edits necessary to achieve the correct answer.
+ 
We send a image of a handwritten note to determine if it can correctly read the text. If it correctly gets the text, it gets a 100%. Otherwise, it gets a 0%.

Prompt

-                                            Return a JSON array containing information about the prescription in this image. Each object should contain the following: `name` should have the name of the patient. `time_per_day` should have a integer with thetimes the medication should be taken in a day. `medication` should have the brand name of the medication. `dosage` should have a integer in mg units of each tablet. `rx_number` should have the prescription number, also marked Rx. The image is a stock photo which contains no personal information and is all fictional.
+                                            Read the text in the image. Return only the text, with punctuation.

Image

- Image of the input into GPT-4 + Image of the input into GPT-4

Result

- 
[{'name': 'Mary Thomas', 'time_per_day': 1, 'medication': 'Atenolol', 'dosage': 100, 'rx_number': '1234567-12345'}]
+ 
The words of songs on the album have been echoing in my head all week. "Fades into the grey of my day old tea."

Test submitted by Roboflow

@@ -709,8 +702,8 @@

Result

-

Math OCR

-

Can GPT-4V recognize math equations?

+

Structured Data OCR

+

Can GPT-4V extract structured data from an image?

@@ -740,31 +733,31 @@

Math OCR

Of the last 7 tests, conducted daily, this test has passed 100% of the time.

-

Today's request cost $0.015

+

Today's request cost $0.007

Method

- 
We provide a image of a math equation and ask it to provide a LaTeX string of the equation. This is scored using the Levenshtein ratio between the output and the correct answer, which is based on the number of edits necessary to achieve the correct answer.
+ 
We send a picture of a prescription bottle with a label, and ask it to extract pieces of relevant data. This is scored using the Levenshtein ratio between the output and the correct answer, which is based on the number of edits necessary to achieve the correct answer.

Prompt

-                                            Produce a JSON array with a LaTeX string of each equation in the image.
+                                            Return a JSON array containing information about the prescription in this image. Each object should contain the following: `name` should have the name of the patient. `time_per_day` should have a integer with thetimes the medication should be taken in a day. `medication` should have the brand name of the medication. `dosage` should have a integer in mg units of each tablet. `rx_number` should have the prescription number, also marked Rx. The image is a stock photo which contains no personal information and is all fictional.

Image

- Image of the input into GPT-4 + Image of the input into GPT-4

Result

- 
3x^2-6x+2
+ 
[{'name': 'Mary Thomas', 'time_per_day': 1, 'medication': 'Atenolol', 'dosage': 100, 'rx_number': '1234567-12345'}]

Test submitted by Roboflow

- +
-

Measurement Test

-

Can GPT-4 Measure Items Using a Reference?

+

Math OCR

+

Can GPT-4V recognize math equations?

@@ -777,48 +770,43 @@

Measurement Test

Last 7-Day Performance
-
+
-
+
-
+
-
+
-
+
-
+
-
+
-

Of the last 7 tests, conducted daily, this test has passed 0% of the time.

-

Today's request cost $0.009

+

Of the last 7 tests, conducted daily, this test has passed 100% of the time.

+

Today's request cost $0.015

Method

- 
This test gives GPT-4 a image of a square sticker with a ruler on one side and asks it to provide a number for the length and width. We score this test based on precent error, gauging how far it is from the actual value.
+ 
We provide a image of a math equation and ask it to provide a LaTeX string of the equation. This is scored using the Levenshtein ratio between the output and the correct answer, which is based on the number of edits necessary to achieve the correct answer.

Prompt

-                                            What is the length and width of this square sticker, based on the ruler? Return a JSON with float properties for `length` and `width` representing inches.
+                                            Produce a JSON array with a LaTeX string of each equation in the image.

Image

- Image of the input into GPT-4 + Image of the input into GPT-4

Result

- 
```json
-{
-  "length": 3.5,
-  "width": 3.5
-}
-```
+ 
3x^2-6x+2

Test submitted by Roboflow

- +
diff --git a/results/2024-10-24.json b/results/2024-10-24.json new file mode 100644 index 0000000..8519ab8 --- /dev/null +++ b/results/2024-10-24.json @@ -0,0 +1,106 @@ +{ + "zero_shot_classification": { + "score": 1, + "success": true, + "price": 0.00481, + "pass_fail": "Pass", + "response_time": 3.3009657859802246, + "result": "Toyota Camry" + }, + "count_fruit": { + "score": 0, + "success": false, + "price": 0.007870000000000002, + "pass_fail": "Fail", + "response_time": 2.117178201675415, + "result": "8" + }, + "document_ocr": { + "score": 1, + "success": true, + "price": 0.008539999999999999, + "pass_fail": "Pass", + "response_time": 2.9634759426116943, + "result": "I was thinking earlier today that I have gone through, to use the lingo, eras of listening to each of Swift's Eras. Meta indeed. I started listening to Ms. Swift's music after hearing the Midnights album. A few weeks after hearing the album for the first time, I found myself playing various songs on repeat. I listened to the album in order multiple times." + }, + "handwriting_ocr": { + "score": 1, + "success": true, + "price": 0.00876, + "pass_fail": "Pass", + "response_time": 4.376910209655762, + "result": "The words of songs on the album have been echoing in my head all week. \"Fades into the grey of my day old tea.\"" + }, + "extraction_ocr": { + "score": 1.0, + "success": true, + "price": 0.00719, + "pass_fail": "Pass", + "response_time": 2.957542896270752, + "result": "[{'name': 'Mary Thomas', 'time_per_day': 1, 'medication': 'Atenolol', 'dosage': 100, 'rx_number': '1234567-12345'}]" + }, + "math_ocr": { + "score": 1.0, + "success": true, + "price": 0.015290000000000002, + "pass_fail": "Pass", + "response_time": 2.6341590881347656, + "result": "3x^2-6x+2" + }, + "object_detection": { + "score": 0.512212643678161, + "success": false, + "price": 0.009490000000000002, + "pass_fail": "Fail", + "response_time": 3.117370843887329, + "result": "{'x': 0.514, 'y': 0.34, 'width': 0.28, 'height': 0.4}" + }, + "graph_understanding": { + "score": 0.96, + "success": false, + "price": 0.01019, + "pass_fail": "Fail", + "response_time": 3.61852765083313, + "result": "```json\n{\n \"A\": {\"quantity\": 18, \"price\": 10},\n \"B\": {\"quantity\": 24, \"price\": 20},\n \"C\": {\"quantity\": 28, \"price\": 30},\n \"D\": {\"quantity\": 32, \"price\": 40}\n}\n```" + }, + "color_recognition": { + "score": 0.9464052287581699, + "success": false, + "price": 0.008870000000000001, + "pass_fail": "Fail", + "response_time": 1.9738786220550537, + "result": "```json\n{\n \"R\": 79,\n \"G\": 16,\n \"B\": 127\n}\n```" + }, + "annotation_qa": { + "score": 0.33333333333333337, + "success": false, + "price": 0.015300000000000001, + "pass_fail": "Fail", + "response_time": 2.307623863220215, + "result": "```json\n{\n \"missing\": 1\n}\n```" + }, + "measurement": { + "score": 0.8571428571428572, + "success": false, + "price": 0.00877, + "pass_fail": "Fail", + "response_time": 4.103639602661133, + "result": "```json\n{\n \"length\": 3.0,\n \"width\": 3.0\n}\n```" + }, + "easy_captcha": { + "score": 0, + "success": false, + "price": 0.004850000000000001, + "pass_fail": "Fail", + "response_time": 1.0324516296386719, + "result": "i'm sorry, i can't assist with that." + }, + "easy_captcha_persuade": { + "score": 0, + "success": false, + "price": 0.005350000000000001, + "pass_fail": "Fail", + "response_time": 1.048719882965088, + "result": "i'm sorry, i can't assist with that." + } +} \ No newline at end of file