index.html

<!DOCTYPE html>
<html lang="en">
  <head>
    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />

    <meta
      name="viewport"
      content="width=device-width, initial-scale=1, shrink-to-fit=no"
    />
    <meta name="description" content="BIRD-bench" />
    <meta name="author" content="" />
    <link rel="icon" href="./logo/bird_fig_main.png" type="image/x-icon" />
    <title>BIRD-bench</title>

    <!-- Bootstrap core CSS -->
    <link href="./style/bootstrap.min.css" rel="stylesheet" />

    <!-- Custom fonts for this template -->
    <link
      href="./style/font-awesome.min.css"
      rel="stylesheet"
      type="text/css"
    />
    <link
      rel="stylesheet"
      href="./style/all.css"
      integrity="sha384-mzrmE5qonljUremFsqc01SB46JvROS7bZs3IO2EmfFsd15uHvIt+Y8vEf7N7fWAU"
      crossorigin="anonymous"
    />

    <!-- Custom styles for this template -->
    <link href="./style/clean-blog.min.css" rel="stylesheet" />
    <link href="./style/bird.css" rel="stylesheet" />
  </head>

  <body>
    <header class="masthead" id="header-color">
      <div style="background-color: #485159; text-align: left">
        <img
          style="margin-top: 5px; padding: 15px"
          height="70px"
          src="./logo/damo.png"
        />
        <img
          style="margin-top: 5px; padding: 15px"
          height="70px"
          src="./logo/hku.png"
        />
        <img
          style="margin-top: 5px; padding: 15px"
          height="70px"
          src="./logo/uiuc.png"
        />
      </div>
      <div class="container">
        <div class="row">
          <div class="col-lg-10 col-md-12 mx-auto">
            <div class="row site-heading flex-box">
              <div class="col-lg-3">
                <img id="logo-img" src="logo/bird_fig_main.png" alt="logo" />
              </div>
              <div class="col-lg-9">
                <div class="align-middle">
                  <h1 id="bird-title">BIRD-SQL</h1>
                  <span class="subheading" id="caption">A Big Bench for Large-Scale Database Grounded
                    Text-to-SQLs</span>
                </div>
              </div>
            </div>
          </div>
        </div>
      </div>
    </header>

    <!-- Main Content -->
    <div class="container" id="main">
      <div class="row">
        <div class="col-lg-5">
          <div class="list-group">
            <div class="list-group-item" id="introduction">
              <!-- <div style="display: grid; grid-template-columns: auto auto; align-items: center;"> -->
              <h4>About BIRD</h4>
              <img
                src="https://badges.toozhao.com/badges/01HFXP6JKNBG5AW3Y5B95DK568/blue.svg"
              />
              <!-- </div> -->
              <p>
                BIRD (<b>BI</b>g Bench for La<b>R</b>ge-scale <b>D</b>atabase
                Grounded Text-to-SQL Evaluation) represents a pioneering,
                cross-domain dataset that examines the impact of extensive
                database contents on text-to-SQL parsing.
                <!-- The dataset includes three extra problems that especially resonates with real-world scenarios:
              <b>1)</b> the necessity to incorporate external knowledge into parsers; <b>2)</b> the need for parsers to
              reason over 'dirty' values;
              and <b>3)</b> the requirment to consider efficiency while executing SQLs.  -->
                BIRD contains over <b>12,751</b> unique question-SQL pairs,
                <b>95</b> big databases with a total size of <b>33.4 GB</b>. It
                also covers more than <b>37</b> professional domains, such as
                blockchain, hockey, healthcare and education, etc.
              </p>
              <div id="button-group">
                <div class="button-row">
                  <a
                    href="https://arxiv.org/pdf/2305.03111.pdf"
                    target="_blank"
                  >
                    <button type="button" class="btn btn-a">Paper</button>
                  </a>
                </div>
                <div class="button-row">
                  <a
                    href="https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/bird"
                    target="_blank"
                  >
                    <button type="button" class="btn btn-a">Code</button>
                  </a>
                </div>
                <div class="button-row">
                  <a
                    href="https://github.com/bird-bench/mini_dev"
                    target="_blank"
                  >
                    <button type="button" class="btn btn-a">
                      Mini-Dev (500)
                    </button>
                  </a>
                </div>
                <div class="button-row">
                  <a
                    href="https://bird-critic.github.io/"
                    target="_blank"
                  >
                  <button type="button" class="btn btn-d">
                    BIRD-CRITIC 1.0 (SQL)
                  </button>
                </div>
                <div class="button-row">
                  <a href="#" onclick="showLiveSQLBenchMessage(); return false;">
                  <button type="button" class="btn btn-e">
                    LiveSQLBench!
                  </button>
                </div>
                <div class="button-row" id="button-row-b">
                  <div class="btn-round">
                    <a
                      href="https://bird-bench.oss-cn-beijing.aliyuncs.com/train.zip"
                      target="_blank"
                    >
                      <button type="button" class="btn btn-b">Train Set</button>
                    </a>
                  </div>
                  <div class="btn-round">
                    <a
                      href="https://bird-bench.oss-cn-beijing.aliyuncs.com/dev.zip"
                      target="_blank"
                    >
                      <button type="button" class="btn btn-b">
                        🔥 Dev Set
                      </button>
                    </a>
                  </div>
                </div>
              </div>
            </div>
            <style>
              .scroll-container {
                position: relative;
                width: 100%;
                max-height: 300px;
                overflow-y: scroll;
              }

              /* Hide the scrollbar by default */
              .scroll-container::-webkit-scrollbar {
                width: 10px;
                background: transparent;
              }

              /* Set the scrollbar thumb color */
              .scroll-container::-webkit-scrollbar-thumb {
                background-color: #ffffff;
                border-radius: 5px;
              }

              /* Show the scrollbar when the mouse is hovering over the scroll container */
              .scroll-container:hover::-webkit-scrollbar {
                width: 10px;
              }

              .scroll-container:hover::-webkit-scrollbar-thumb:hover {
                background-color: #555;
              }
            </style>
            <script>
              // A new function for BIRD-CRITIC
              function showBirdCriticMessage() {
                alert(
                  "BIRD-CRITIC is the first SQL debugging benchmark investigating whether LLMs can effectively resolve user issues in real-world database applications. It contains 600 tasks in dev and 200 OOD hidden tests. The full challenges in BIRD-SQL 2025 are under collaboration with Google Cloud.\n\n" +
                  "Key Highlights:\n" +
                  "• 5 popular SQL dialects: MySQL, PostgreSQL, SQLServer, Oracle, and one OOD dialect\n" +
                  "• Challenging tasks that go beyond SELECT (e.g., CREATE TRIGGER, UPDATE constraints, SQL Optimization)\n" +
                  "• Each task is derived from real-world user cases and reproduced within the BIRD environment, verified by human experts\n" +
                  "• Multi-thread sandbox enabling fast and robust evaluation\n" +
                  "• Professionally curated test cases for each task.\n" +
                  "We will release a PREVIEW set in this month and full set next month. Email us if you have some suggestions or new contents that you want to see!\n\n" +
                  "Other exciting Sibling benchmarks you may expect in BIRD-SQL 2025:\n" +
                  "• BIRD-CRITIC 1.5 (Repo-level Code Debugging for Real-World DB Applications)\n" +
                  "• BIRD-Interact\n" +
                  "• BIRD-Vision\n" +
                  "• BIRD-Pro & BIRD-Pro Max: The second generation of BIRD-SQL covering comprehensive DB operations / BI querying with new open-source enterprise-level databases with thousands of columns, advanced constrains and TB-level data."
                );
              }
            </script>
            <script>
                function showLiveSQLBenchMessage() {
                  alert(
                    "Inspired by great LiveCodeBench, we're developing LiveSQLBench to provide a robust and contamination-free evaluation for text-to-SQL models on challenging, comprehensive benchmark, which will be updated periodically.\n\n" +
                    "LiveSQLBench (BIRD-SQL Pro v0.5) will feature diverse real-world user queries, including Business Intelligence (BI), CRUD operations, etc.\n\n" +
                    "Each release will include 50 new, fully open-source DBs curated by the BIRD team through expert collaboration and continuous improvement. It will cover a wide range of database sizes, from end-user level (around 39 columns) to industrial level (1340+ columns), catering to various query types.\n\n" +
                    "Look for LiveSQLBench to launch in March! And more exciting text-to-SQL evaluations on the way~ :) "
                  );
                }
            </script>


            <div class="list-group-item" id="news">
              <h4>News</h4>

              <div class="scroll-container">
                <div
                  class="card card-outline-secondary mb-4"
                  style="text-align: left"
                >
                  <div class="card-body" style="background-color: #f1f6f9">
                    <ul style="padding-left: 0">
                      <li style="list-style-type: none">
                         <span
                           style="
                           background-color: #B03052;
                           color: white;
                           padding: 2px 4px;
                           border-radius: 5px;
                           "
                         ><strong style="color: #ffe7ce">Apr. 20, 2025:</strong></span> We have released bird-critic-1.0-open (600 tasks by 4 dialects). Check out the data in <a href="https://huggingface.co/datasets/birdsql/bird-critic-1.0-open">Hugging Face</a> and the newest code in <a href="https://github.com/bird-bench/BIRD-CRITIC-1">GitHub</a>. The full set of PostgreSQL will be released 1 week later. 
                        It seems that bird-critic is a challenging reasoning tasks for text-to-SQL since all top-peforming models are reasoning-based models. Have fun! Thanks!
                       </li>

                      <li style="list-style-type: none">
                        <span
                          style="
                            background-color: #B03052;
                            color: white;
                            padding: 2px 4px;
                            border-radius: 5px;
                          "
                          ><strong style="color: #ffe7ce">Feb. 4, 2025:</strong></span> We've launched <a href="https://bird-critic.github.io/">BIRD-Critic (a.k.a SWE-SQL)</a>, a brand new text-to-SQL benchmark that really digs into reasoning challenges! A lite version is ready for exploration. Full sets are coming soon! Feel free for any feedbacks! Your inputs and suggestions are much appreciated!

                      </li>
                      <!--                           <li style="list-style-type: none;"><span style="background-color: #22A699; color: white; padding: 2px 4px; border-radius: 5px;"><strong style="color: #FFE7CE">May. 27, 2024:</strong></span> Glad to share our paper <a href="https://arxiv.org/abs/2405.15307">Before Generation, Align it! A Novel and Effective Strategy for Mitigating Hallucinations in Text-to-SQL Generation (TA-SQL)</a>, which will appear at <code><strong>ACL 2024</strong></code> Findings. This is a light but effective method to calbriate hallucinations of text-to-SQL generation. Additionally, we firstly introduce Pythonic prompts for text-to-SQL and include results on open-source LLMs.</li> -->
                      <li style="list-style-type: none">
                        <span
                          style="
                            background-color: #B03052;
                            color: white;
                            padding: 2px 4px;
                            border-radius: 5px;
                          "
                          ><strong style="color: #ffe7ce">Nov. 26, 2024:</strong></span> Thanks the support of <code>BIRD-SQL 2023</code>!
                             Now we are pleased to share that the project <strong><code>BIRD 2025</code></strong> has been started.
                             It will contains 4-6 new benchmarks with each covering its special focus of professional databases and their knowledge in the wild applications.
                             <strong>We will release the first benchmark by early Jan</strong>. Feel free to let us know your needs or suggestions for cooking new generations of Text-to-SQL challenges. Thanks!</a>
                      </li>
                      <li style="list-style-type: none">
                        <span
                          style="
                            background-color: #22a699;
                            color: white;
                            padding: 2px 4px;
                            border-radius: 5px;
                          "
                          ><strong style="color: #ffe7ce"
                            >Aug. 4, 2024:</strong></span> The Reward-based Valid Efficiency Score (R-VES) will be used as the efficiency metric for future test submissions.
                        The rationale and formula for R-VES can be found in the <a href="https://github.com/bird-bench/mini_dev">Mini-Dev repository</a>.
                        You can check the legacy VES scores for previous submissions <a href="https://abundant-dawn-033.notion.site/BIRD-VES-Leaderboard-Legacy-7aea622573dd42c59a1bcdb122d2cf44?pvs=4">here</a>.
                      </li>
                      <li style="list-style-type: none">
                        <span
                          style="
                          background-color: #4f709c;
                          color: white;
                          padding: 2px 4px;
                          border-radius: 5px;
                          "
                          ><strong style="color: #ffe7ce"
                            >Jun. 30, 2024:</strong
                          ></span
                        > Due to large requests of test submissions about mixed
                        models (open-source + GPU-based closed source), we
                        update the submission instructions to accelerate your
                        waiting time. Please check it out!
                      </li>
                      <li style="list-style-type: none">
                        <span
                          style="
                          background-color: #4f709c;
                          color: white;
                          padding: 2px 4px;
                          border-radius: 5px;
                          "
                          ><strong style="color: #ffe7ce"
                            >Jun. 30, 2024:</strong
                          ></span
                        >
                        If you are interested in code agent, please do not miss a SOTA code agent implementation by <a href="https://github.com/All-Hands-AI/OpenHands/tree/main/evaluation/bird">OpenDevin</a> for BIRD Dev!
                      </li>
                      <li style="list-style-type: none">
                        <span
                          style="
                          background-color: #4f709c;
                          color: white;
                          padding: 2px 4px;
                          border-radius: 5px;
                          "
                          ><strong style="color: #ffe7ce"
                            >Jun. 27, 2024:</strong
                          ></span
                        >
                        Excited to announce the release of our
                        <a
                          ref="https://github.com/bird-bench/mini_dev/tree/main"
                          >BIRD Mini-Dev dataset</a
                        >
                        with 500 high-quality examples. This dataset includes
                        all BIRD keywords, with modifications for questions such
                        as the addition of <code>window function</code>. We are
                        the first to deliver it in not only
                        <strong>SQLite</strong>, but also
                        <strong>MySQL</strong>, and <strong>PostgreSQL</strong>.
                        We include Soft-F1 and R-VES metrics to reduce bias.
                        Don't miss the <code>column_meaning.json</code> file,
                        preprocessed by TA-SQL. Available for dev and testing
                        set. Check out our work here:
                        <a href="https://arxiv.org/abs/2405.15307"
                          >Before Generation, Align it! A Novel and Effective
                          Strategy for Mitigating Hallucinations in Text-to-SQL
                          Generation (TA-SQL)</a
                        >, appearing at
                        <code><strong>ACL 2024</strong></code> Findings.
                      </li>
                      <li style="list-style-type: none">
                        <span
                          style="
                          background-color: #4f709c;
                          color: white;
                          padding: 2px 4px;
                          border-radius: 5px;
                          "
                          ><strong style="color: #ffe7ce"
                            >Apr. 27, 2024:</strong
                          ></span
                        >
                        Due to large volume of requests, we now modify the
                        license of our data to
                        <code><strong>CC BY-SA 4.0</strong></code
                        >. However, we will not take responsibility of any bad
                        purposes by using our data. Since we develop this
                        benchmark for research and healthy application only.
                      </li>
                      <li style="list-style-type: none">
                        <span
                          style="
                            background-color: #4f709c;
                            color: white;
                            padding: 2px 4px;
                            border-radius: 5px;
                          "
                          ><strong style="color: #ffe7ce"
                            >Mar 13, 2024:</strong
                          ></span
                        >
                        Please also take a look at our related work:
                        <a href="https://arxiv.org/abs/2403.05307"
                          >Tapilot-Crossing</a
                        >, which is the first challenging and more realistic
                        benchmark designed to evaluate Large Language Model
                        (LLM) agents on interactive data analysis tasks. The
                        code includes Python and Private Library. And it covers
                        6 common agent actions in evaluation.
                      </li>
                      <li style="list-style-type: none">
                        <span
                          style="
                            background-color: #4f709c;
                            color: white;
                            padding: 2px 4px;
                            border-radius: 5px;
                          "
                          ><strong style="color: #ffe7ce"
                            >Sept 25, 2023:</strong
                          ></span
                        >
                        We have released a cleaner version of
                        <code>dev set</code>. Please download dev set again. We
                        checked all cases of dev set and fixed all errors that
                        we found. After cleaning, the ChatGPT (gpt-3.5-turbo)
                        and GPT4 (gpt-4-32k) EX scores have improved to
                        <strong>42.24</strong> (from 37.22) and
                        <strong>49.15</strong> (from 46.35), respectively.
                        Thanks for all feedbacks!
                      </li>
                      <li style="list-style-type: none">
                        <span
                          style="
                            background-color: #4f709c;
                            color: white;
                            padding: 2px 4px;
                            border-radius: 5px;
                          "
                          ><strong style="color: #ffe7ce"
                            >Sept 21, 2023:</strong
                          ></span
                        >
                        Our paper has been accepted by
                        <code><strong>NeurIPS 2023</strong></code> as a
                        <code><strong>Spotlight</strong></code
                        >!!! Thanks for all the efforts and suggestions of
                        co-authors, anonymous reviewers, awesome
                        researchers/users in github or emails.
                      </li>
                      <li style="list-style-type: none">
                        <span
                          style="
                            background-color: #4f709c;
                            color: white;
                            padding: 2px 4px;
                            border-radius: 5px;
                          "
                          ><strong style="color: #f6f1f1"
                            >July 17, 2023:</strong
                          ></span
                        >
                        We update newest results of GPT-4, Claude-2 and Palm-2.
                      </li>
                      <li style="list-style-type: none">
                        <span
                          style="
                            background-color: #4f709c;
                            color: white;
                            padding: 2px 4px;
                            border-radius: 5px;
                          "
                          ><strong style="color: #f6f1f1"
                            >July 14, 2023:</strong
                          ></span
                        >
                        The data link has been updated, fixing the schema names
                        in the CSV files. Additionally, tied results caused by
                        <code>order_by limit 1</code> are now considered. Both
                        SQL queries - with and without accounting for tied
                        results - are valid at this time.
                      </li>
                      <li style="list-style-type: none">
                        <span
                          style="
                            background-color: #4f709c;
                            color: white;
                            padding: 2px 4px;
                            border-radius: 5px;
                          "
                          ><strong style="color: #f6f1f1"
                            >Jun 12, 2023:</strong
                          ></span
                        >
                        We are welcome to any suggestions and reported gold
                        errors in
                        <a
                          href="https://github.com/AlibabaResearch/DAMO-ConvAI/issues/39"
                          >help_wanted</a
                        >. Any of your help is appreciated!
                      </li>
                      <li style="list-style-type: none">
                        <span
                          style="
                            background-color: #4f709c;
                            color: white;
                            padding: 2px 4px;
                            border-radius: 5px;
                          "
                          ><strong style="color: #f6f1f1"
                            >Jun 5, 2023:</strong
                          ></span
                        >
                        We open-sourced our
                        <a href="https://arxiv.org/abs/2301.07507">Graphix-T5</a
                        >, a graph-aware semi-pretrained text-to-text PLM
                        specifically designed to improve multi-hop reasoning for
                        the complex text-to-SQL task.
                      </li>
                      <li style="list-style-type: none">
                        <span
                          style="
                            background-color: #4f709c;
                            color: white;
                            padding: 2px 4px;
                            border-radius: 5px;
                          "
                          ><strong style="color: #f6f1f1"
                            >May 30, 2023:</strong
                          ></span
                        >
                        If you are interested in ICL, please check out our
                        interesting work
                        <a href="https://arxiv.org/pdf/2305.13016.pdf"
                          >deep-thinking🤔.</a
                        >
                        Generate 1000 models for 1000 people smoothly!
                      </li>
                    </ul>
                  </div>
                </div>
              </div>
            </div>
            <div class="list-group-item" id="illustration">
              <h4>Surprise from BIRD</h4>
              <p>
                <b>1. Large and Dirty values:</b> Due to the nature of the
                real-world scenarios from which BIRD's database values were
                collected, they typically retain their original and frequently
                "dirty" format. Hence, text-to-SQL parsers must first analyze
                these values to account for their non-standard format before
                engaging in reasoning.
              </p>
              <img
                src="./img/dirty_1.png"
                class="column-img"
                id="dirty-img-1"
                style="margin: 0 auto"
              />
              <img
                src="./img/dirty_2.png"
                class="column-img"
                id="dirty-img-2"
                style="margin: 0 auto"
              />
              <p>
                <b>2. External Knowledge:</b> "account.type = 'OWNER'" can be
                inferred by the knowledge evidence: "The condition of the loans
                require the account type should be the owner."
              </p>
              <img
                src="./img/ex_kg_2.png"
                class="column-img"
                id="kg-img"
                style="margin: 0 auto"
              />
              <p>
                <b>3. Text-to-Efficient-SQL:</b> BIRD is the first text-to-SQL
                benchmark designed to encourage semantic parsers to produce SQL
                queries that are not only correct but also efficient. This
                emphasis on efficiency is especially valuable in real-world data
                / business analysis circumstances.
              </p>
              <img
                src="./img/efficient_sql.png"
                class="column-img"
                id="eff-img"
                style="margin: 0 auto"
              />
            </div>
            <!-- <div class="list-group-item">
            <h4>Download Dataset</h4>
            <p>We are cleaning and checking our databases, SQLs, Question, and Knowledge carefully. Will come up soon!</p>
            <p class="text-center">
              <a href="https://bird-bench.github.io/"><button type="button" class="btn btn-success", id="code">BIRD Code Nest</button></a>
            </p>
          </div> -->
            <div class="list-group-item">
              <h4>Submission</h4>
              <p>
                Please follow the Submission Guideline (below) and contact
                <code>bird.bench23@gmail.com</code> for test evaluation.
                Ususally, we will return your results in 10 days!
              </p>
              <style>
                .btn-a {
                  border-radius: 25px; /* Adjust this value to change the roundness of the button */
                }
                .btn-a:hover {
                  background-color: #22a699; /* Change the background color when the cursor is over the button */
                }
              </style>
              <div class="button-row">
                <a
                  href="https://docs.google.com/document/d/1Rs6d_pcs2vfqW4Ymub7Wb1XtBNlrc-WfH3T7U1ktuBo/edit?usp=sharing"
                  target="_blank"
                >
                  <button type="button" class="btn btn-a">
                    Submission Guidelines
                  </button>
                </a>
              </div>
            </div>

            <div class="list-group-item">
              <h4>Subscribe to BIRD Update</h4>
              <p>
                Bird is a long-term research project aimed at bridging the gap
                between semantic parsing models and the success of database
                applications. To receive the latest updates of the dataset, you
                can leave your email address.
              </p>
              <form onsubmit="return submitForm();">
                <input
                  type="email"
                  id="email"
                  name="email"
                  placeholder="Email address"
                  required
                />
                <button type="submit">Subscribe</button>
              </form>
              <p id="message"></p>
            </div>

            <head>
              <meta charset="utf-8" />
              <title>Email Subscription</title>
              <script>
                function submitForm() {
                  var email = document.getElementById("email").value;
                  var url =
                    "https://script.google.com/macros/s/AKfycbynv6Ibnl9gJosprkRmXhbKtllZX8dqrbj_xQkcbZ_NgEDBxwyjz1xyvIvnm_4yPh3ZPA/exec";
                  var data = "email=" + encodeURIComponent(email);

                  fetch(url, {
                    method: "POST",
                    body: data,
                    headers: {
                      "Content-Type": "application/x-www-form-urlencoded",
                    },
                  })
                    .then((response) => response.text())
                    .then((text) => {
                      document.getElementById("message").innerHTML =
                        "<i>Subscribe Successfully! Thanks for your attention!</i>";
                    })
                    .catch((error) => {
                      console.error("Error:", error);
                    });

                  return false;
                }
              </script>
            </head>
            <div class="list-group-item">
              <h4>Citation</h4>
              <!-- <pre id="citation">
@misc{li2023llm,
  title={Can LLM Already Serve as A Database Interface? A BIg Bench for Large-Scale Database Grounded Text-to-SQLs},
  author={Jinyang Li and Binyuan Hui and Ge Qu and Binhua Li and Jiaxi Yang and Bowen Li and Bailin Wang and Bowen Qin and Rongyu Cao and Ruiying Geng and Nan Huo and Chenhao Ma and Kevin C. C. Chang and Fei Huang and Reynold Cheng and Yongbin Li},
  year={2023},
  eprint={2305.03111},
  archivePrefix={arXiv},
  primaryClass={cs.CL}
}</pre> -->
              <pre id="citation">
@article{li2024can,
  title={Can llm already serve as a database interface? a big bench for large-scale database grounded text-to-sqls},
  author={Li, Jinyang and Hui, Binyuan and Qu, Ge and Yang, Jiaxi and Li, Binhua and Li, Bowen and Wang, Bailin and Qin, Bowen and Geng, Ruiying and Huo, Nan and others},
  journal={Advances in Neural Information Processing Systems},
  volume={36},
  year={2024}
}</pre
              >
            </div>
          </div>
        </div>

        <div class="col-lg-7">
          <div class="btn-group mb-3 w-100">
            <button id="overall-leaderboard-btn" class="btn btn-f">Overall Leaderboard</button>
            <button id="single-model-leaderboard-btn" class="btn btn-f">Single-Model Leaderboard</button>
          </div>
  
          <div id="overall-leaderboard">
            <div class="card card-outline-secondary">
              <div class="card-header">Leaderboard - Execution Accuracy (EX)</div>
              <div class="table-wrapper">
                <table class="table" id="ex-table">
                  <thead class="thead-light">
                    <tr>
                      <th scope="col" rowspan="1"></th>
                      <th
                        scope="col"
                        rowspan="1"
                        class="align-middle text-center"
                      >
                        Model
                      </th>
                      <th
                        scope="col"
                        rowspan="1"
                        class="align-middle text-center"
                      >
                        Code
                      </th>
                      <th
                        scope="col"
                        rowspan="1"
                        class="align-middle text-center"
                      >
                        Size
                      </th>
                      <th
                        scope="col"
                        rowspan="1"
                        class="align-middle text-center"
                      >
                        Oracle Knowledge
                      </th>
                      <th scope="col" colspan="1" class="text-center">Dev (%)</th>
                      <th scope="col" colspan="1" class="text-center">
                        Test (%)
                      </th>
                    </tr>
                  </thead>
                  <tbody>
                    <tr class="table-success">
                      <td scope="row" class="align-middle text-center"><br /></td>
                      <td class="align-middle text-center">
                        Human Performance<br />
                        <span class="affiliation"
                          >Data Engineers + DB Students</span
                        ><br /><a href="" class="paperlink"></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center"><b>92.96</b></td>
                    </tr>
  
                    <!-- <tr class="table-success">
                    <td scope="row" class="align-middle text-center"><br>
                    </td>
                    <td class="align-middle text-center">Human Performance<br>
                      <span class="affiliation">Data Engineers + DB Students</span><br><a href="" class="paperlink"></a>
                    </td>
                    <td class="align-middle text-center"></td>
                    <td class="align-middle text-center">✔️</td>
                    <td class="align-middle text-center"></td>
                    <td class="align-middle text-center"></td>
                    <td class="align-middle text-center"><b>72.37</b></td>
                  </tr> -->
  
                  <tr>
                    <td scope="row" class="align-middle text-center counter-cell">
                      <span class="badge badge-secondary">Mar 11, 2025</span>
                    </td>
                    <td class="align-middle text-center">AskData + GPT-4o<br>
                      <span class="affiliation">AT&T CDO - DSAIR</span>
                    </td>
                    <td class="align-middle text-center"></td>
                    <td class="align-middle text-center">UNK</td>
                    <td class="align-middle text-center">✔️</td>
                    <td class="align-middle text-center">75.36</td>
                    <td class="align-middle text-center"><b>77.14</b></td>
                  </tr>
  
  
                  <tr>
                    <td scope="row" class="align-middle text-center counter-cell">
                      <span class="badge badge-secondary">Apr 16, 2025</span>
                    </td>
                    <td class="align-middle text-center">CHASE-SQL + Gemini<br>
                      <span class="affiliation">Google Cloud</span><br><a href="https://arxiv.org/abs/2410.01943" class="paperlink">[Pourreza et al. '24]</a>
                    </td>
                    <td class="align-middle text-center"></td>
                    <td class="align-middle text-center">UNK</td>
                    <td class="align-middle text-center">✔️</td>
                    <td class="align-middle text-center">74.90</td>
                    <td class="align-middle text-center">76.02</td>
                  </tr>
  
                  <tr>
                    <td scope="row" class="align-middle text-center counter-cell">
                      <span class="badge badge-secondary">Feb 27, 2025</span>
                    </td>
                    <td class="align-middle text-center">Contextual-SQL<br>
                      <span class="affiliation">Contextual AI</span>
                    </td>
                    <td class="align-middle text-center"></td>
                    <td class="align-middle text-center">UNK</td>
                    <td class="align-middle text-center">✔️</td>
                    <td class="align-middle text-center">73.50</td>
                    <td class="align-middle text-center">75.63</td>
                  </tr>
  
                  <tr>
                    <td scope="row" class="align-middle text-center counter-cell">
                      <span class="badge badge-secondary">Dec 17, 2024</span>
                    </td>
                    <td class="align-middle text-center">XiYan-SQL<br>
                      <span class="affiliation">Alibaba Cloud</span><br><a href="https://arxiv.org/abs/2411.08599" class="paperlink">[Yifu Liu et al. '24]</a>
                    </td>
                    <td class="align-middle text-center"><a href="https://github.com/XGenerationLab/XiYan-SQL">[link]</a></td>
                    <td class="align-middle text-center">UNK</td>
                    <td class="align-middle text-center">✔️</td>
                    <td class="align-middle text-center">73.34</td>
                    <td class="align-middle text-center">75.63</td>
                  </tr>
  
                  
  <!--                 <tr>
                    <td scope="row" class="align-middle text-center counter-cell">
                      <span class="badge badge-secondary">Nov 3, 2024</span>
                    </td>
                    <td class="align-middle text-center">CHASE-SQL + Gemini<br>
                      <span class="affiliation">Google Cloud</span><br><a href="https://arxiv.org/abs/2410.01943" class="paperlink">[Pourreza et al. '24]</a>
                    </td>
                    <td class="align-middle text-center"></td>
                    <td class="align-middle text-center">UNK</td>
                    <td class="align-middle text-center">✔️</td>
                    <td class="align-middle text-center">73.14</td>
                    <td class="align-middle text-center">74.06</td>
                  </tr> -->
                  <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Oct 27, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        ExSL + granite-34b-code<br />
                        <span class="affiliation">IBM Research AI</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">34B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">72.43</td>
                      <td class="align-middle text-center">73.17</td>
                  </tr>
                  <tr>
                    <td scope="row" class="align-middle text-center counter-cell">
                      <span class="badge badge-secondary">Mar 31, 2025</span>
                    </td>
                    <td class="align-middle text-center">Reasoning-SQL 14B<br>
                      <span class="affiliation">Google Cloud / Stanford</span><br><a href="https://arxiv.org/pdf/2503.23157" class="paperlink">[Pourreza et al. '25]</a>
                    </td>
                    <td class="align-middle text-center"></td>
                    <td class="align-middle text-center">14 B</td>
                    <td class="align-middle text-center">✔️</td>
                    <td class="align-middle text-center">72.29</td>
                    <td class="align-middle text-center">72.78</td>
                  </tr>
                  <tr>
                    <td scope="row" class="align-middle text-center counter-cell">
                      <span class="badge badge-secondary">Aug 21, 2024</span>
                    </td>
                    <td class="align-middle text-center">OpenSearch-SQL, v2 + GPT-4o<br>
                      <span class="affiliation">Alibaba Cloud</span><br><a href="https://arxiv.org/abs/2502.14913" class="paperlink">[Xiangjin Xie et al. '25]</a>
                    </td>
                    <td class="align-middle text-center"><a href="https://github.com/OpenSearch-AI/OpenSearch-SQL">[link]</a></td>
                    <td class="align-middle text-center">UNK</td>
                    <td class="align-middle text-center">✔️</td>
                    <td class="align-middle text-center">69.30</td>
                    <td class="align-middle text-center">72.28</td>
                  </tr>
  
                  <tr>
                    <td scope="row" class="align-middle text-center counter-cell">
                      <span class="badge badge-secondary">Mar 4, 2025</span>
                    </td>
                    <td class="align-middle text-center">OmniSQL-32B<br>
                      <span class="affiliation">Renmin University of China + ByteDance Infra Lab</span><br><a href="https://arxiv.org/abs/2503.02240" class="paperlink">[Li et al. '25]</a>
                    </td>
                    <td class="align-middle text-center"><a href="https://github.com/RUCKBReasoning/OmniSQL">[link]</a></td>
                    <td class="align-middle text-center">32B</td>
                    <td class="align-middle text-center">✔️</td>
                    <td class="align-middle text-center">69.23</td>
                    <td class="align-middle text-center">72.05</td>
                  </tr>
                    
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Jul 22, 2024</span>
                      </td>
                      <td class="align-middle text-center"> Distillery + GPT-4o<br>
                        <span class="affiliation">Distyl AI Research</span><br><a href="https://arxiv.org/abs/2408.07702" class="paperlink">[Maamari et al. '24]</a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">67.21</td>
                      <td class="align-middle text-center">71.83</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Apr 18, 2025</span>
                      </td>
                      <td class="align-middle text-center">
                        GenaSQL<br />
                        <span class="affiliation">Gena AI</span><br />
                        <a href="https://gena.co/" target="_blank"><span class="affiliation">gena.co</span></a>
                      </td>
                      <td class="align-middle text-center">
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">69.25</td>
                      <td class="align-middle text-center">71.38</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Apr 10, 2025</span>
                      </td>
                      <td class="align-middle text-center">
                        Queryosity<br />
                        <span class="affiliation">Conrad Labs</span><br />
                        <a href="https://queryosity.io" target="_blank"><span class="affiliation">queryosity.io</span></a>
                      </td>
                      <td class="align-middle text-center">
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">69.40</td>
                      <td class="align-middle text-center">71.16</td>
                    </tr>
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">May 21, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        CHESS<sub>IR +</sub><sub>CG +</sub><sub>UT</sub><br />
                        <span class="affiliation">Stanford</span><br /><a
                          href="https://arxiv.org/abs/2405.16755"
                          class="paperlink"
                          >[Talaei et al.'24]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/ShayanTalaei/CHESS">[link]</a>
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">68.31</td>
                      <td class="align-middle text-center">71.10</td>
                    </tr>
  
                    <!--
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Aug 1, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        ExSL + granite-34b-code<br />
                        <span class="affiliation">IBM Research AI</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">34B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">67.47</td>
                      <td class="align-middle text-center">70.37</td>
                    </tr>
                    -->
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                      <span class="badge badge-secondary">Aug 28, 2024</span>
                      </td>
                      <td class="align-middle text-center">Insights AI<br>
                      <span class="affiliation">Uber Freight</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">72.16</td>
                      <td class="align-middle text-center">70.26</td>
                      </tr>
  
                    <tr>
                        <td scope="row" class="align-middle text-center counter-cell">
                            <span class="badge badge-secondary">Aug 30, 2024</span>
                        </td>
                        <td class="align-middle text-center">PURPLE + RED + GPT-4o<br>
                            <span class="affiliation">Fudan University + Transwarp Technology </span>
                        </td>
                        <td class="align-middle text-center"></td>
                        <td class="align-middle text-center">UNK</td>
                        <td class="align-middle text-center">✔️</td>
                        <td class="align-middle text-center">68.12</td>
                        <td class="align-middle text-center">70.21</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Nov 10, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        PB-SQL, GPT-4o<br />
                        <span class="affiliation">Seoul National University</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">68.64</td>
                      <td class="align-middle text-center">69.26</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Jan 9, 2025</span>
                      </td>
                      <td class="align-middle text-center">
                        GSR<br />
                        <span class="affiliation">Anonymous</span>
                      </td>
                      <td class="align-middle text-center"><a href="https://github.com/GSR-SQL/GSR">[link]</a></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">66.88</td>
                      <td class="align-middle text-center">69.26</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Dec 18, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        TC-SQL<br />
                        <span class="affiliation">mindflow.ai</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">70.93</td>
                      <td class="align-middle text-center">69.20</td>
                    </tr>
  
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Jan 09, 2025</span>
                      </td>
                      <td class="align-middle text-center">XiYanSQL-QwenCoder-32B<br>
                        <span class="affiliation">Alibaba Cloud</span><br><a href="https://arxiv.org/abs/2411.08599" class="paperlink">[Yifu Liu et al. '24]</a>
                      </td>
                      <td class="align-middle text-center"><a href="https://github.com/XGenerationLab/XiYanSQL-QwenCoder-32B">[link]</a></td>
                      <td class="align-middle text-center">32B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">67.01</td>
                      <td class="align-middle text-center">69.03</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Jul 14, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        RECAP + Gemini<br />
                        <span class="affiliation">Google Cloud</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">66.95</td>
                      <td class="align-middle text-center">69.03</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Jul 2, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        ByteBrain<br />
                        <span class="affiliation">ByteDance Infra Lab</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">33B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">65.45</td>
                      <td class="align-middle text-center">68.87</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Nov 18, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        RSL-SQL + GPT-4o<br />
                        <span class="affiliation">Anonymous</span><br /><a
                          href="https://arxiv.org/abs/2411.00073"
                          class="paperlink"
                          >[Cao et al.'24]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/Laqcce-cao/RSL-SQL">[link]</a>
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">67.21</td>
                      <td class="align-middle text-center">68.70</td>
                    </tr>
  
                  <tr>
                    <td scope="row" class="align-middle text-center counter-cell">
                      <span class="badge badge-secondary">Mar 4, 2025</span>
                    </td>
                    <td class="align-middle text-center">OmniSQL-7B<br>
                      <span class="affiliation">Renmin University of China + ByteDance Infra Lab</span><br><a href="https://arxiv.org/abs/2503.02240" class="paperlink">[Li et al. '25]</a>
                    </td>
                    <td class="align-middle text-center"><a href="https://github.com/RUCKBReasoning/OmniSQL">[link]</a></td>
                    <td class="align-middle text-center">7B</td>
                    <td class="align-middle text-center">✔️</td>
                    <td class="align-middle text-center">69.04</td>
                    <td class="align-middle text-center">67.97</td>
                  </tr>
                    
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">May 14, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        ExSL + granite-20b-code<br />
                        <span class="affiliation">IBM Research AI</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">20B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">65.38</td>
                      <td class="align-middle text-center">67.86</td>
                    </tr>
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Nov 11, 2024</span>
                      </td>
                      <td class="align-middle text-center">AskData + GPT-4o<br>
                        <span class="affiliation">AT&T CDO - DSAIR</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">65.91</td>
                      <td class="align-middle text-center">67.41</td>
                    </tr>
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">May 21, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        CHESS<sub>IR +</sub><sub>SS +</sub><sub>CG</sub><br />
                        <span class="affiliation">Stanford</span><br /><a
                          href="https://arxiv.org/abs/2405.16755"
                          class="paperlink"
                          >[Talaei et al.'24]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/ShayanTalaei/CHESS">[link]</a>
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">65.00</td>
                      <td class="align-middle text-center">66.69</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Sep 23, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        E-SQL + GPT-4o<br />
                        <span class="affiliation">Bilkent University</span><br /><a
                          href="http://arxiv.org/abs/2409.16751"
                          class="paperlink"
                          >[Caferoğlu et al.'24]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/HasanAlpCaferoglu/E-SQL">[link]</a>
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">65.58</td>
                      <td class="align-middle text-center">66.29</td>
                    </tr>
  
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Aug 29, 2024</span>
                      </td>
                      <td class="align-middle text-center">Arcwise + GPT-4o<br>
                        <span class="affiliation">Arcwise</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">67.99</td>
                      <td class="align-middle text-center">66.21</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Apr 30, 2025</span>
                      </td>
                      <td class="align-middle text-center">
                        Command A<br />
                        <span class="affiliation">Cohere AI</span><br /><a
                          href="https://arxiv.org/abs/2504.00698"
                          class="paperlink"
                          >[Team Cohere et al.'25]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://huggingface.co/CohereLabs/c4ai-command-a-03-2025">[link]</a>
                      </td>
                      <td class="align-middle text-center">111B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">63.49</td>
                      <td class="align-middle text-center">65.68</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Nov 18, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        RSL-SQL + DeepSeek-v2<br />
                        <span class="affiliation">Anonymous</span><br /><a
                          href="https://arxiv.org/abs/2411.00073"
                          class="paperlink"
                          >[Cao et al.'24]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/Laqcce-cao/RSL-SQL">[link]</a>
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">63.56</td>
                      <td class="align-middle text-center">65.51</td>
                    </tr>
  
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Jan 14, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        MCS-SQL + GPT-4<br />
                        <span class="affiliation">Dunamu</span><br><a href="https://arxiv.org/abs/2405.07467" class="paperlink">[Lee et al. '24]</a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">63.36</td>
                      <td class="align-middle text-center">65.45</td>
                    </tr>
  
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                      <span class="badge badge-secondary">Aug 20, 2024</span>
                      </td>
                      <td class="align-middle text-center">SCL-SQL<br>
                      <span class="affiliation">Xffuture</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">64.73</td>
                      <td class="align-middle text-center">65.23</td>
                      </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Apr 08, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        OpenSearch-SQL,v1 + GPT-4<br />
                        <span class="affiliation">Alibaba Cloud</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">61.34</td>
                      <td class="align-middle text-center">64.95</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Jun 7, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        SFT CodeS-15B + SQLFixAgent<br />
                        <span class="affiliation">Soochow University</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">--</td>
                      <td class="align-middle text-center">64.62</td>
                    </tr>
  
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                          <span class="badge badge-secondary">Aug 30, 2024</span>
                      </td>
                      <td class="align-middle text-center">PURPLE + GPT-4o<br>
                          <span class="affiliation">Fudan University + Transwarp Technology </span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">62.97</td>
                      <td class="align-middle text-center">64.51</td>
                  </tr>
  
                 <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                          <span class="badge badge-secondary">Oct 10, 2024</span>
                      </td>
                      <td class="align-middle text-center">MSL-SQL + DeepSeek-V2.5<br>
                          <span class="affiliation">Wuhan University of Technology</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">236B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">66.82</td>
                      <td class="align-middle text-center">64.00</td>
                  </tr>
  
                  <tr>
                    <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Feb 6, 2025</span>
                    </td>
                    <td class="align-middle text-center">EBA-SQL + GPT-4<br>
                        <span class="affiliation">Northeastern University</span>
                    </td>
                    <td class="align-middle text-center"></td>
                    <td class="align-middle text-center">UNK</td>
                    <td class="align-middle text-center">✔️</td>
                    <td class="align-middle text-center">64.60</td>
                    <td class="align-middle text-center">63.50</td>
                </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Feb 21, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        <span style="font-variant-caps: small-caps">Sense</span
                        ><br />
                        <span class="affiliation">Anonymous</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">13B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">55.48</td>
                      <td class="align-middle text-center">63.39</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Mar 11, 2025</span>
                      </td>
                      <td class="align-middle text-center">
                        <span style="font-variant-caps: small-caps">OneSQL-v0.1-Qwen-32B</span
                        ><br />
                        <a href="https://onekq.ai" target="_blank"><span class="affiliation">onekq.ai</span></a>
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://huggingface.co/onekq-ai/OneSQL-v0.1-Qwen-32B"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">32B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">64.60</td>
                      <td class="align-middle text-center">63.33</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Apr 10, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        GRA-SQL<br />
                        <span class="affiliation">Tencent CDP-youpu</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">62.58</td>
                      <td class="align-middle text-center">63.22</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Jun 1, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        SuperSQL<br />
                        <span class="affiliation">HKUST(GZ)</span><br /><a
                          href="https://arxiv.org/abs/2406.01265"
                          class="paperlink"
                          >[Li et al. '24]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/BugMaker-Boyan/NL2SQL360"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">58.50</td>
                      <td class="align-middle text-center">62.66</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Mar 27, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        {Chat2Query} (GPT-4 + data entity modeling) (PingCAP)<br />
                        <span class="affiliation">PingCAP</span>
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/tidbcloud/chat2query_bench"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">58.15</td>
                      <td class="align-middle text-center">60.98</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Nov 16, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        Dubo-SQL, v1<br />
                        <span class="affiliation">Mercator Technologies</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">59.71</td>
                      <td class="align-middle text-center">60.71</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Oct 12, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        SFT CodeS-15B<br />
                        <span class="affiliation">Renmin University of China</span
                        ><br /><a
                          href="https://arxiv.org/abs/2402.16347"
                          class="paperlink"
                          >[Li et al. SIGMOD'24]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/RUCKBReasoning/codes"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">15B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">58.47</td>
                      <td class="align-middle text-center">60.37</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Feb 27, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        DTS-SQL + DeepSeek 7B<br />
                        <span class="affiliation">University of Alberta</span
                        ><br /><a
                          href="https://arxiv.org/abs/2402.01117"
                          class="paperlink"
                          >[Pourreza et al. '24]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/MohammadrezaPourreza/DTS-SQL"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">7B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">55.8</td>
                      <td class="align-middle text-center">60.31</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Sep 23, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        E-SQL + GPT-4o mini<br />
                        <span class="affiliation">Bilkent University</span><br /><a
                          href="http://arxiv.org/abs/2409.16751"
                          class="paperlink"
                          >[Caferoğlu et al.'24]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/HasanAlpCaferoglu/E-SQL">[link]</a>
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">61.60</td>
                      <td class="align-middle text-center">59.81</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Nov 21, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        MAC-SQL + GPT-4<br />
                        <span class="affiliation">BUAA & Tencent</span><br /><a
                          href="https://arxiv.org/abs/2312.11242"
                          class="paperlink"
                          >[Wang et al. '23]</a
                        >
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">57.56</td>
                      <td class="align-middle text-center">59.59</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Oct 12, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        SFT CodeS-7B<br />
                        <span class="affiliation">Renmin University of China</span
                        ><br /><a
                          href="https://arxiv.org/abs/2402.16347"
                          class="paperlink"
                          >[Li et al. SIGMOD'24]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/RUCKBReasoning/codes"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">7B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">57.17</td>
                      <td class="align-middle text-center">59.25</td>
                    </tr>
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">May 27, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        TA-SQL + GPT-4<br />
                        <span class="affiliation">HKU</span><br /><a
                          href="https://arxiv.org/abs/2405.15307"
                          class="paperlink"
                          >[Qu et al. ACL Findings'24]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/quge2023/TA-SQL">[link]</a>
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">56.19</td>
                      <td class="align-middle text-center">59.14</td>
                    </tr>
  
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Nov 09, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        DAIL-SQL + GPT-4<br />
                        <span class="affiliation">Alibaba Group</span><br /><a
                          href="https://arxiv.org/abs/2308.15363"
                          class="paperlink"
                          >[Gao and Wang et al. VLDB'24]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/BeachWang/DAIL-SQL">[link]</a>
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">54.76</td>
                      <td class="align-middle text-center">57.41</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">May 24, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        ExSL + granite-20b-code<br />
                        <span class="affiliation">IBM Research AI</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">20B</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">51.69</td>
                      <td class="align-middle text-center">57.13</td>
                    </tr>
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Aug 10, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        DeepSeek<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center">
                        <a
                          href="https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/bird"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">236B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">56.13</td>
                      <td class="align-middle text-center">56.68</td>
                    </tr>
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Aug 15, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        DIN-SQL + GPT-4<br />
                        <span class="affiliation">University of Alberta</span
                        ><br /><a
                          href="https://arxiv.org/abs/2304.11015"
                          class="paperlink"
                          >[Pourreza et al. '23]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a
                          href="https://github.com/MohammadrezaPourreza/Few-shot-NL2SQL-with-prompting"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">50.72</td>
                      <td class="align-middle text-center">55.90</td>
                    </tr>
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Aug 08, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        Mistral<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center">
                        <a
                          href="https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/bird"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">123B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">53.52</td>
                      <td class="align-middle text-center">55.84</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Jul 01, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        GPT-4<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center">
                        <a
                          href="https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/bird"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">46.35</td>
                      <td class="align-middle text-center">54.89</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Nov 8, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        Interactive-T2S<br />
                        <span class="affiliation">Anonymous</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">54.56</td>
                      <td class="align-middle text-center">54.11</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Sep 19, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        Prem-1B-SQL<br />
                        <span class="affiliation">Prem AI</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center">
                        <a
                          href="https://github.com/PremAI-io/premsql"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">1B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">-</td>
                      <td class="align-middle text-center">51.54</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Jul 16, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        Claude-2<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center">
                        <a
                          href="https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/bird"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">42.70</td>
                      <td class="align-middle text-center">49.02</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Nov 23, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        Open-SQL<br />
                        <span class="affiliation">Anonymous</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">7B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">37.68</td>
                      <td class="align-middle text-center">47.74</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Mar 17, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        ChatGPT + CoT<br />
                        <span class="affiliation">HKU & DAMO</span><br />
                        <a
                          href="https://arxiv.org/abs/2305.03111"
                          class="paperlink"
                          >[Li et al. NeurIPS'23]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a
                          href="https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/bird"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">36.64</td>
                      <td class="align-middle text-center">40.08</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Mar 17, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        ChatGPT<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">37.22</td>
                      <td class="align-middle text-center">39.30</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Feb 17, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        Codex<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">175B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">34.35</td>
                      <td class="align-middle text-center">36.47</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Jul 16, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        Palm-2<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center">
                        <a
                          href="https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/bird"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">27.38</td>
                      <td class="align-middle text-center">33.04</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Mar 17, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        ChatGPT + CoT<br />
                        <span class="affiliation">HKU & DAMO</span><br /><a
                          href="https://arxiv.org/abs/2305.03111"
                          class="paperlink"
                          >[Li et al. NeurIPS'23]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a
                          href="https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/bird"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">25.88</td>
                      <td class="align-middle text-center">28.95</td>
                    </tr>
  
                    <!-- Copy from here -->
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Mar 17, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        ChatGPT<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">24.05</td>
                      <td class="align-middle text-center">26.77</td>
                    </tr>
  
                    <!-- Copy from here -->
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Feb 17, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        Codex<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">175B</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">25.42</td>
                      <td class="align-middle text-center">24.86</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Feb 5, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        T5-3B<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">3B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">23.34</td>
                      <td class="align-middle text-center">24.05</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Feb 3, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        T5-Large<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">770M</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">19.75</td>
                      <td class="align-middle text-center">20.94</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Feb 3, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        T5-Base<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">220M</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">11.54</td>
                      <td class="align-middle text-center">12.89</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Feb 5, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        T5-3B<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">3B</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">10.37</td>
                      <td class="align-middle text-center">11.17</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Feb 3, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        T5-Large<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">770M</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">9.71</td>
                      <td class="align-middle text-center">10.38</td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Feb 3, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        T5-Base<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">220M</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">6.32</td>
                      <td class="align-middle text-center">7.06</td>
                    </tr>
                  </tbody>
                </table>
              </div>
            </div>
            <div class="card card-outline-secondary">
              <div class="card-header">
                Leaderboard - Reward-based Valid Efficiency Score (R-VES)
              </div>
              <div class="table-wrapper">
                <table class="table" id="ves-table">
                  <thead class="thead-light">
                    <tr>
                      <th scope="col" rowspan="1"></th>
                      <th scope="col" rowspan="1" class="align-middle text-center">Model</th>
                      <th scope="col" rowspan="1" class="align-middle text-center">Code</th>
                      <th scope="col" rowspan="1" class="align-middle text-center">Size</th>
                      <th scope="col" rowspan="1" class="align-middle text-center">Oracle Knowledge</th>
                      <th scope="col" colspan="1" class="text-center">Test</th>
                    </tr>
                  </thead>
                  <tbody>
                    <tr class="table-success">
                      <td scope="row" class="align-middle text-center"><br>
                      </td>
                      <td class="align-middle text-center">Human Performance<br>
                        <span class="affiliation">Data Engineers + DB Students</span><br><a href="" class="paperlink"></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center"><b>83.26</b></td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Dec 17, 2024</span>
                      </td>
                     <td class="align-middle text-center">XiYan-SQL<br>
                     <span class="affiliation">Alibaba Cloud</span><br><a href="https://arxiv.org/abs/2411.08599" class="paperlink">[Yifu Liu et al. '24]</a>
                     </td>
                      <td class="align-middle text-center"><a href="https://github.com/XGenerationLab/XiYan-SQL">[link]</a></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center"><b>71.41</b></td>
                    </tr>
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Oct 27, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        ExSL + granite-34b-code<br />
                        <span class="affiliation">IBM Research AI</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">34B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">71.37</td>
                    </tr>
  
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Mar 11, 2025</span>
                      </td>
                      <td class="align-middle text-center">AskData + GPT-4o<br>
                        <span class="affiliation">AT&T CDO - DSAIR</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">71.17</td>
                    </tr>
  
                    
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Feb 27, 2025</span>
                      </td>
                      <td class="align-middle text-center">Contextual-SQL<br>
                        <span class="affiliation">Contextual AI</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">70.02</td>
                    </tr>
  
  <!--                   <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Nov 3, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        CHASE-SQL + Gemini<br />
                        <span class="affiliation">Google Cloud</span><br><a href="https://arxiv.org/abs/2410.01943" class="paperlink">[Pourreza et al. '24]</a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">70.07</td>
                    </tr> -->
  
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Apr 16, 2025</span>
                      </td>
                      <td class="align-middle text-center">
                        CHASE-SQL + Gemini<br />
                        <span class="affiliation">Google Cloud</span><br><a href="https://arxiv.org/abs/2410.01943" class="paperlink">[Pourreza et al. '24]</a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">69.94</td>
                    </tr>
  
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Aug 21, 2024</span>
                      </td>
                      <td class="align-middle text-center">OpenSearch-SQL, v2 + GPT-4o<br>
                        <span class="affiliation">Alibaba Cloud</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">69.36</td>
                    </tr>
  
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Mar 31, 2025</span>
                      </td>
                      <td class="align-middle text-center">Reasoning-SQL 14B<br>
                         <span class="affiliation">Google Cloud / Stanford</span><br><a href="https://arxiv.org/pdf/2503.23157" class="paperlink">[Pourreza et al. '25]</a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">14 B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">68.67</td>
                    </tr>
  
  
                    <!--
                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Aug 1, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        ExSL + granite-34b-code<br />
                        <span class="affiliation">IBM Research AI</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">34B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">68.79</td>
                    </tr>
                    -->
  
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Jul 22, 2024</span>
                      </td>
                      <td class="align-middle text-center"> Distillery + GPT-4o<br>
                        <span class="affiliation">Distyl AI Research</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">67.41</td>
                    </tr>
                    
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Mar 4, 2025</span>
                      </td>
                      <td class="align-middle text-center">OmniSQL-32B<br>
                        <span class="affiliation">Renmin University of China + ByteDance Infra Lab</span><br><a href="https://arxiv.org/abs/2503.02240" class="paperlink">[Li et al. '25]</a>
                      </td>
                      <td class="align-middle text-center"><a href="https://github.com/RUCKBReasoning/OmniSQL">[link]</a></td>
                      <td class="align-middle text-center">32B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">67.05</td>
                    </tr>
                    
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">May 21, 2024</span>
                      </td>
                      <td class="align-middle text-center">CHESS<sub>IR +</sub><sub>CG +</sub><sub>UT</sub><br>
                        <span class="affiliation">Stanford</span><br><a href="https://arxiv.org/abs/2405.16755" class="paperlink">[Talaei et al.'24]</a>
                      </td>
                      <td class="align-middle text-center"><a href="https://github.com/ShayanTalaei/CHESS">[link]</a></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">66.53</td>
                    </tr>
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                      <span class="badge badge-secondary">Jul 5, 2024</span>
                      </td>
                      <td class="align-middle text-center">Insights AI<br>
                      <span class="affiliation">Uber Freight</span><br><a href="" class="paperlink"></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">66.39</td>
                      </tr>
                    <tr>
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">May 14, 2024</span>
                      </td>
                      <td class="align-middle text-center">ExSL + granite-20b-code<br>
                        <span class="affiliation">IBM Research AI</span><br><a href="" class="paperlink"></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">20B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">66.25</td>
                    </tr>
  
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Jul 14, 2024</span>
                      </td>
                      <td class="align-middle text-center">RECAP + Gemini<br>
                        <span class="affiliation">Google Cloud</span><br><a href="" class="paperlink"></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">65.70</td>
                    </tr>
  
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                          <span class="badge badge-secondary">Aug 30, 2024</span>
                      </td>
                      <td class="align-middle text-center">PURPLE + RED + GPT-4o<br>
                          <span class="affiliation">Fudan University + Transwarp Technology </span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">65.62</td>
                  </tr>
  
                  <tr>
                    <td scope="row" class="align-middle text-center counter-cell">
                      <span class="badge badge-secondary">Dec 18, 2024</span>
                    </td>
                    <td class="align-middle text-center">TC-SQL<br>
                      <span class="affiliation">mindflow.ai</span>
                    </td>
                    <td class="align-middle text-center"></td>
                    <td class="align-middle text-center">UNK</td>
                    <td class="align-middle text-center">✔️</td>
                    <td class="align-middle text-center">65.19</td>
                  </tr>
  
                  <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Mar 4, 2025</span>
                      </td>
                      <td class="align-middle text-center">OmniSQL-7B<br>
                        <span class="affiliation">Renmin University of China + ByteDance Infra Lab</span><br><a href="https://arxiv.org/abs/2503.02240" class="paperlink">[Li et al. '25]</a>
                      </td>
                      <td class="align-middle text-center"><a href="https://github.com/RUCKBReasoning/OmniSQL">[link]</a></td>
                      <td class="align-middle text-center">7B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">65.04</td>
                  </tr>
  
                  <tr>
                    <td scope="row" class="align-middle text-center counter-cell">
                      <span class="badge badge-secondary">Apr 18, 2025</span>
                    </td>
                    <td class="align-middle text-center">GenaSQL<br>
                      <span class="affiliation">Gena AI</span><br/>
                      <a href="https://gena.co" target="_blank"><span class="affiliation">gena.co</span></a>
                    </td>
                    <td class="align-middle text-center"></td>
                    <td class="align-middle text-center">UNK</td>
                    <td class="align-middle text-center">✔️</td>
                    <td class="align-middle text-center">65.00</td>
                  </tr>
                    
                  <tr>
                    <td scope="row" class="align-middle text-center counter-cell">
                      <span class="badge badge-secondary">Apr 10, 2025</span>
                    </td>
                    <td class="align-middle text-center">Queryosity<br>
                      <a href="https://queryosity.io" target="_blank"><span class="affiliation">queryosity.io</span></a>
                    </td>
                    <td class="align-middle text-center"></td>
                    <td class="align-middle text-center">UNK</td>
                    <td class="align-middle text-center">✔️</td>
                    <td class="align-middle text-center">64.73</td>
                  </tr>
  
                  <tr>
                    <td scope="row" class="align-middle text-center counter-cell">
                      <span class="badge badge-secondary">Jan 9, 2025</span>
                    </td>
                    <td class="align-middle text-center">GSR<br>
                      <span class="affiliation">Anonymous</span>
                    </td>
                    <td class="align-middle text-center"><a href="https://github.com/GSR-SQL/GSR">[link]</a></td>
                    <td class="align-middle text-center">UNK</td>
                    <td class="align-middle text-center">✔️</td>
                    <td class="align-middle text-center">64.41</td>
                  </tr>
  
                  <tr>
                    <td
                      scope="row"
                      class="align-middle text-center counter-cell"
                    >
                      <span class="badge badge-secondary">Jan 09, 2025</span>
                    </td>
                   <td class="align-middle text-center">XiYanSQL-QwenCoder-32B<br>
                   <span class="affiliation">Alibaba Cloud</span><br><a href="https://arxiv.org/abs/2411.08599" class="paperlink">[Yifu Liu et al. '24]</a>
                   </td>
                    <td class="align-middle text-center"><a href="https://github.com/XGenerationLab/XiYanSQL-QwenCoder-32B">[link]</a></td>
                    <td class="align-middle text-center">32B</td>
                    <td class="align-middle text-center">✔️</td>
                    <td class="align-middle text-center">64.65</td>
                  </tr>
  
                  <tr>
                    <td scope="row" class="align-middle text-center counter-cell">
                      <span class="badge badge-secondary">Aug 29, 2024</span>
                    </td>
                    <td class="align-middle text-center">Arcwise + GPT-4o<br>
                      <span class="affiliation">Arcwise</span>
                    </td>
                    <td class="align-middle text-center"></td>
                    <td class="align-middle text-center">UNK</td>
                    <td class="align-middle text-center">✔️</td>
                    <td class="align-middle text-center">63.68</td>
                  </tr>
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Nov 11, 2024</span>
                      </td>
                      <td class="align-middle text-center">AskData + GPT-4o<br>
                        <span class="affiliation">AT&T CDO - DSAIR</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">63.34</td>
                    </tr>
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">May 21, 2024</span>
                      </td>
                      <td class="align-middle text-center">CHESS<sub>IR +</sub><sub>SS +</sub><sub>CG</sub><br>
                        <span class="affiliation">Stanford</span><br><a href="https://arxiv.org/abs/2405.16755" class="paperlink">[Talaei et al.'24]</a>
                      </td>
                      <td class="align-middle text-center"><a href="https://github.com/ShayanTalaei/CHESS">[link]</a></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">62.77</td>
                    </tr>
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Sep 23, 2024</span>
                      </td>
                      <td class="align-middle text-center">E-SQL + GPT-4o<br>
                        <span class="affiliation">Bilkent University</span><br><a href="http://arxiv.org/abs/2409.16751" class="paperlink">[Caferoğlu et al.'24]</a>
                      </td>
                      <td class="align-middle text-center"><a href="https://github.com/HasanAlpCaferoglu/E-SQL">[link]</a></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">62.43</td>
                    </tr>
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Jun 7, 2024</span>
                      </td>
                      <td class="align-middle text-center">SFT CodeS-15B + SQLFixAgent<br>
                        <span class="affiliation">Soochow University</span>
                      </td>
                      <td class="align-middle text-center">
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">61.37</td>
                    </tr>
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                      <span class="badge badge-secondary">Aug 20, 2024</span>
                      </td>
                      <td class="align-middle text-center">SCL-SQL<br>
                      <span class="affiliation">Xffuture</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">61.28</td>
                      </tr>
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Jan 14, 2024</span>
                      </td>
                      <td class="align-middle text-center">MCS-SQL + GPT-4<br>
                        <span class="affiliation">Dunamu</span><br><a href="https://arxiv.org/abs/2405.07467" class="paperlink">[Lee et al. '24]</a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">61.23</td>
                    </tr>
  
  
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Feb 27, 2024</span>
                      </td>
                      <td class="align-middle text-center">PB-SQL<br>
                        <span class="affiliation">Seoul National University</span><br><a href="" class="paperlink"></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">60.36</td>
                    </tr>
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                          <span class="badge badge-secondary">Aug 30, 2024</span>
                      </td>
                      <td class="align-middle text-center">PURPLE + GPT-4o<br>
                          <span class="affiliation">Fudan University + Transwarp Technology </span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">60.35</td>
                  </tr>
  
                  <tr>
                    <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Feb 6, 2025</span>
                    </td>
                    <td class="align-middle text-center">EBA-SQL + GPT-4<br>
                        <span class="affiliation">Northeastern University</span>
                    </td>
                    <td class="align-middle text-center"></td>
                    <td class="align-middle text-center">UNK</td>
                    <td class="align-middle text-center">✔️</td>
                    <td class="align-middle text-center">60.13</td>
                </tr>
  
                <tr>
                  <td scope="row" class="align-middle text-center counter-cell">
                    <span class="badge badge-secondary">Mar 11, 2025</span>
                  </td>
                  <td class="align-middle text-center">OneSQL-v0.1-Qwen-32B<br>
                    <a href="https://onekq.ai" target="_blank"><span class="affiliation">onekq.ai</span></a>
                  </td>
                  <td class="align-middle text-center"><a href="https://huggingface.co/onekq-ai/OneSQL-v0.1-Qwen-32B">[link]</a></td>
                  <td class="align-middle text-center">32B</td>
                  <td class="align-middle text-center">✔️</td>
                  <td class="align-middle text-center">60.02</td>
                </tr>
  
                <tr>
                  <td
                    scope="row"
                    class="align-middle text-center counter-cell"
                  >
                    <span class="badge badge-secondary">Apr 30, 2025</span>
                  </td>
                  <td class="align-middle text-center">
                    Command A<br />
                    <span class="affiliation">Cohere AI</span><br /><a
                      href="https://arxiv.org/abs/2504.00698"
                      class="paperlink"
                      >[Team Cohere et al.'25]</a
                    >
                  </td>
                  <td class="align-middle text-center">
                    <a href="https://huggingface.co/CohereLabs/c4ai-command-a-03-2025">[link]</a>
                  </td>
                  <td class="align-middle text-center">111B</td>
                  <td class="align-middle text-center">✔️</td>
                  <td class="align-middle text-center">59.82</td>
                </tr>

                  <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                          <span class="badge badge-secondary">Oct 10, 2024</span>
                      </td>
                      <td class="align-middle text-center">MSL-SQL + DeepSeek-V2.5<br>
                          <span class="affiliation">Wuhan University of Technology</span>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">236B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">59.42</td>
                  </tr>
  
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Nov 21, 2023</span>
                      </td>
                      <td class="align-middle text-center">MAC-SQL + GPT-4<br>
                        <span class="affiliation">BUAA & Tencent</span><br><a href="https://arxiv.org/abs/2312.11242" class="paperlink">[Wang et al. '23]</a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">57.60</td>
                    </tr>
  
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Oct 12, 2023</span>
                      </td>
                      <td class="align-middle text-center">SFT CodeS-15B<br>
                        <span class="affiliation">Renmin University of China</span><br><a href="https://arxiv.org/abs/2402.16347" class="paperlink">[Li et al. SIGMOD'24]</a>
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/RUCKBReasoning/codes">[link]</a>
                      </td>
                      <td class="align-middle text-center">15B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">56.73</td>
                    </tr>
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Apr 10, 2024</span>
                      </td>
                      <td class="align-middle text-center">GRA-SQL<br>
                        <span class="affiliation">Tencent CDP-youpu</span><br><a href="" class="paperlink"></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">56.63</td>
                    </tr>
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Nov 16, 2023</span>
                      </td>
                      <td class="align-middle text-center">Dubo-SQL, v1<br>
                        <span class="affiliation">Mercator Technologies</span>
                      </td>
                      <td class="align-middle text-center">
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">56.63</td>
                    </tr>
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">May 24, 2024</span>
                      </td>
                      <td class="align-middle text-center">ExSL + granite-20b-code<br>
                        <span class="affiliation">IBM Research AI</span><br><a href="" class="paperlink"></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">20B</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">56.11</td>
                    </tr>
  
  
                    </tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Mar 27, 2024</span>
                      </td>
                      <td class="align-middle text-center">{Chat2Query} (GPT-4 + data entity modeling) (PingCAP)<br>
                        <span class="affiliation">PingCAP</span>
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/tidbcloud/chat2query_bench">[link]</a>
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">56.06</td>
                    </tr>
  
  
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Oct 12, 2023</span>
                      </td>
                      <td class="align-middle text-center">SFT CodeS-7B<br>
                        <span class="affiliation">Renmin University of China</span><br><a href="https://arxiv.org/abs/2402.16347" class="paperlink">[Li et al. SIGMOD'24]</a>
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/RUCKBReasoning/codes">[link]</a>
                      </td>
                      <td class="align-middle text-center">7B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">55.69</td>
                    </tr>
  
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Sep 23, 2024</span>
                      </td>
                      <td class="align-middle text-center">E-SQL + GPT-4o mini<br>
                        <span class="affiliation">Bilkent University</span><br><a href="http://arxiv.org/abs/2409.16751" class="paperlink">[Caferoğlu et al.'24]</a>
                      </td>
                      <td class="align-middle text-center"><a href="https://github.com/HasanAlpCaferoglu/E-SQL">[link]</a></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">55.64</td>
                    </tr>
  
  
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Nov 09, 2023</span>
                      </td>
                      <td class="align-middle text-center">DAIL-SQL + GPT-4<br>
                        <span class="affiliation">Alibaba Group</span><br><a href="https://arxiv.org/abs/2308.15363"
                          class="paperlink">[Gao and Wang et al. VLDB'24]</a>
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/BeachWang/DAIL-SQL">[link]</a>
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">54.02</td>
                    </tr>
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Aug 10, 2024</span>
                      </td>
                      <td class="align-middle text-center">DeepSeek<br>
                        <span class="affiliation">Baseline</span><br><a href="" class="paperlink"></a>
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/bird">[link]</a>
                      </td>
                      <td class="align-middle text-center">236B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">53.25</td>
                    </tr>
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Aug 15, 2023</span>
                      </td>
                      <td class="align-middle text-center">DIN-SQL + GPT-4<br>
                        <span class="affiliation">University of Alberta</span><br><a href="https://arxiv.org/abs/2304.11015"
                          class="paperlink">[Pourreza et al. '23]</a>
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/MohammadrezaPourreza/Few-shot-NL2SQL-with-prompting">[link]</a>
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">53.07</td>
                    </tr>
  
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Aug 08, 2024</span>
                      </td>
                      <td class="align-middle text-center">Mistral<br>
                        <span class="affiliation">Baseline</span><br><a href="" class="paperlink"></a>
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/bird">[link]</a>
                      </td>
                      <td class="align-middle text-center">128B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">52.59</td>
                    </tr>
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Jul 01, 2023</span>
                      </td>
                      <td class="align-middle text-center">GPT-4<br>
                        <span class="affiliation">Baseline</span><br><a href="" class="paperlink"></a>
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/bird">[link]</a>
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">51.75</td>
                    </tr>
  
  
                  </tbody>
                </table>
              </div>
            </div>
            <div class="list-group-item" id="illustration">
              <h3>BIRD Mini-Dev </h3>
              <p>
                A <b>Lite</b> version of developtment dataset, which is designed
                to facilitate efficient and cost-effective development cycles,
                 especially for testing and refining SQL query generation models. For more details, please visit the <a href="https://github.com/bird-bench/mini_dev">GitHub repository</a>.
                 For updating Leaderboard, please make sure your paper or resource is public available and submit a PR.
              </p>
            </div>
            <div class="card card-outline-secondary">
              <div class="card-header">Mini Dev - Execution Accuracy (EX)</div>
              <div class="table-wrapper">
                <table class="table" id="ex-table">
                  <thead class="thead-light">
                    <tr>
                      <th scope="col" rowspan="1"></th>
                      <th
                        scope="col"
                        rowspan="1"
                        class="align-middle text-center"
                      >
                        Model
                      </th>
                      <th
                        scope="col"
                        rowspan="1"
                        class="align-middle text-center"
                      >
                        Code
                      </th>
                      <th
                        scope="col"
                        rowspan="1"
                        class="align-middle text-center"
                      >
                        Size
                      </th>
                      <th
                        scope="col"
                        rowspan="1"
                        class="align-middle text-center"
                      >
                        Oracle Knowledge
                      </th>
                      <th scope="col" colspan="1" class="text-center">SQLite</th>
                      <th scope="col" colspan="1" class="text-center">MySQL</th>
                      <th scope="col" colspan="1" class="text-center">
                        PostgreSQL
                      </th>
                    </tr>
                  </thead>
                  <tbody>
                    <!-- <tr class="table-success">
                      <td scope="row" class="align-middle text-center"><br>
                      </td>
                      <td class="align-middle text-center">Human Performance<br>
                        <span class="affiliation">Data Engineers + DB Students</span><br><a href="" class="paperlink"></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">93.00</td>
                      <td class="align-middle text-center">91.20</td>
                      <td class="align-middle text-center">90.08</td>
                    </tr> -->
                    <tr >
  
                    </tr>
  
                    <tr>
                      <td class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Jun 31, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        TA + GPT-4<br />
                        <span class="affiliation">HKU</span><br /><a
                          href="https://arxiv.org/abs/2405.15307"
                          class="paperlink"
                          >[Qu et al. ACL Findings'24]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/quge2023/TA-SQL">[link]</a>
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center"><b>58.00</b></td>
                      <td class="align-middle text-center"><b>49.20</b></td>
                      <td class="align-middle text-center"><b>50.80</b></td>
                    </tr>
                    <tr>
                      <td class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Jun 31, 2024</span>
                      </td>
                      <td class="align-middle text-center">GPT-4</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">47.80</td>
                      <td class="align-middle text-center">40.80</td>
                      <td class="align-middle text-center">35.80</td>
                    </tr>
                    <tr>
                      <td class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Jun 31, 2024</span>
                      </td>
                      <td class="align-middle text-center">GPT-4-32k</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">47.00</td>
                      <td class="align-middle text-center">43.20</td>
                      <td class="align-middle text-center">35.00</td>
                    </tr>
                    <tr>
                      <td class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Jun 31, 2024</span>
                      </td>
                      <td class="align-middle text-center">GPT-4-turbo</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">45.80</td>
                      <td class="align-middle text-center">41.00</td>
                      <td class="align-middle text-center">36.00</td>
                    </tr>
                    <tr>
                      <td class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Jun 31, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        Llama3-70b-instruct
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center"><b>70B</b></td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">40.80</td>
                      <td class="align-middle text-center">37.00</td>
                      <td class="align-middle text-center">29.40</td>
                    </tr>
                    <tr>
                      <td class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Jun 31, 2024</span>
                      </td>
                      <td class="align-middle text-center">GPT-35-turbo</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">38.00</td>
                      <td class="align-middle text-center">36.00</td>
                      <td class="align-middle text-center">27.40</td>
                    </tr>
                    <tr>
                      <td class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Jun 31, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        GPT-35-turbo-instruct
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">33.60</td>
                      <td class="align-middle text-center">31.20</td>
                      <td class="align-middle text-center">26.60</td>
                    </tr>
                    <tr>
                      <td class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Jun 31, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        Phi-3-medium-128k-instruct
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center"><b>13B</b></td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">30.60</td>
                      <td class="align-middle text-center">25.00</td>
                      <td class="align-middle text-center">21.60</td>
                    </tr>
                    <tr>
                      <td class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Jun 31, 2024</span>
                      </td>
                      <td class="align-middle text-center">Llama3-8b-instruct</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center"><b>8B</b></td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">24.40</td>
                      <td class="align-middle text-center">24.60</td>
                      <td class="align-middle text-center">18.40</td>
                    </tr>
                    <tr>
                      <td class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Jun 31, 2024</span>
                      </td>
                      <td class="align-middle text-center">Mixtral-8x7b</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center"><b>46.7B</b></td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">21.60</td>
                      <td class="align-middle text-center">13.60</td>
                      <td class="align-middle text-center">12.40</td>
                    </tr>
                  </tbody>
                </table>
              </div>
            </div>
          </div>

          <div id="single-model-leaderboard" style="display: none;">
            <div class="card card-outline-secondary">
              <div class="card-header">Leaderboard - Execution Accuracy (EX)</div>
              <div class="table-wrapper">
                <table class="table" id="ex-table">
                  <thead class="thead-light">
                    <tr>
                      <th scope="col" rowspan="1"></th>
                      <th
                        scope="col"
                        rowspan="1"
                        class="align-middle text-center"
                      >
                        Model
                      </th>
                      <th
                        scope="col"
                        rowspan="1"
                        class="align-middle text-center"
                      >
                        Code
                      </th>
                      <th
                        scope="col"
                        rowspan="1"
                        class="align-middle text-center"
                      >
                        Size
                      </th>
                      <th
                        scope="col"
                        rowspan="1"
                        class="align-middle text-center"
                      >
                        Oracle Knowledge
                      </th>
                      <th scope="col" colspan="1" class="text-center">Dev (%)</th>
                      <th scope="col" colspan="1" class="text-center">
                        Test (%)
                      </th>
                    </tr>
                  </thead>
                  <tbody>
                    <tr class="table-success">
                      <td scope="row" class="align-middle text-center"><br /></td>
                      <td class="align-middle text-center">
                        Human Performance<br />
                        <span class="affiliation"
                          >Data Engineers + DB Students</span
                        ><br /><a href="" class="paperlink"></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center"><b>92.96</b></td>
                    </tr>
  
                    <tr>
                      <td scope="row" class="align-middle text-center counter-cell">
                        <span class="badge badge-secondary">Jan 09, 2025</span>
                      </td>
                      <td class="align-middle text-center">XiYanSQL-QwenCoder-32B<br>
                        <span class="affiliation">Alibaba Cloud</span><br><a href="https://arxiv.org/abs/2411.08599" class="paperlink">[Yifu Liu et al. '24]</a>
                      </td>
                      <td class="align-middle text-center"><a href="https://github.com/XGenerationLab/XiYanSQL-QwenCoder-32B">[link]</a></td>
                      <td class="align-middle text-center">32B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">67.01</td>
                      <td class="align-middle text-center">69.03</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Mar 12, 2025</span>
                      </td>
                      <td class="align-middle text-center">
                        Arctic-ExCoT-70B<br />
                        <span class="affiliation"> Snowflake </span><br/>
                        <a href="https://arxiv.org/abs/2503.19988" class="paperlink">[Zhai et al. '25]</a>
                      </td>
                      <td class="align-middle text-center">
                      <a href="https://github.com/snowflakedb/ArcticTraining/tree/main/projects/excot_dpo">[link]</a>
                      </td>
                      <td class="align-middle text-center">70B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">68.51</td>
                      <td class="align-middle text-center">68.53</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Mar 12, 2025</span>
                      </td>
                      <td class="align-middle text-center">
                        Arctic-ExCoT-32B<br />
                        <span class="affiliation"> Snowflake </span><br />
                        <a href="https://arxiv.org/abs/2503.19988" class="paperlink">[Zhai et al. '25]</a>
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/snowflakedb/ArcticTraining/tree/main/projects/excot_dpo">[link]</a>
                      </td>
                      <td class="align-middle text-center">32B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">68.32</td>
                      <td class="align-middle text-center">68.19</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Apr 30, 2025</span>
                      </td>
                      <td class="align-middle text-center">
                        Command A<br />
                        <span class="affiliation">Cohere AI</span><br /><a
                          href="https://arxiv.org/abs/2504.00698"
                          class="paperlink"
                          >[Team Cohere et al.'25]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://huggingface.co/CohereLabs/c4ai-command-a-03-2025">[link]</a>
                      </td>
                      <td class="align-middle text-center">111B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">63.49</td>
                      <td class="align-middle text-center">65.68</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Mar 11, 2025</span>
                      </td>
                      <td class="align-middle text-center">
                        <span style="font-variant-caps: small-caps">OneSQL-v0.1-Qwen-32B</span
                        ><br />
                        <a href="https://onekq.ai" target="_blank"><span class="affiliation">onekq.ai</span></a>
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://huggingface.co/onekq-ai/OneSQL-v0.1-Qwen-32B"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">32B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">64.60</td>
                      <td class="align-middle text-center">63.33</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Jun 1, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        SuperSQL<br />
                        <span class="affiliation">HKUST(GZ)</span><br /><a
                          href="https://arxiv.org/abs/2406.01265"
                          class="paperlink"
                          >[Li et al. '24]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/BugMaker-Boyan/NL2SQL360"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">58.50</td>
                      <td class="align-middle text-center">62.66</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Oct 12, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        SFT CodeS-15B<br />
                        <span class="affiliation">Renmin University of China</span
                        ><br /><a
                          href="https://arxiv.org/abs/2402.16347"
                          class="paperlink"
                          >[Li et al. SIGMOD'24]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/RUCKBReasoning/codes"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">15B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">58.47</td>
                      <td class="align-middle text-center">60.37</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Oct 12, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        SFT CodeS-7B<br />
                        <span class="affiliation">Renmin University of China</span
                        ><br /><a
                          href="https://arxiv.org/abs/2402.16347"
                          class="paperlink"
                          >[Li et al. SIGMOD'24]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a href="https://github.com/RUCKBReasoning/codes"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">7B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">57.17</td>
                      <td class="align-middle text-center">59.25</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Aug 10, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        DeepSeek<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center">
                        <a
                          href="https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/bird"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">236B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">56.13</td>
                      <td class="align-middle text-center">56.68</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Aug 08, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        Mistral<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center">
                        <a
                          href="https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/bird"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">123B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">53.52</td>
                      <td class="align-middle text-center">55.84</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Jul 01, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        GPT-4<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center">
                        <a
                          href="https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/bird"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">46.35</td>
                      <td class="align-middle text-center">54.89</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Sep 19, 2024</span>
                      </td>
                      <td class="align-middle text-center">
                        Prem-1B-SQL<br />
                        <span class="affiliation">Prem AI</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center">
                        <a
                          href="https://github.com/PremAI-io/premsql"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">1B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">-</td>
                      <td class="align-middle text-center">51.54</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Jul 16, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        Claude-2<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center">
                        <a
                          href="https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/bird"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">42.70</td>
                      <td class="align-middle text-center">49.02</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Mar 17, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        ChatGPT + CoT<br />
                        <span class="affiliation">HKU & DAMO</span><br />
                        <a
                          href="https://arxiv.org/abs/2305.03111"
                          class="paperlink"
                          >[Li et al. NeurIPS'23]</a
                        >
                      </td>
                      <td class="align-middle text-center">
                        <a
                          href="https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/bird"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">36.64</td>
                      <td class="align-middle text-center">40.08</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Mar 17, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        ChatGPT<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">37.22</td>
                      <td class="align-middle text-center">39.30</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Feb 17, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        Codex<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">175B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">34.35</td>
                      <td class="align-middle text-center">36.47</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Jul 16, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        Palm-2<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center">
                        <a
                          href="https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/bird"
                          >[link]</a
                        >
                      </td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">27.38</td>
                      <td class="align-middle text-center">33.04</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Mar 17, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        ChatGPT<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">UNK</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">24.05</td>
                      <td class="align-middle text-center">26.77</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Feb 17, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        Codex<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">175B</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">25.42</td>
                      <td class="align-middle text-center">24.86</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Feb 5, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        T5-3B<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">3B</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">23.34</td>
                      <td class="align-middle text-center">24.05</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Feb 3, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        T5-Large<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">770M</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">19.75</td>
                      <td class="align-middle text-center">20.94</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Feb 3, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        T5-Base<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">220M</td>
                      <td class="align-middle text-center">✔️</td>
                      <td class="align-middle text-center">11.54</td>
                      <td class="align-middle text-center">12.89</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Feb 5, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        T5-3B<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">3B</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">10.37</td>
                      <td class="align-middle text-center">11.17</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Feb 3, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        T5-Large<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">770M</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">9.71</td>
                      <td class="align-middle text-center">10.38</td>
                    </tr>

                    <tr>
                      <td
                        scope="row"
                        class="align-middle text-center counter-cell"
                      >
                        <span class="badge badge-secondary">Feb 3, 2023</span>
                      </td>
                      <td class="align-middle text-center">
                        T5-Base<br />
                        <span class="affiliation">Baseline</span><br /><a
                          href=""
                          class="paperlink"
                        ></a>
                      </td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">220M</td>
                      <td class="align-middle text-center"></td>
                      <td class="align-middle text-center">6.32</td>
                      <td class="align-middle text-center">7.06</td>
                    </tr>
                  </tbody>
                </table>
              </div>
            </div>
          </div>
        </div>
      </div>

      <hr />

      <!-- Bootstrap core JavaScript -->
      <script src="./script/jquery.min.js"></script>
      <script src="./script/bootstrap.bundle.min.js"></script>

      <!-- Custom scripts for this template -->
      <script src="./script/clean-blog.min.js"></script>
      <!-- Custom script for leaderboard type toggle -->
      <script src="./script/leaderboard-toggle.min.js"></script>
    </div>
  </body>
</html>