LLM operator

evadb/binder/function_expression_binder.py

@@ -48,6 +48,12 @@ def bind_func_expr(binder: StatementBinder, node: FunctionExpression):
         handle_bind_extract_object_function(node, binder)
         return
 
+    # handle the special case of "completion or chatgpt"
+    if string_comparison_case_insensitive(
+        node.name, "chatgpt"
+    ) or string_comparison_case_insensitive(node.name, "completion"):
+        handle_bind_llm_function(node, binder)
+
     # Handle Func(*)
     if (
         len(node.children) == 1
@@ -106,6 +112,7 @@ def bind_func_expr(binder: StatementBinder, node: FunctionExpression):
             )
             # certain functions take additional inputs like yolo needs the model_name
             # these arguments are passed by the user as part of metadata
+
             # we also handle the special case of ChatGPT where we need to send the
             # OpenAPI key as part of the parameter if not provided by the user
             properties = get_metadata_properties(function_obj)
@@ -143,6 +150,18 @@ def bind_func_expr(binder: StatementBinder, node: FunctionExpression):
 
     resolve_alias_table_value_expression(node)
 
+def handle_bind_llm_function(node, binder):
+    # we also handle the special case of ChatGPT where we need to send the
+    # OpenAPI key as part of the parameter if not provided by the user
+    function_obj = binder._catalog().get_function_catalog_entry_by_name(node.name)
+    properties = get_metadata_properties(function_obj)
+    # if the user didn't provide any API_KEY, check if we have one in the catalog
+    if "OPENAI_API_KEY" not in properties.keys():
+        openapi_key = binder._catalog().get_configuration_catalog_value(
+            "OPENAI_API_KEY"
+        )
+        properties["openai_api_key"] = openapi_key
+
 
 def handle_bind_extract_object_function(
     node: FunctionExpression, binder_context: StatementBinder
@@ -154,9 +173,11 @@ def handle_bind_extract_object_function(
             Its inputs are id, data, output of detector.
         4. Bind the EXTRACT_OBJECT function expression and append the new children.
         5. Handle the alias and populate the outputs of the EXTRACT_OBJECT function
+
     Args:
         node (FunctionExpression): The function expression representing the extract object operation.
         binder_context (StatementBinder): The binder object used to bind expressions in the statement.
+
     Raises:
         AssertionError: If the number of children in the `node` is not equal to 3.
     """

evadb/constants.py

@@ -21,3 +21,4 @@
 IFRAMES = "IFRAMES"
 AUDIORATE = "AUDIORATE"
 DEFAULT_FUNCTION_EXPRESSION_COST = 100
+LLM_FUNCTIONS = ["chatgpt", "completion"]

evadb/executor/llm_executor.py

@@ -0,0 +1,36 @@
+# coding=utf-8
+# Copyright 2018-2023 EvaDB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Iterator
+
+from evadb.database import EvaDBDatabase
+from evadb.executor.abstract_executor import AbstractExecutor
+from evadb.models.storage.batch import Batch
+from evadb.plan_nodes.llm_plan import LLMPlan
+
+
+class LLMExecutor(AbstractExecutor):
+    def __init__(self, db: EvaDBDatabase, node: LLMPlan):
+        super().__init__(db, node)
+        self.llm_expr = node.llm_expr
+        self.alias = node.alias
+
+    def exec(self, *args, **kwargs) -> Iterator[Batch]:
+        child_executor = self.children[0]
+        for batch in child_executor.exec(**kwargs):
+            llm_result = self.llm_expr.evaluate(batch)
+
+            output = Batch.merge_column_wise([batch, llm_result])
+
+            yield output

evadb/executor/plan_executor.py

@@ -33,6 +33,7 @@
 from evadb.executor.insert_executor import InsertExecutor
 from evadb.executor.join_build_executor import BuildJoinExecutor
 from evadb.executor.limit_executor import LimitExecutor
+from evadb.executor.llm_executor import LLMExecutor
 from evadb.executor.load_executor import LoadDataExecutor
 from evadb.executor.nested_loop_join_executor import NestedLoopJoinExecutor
 from evadb.executor.orderby_executor import OrderByExecutor
@@ -152,6 +153,8 @@ def _build_execution_tree(
             executor_node = CreateIndexExecutor(db=self._db, node=plan)
         elif plan_opr_type == PlanOprType.APPLY_AND_MERGE:
             executor_node = ApplyAndMergeExecutor(db=self._db, node=plan)
+        elif plan_opr_type == PlanOprType.LLM:
+            executor_node = LLMExecutor(db=self._db, node=plan)
         elif plan_opr_type == PlanOprType.VECTOR_INDEX_SCAN:
             executor_node = VectorIndexScanExecutor(db=self._db, node=plan)
         elif plan_opr_type == PlanOprType.DELETE:

evadb/expression/expression_utils.py

@@ -15,9 +15,11 @@
 
 from typing import List, Set
 
+from evadb.constants import LLM_FUNCTIONS
 from evadb.expression.abstract_expression import AbstractExpression, ExpressionType
 from evadb.expression.comparison_expression import ComparisonExpression
 from evadb.expression.constant_value_expression import ConstantValueExpression
+from evadb.expression.function_expression import FunctionExpression
 from evadb.expression.logical_expression import LogicalExpression
 from evadb.expression.tuple_value_expression import TupleValueExpression
 
@@ -296,3 +298,20 @@ def _has_simple_expressions(expr):
     ]
 
     return _has_simple_expressions(predicate) and contains_single_column(predicate)
+
+
+def is_llm_expression(expr: AbstractExpression):
+    if isinstance(expr, FunctionExpression) and expr.name.lower() in LLM_FUNCTIONS:
+        return True
+    return False
+
+
+def extract_llm_expressions_from_project(exprs: List[AbstractExpression]):
+    remaining_exprs = []
+    llm_exprs = []
+    for expr in exprs:
+        if is_llm_expression(expr):
+            llm_exprs.append(expr.copy())
+        else:
+            remaining_exprs.append(expr)
+    return llm_exprs, remaining_exprs

evadb/functions/llms/base.py

@@ -0,0 +1,110 @@
+# coding=utf-8
+# Copyright 2018-2023 EvaDB
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import json
+import os
+from abc import abstractmethod
+from typing import List
+
+import pandas as pd
+
+from evadb.catalog.catalog_type import NdArrayType
+from evadb.functions.abstract.abstract_function import AbstractFunction
+from evadb.functions.decorators.decorators import forward, setup
+from evadb.functions.decorators.io_descriptors.data_types import PandasDataframe
+
+
+class BaseLLM(AbstractFunction):
+    """ """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.model_stats = None
+
+    @setup(cacheable=True, function_type="chat-completion", batchable=True)
+    def setup(self, *args, **kwargs) -> None:
+        super().setup(*args, **kwargs)
+
+    @forward(
+        input_signatures=[
+            PandasDataframe(
+                columns=["query", "content", "prompt"],
+                column_types=[
+                    NdArrayType.STR,
+                    NdArrayType.STR,
+                    NdArrayType.STR,
+                ],
+                column_shapes=[(1,), (1,), (None,)],
+            )
+        ],
+        output_signatures=[
+            PandasDataframe(
+                columns=["response", "model"],
+                column_types=[
+                    NdArrayType.STR,
+                    NdArrayType.STR,
+                ],
+                column_shapes=[(1,), (1,)],
+            )
+        ],
+    )
+    def forward(self, text_df):
+        queries = text_df[text_df.columns[0]]
+        contents = text_df[text_df.columns[0]]
+        if len(text_df.columns) > 1:
+            queries = text_df.iloc[:, 0]
+            contents = text_df.iloc[:, 1]
+
+        prompt = None
+        if len(text_df.columns) > 2:
+            prompt = text_df.iloc[0, 2]
+
+        responses, models = self.generate(queries, contents, prompt)
+        return pd.DataFrame({"response": responses, "model": models})
+
+    @abstractmethod
+    def generate(self, queries: List[str], contents: List[str], prompt: str) -> List[str]:
+        """
+        All the child classes should overload this function
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_cost(self, prompt: str, query: str, content: str, response: str = "") -> tuple[float]:
+        """
+        Return the token usage as tuple of input_token_usage, output_token_usage, and dollar cost of running the LLM on the prompt and the getting the provided response.
+        """
+        pass
+
+    @abstractmethod
+    def get_max_cost(self, prompt: str, query: str, content: str) -> tuple[float]:
+        """
+        Return the token usage as tuple of input_token_usage, output_token_usage, and dollar cost of running the LLM on the prompt and the getting the provided response.
+        """
+        pass
+
+    def get_model_stats(self, model_name: str):
+        # read the statistics if not already read
+        if self.model_stats is None:
+            current_file_path = os.path.dirname(os.path.realpath(__file__))
+            with open(f"{current_file_path}/llm_stats.json") as f:
+                self.model_stats = json.load(f)
+
+        assert (
+            model_name in self.model_stats
+        ), f"we do not have statistics for the model {model_name}"
+
+        return self.model_stats[model_name]

evadb/functions/llms/llm_stats.json

@@ -0,0 +1,128 @@
+{
+    "gpt-4": {
+        "max_token_context": 8192,
+        "input_cost_per_token": 0.00003,
+        "output_cost_per_token": 0.00006,
+        "provider": "openai",
+        "mode": "chat"
+    },
+    "gpt-4-0314": {
+        "max_token_context": 8192,
+        "input_cost_per_token": 0.00003,
+        "output_cost_per_token": 0.00006,
+        "provider": "openai",
+        "mode": "chat"
+    },
+    "gpt-4-0613": {
+        "max_token_context": 8192,
+        "input_cost_per_token": 0.00003,
+        "output_cost_per_token": 0.00006,
+        "provider": "openai",
+        "mode": "chat"
+    },
+    "gpt-4-32k": {
+        "max_token_context": 32768,
+        "input_cost_per_token": 0.00006,
+        "output_cost_per_token": 0.00012,
+        "provider": "openai",
+        "mode": "chat"
+    },
+    "gpt-4-32k-0314": {
+        "max_token_context": 32768,
+        "input_cost_per_token": 0.00006,
+        "output_cost_per_token": 0.00012,
+        "provider": "openai",
+        "mode": "chat"
+    },
+    "gpt-4-32k-0613": {
+        "max_token_context": 32768,
+        "input_cost_per_token": 0.00006,
+        "output_cost_per_token": 0.00012,
+        "provider": "openai",
+        "mode": "chat"
+    },
+    "gpt-3.5-turbo": {
+        "max_token_context": 4097,
+        "input_cost_per_token": 0.0000015,
+        "output_cost_per_token": 0.000002,
+        "provider": "openai",
+        "mode": "chat"
+    },
+    "gpt-3.5-turbo-0301": {
+        "max_token_context": 4097,
+        "input_cost_per_token": 0.0000015,
+        "output_cost_per_token": 0.000002,
+        "provider": "openai",
+        "mode": "chat"
+    },
+    "gpt-3.5-turbo-0613": {
+        "max_token_context": 4097,
+        "input_cost_per_token": 0.0000015,
+        "output_cost_per_token": 0.000002,
+        "provider": "openai",
+        "mode": "chat"
+    },
+    "gpt-3.5-turbo-16k": {
+        "max_token_context": 16385,
+        "input_cost_per_token": 0.000003,
+        "output_cost_per_token": 0.000004,
+        "provider": "openai",
+        "mode": "chat"
+    },
+    "gpt-3.5-turbo-16k-0613": {
+        "max_token_context": 16385,
+        "input_cost_per_token": 0.000003,
+        "output_cost_per_token": 0.000004,
+        "provider": "openai",
+        "mode": "chat"
+    },
+    "text-davinci-003": {
+        "max_token_context": 4097,
+        "input_cost_per_token": 0.000002,
+        "output_cost_per_token": 0.000002,
+        "provider": "openai",
+        "mode": "completion"
+    },
+    "text-curie-001": {
+        "max_token_context": 2049,
+        "input_cost_per_token": 0.000002,
+        "output_cost_per_token": 0.000002,
+        "provider": "openai",
+        "mode": "completion"
+    },
+    "text-babbage-001": {
+        "max_token_context": 2049,
+        "input_cost_per_token": 0.0000004,
+        "output_cost_per_token": 0.0000004,
+        "provider": "openai",
+        "mode": "completion"
+    },
+    "text-ada-001": {
+        "max_token_context": 2049,
+        "input_cost_per_token": 0.0000004,
+        "output_cost_per_token": 0.0000004,
+        "provider": "openai",
+        "mode": "completion"
+    },
+    "babbage-002": {
+        "max_token_context": 16384,
+        "input_cost_per_token": 0.0000004,
+        "output_cost_per_token": 0.0000004,
+        "provider": "openai",
+        "mode": "completion"
+    },
+    "davinci-002": {
+        "max_token_context": 16384,
+        "input_cost_per_token": 0.000002,
+        "output_cost_per_token": 0.000002,
+        "provider": "openai",
+        "mode": "completion"
+    },
+    "gpt-3.5-turbo-instruct": {
+        "max_token_context": 8192,
+        "input_cost_per_token": 0.0000015,
+        "output_cost_per_token": 0.000002,
+        "provider": "openai",
+        "mode": "completion"
+    }
+}