From c8c2d47b9b17467c5fb4a97638703b5e1666045f Mon Sep 17 00:00:00 2001
From: Grant Linville <grant@acorn.io>
Date: Mon, 4 Nov 2024 15:35:38 -0500
Subject: [PATCH 1/8] chore: update for dataset rewrite
Signed-off-by: Grant Linville <grant@acorn.io>
---
datasets.go | 184 ++++++++++++-----------------------------------
datasets_test.go | 69 ++++++++++--------
opts.go | 4 +-
3 files changed, 88 insertions(+), 169 deletions(-)
diff --git a/datasets.go b/datasets.go
index 0b39edc..0a4a1c0 100644
--- a/datasets.go
+++ b/datasets.go
@@ -2,7 +2,6 @@ package gptscript
import (
"context"
- "encoding/base64"
"encoding/json"
"fmt"
"os"
@@ -15,170 +14,82 @@ type DatasetElementMeta struct {
type DatasetElement struct {
DatasetElementMeta `json:",inline"`
- Contents []byte `json:"contents"`
-}
-
-type DatasetMeta struct {
- ID string `json:"id"`
- Name string `json:"name"`
- Description string `json:"description"`
+ Contents string `json:"contents"`
+ BinaryContents []byte `json:"binaryContents"`
}
type Dataset struct {
- DatasetMeta `json:",inline"`
- BaseDir string `json:"baseDir,omitempty"`
- Elements map[string]DatasetElementMeta `json:"elements"`
+ ID string `json:"id"`
+ BaseDir string `json:"baseDir,omitempty"`
+ Elements map[string]DatasetElementMeta `json:"elements"`
}
type datasetRequest struct {
- Input string `json:"input"`
- WorkspaceID string `json:"workspaceID"`
- DatasetToolRepo string `json:"datasetToolRepo"`
- Env []string `json:"env"`
-}
-
-type createDatasetArgs struct {
- Name string `json:"datasetName"`
- Description string `json:"datasetDescription"`
-}
-
-type addDatasetElementArgs struct {
- DatasetID string `json:"datasetID"`
- ElementName string `json:"elementName"`
- ElementDescription string `json:"elementDescription"`
- ElementContent string `json:"elementContent"`
+ Input string `json:"input"`
+ DatasetTool string `json:"datasetTool"`
+ Env []string `json:"env"`
}
type addDatasetElementsArgs struct {
- DatasetID string `json:"datasetID"`
- Elements []DatasetElement `json:"elements"`
+ WorkspaceID string `json:"workspaceID"`
+ DatasetID string `json:"datasetID"`
+ Elements []DatasetElement `json:"elements"`
}
type listDatasetElementArgs struct {
- DatasetID string `json:"datasetID"`
+ WorkspaceID string `json:"workspaceID"`
+ DatasetID string `json:"datasetID"`
}
type getDatasetElementArgs struct {
- DatasetID string `json:"datasetID"`
- Element string `json:"element"`
+ WorkspaceID string `json:"workspaceID"`
+ DatasetID string `json:"datasetID"`
+ Element string `json:"name"`
}
-func (g *GPTScript) ListDatasets(ctx context.Context, workspaceID string) ([]DatasetMeta, error) {
- if workspaceID == "" {
- workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID")
- }
-
+func (g *GPTScript) ListDatasets(ctx context.Context) ([]string, error) {
out, err := g.runBasicCommand(ctx, "datasets", datasetRequest{
- Input: "{}",
- WorkspaceID: workspaceID,
- DatasetToolRepo: g.globalOpts.DatasetToolRepo,
- Env: g.globalOpts.Env,
+ Input: fmt.Sprintf(`{"workspaceID": %q}`, os.Getenv("GPTSCRIPT_WORKSPACE_ID")),
+ DatasetTool: g.globalOpts.DatasetTool,
+ Env: g.globalOpts.Env,
})
if err != nil {
return nil, err
}
- var datasets []DatasetMeta
+ var datasets []string
if err = json.Unmarshal([]byte(out), &datasets); err != nil {
return nil, err
}
return datasets, nil
}
-func (g *GPTScript) CreateDataset(ctx context.Context, workspaceID, name, description string) (Dataset, error) {
- if workspaceID == "" {
- workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID")
- }
-
- args := createDatasetArgs{
- Name: name,
- Description: description,
- }
- argsJSON, err := json.Marshal(args)
- if err != nil {
- return Dataset{}, fmt.Errorf("failed to marshal dataset args: %w", err)
- }
-
- out, err := g.runBasicCommand(ctx, "datasets/create", datasetRequest{
- Input: string(argsJSON),
- WorkspaceID: workspaceID,
- DatasetToolRepo: g.globalOpts.DatasetToolRepo,
- Env: g.globalOpts.Env,
- })
- if err != nil {
- return Dataset{}, err
- }
-
- var dataset Dataset
- if err = json.Unmarshal([]byte(out), &dataset); err != nil {
- return Dataset{}, err
- }
- return dataset, nil
-}
-
-func (g *GPTScript) AddDatasetElement(ctx context.Context, workspaceID, datasetID, elementName, elementDescription string, elementContent []byte) (DatasetElementMeta, error) {
- if workspaceID == "" {
- workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID")
- }
-
- args := addDatasetElementArgs{
- DatasetID: datasetID,
- ElementName: elementName,
- ElementDescription: elementDescription,
- ElementContent: base64.StdEncoding.EncodeToString(elementContent),
- }
- argsJSON, err := json.Marshal(args)
- if err != nil {
- return DatasetElementMeta{}, fmt.Errorf("failed to marshal element args: %w", err)
- }
-
- out, err := g.runBasicCommand(ctx, "datasets/add-element", datasetRequest{
- Input: string(argsJSON),
- WorkspaceID: workspaceID,
- DatasetToolRepo: g.globalOpts.DatasetToolRepo,
- Env: g.globalOpts.Env,
- })
- if err != nil {
- return DatasetElementMeta{}, err
- }
-
- var element DatasetElementMeta
- if err = json.Unmarshal([]byte(out), &element); err != nil {
- return DatasetElementMeta{}, err
- }
- return element, nil
+func (g *GPTScript) CreateDatasetWithElements(ctx context.Context, elements []DatasetElement) (string, error) {
+ return g.AddDatasetElements(ctx, "", elements)
}
-func (g *GPTScript) AddDatasetElements(ctx context.Context, workspaceID, datasetID string, elements []DatasetElement) error {
- if workspaceID == "" {
- workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID")
- }
-
+func (g *GPTScript) AddDatasetElements(ctx context.Context, datasetID string, elements []DatasetElement) (string, error) {
args := addDatasetElementsArgs{
- DatasetID: datasetID,
- Elements: elements,
+ WorkspaceID: os.Getenv("GPTSCRIPT_WORKSPACE_ID"),
+ DatasetID: datasetID,
+ Elements: elements,
}
argsJSON, err := json.Marshal(args)
if err != nil {
- return fmt.Errorf("failed to marshal element args: %w", err)
+ return "", fmt.Errorf("failed to marshal element args: %w", err)
}
- _, err = g.runBasicCommand(ctx, "datasets/add-elements", datasetRequest{
- Input: string(argsJSON),
- WorkspaceID: workspaceID,
- DatasetToolRepo: g.globalOpts.DatasetToolRepo,
- Env: g.globalOpts.Env,
+ return g.runBasicCommand(ctx, "datasets/add-elements", datasetRequest{
+ Input: string(argsJSON),
+ DatasetTool: g.globalOpts.DatasetTool,
+ Env: g.globalOpts.Env,
})
- return err
}
-func (g *GPTScript) ListDatasetElements(ctx context.Context, workspaceID, datasetID string) ([]DatasetElementMeta, error) {
- if workspaceID == "" {
- workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID")
- }
-
+func (g *GPTScript) ListDatasetElements(ctx context.Context, datasetID string) ([]DatasetElementMeta, error) {
args := listDatasetElementArgs{
- DatasetID: datasetID,
+ WorkspaceID: os.Getenv("GPTSCRIPT_WORKSPACE_ID"),
+ DatasetID: datasetID,
}
argsJSON, err := json.Marshal(args)
if err != nil {
@@ -186,10 +97,9 @@ func (g *GPTScript) ListDatasetElements(ctx context.Context, workspaceID, datase
}
out, err := g.runBasicCommand(ctx, "datasets/list-elements", datasetRequest{
- Input: string(argsJSON),
- WorkspaceID: workspaceID,
- DatasetToolRepo: g.globalOpts.DatasetToolRepo,
- Env: g.globalOpts.Env,
+ Input: string(argsJSON),
+ DatasetTool: g.globalOpts.DatasetTool,
+ Env: g.globalOpts.Env,
})
if err != nil {
return nil, err
@@ -202,14 +112,11 @@ func (g *GPTScript) ListDatasetElements(ctx context.Context, workspaceID, datase
return elements, nil
}
-func (g *GPTScript) GetDatasetElement(ctx context.Context, workspaceID, datasetID, elementName string) (DatasetElement, error) {
- if workspaceID == "" {
- workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID")
- }
-
+func (g *GPTScript) GetDatasetElement(ctx context.Context, datasetID, elementName string) (DatasetElement, error) {
args := getDatasetElementArgs{
- DatasetID: datasetID,
- Element: elementName,
+ WorkspaceID: os.Getenv("GPTSCRIPT_WORKSPACE_ID"),
+ DatasetID: datasetID,
+ Element: elementName,
}
argsJSON, err := json.Marshal(args)
if err != nil {
@@ -217,10 +124,9 @@ func (g *GPTScript) GetDatasetElement(ctx context.Context, workspaceID, datasetI
}
out, err := g.runBasicCommand(ctx, "datasets/get-element", datasetRequest{
- Input: string(argsJSON),
- WorkspaceID: workspaceID,
- DatasetToolRepo: g.globalOpts.DatasetToolRepo,
- Env: g.globalOpts.Env,
+ Input: string(argsJSON),
+ DatasetTool: g.globalOpts.DatasetTool,
+ Env: g.globalOpts.Env,
})
if err != nil {
return DatasetElement{}, err
diff --git a/datasets_test.go b/datasets_test.go
index 6fc7ed9..8ece66a 100644
--- a/datasets_test.go
+++ b/datasets_test.go
@@ -2,6 +2,7 @@ package gptscript
import (
"context"
+ "os"
"testing"
"github.com/stretchr/testify/require"
@@ -11,66 +12,78 @@ func TestDatasets(t *testing.T) {
workspaceID, err := g.CreateWorkspace(context.Background(), "directory")
require.NoError(t, err)
+ require.NoError(t, os.Setenv("GPTSCRIPT_WORKSPACE_ID", workspaceID))
+
defer func() {
_ = g.DeleteWorkspace(context.Background(), workspaceID)
}()
- // Create a dataset
- dataset, err := g.CreateDataset(context.Background(), workspaceID, "test-dataset", "This is a test dataset")
- require.NoError(t, err)
- require.Equal(t, "test-dataset", dataset.Name)
- require.Equal(t, "This is a test dataset", dataset.Description)
- require.Equal(t, 0, len(dataset.Elements))
-
- // Add an element
- elementMeta, err := g.AddDatasetElement(context.Background(), workspaceID, dataset.ID, "test-element", "This is a test element", []byte("This is the content"))
+ datasetID, err := g.CreateDatasetWithElements(context.Background(), []DatasetElement{
+ {
+ DatasetElementMeta: DatasetElementMeta{
+ Name: "test-element-1",
+ Description: "This is a test element 1",
+ },
+ Contents: "This is the content 1",
+ },
+ })
require.NoError(t, err)
- require.Equal(t, "test-element", elementMeta.Name)
- require.Equal(t, "This is a test element", elementMeta.Description)
- // Add two more
- err = g.AddDatasetElements(context.Background(), workspaceID, dataset.ID, []DatasetElement{
+ // Add two more elements
+ _, err = g.AddDatasetElements(context.Background(), datasetID, []DatasetElement{
{
DatasetElementMeta: DatasetElementMeta{
Name: "test-element-2",
Description: "This is a test element 2",
},
- Contents: []byte("This is the content 2"),
+ Contents: "This is the content 2",
},
{
DatasetElementMeta: DatasetElementMeta{
Name: "test-element-3",
Description: "This is a test element 3",
},
- Contents: []byte("This is the content 3"),
+ Contents: "This is the content 3",
+ },
+ {
+ DatasetElementMeta: DatasetElementMeta{
+ Name: "binary-element",
+ Description: "this element has binary contents",
+ },
+ BinaryContents: []byte("binary contents"),
},
})
require.NoError(t, err)
// Get the first element
- element, err := g.GetDatasetElement(context.Background(), workspaceID, dataset.ID, "test-element")
+ element, err := g.GetDatasetElement(context.Background(), datasetID, "test-element-1")
require.NoError(t, err)
- require.Equal(t, "test-element", element.Name)
- require.Equal(t, "This is a test element", element.Description)
- require.Equal(t, []byte("This is the content"), element.Contents)
+ require.Equal(t, "test-element-1", element.Name)
+ require.Equal(t, "This is a test element 1", element.Description)
+ require.Equal(t, "This is the content 1", element.Contents)
// Get the third element
- element, err = g.GetDatasetElement(context.Background(), workspaceID, dataset.ID, "test-element-3")
+ element, err = g.GetDatasetElement(context.Background(), datasetID, "test-element-3")
require.NoError(t, err)
require.Equal(t, "test-element-3", element.Name)
require.Equal(t, "This is a test element 3", element.Description)
- require.Equal(t, []byte("This is the content 3"), element.Contents)
+ require.Equal(t, "This is the content 3", element.Contents)
+
+ // Get the binary element
+ element, err = g.GetDatasetElement(context.Background(), datasetID, "binary-element")
+ require.NoError(t, err)
+ require.Equal(t, "binary-element", element.Name)
+ require.Equal(t, "this element has binary contents", element.Description)
+ require.Equal(t, []byte("binary contents"), element.BinaryContents)
// List elements in the dataset
- elements, err := g.ListDatasetElements(context.Background(), workspaceID, dataset.ID)
+ elements, err := g.ListDatasetElements(context.Background(), datasetID)
require.NoError(t, err)
- require.Equal(t, 3, len(elements))
+ require.Equal(t, 4, len(elements))
// List datasets
- datasets, err := g.ListDatasets(context.Background(), workspaceID)
+ datasetIDs, err := g.ListDatasets(context.Background())
require.NoError(t, err)
- require.Equal(t, 1, len(datasets))
- require.Equal(t, "test-dataset", datasets[0].Name)
- require.Equal(t, "This is a test dataset", datasets[0].Description)
- require.Equal(t, dataset.ID, datasets[0].ID)
+ require.Equal(t, 1, len(datasetIDs))
+ require.Equal(t, datasetID, datasetIDs[0])
}
diff --git a/opts.go b/opts.go
index e08d217..07507e2 100644
--- a/opts.go
+++ b/opts.go
@@ -11,7 +11,7 @@ type GlobalOptions struct {
DefaultModelProvider string `json:"DefaultModelProvider"`
CacheDir string `json:"CacheDir"`
Env []string `json:"env"`
- DatasetToolRepo string `json:"DatasetToolRepo"`
+ DatasetTool string `json:"DatasetTool"`
WorkspaceTool string `json:"WorkspaceTool"`
}
@@ -46,7 +46,7 @@ func completeGlobalOptions(opts ...GlobalOptions) GlobalOptions {
result.OpenAIBaseURL = firstSet(opt.OpenAIBaseURL, result.OpenAIBaseURL)
result.DefaultModel = firstSet(opt.DefaultModel, result.DefaultModel)
result.DefaultModelProvider = firstSet(opt.DefaultModelProvider, result.DefaultModelProvider)
- result.DatasetToolRepo = firstSet(opt.DatasetToolRepo, result.DatasetToolRepo)
+ result.DatasetTool = firstSet(opt.DatasetTool, result.DatasetTool)
result.WorkspaceTool = firstSet(opt.WorkspaceTool, result.WorkspaceTool)
result.Env = append(result.Env, opt.Env...)
}
From 3872d0d519e9bc312e8110106851dfc5e24212d0 Mon Sep 17 00:00:00 2001
From: Grant Linville <grant@acorn.io>
Date: Mon, 4 Nov 2024 15:38:55 -0500
Subject: [PATCH 2/8] fix comment
Signed-off-by: Grant Linville <grant@acorn.io>
---
datasets_test.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/datasets_test.go b/datasets_test.go
index 8ece66a..0d74647 100644
--- a/datasets_test.go
+++ b/datasets_test.go
@@ -29,7 +29,7 @@ func TestDatasets(t *testing.T) {
})
require.NoError(t, err)
- // Add two more elements
+ // Add three more elements
_, err = g.AddDatasetElements(context.Background(), datasetID, []DatasetElement{
{
DatasetElementMeta: DatasetElementMeta{
From 3e61e9007abeb23ad01e056197b994e09f142216 Mon Sep 17 00:00:00 2001
From: Grant Linville <grant@acorn.io>
Date: Mon, 4 Nov 2024 16:42:10 -0500
Subject: [PATCH 3/8] remove unused type
Signed-off-by: Grant Linville <grant@acorn.io>
---
datasets.go | 6 ------
1 file changed, 6 deletions(-)
diff --git a/datasets.go b/datasets.go
index 0a4a1c0..f33aa7a 100644
--- a/datasets.go
+++ b/datasets.go
@@ -18,12 +18,6 @@ type DatasetElement struct {
BinaryContents []byte `json:"binaryContents"`
}
-type Dataset struct {
- ID string `json:"id"`
- BaseDir string `json:"baseDir,omitempty"`
- Elements map[string]DatasetElementMeta `json:"elements"`
-}
-
type datasetRequest struct {
Input string `json:"input"`
DatasetTool string `json:"datasetTool"`
From 5cd6aaec9bf07275d08decd96edbf22b37a19372 Mon Sep 17 00:00:00 2001
From: Grant Linville <grant@acorn.io>
Date: Tue, 5 Nov 2024 14:23:46 -0500
Subject: [PATCH 4/8] workspaceID is no longer part of tool input
Signed-off-by: Grant Linville <grant@acorn.io>
---
datasets.go | 33 ++++++++++++++++-----------------
1 file changed, 16 insertions(+), 17 deletions(-)
diff --git a/datasets.go b/datasets.go
index f33aa7a..4270a4f 100644
--- a/datasets.go
+++ b/datasets.go
@@ -19,31 +19,30 @@ type DatasetElement struct {
}
type datasetRequest struct {
+ WorkspaceID string `json:"workspaceID"`
Input string `json:"input"`
DatasetTool string `json:"datasetTool"`
Env []string `json:"env"`
}
type addDatasetElementsArgs struct {
- WorkspaceID string `json:"workspaceID"`
- DatasetID string `json:"datasetID"`
- Elements []DatasetElement `json:"elements"`
+ DatasetID string `json:"datasetID"`
+ Elements []DatasetElement `json:"elements"`
}
type listDatasetElementArgs struct {
- WorkspaceID string `json:"workspaceID"`
- DatasetID string `json:"datasetID"`
+ DatasetID string `json:"datasetID"`
}
type getDatasetElementArgs struct {
- WorkspaceID string `json:"workspaceID"`
- DatasetID string `json:"datasetID"`
- Element string `json:"name"`
+ DatasetID string `json:"datasetID"`
+ Element string `json:"name"`
}
func (g *GPTScript) ListDatasets(ctx context.Context) ([]string, error) {
out, err := g.runBasicCommand(ctx, "datasets", datasetRequest{
- Input: fmt.Sprintf(`{"workspaceID": %q}`, os.Getenv("GPTSCRIPT_WORKSPACE_ID")),
+ Input: "{}",
+ WorkspaceID: os.Getenv("GPTSCRIPT_WORKSPACE_ID"),
DatasetTool: g.globalOpts.DatasetTool,
Env: g.globalOpts.Env,
})
@@ -64,9 +63,8 @@ func (g *GPTScript) CreateDatasetWithElements(ctx context.Context, elements []Da
func (g *GPTScript) AddDatasetElements(ctx context.Context, datasetID string, elements []DatasetElement) (string, error) {
args := addDatasetElementsArgs{
- WorkspaceID: os.Getenv("GPTSCRIPT_WORKSPACE_ID"),
- DatasetID: datasetID,
- Elements: elements,
+ DatasetID: datasetID,
+ Elements: elements,
}
argsJSON, err := json.Marshal(args)
if err != nil {
@@ -75,6 +73,7 @@ func (g *GPTScript) AddDatasetElements(ctx context.Context, datasetID string, el
return g.runBasicCommand(ctx, "datasets/add-elements", datasetRequest{
Input: string(argsJSON),
+ WorkspaceID: os.Getenv("GPTSCRIPT_WORKSPACE_ID"),
DatasetTool: g.globalOpts.DatasetTool,
Env: g.globalOpts.Env,
})
@@ -82,8 +81,7 @@ func (g *GPTScript) AddDatasetElements(ctx context.Context, datasetID string, el
func (g *GPTScript) ListDatasetElements(ctx context.Context, datasetID string) ([]DatasetElementMeta, error) {
args := listDatasetElementArgs{
- WorkspaceID: os.Getenv("GPTSCRIPT_WORKSPACE_ID"),
- DatasetID: datasetID,
+ DatasetID: datasetID,
}
argsJSON, err := json.Marshal(args)
if err != nil {
@@ -92,6 +90,7 @@ func (g *GPTScript) ListDatasetElements(ctx context.Context, datasetID string) (
out, err := g.runBasicCommand(ctx, "datasets/list-elements", datasetRequest{
Input: string(argsJSON),
+ WorkspaceID: os.Getenv("GPTSCRIPT_WORKSPACE_ID"),
DatasetTool: g.globalOpts.DatasetTool,
Env: g.globalOpts.Env,
})
@@ -108,9 +107,8 @@ func (g *GPTScript) ListDatasetElements(ctx context.Context, datasetID string) (
func (g *GPTScript) GetDatasetElement(ctx context.Context, datasetID, elementName string) (DatasetElement, error) {
args := getDatasetElementArgs{
- WorkspaceID: os.Getenv("GPTSCRIPT_WORKSPACE_ID"),
- DatasetID: datasetID,
- Element: elementName,
+ DatasetID: datasetID,
+ Element: elementName,
}
argsJSON, err := json.Marshal(args)
if err != nil {
@@ -119,6 +117,7 @@ func (g *GPTScript) GetDatasetElement(ctx context.Context, datasetID, elementNam
out, err := g.runBasicCommand(ctx, "datasets/get-element", datasetRequest{
Input: string(argsJSON),
+ WorkspaceID: os.Getenv("GPTSCRIPT_WORKSPACE_ID"),
DatasetTool: g.globalOpts.DatasetTool,
Env: g.globalOpts.Env,
})
From aa62b408db2e89d8899ae0c7bb60953c3bb7b4f9 Mon Sep 17 00:00:00 2001
From: Grant Linville <grant@acorn.io>
Date: Tue, 5 Nov 2024 14:43:08 -0500
Subject: [PATCH 5/8] include workspaceID in env
Signed-off-by: Grant Linville <grant@acorn.io>
---
datasets.go | 13 ++++---------
1 file changed, 4 insertions(+), 9 deletions(-)
diff --git a/datasets.go b/datasets.go
index 4270a4f..2b5e95c 100644
--- a/datasets.go
+++ b/datasets.go
@@ -19,7 +19,6 @@ type DatasetElement struct {
}
type datasetRequest struct {
- WorkspaceID string `json:"workspaceID"`
Input string `json:"input"`
DatasetTool string `json:"datasetTool"`
Env []string `json:"env"`
@@ -42,9 +41,8 @@ type getDatasetElementArgs struct {
func (g *GPTScript) ListDatasets(ctx context.Context) ([]string, error) {
out, err := g.runBasicCommand(ctx, "datasets", datasetRequest{
Input: "{}",
- WorkspaceID: os.Getenv("GPTSCRIPT_WORKSPACE_ID"),
DatasetTool: g.globalOpts.DatasetTool,
- Env: g.globalOpts.Env,
+ Env: append(g.globalOpts.Env, "GPTSCRIPT_WORKSPACE_ID="+os.Getenv("GPTSCRIPT_WORKSPACE_ID")),
})
if err != nil {
return nil, err
@@ -73,9 +71,8 @@ func (g *GPTScript) AddDatasetElements(ctx context.Context, datasetID string, el
return g.runBasicCommand(ctx, "datasets/add-elements", datasetRequest{
Input: string(argsJSON),
- WorkspaceID: os.Getenv("GPTSCRIPT_WORKSPACE_ID"),
DatasetTool: g.globalOpts.DatasetTool,
- Env: g.globalOpts.Env,
+ Env: append(g.globalOpts.Env, "GPTSCRIPT_WORKSPACE_ID="+os.Getenv("GPTSCRIPT_WORKSPACE_ID")),
})
}
@@ -90,9 +87,8 @@ func (g *GPTScript) ListDatasetElements(ctx context.Context, datasetID string) (
out, err := g.runBasicCommand(ctx, "datasets/list-elements", datasetRequest{
Input: string(argsJSON),
- WorkspaceID: os.Getenv("GPTSCRIPT_WORKSPACE_ID"),
DatasetTool: g.globalOpts.DatasetTool,
- Env: g.globalOpts.Env,
+ Env: append(g.globalOpts.Env, "GPTSCRIPT_WORKSPACE_ID="+os.Getenv("GPTSCRIPT_WORKSPACE_ID")),
})
if err != nil {
return nil, err
@@ -117,9 +113,8 @@ func (g *GPTScript) GetDatasetElement(ctx context.Context, datasetID, elementNam
out, err := g.runBasicCommand(ctx, "datasets/get-element", datasetRequest{
Input: string(argsJSON),
- WorkspaceID: os.Getenv("GPTSCRIPT_WORKSPACE_ID"),
DatasetTool: g.globalOpts.DatasetTool,
- Env: g.globalOpts.Env,
+ Env: append(g.globalOpts.Env, "GPTSCRIPT_WORKSPACE_ID="+os.Getenv("GPTSCRIPT_WORKSPACE_ID")),
})
if err != nil {
return DatasetElement{}, err
From eda15f3e27da420e201cabbac2862ebee03ca5b1 Mon Sep 17 00:00:00 2001
From: Grant Linville <grant@acorn.io>
Date: Tue, 5 Nov 2024 15:13:17 -0500
Subject: [PATCH 6/8] re-add dataset name and description
Signed-off-by: Grant Linville <grant@acorn.io>
---
datasets.go | 36 +++++++++++++++++++++++++++++-------
datasets_test.go | 11 ++++++++---
2 files changed, 37 insertions(+), 10 deletions(-)
diff --git a/datasets.go b/datasets.go
index 2b5e95c..453188a 100644
--- a/datasets.go
+++ b/datasets.go
@@ -18,6 +18,12 @@ type DatasetElement struct {
BinaryContents []byte `json:"binaryContents"`
}
+type DatasetMeta struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ Description string `json:"description"`
+}
+
type datasetRequest struct {
Input string `json:"input"`
DatasetTool string `json:"datasetTool"`
@@ -25,8 +31,10 @@ type datasetRequest struct {
}
type addDatasetElementsArgs struct {
- DatasetID string `json:"datasetID"`
- Elements []DatasetElement `json:"elements"`
+ DatasetID string `json:"datasetID"`
+ Name string `json:"name"`
+ Description string `json:"description"`
+ Elements []DatasetElement `json:"elements"`
}
type listDatasetElementArgs struct {
@@ -38,7 +46,7 @@ type getDatasetElementArgs struct {
Element string `json:"name"`
}
-func (g *GPTScript) ListDatasets(ctx context.Context) ([]string, error) {
+func (g *GPTScript) ListDatasets(ctx context.Context) ([]DatasetMeta, error) {
out, err := g.runBasicCommand(ctx, "datasets", datasetRequest{
Input: "{}",
DatasetTool: g.globalOpts.DatasetTool,
@@ -48,22 +56,36 @@ func (g *GPTScript) ListDatasets(ctx context.Context) ([]string, error) {
return nil, err
}
- var datasets []string
+ var datasets []DatasetMeta
if err = json.Unmarshal([]byte(out), &datasets); err != nil {
return nil, err
}
return datasets, nil
}
-func (g *GPTScript) CreateDatasetWithElements(ctx context.Context, elements []DatasetElement) (string, error) {
- return g.AddDatasetElements(ctx, "", elements)
+type DatasetOptions struct {
+ Name, Description string
}
-func (g *GPTScript) AddDatasetElements(ctx context.Context, datasetID string, elements []DatasetElement) (string, error) {
+func (g *GPTScript) CreateDatasetWithElements(ctx context.Context, elements []DatasetElement, options ...DatasetOptions) (string, error) {
+ return g.AddDatasetElements(ctx, "", elements, options...)
+}
+
+func (g *GPTScript) AddDatasetElements(ctx context.Context, datasetID string, elements []DatasetElement, options ...DatasetOptions) (string, error) {
args := addDatasetElementsArgs{
DatasetID: datasetID,
Elements: elements,
}
+
+ for _, opt := range options {
+ if opt.Name != "" {
+ args.Name = opt.Name
+ }
+ if opt.Description != "" {
+ args.Description = opt.Description
+ }
+ }
+
argsJSON, err := json.Marshal(args)
if err != nil {
return "", fmt.Errorf("failed to marshal element args: %w", err)
diff --git a/datasets_test.go b/datasets_test.go
index 0d74647..8fb10c5 100644
--- a/datasets_test.go
+++ b/datasets_test.go
@@ -26,6 +26,9 @@ func TestDatasets(t *testing.T) {
},
Contents: "This is the content 1",
},
+ }, DatasetOptions{
+ Name: "test-dataset",
+ Description: "this is a test dataset",
})
require.NoError(t, err)
@@ -82,8 +85,10 @@ func TestDatasets(t *testing.T) {
require.Equal(t, 4, len(elements))
// List datasets
- datasetIDs, err := g.ListDatasets(context.Background())
+ datasets, err := g.ListDatasets(context.Background())
require.NoError(t, err)
- require.Equal(t, 1, len(datasetIDs))
- require.Equal(t, datasetID, datasetIDs[0])
+ require.Equal(t, 1, len(datasets))
+ require.Equal(t, datasetID, datasets[0].ID)
+ require.Equal(t, "test-dataset", datasets[0].Name)
+ require.Equal(t, "this is a test dataset", datasets[0].Description)
}
From 11455255a1d8c05850412c97f3eaf22af7899047 Mon Sep 17 00:00:00 2001
From: Grant Linville <grant@acorn.io>
Date: Tue, 5 Nov 2024 15:54:41 -0500
Subject: [PATCH 7/8] use a second client in the test
Signed-off-by: Grant Linville <grant@acorn.io>
---
datasets.go | 9 ++++-----
datasets_test.go | 20 ++++++++++++--------
2 files changed, 16 insertions(+), 13 deletions(-)
diff --git a/datasets.go b/datasets.go
index 453188a..fb408c4 100644
--- a/datasets.go
+++ b/datasets.go
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"fmt"
- "os"
)
type DatasetElementMeta struct {
@@ -50,7 +49,7 @@ func (g *GPTScript) ListDatasets(ctx context.Context) ([]DatasetMeta, error) {
out, err := g.runBasicCommand(ctx, "datasets", datasetRequest{
Input: "{}",
DatasetTool: g.globalOpts.DatasetTool,
- Env: append(g.globalOpts.Env, "GPTSCRIPT_WORKSPACE_ID="+os.Getenv("GPTSCRIPT_WORKSPACE_ID")),
+ Env: g.globalOpts.Env,
})
if err != nil {
return nil, err
@@ -94,7 +93,7 @@ func (g *GPTScript) AddDatasetElements(ctx context.Context, datasetID string, el
return g.runBasicCommand(ctx, "datasets/add-elements", datasetRequest{
Input: string(argsJSON),
DatasetTool: g.globalOpts.DatasetTool,
- Env: append(g.globalOpts.Env, "GPTSCRIPT_WORKSPACE_ID="+os.Getenv("GPTSCRIPT_WORKSPACE_ID")),
+ Env: g.globalOpts.Env,
})
}
@@ -110,7 +109,7 @@ func (g *GPTScript) ListDatasetElements(ctx context.Context, datasetID string) (
out, err := g.runBasicCommand(ctx, "datasets/list-elements", datasetRequest{
Input: string(argsJSON),
DatasetTool: g.globalOpts.DatasetTool,
- Env: append(g.globalOpts.Env, "GPTSCRIPT_WORKSPACE_ID="+os.Getenv("GPTSCRIPT_WORKSPACE_ID")),
+ Env: g.globalOpts.Env,
})
if err != nil {
return nil, err
@@ -136,7 +135,7 @@ func (g *GPTScript) GetDatasetElement(ctx context.Context, datasetID, elementNam
out, err := g.runBasicCommand(ctx, "datasets/get-element", datasetRequest{
Input: string(argsJSON),
DatasetTool: g.globalOpts.DatasetTool,
- Env: append(g.globalOpts.Env, "GPTSCRIPT_WORKSPACE_ID="+os.Getenv("GPTSCRIPT_WORKSPACE_ID")),
+ Env: g.globalOpts.Env,
})
if err != nil {
return DatasetElement{}, err
diff --git a/datasets_test.go b/datasets_test.go
index 8fb10c5..14c84a6 100644
--- a/datasets_test.go
+++ b/datasets_test.go
@@ -12,13 +12,17 @@ func TestDatasets(t *testing.T) {
workspaceID, err := g.CreateWorkspace(context.Background(), "directory")
require.NoError(t, err)
- require.NoError(t, os.Setenv("GPTSCRIPT_WORKSPACE_ID", workspaceID))
+ client, err := NewGPTScript(GlobalOptions{
+ OpenAIAPIKey: os.Getenv("OPENAI_API_KEY"),
+ Env: []string{"GPTSCRIPT_WORKSPACE_ID=" + workspaceID},
+ })
+ require.NoError(t, err)
defer func() {
_ = g.DeleteWorkspace(context.Background(), workspaceID)
}()
- datasetID, err := g.CreateDatasetWithElements(context.Background(), []DatasetElement{
+ datasetID, err := client.CreateDatasetWithElements(context.Background(), []DatasetElement{
{
DatasetElementMeta: DatasetElementMeta{
Name: "test-element-1",
@@ -33,7 +37,7 @@ func TestDatasets(t *testing.T) {
require.NoError(t, err)
// Add three more elements
- _, err = g.AddDatasetElements(context.Background(), datasetID, []DatasetElement{
+ _, err = client.AddDatasetElements(context.Background(), datasetID, []DatasetElement{
{
DatasetElementMeta: DatasetElementMeta{
Name: "test-element-2",
@@ -59,33 +63,33 @@ func TestDatasets(t *testing.T) {
require.NoError(t, err)
// Get the first element
- element, err := g.GetDatasetElement(context.Background(), datasetID, "test-element-1")
+ element, err := client.GetDatasetElement(context.Background(), datasetID, "test-element-1")
require.NoError(t, err)
require.Equal(t, "test-element-1", element.Name)
require.Equal(t, "This is a test element 1", element.Description)
require.Equal(t, "This is the content 1", element.Contents)
// Get the third element
- element, err = g.GetDatasetElement(context.Background(), datasetID, "test-element-3")
+ element, err = client.GetDatasetElement(context.Background(), datasetID, "test-element-3")
require.NoError(t, err)
require.Equal(t, "test-element-3", element.Name)
require.Equal(t, "This is a test element 3", element.Description)
require.Equal(t, "This is the content 3", element.Contents)
// Get the binary element
- element, err = g.GetDatasetElement(context.Background(), datasetID, "binary-element")
+ element, err = client.GetDatasetElement(context.Background(), datasetID, "binary-element")
require.NoError(t, err)
require.Equal(t, "binary-element", element.Name)
require.Equal(t, "this element has binary contents", element.Description)
require.Equal(t, []byte("binary contents"), element.BinaryContents)
// List elements in the dataset
- elements, err := g.ListDatasetElements(context.Background(), datasetID)
+ elements, err := client.ListDatasetElements(context.Background(), datasetID)
require.NoError(t, err)
require.Equal(t, 4, len(elements))
// List datasets
- datasets, err := g.ListDatasets(context.Background())
+ datasets, err := client.ListDatasets(context.Background())
require.NoError(t, err)
require.Equal(t, 1, len(datasets))
require.Equal(t, datasetID, datasets[0].ID)
From 9dae54cb6bf8c63c99854cc56bd9536b12c6f8eb Mon Sep 17 00:00:00 2001
From: Grant Linville <grant@acorn.io>
Date: Wed, 6 Nov 2024 11:24:29 -0500
Subject: [PATCH 8/8] fix environment
Signed-off-by: Grant Linville <grant@acorn.io>
---
datasets_test.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/datasets_test.go b/datasets_test.go
index 14c84a6..c1f1a92 100644
--- a/datasets_test.go
+++ b/datasets_test.go
@@ -14,7 +14,7 @@ func TestDatasets(t *testing.T) {
client, err := NewGPTScript(GlobalOptions{
OpenAIAPIKey: os.Getenv("OPENAI_API_KEY"),
- Env: []string{"GPTSCRIPT_WORKSPACE_ID=" + workspaceID},
+ Env: append(os.Environ(), "GPTSCRIPT_WORKSPACE_ID="+workspaceID),
})
require.NoError(t, err)