Skip to content

Commit bb33337

Browse files
orangetinorangetin
authored
Merge pull request #50 from togethercomputer/orangetin/tabulate
Tabulate outputs for JSON outputs
2 parents 9a37253 + 3810a8f commit bb33337

File tree

7 files changed

+188
-118
lines changed

7 files changed

+188
-118
lines changed

‎pyproject.toml‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,18 @@ dependencies = [
2323
"requests==2.31.0",
2424
"tqdm==4.66.1",
2525
"sseclient-py==1.7.2",
26+
"tabulate==0.9.0",
2627
]
2728

2829
[project.optional-dependencies]
29-
quality = ["black~=23.1", "ruff>=0.0.241,<=0.0.259", "mypy>=1.3.0", "types-requests>=2.31.0.1", "types-tqdm>=4.65.0.0",]
30+
quality = [
31+
"black~=23.1",
32+
"ruff>=0.0.241,<=0.0.259",
33+
"mypy>=1.3.0",
34+
"types-requests>=2.31.0.1",
35+
"types-tqdm>=4.65.0.0",
36+
"types-tabulate==0.9.0.3"
37+
]
3038
tokenize = ["transformers>=4.33.2"]
3139

3240
[project.urls]

‎src/together/commands/files.py‎

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@
33
import argparse
44
import json
55

6+
from tabulate import tabulate
7+
68
from together import Files
7-
from together.utils import extract_time
9+
from together.utils import bytes_to_human_readable, extract_time
810

911

1012
def add_parser(subparsers: argparse._SubParsersAction[argparse.ArgumentParser]) -> None:
@@ -23,6 +25,12 @@ def add_parser(subparsers: argparse._SubParsersAction[argparse.ArgumentParser])
2325

2426
def _add_list(parser: argparse._SubParsersAction[argparse.ArgumentParser]) -> None:
2527
subparser = parser.add_parser("list")
28+
subparser.add_argument(
29+
"--raw",
30+
help="Raw JSON dump of response",
31+
default=False,
32+
action="store_true",
33+
)
2634
subparser.set_defaults(func=_run_list)
2735

2836

@@ -82,6 +90,12 @@ def _add_retrieve(parser: argparse._SubParsersAction[argparse.ArgumentParser]) -
8290
help="File ID of remote file",
8391
type=str,
8492
)
93+
subparser.add_argument(
94+
"--raw",
95+
help="Raw JSON dump of response",
96+
default=False,
97+
action="store_true",
98+
)
8599
subparser.set_defaults(func=_run_retrieve)
86100

87101

@@ -111,7 +125,24 @@ def _add_retrieve_content(
111125
def _run_list(args: argparse.Namespace) -> None:
112126
response = Files.list()
113127
response["data"].sort(key=extract_time)
114-
print(json.dumps(response, indent=4))
128+
if args.raw:
129+
print(json.dumps(response, indent=4))
130+
else:
131+
display_list = []
132+
for i in response["data"]:
133+
display_list.append(
134+
{
135+
"File name": i.get("filename"),
136+
"File ID": i.get("id"),
137+
"Size": bytes_to_human_readable(
138+
float(str(i.get("bytes")))
139+
), # convert to string for mypy typing
140+
"Created At": i.get("created_at"),
141+
"Line Count": i.get("LineCount"),
142+
}
143+
)
144+
table = tabulate(display_list, headers="keys", tablefmt="grid", showindex=True)
145+
print(table)
115146

116147

117148
def _run_check(args: argparse.Namespace) -> None:
@@ -131,7 +162,12 @@ def _run_delete(args: argparse.Namespace) -> None:
131162

132163
def _run_retrieve(args: argparse.Namespace) -> None:
133164
response = Files.retrieve(args.file_id)
134-
print(json.dumps(response, indent=4))
165+
if args.raw:
166+
print(json.dumps(response, indent=4))
167+
else:
168+
table_data = [{"Key": key, "Value": value} for key, value in response.items()]
169+
table = tabulate(table_data, tablefmt="grid")
170+
print(table)
135171

136172

137173
def _run_retrieve_content(args: argparse.Namespace) -> None:

‎src/together/commands/finetune.py‎

Lines changed: 74 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,10 @@
44
import json
55
import os
66

7+
from tabulate import tabulate
8+
79
from together import Finetune
8-
from together.utils import parse_timestamp
10+
from together.utils import finetune_price_to_dollars, parse_timestamp
911

1012

1113
def add_parser(subparsers: argparse._SubParsersAction[argparse.ArgumentParser]) -> None:
@@ -43,13 +45,6 @@ def _add_create(parser: argparse._SubParsersAction[argparse.ArgumentParser]) ->
4345
required=False,
4446
action="store_true",
4547
)
46-
# subparser.add_argument(
47-
# "--validation-file",
48-
# "-v",
49-
# default=None,
50-
# help="The ID of an uploaded file that contains validation data.",
51-
# type=str,
52-
# )
5348
subparser.add_argument(
5449
"--model",
5550
"-m",
@@ -90,46 +85,6 @@ def _add_create(parser: argparse._SubParsersAction[argparse.ArgumentParser]) ->
9085
help="The learning rate multiplier to use for training. Default=0.00001",
9186
type=float,
9287
)
93-
# subparser.add_argument(
94-
# "--warmup-steps",
95-
# "-ws",
96-
# default=0,
97-
# help="Warmup steps",
98-
# type=int,
99-
# )
100-
# subparser.add_argument(
101-
# "--train-warmup-steps",
102-
# "-tws",
103-
# default=0,
104-
# help="Train warmup steps",
105-
# type=int,
106-
# )
107-
# subparser.add_argument(
108-
# "--sequence-length",
109-
# "-sl",
110-
# default=2048,
111-
# help="Max sequence length",
112-
# type=int,
113-
# )
114-
# subparser.add_argument(
115-
# "--seed",
116-
# default=42,
117-
# help="Training seed",
118-
# type=int,
119-
# )
120-
# subparser.add_argument(
121-
# "--fp32",
122-
# help="Enable FP32 training. Defaults to false (FP16 training).",
123-
# default=False,
124-
# action="store_true",
125-
# )
126-
# subparser.add_argument(
127-
# "--checkpoint-steps",
128-
# "-b",
129-
# default=0,
130-
# help="Number of steps between each checkpoint. Defaults to 0 = checkpoints per epoch.",
131-
# type=int,
132-
# )
13388
subparser.add_argument(
13489
"--suffix",
13590
"-s",
@@ -162,53 +117,69 @@ def _add_create(parser: argparse._SubParsersAction[argparse.ArgumentParser]) ->
162117

163118
subparser.set_defaults(func=_run_create)
164119

165-
# End of create_finetune
166-
167120

168121
def _add_list(parser: argparse._SubParsersAction[argparse.ArgumentParser]) -> None:
169122
# List_Finetune
170-
list_parser = parser.add_parser("list")
171-
list_parser.set_defaults(func=_run_list)
123+
subparser = parser.add_parser("list")
124+
subparser.add_argument(
125+
"--raw",
126+
help="Raw JSON dump of response",
127+
default=False,
128+
action="store_true",
129+
)
130+
subparser.set_defaults(func=_run_list)
172131

173132

174133
def _add_retrieve(parser: argparse._SubParsersAction[argparse.ArgumentParser]) -> None:
175-
retrieve_finetune_parser = parser.add_parser("retrieve")
176-
retrieve_finetune_parser.add_argument(
134+
subparser = parser.add_parser("retrieve")
135+
subparser.add_argument(
177136
"fine_tune_id",
178137
metavar="FINETUNE-ID",
179138
default=None,
180139
help="Fine-tuning ID",
181140
type=str,
182141
)
183-
retrieve_finetune_parser.set_defaults(func=_run_retrieve)
142+
subparser.add_argument(
143+
"--raw",
144+
help="Raw JSON dump of response",
145+
default=False,
146+
action="store_true",
147+
)
148+
subparser.set_defaults(func=_run_retrieve)
184149

185150

186151
def _add_cancel(parser: argparse._SubParsersAction[argparse.ArgumentParser]) -> None:
187152
# Cancel Finetune
188-
cancel_finetune_parser = parser.add_parser("cancel")
189-
cancel_finetune_parser.add_argument(
153+
subparser = parser.add_parser("cancel")
154+
subparser.add_argument(
190155
"fine_tune_id",
191156
metavar="FINETUNE-ID",
192157
default=None,
193158
help="Fine-tuning ID",
194159
type=str,
195160
)
196-
cancel_finetune_parser.set_defaults(func=_run_cancel)
161+
subparser.set_defaults(func=_run_cancel)
197162

198163

199164
def _add_list_events(
200165
parser: argparse._SubParsersAction[argparse.ArgumentParser],
201166
) -> None:
202167
# List finetune events
203-
list_finetune_events_parser = parser.add_parser("list-events")
204-
list_finetune_events_parser.add_argument(
168+
subparser = parser.add_parser("list-events")
169+
subparser.add_argument(
205170
"fine_tune_id",
206171
metavar="FINETUNE-ID",
207172
default=None,
208173
help="Fine-tuning ID",
209174
type=str,
210175
)
211-
list_finetune_events_parser.set_defaults(func=_run_list_events)
176+
subparser.add_argument(
177+
"--raw",
178+
help="Raw JSON dump of response",
179+
default=False,
180+
action="store_true",
181+
)
182+
subparser.set_defaults(func=_run_list_events)
212183

213184

214185
def _add_download(parser: argparse._SubParsersAction[argparse.ArgumentParser]) -> None:
@@ -269,22 +240,6 @@ def _add_checkpoints(
269240
checkpoint_parser.set_defaults(func=_run_checkpoint)
270241

271242

272-
# def _add_delete_model(
273-
# parser: argparse._SubParsersAction[argparse.ArgumentParser],
274-
# ) -> None:
275-
# # Delete finetune model
276-
# delete_finetune_model_parser = parser.add_parser("delete-model")
277-
# delete_finetune_model_parser.add_argument(
278-
# "--model",
279-
# "-m",
280-
# default=None,
281-
# help="Model name",
282-
# type=str,
283-
# required=True,
284-
# )
285-
# delete_finetune_model_parser.set_defaults(func=_run_delete_model)
286-
287-
288243
def _run_create(args: argparse.Namespace) -> None:
289244
finetune = Finetune()
290245

@@ -300,18 +255,11 @@ def _run_create(args: argparse.Namespace) -> None:
300255

301256
response = finetune.create(
302257
training_file=args.training_file, # training file_id
303-
# validation_file=args.validation_file, # validation file_id
304258
model=args.model,
305259
n_epochs=args.n_epochs,
306260
n_checkpoints=args.n_checkpoints,
307261
batch_size=args.batch_size,
308262
learning_rate=args.learning_rate,
309-
# warmup_steps=args.warmup_steps,
310-
# train_warmup_steps=args.train_warmup_steps,
311-
# seq_length=args.sequence_length,
312-
# seed=args.seed,
313-
# fp16=not args.fp32,
314-
# checkpoint_steps=args.checkpoint_steps,
315263
suffix=args.suffix,
316264
estimate_price=args.estimate_price,
317265
wandb_api_key=args.wandb_api_key if not args.no_wandb_api_key else None,
@@ -323,15 +271,38 @@ def _run_create(args: argparse.Namespace) -> None:
323271

324272
def _run_list(args: argparse.Namespace) -> None:
325273
response = Finetune.list()
326-
data_list = response["data"]
327-
sorted_data = sorted(data_list, key=lambda x: parse_timestamp(x["created_at"]))
328-
response["data"] = sorted_data
329-
print(json.dumps(response, indent=4))
274+
response["data"].sort(key=lambda x: parse_timestamp(x["created_at"]))
275+
if args.raw:
276+
print(json.dumps(response, indent=4))
277+
else:
278+
display_list = []
279+
for i in response["data"]:
280+
display_list.append(
281+
{
282+
"Fine-tune ID": i.get("id"),
283+
"Model Output Name": i.get("model_output_name"),
284+
"Price": finetune_price_to_dollars(
285+
float(str(i.get("total_price")))
286+
), # convert to string for mypy typing
287+
"Created At": i.get("created_at"),
288+
}
289+
)
290+
table = tabulate(display_list, headers="keys", tablefmt="grid", showindex=True)
291+
print(table)
330292

331293

332294
def _run_retrieve(args: argparse.Namespace) -> None:
333295
response = Finetune.retrieve(args.fine_tune_id)
334-
print(json.dumps(response, indent=4))
296+
if args.raw:
297+
print(json.dumps(response, indent=4))
298+
else:
299+
table_data = [
300+
{"Key": key, "Value": value}
301+
for key, value in response.items()
302+
if key not in ["events", "model_output_path"]
303+
]
304+
table = tabulate(table_data, tablefmt="grid")
305+
print(table)
335306

336307

337308
def _run_cancel(args: argparse.Namespace) -> None:
@@ -341,7 +312,20 @@ def _run_cancel(args: argparse.Namespace) -> None:
341312

342313
def _run_list_events(args: argparse.Namespace) -> None:
343314
response = Finetune.list_events(args.fine_tune_id)
344-
print(json.dumps(response, indent=4))
315+
if args.raw:
316+
print(json.dumps(response, indent=4))
317+
else:
318+
display_list = []
319+
for i in response["data"]:
320+
display_list.append(
321+
{
322+
"Message": i.get("message"),
323+
"Type": i.get("type"),
324+
"Hash": i.get("hash"),
325+
}
326+
)
327+
table = tabulate(display_list, headers="keys", tablefmt="grid", showindex=True)
328+
print(table)
345329

346330

347331
def _run_download(args: argparse.Namespace) -> None:
@@ -358,9 +342,3 @@ def _run_checkpoint(args: argparse.Namespace) -> None:
358342
checkpoints = Finetune.get_checkpoints(args.fine_tune_id)
359343
print(json.dumps(checkpoints, indent=4))
360344
print(f"\n{len(checkpoints)} checkpoints found")
361-
362-
363-
# def _run_delete_model(args: argparse.Namespace) -> None:
364-
# finetune = Finetune(args.endpoint)
365-
# response = finetune.delete_finetune_model(args.model)
366-
# print(json.dumps(response))

0 commit comments

Comments
 (0)