Skip to content

Commit 93dc463

Browse files
committed
feat(cli): Add Analytic tracking to CLI commands
1 parent 57dcc89 commit 93dc463

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+487
-24
lines changed

pyproject.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ dependencies = [
2525
"filelock>=3.13.1",
2626
"types-pyyaml>=6.0.12.20250915",
2727
"tomli>=2.0.0; python_version < '3.11'",
28+
"py-machineid>=1.0.0",
29+
"detect-agent>=0.1.0",
2830
]
2931

3032
requires-python = ">= 3.9"

src/together/lib/cli/__init__.py

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,16 @@
99

1010
import together
1111
from together._version import __version__
12+
from together.lib.utils import log_debug
1213
from together._constants import DEFAULT_TIMEOUT
1314
from together._utils._logs import setup_logging
1415
from together.lib.cli.api.beta import beta
1516
from together.lib.cli.api.evals import evals
1617
from together.lib.cli.api.files import files
18+
from together.lib.cli._track_cli import CliTrackingEvents, track_cli
1719
from together.lib.cli.api.models import models
1820
from together.lib.cli.api.endpoints import endpoints
21+
from together.lib.cli.api.telemetry import telemetry
1922
from together.lib.cli.api.fine_tuning import fine_tuning
2023

2124

@@ -58,13 +61,13 @@ def main(
5861
debug: bool | None,
5962
max_retries: int | None,
6063
) -> None:
61-
"""This is a sample CLI tool."""
64+
"""Together AI command-line interface."""
6265
if debug:
6366
os.environ.setdefault("TOGETHER_LOG", "debug")
6467
setup_logging() # Must run this again here to allow the new logging configuration to take effect
6568

6669
try:
67-
ctx.obj = together.Together(
70+
client = together.Together(
6871
api_key=api_key,
6972
base_url=base_url,
7073
timeout=timeout,
@@ -79,7 +82,7 @@ def main(
7982
# Instead we opt to create a dummy client and hook into any requests performed by the client. We take that moment to print the error and exit.
8083
except Exception as e:
8184
if "api_key" in str(e):
82-
ctx.obj = together.Together(
85+
client = together.Together(
8386
api_key="0000000000000000000000000000000000000000",
8487
base_url=base_url,
8588
timeout=timeout,
@@ -98,12 +101,26 @@ def block_requests_for_api_key(_: httpx.Request) -> None:
98101
click.secho(f"\nUsage: together --api-key <your-api-key> {invoked_command_name}", fg="yellow")
99102
sys.exit(1)
100103

101-
ctx.obj._client.event_hooks["request"].append(block_requests_for_api_key)
102-
return
104+
client._client.event_hooks["request"].append(block_requests_for_api_key)
105+
else:
106+
raise e
103107

104-
raise e
108+
# Wrap the client's httpx requests to track the parameters sent on api requests
109+
def track_request(request: httpx.Request) -> None:
110+
try:
111+
track_cli(
112+
CliTrackingEvents.ApiRequest,
113+
{"url": str(request.url), "method": request.method, "body": request.content.decode("utf-8")},
114+
)
115+
except Exception as e:
116+
log_debug("Error tracking api request", error=e)
117+
118+
client._client.event_hooks["request"].append(track_request)
105119

120+
ctx.obj = client
106121

122+
123+
main.add_command(telemetry)
107124
main.add_command(files)
108125
main.add_command(fine_tuning)
109126
main.add_command(models)
@@ -112,4 +129,12 @@ def block_requests_for_api_key(_: httpx.Request) -> None:
112129
main.add_command(beta)
113130

114131
if __name__ == "__main__":
115-
main()
132+
# When running the script, call the command with standalone_mode=False
133+
# to prevent Click's default top-level exception handling from suppressing
134+
# your try/except block.
135+
try:
136+
main(standalone_mode=False)
137+
except SystemExit as e:
138+
# Re-raise SystemExit if it's not a success exit (code 0)
139+
if e.code != 0:
140+
raise

src/together/lib/cli/_track_cli.py

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
from __future__ import annotations
2+
3+
import os
4+
import re
5+
import sys
6+
import json
7+
import time
8+
import uuid
9+
import platform
10+
import threading
11+
from enum import Enum
12+
from typing import Any, TypeVar, Callable, cast
13+
from pathlib import Path
14+
from functools import wraps
15+
16+
import httpx
17+
import machineid
18+
from detect_agent import determine_agent
19+
20+
from together import __version__
21+
from together.lib.utils import log_debug
22+
23+
F = TypeVar("F", bound=Callable[..., Any])
24+
25+
SESSION_ID = int(str(uuid.uuid4().int)[0:13])
26+
27+
_ENV_TELEMETRY_OFF = frozenset({"1", "true", "yes"})
28+
_ERROR_MESSAGE_MAX_LEN = 500
29+
_CONFIG_DIR_NAME = "together"
30+
_CONFIG_FILE_NAME = "cli.json"
31+
32+
33+
def telemetry_config_path() -> Path:
34+
if sys.platform == "win32":
35+
appdata = os.environ.get("APPDATA")
36+
if appdata:
37+
return Path(appdata) / "Together" / _CONFIG_FILE_NAME
38+
xdg = os.environ.get("XDG_CONFIG_HOME")
39+
if xdg:
40+
return Path(xdg) / _CONFIG_DIR_NAME / _CONFIG_FILE_NAME
41+
return Path.home() / ".config" / _CONFIG_DIR_NAME / _CONFIG_FILE_NAME
42+
43+
44+
def load_telemetry_config() -> dict[str, Any]:
45+
path = telemetry_config_path()
46+
if not path.is_file():
47+
return {}
48+
try:
49+
raw = path.read_text(encoding="utf-8")
50+
data = json.loads(raw)
51+
if not isinstance(data, dict):
52+
return {}
53+
return cast(dict[str, Any], data)
54+
except (OSError, json.JSONDecodeError):
55+
return {}
56+
57+
58+
def save_telemetry_config(data: dict[str, Any]) -> None:
59+
path = telemetry_config_path()
60+
path.parent.mkdir(parents=True, exist_ok=True)
61+
tmp = path.with_name(path.name + ".tmp")
62+
tmp.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
63+
tmp.replace(path)
64+
if sys.platform != "win32":
65+
try:
66+
path.chmod(0o600)
67+
except OSError:
68+
pass
69+
70+
71+
def is_tracking_enabled() -> bool:
72+
if _env_telemetry_disabled():
73+
log_debug("Analytics tracking disabled by environment variable")
74+
return False
75+
if _config_telemetry_disabled():
76+
log_debug("Analytics tracking disabled by config file")
77+
return False
78+
return True
79+
80+
class CliTrackingEvents(Enum):
81+
CommandStarted = "cli_command_started"
82+
CommandCompleted = "cli_command_completed"
83+
CommandFailed = "cli_command_failed"
84+
CommandUserAborted = "cli_command_user_aborted"
85+
ApiRequest = "cli_command_api_request"
86+
87+
88+
def track_cli(event_name: CliTrackingEvents, args: dict[str, Any]) -> None:
89+
"""Track a CLI event. Non-Blocking."""
90+
if not is_tracking_enabled():
91+
return
92+
93+
def send_event() -> None:
94+
analytics_api_env = os.getenv("TOGETHER_TELEMETRY_API")
95+
analytics_api = (
96+
analytics_api_env
97+
if analytics_api_env
98+
else "https://api.together.ai/together/gateway/pub/v1/httpRequest"
99+
)
100+
101+
try:
102+
agent_info = determine_agent()
103+
agent_name = ""
104+
if agent_info["agent"]:
105+
agent_name = agent_info["agent"]["name"]
106+
107+
log_debug("Analytics event sending", event_name=event_name.value, args=args)
108+
109+
payload = {
110+
"event_source": "cli",
111+
"event_type": event_name.value,
112+
"event_properties": {
113+
"is_ci": os.getenv("CI") is not None,
114+
"is_agent": agent_info["is_agent"],
115+
"agent_name": agent_name,
116+
"os": platform.system(),
117+
"arch": platform.machine() or "",
118+
**args,
119+
},
120+
"context": {
121+
"session_id": str(SESSION_ID),
122+
"runtime": {
123+
"name": "together-cli",
124+
"version": __version__,
125+
},
126+
},
127+
"identity": {
128+
"device_id": machineid.id().lower(),
129+
},
130+
"event_options": {
131+
"time": int(time.time() * 1000),
132+
},
133+
}
134+
body = json.dumps(payload)
135+
with httpx.Client() as client:
136+
client.post(
137+
analytics_api,
138+
headers={
139+
"content-type": "application/json",
140+
"user-agent": f"together-cli:{__version__}",
141+
},
142+
content=body,
143+
)
144+
except Exception as e:
145+
log_debug("Error sending analytics event", error=e)
146+
147+
threading.Thread(target=send_event).start()
148+
149+
150+
def auto_track_command(command: str) -> Callable[[F], F]:
151+
"""Decorator for click commands to automatically track CLI commands start/completion/failure."""
152+
153+
def decorator(f: F) -> F:
154+
@wraps(f)
155+
def wrapper(*args: Any, **kwargs: Any) -> Any:
156+
track_cli(CliTrackingEvents.CommandStarted, {"command": command, "arguments": kwargs})
157+
try:
158+
result = f(*args, **kwargs)
159+
except KeyboardInterrupt as e:
160+
track_cli(
161+
CliTrackingEvents.CommandUserAborted,
162+
{"command": command, "arguments": kwargs},
163+
)
164+
raise e
165+
166+
except Exception as e:
167+
track_cli(
168+
CliTrackingEvents.CommandFailed,
169+
{
170+
"command": command,
171+
"arguments": kwargs,
172+
"error": _sanitize_cli_error_message(str(e)),
173+
},
174+
)
175+
raise e
176+
177+
track_cli(CliTrackingEvents.CommandCompleted, {"command": command, "arguments": kwargs})
178+
return result
179+
180+
return wrapper # type: ignore
181+
182+
return decorator # type: ignore
183+
184+
185+
def _sanitize_cli_error_message(msg: str) -> str:
186+
s = msg.strip()
187+
if len(s) > _ERROR_MESSAGE_MAX_LEN:
188+
s = s[: _ERROR_MESSAGE_MAX_LEN] + "…"
189+
s = re.sub(r"(?i)(bearer\s+)[A-Za-z0-9._\-/+]{20,}", r"\1<redacted>", s)
190+
s = re.sub(
191+
r"(?i)(api[_-]?key\s*[\"':=]\s*|api[_-]?key\s+)([A-Za-z0-9._\-]{20,})",
192+
r"\1<redacted>",
193+
s,
194+
)
195+
s = re.sub(r"(?i)(Authorization:\s*)([^\s]+)", r"\1<redacted>", s)
196+
return s
197+
198+
def _env_telemetry_disabled() -> bool:
199+
v = os.getenv("TOGETHER_TELEMETRY_DISABLED", "").strip().lower()
200+
return v in _ENV_TELEMETRY_OFF
201+
202+
203+
def _config_telemetry_disabled() -> bool:
204+
return load_telemetry_config().get("telemetry_enabled") is False

src/together/lib/cli/api/beta/clusters/create.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
from rich import print
99

1010
from together import Together
11+
from together.lib.cli._track_cli import auto_track_command
1112
from together.lib.cli.api._utils import handle_api_errors
1213
from together.types.beta.cluster_create_params import SharedVolume, ClusterCreateParams
1314

@@ -61,6 +62,7 @@
6162
)
6263
@click.pass_context
6364
@handle_api_errors("Clusters")
65+
@auto_track_command("clusters create")
6466
def create(
6567
ctx: click.Context,
6668
name: str | None = None,

src/together/lib/cli/api/beta/clusters/delete.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import click
44

55
from together import Together
6+
from together.lib.cli._track_cli import auto_track_command
67
from together.lib.cli.api._utils import handle_api_errors
78

89

@@ -15,6 +16,7 @@
1516
)
1617
@click.pass_context
1718
@handle_api_errors("Clusters")
19+
@auto_track_command("clusters delete")
1820
def delete(ctx: click.Context, cluster_id: str, json: bool) -> None:
1921
"""Delete a cluster by ID"""
2022
client: Together = ctx.obj

src/together/lib/cli/api/beta/clusters/get_credentials.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import click
1111

1212
from together import Together, TogetherError
13+
from together.lib.cli._track_cli import auto_track_command
1314
from together.lib.cli.api._utils import handle_api_errors
1415

1516

@@ -39,6 +40,7 @@
3940
)
4041
@click.pass_context
4142
@handle_api_errors("Clusters")
43+
@auto_track_command("clusters get-credentials")
4244
def get_credentials(
4345
ctx: click.Context,
4446
cluster_id: str,

src/together/lib/cli/api/beta/clusters/list.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import click
44

55
from together import Together
6+
from together.lib.cli._track_cli import auto_track_command
67

78

89
@click.command()
@@ -12,6 +13,7 @@
1213
help="Output in JSON format",
1314
)
1415
@click.pass_context
16+
@auto_track_command("clusters list")
1517
def list(ctx: click.Context, json: bool) -> None:
1618
"""List clusters"""
1719
client: Together = ctx.obj

src/together/lib/cli/api/beta/clusters/list_regions.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from tabulate import tabulate
66

77
from together import Together
8+
from together.lib.cli._track_cli import auto_track_command
89
from together.lib.cli.api._utils import handle_api_errors
910

1011

@@ -16,6 +17,7 @@
1617
)
1718
@click.pass_context
1819
@handle_api_errors("Clusters")
20+
@auto_track_command("clusters list-regions")
1921
def list_regions(ctx: click.Context, json: bool) -> None:
2022
"""List regions"""
2123
client: Together = ctx.obj

src/together/lib/cli/api/beta/clusters/retrieve.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
from rich import print
55

66
from together import Together
7+
from together.lib.cli._track_cli import auto_track_command
78
from together.lib.cli.api._utils import handle_api_errors
89

910

@@ -16,6 +17,7 @@
1617
)
1718
@click.pass_context
1819
@handle_api_errors("Clusters")
20+
@auto_track_command("clusters retrieve")
1921
def retrieve(ctx: click.Context, cluster_id: str, json: bool) -> None:
2022
"""Retrieve a cluster by ID"""
2123
client: Together = ctx.obj

0 commit comments

Comments
 (0)