Coverage for langsmith/client.py: 20%
2427 statements
« prev ^ index » next coverage.py v7.10.1, created at 2025-12-11 16:15 -0800
1"""Client for interacting with the LangSmith API.
3Use the client to customize API keys / workspace connections, SSL certs,
4etc. for tracing.
6Also used to create, read, update, and delete LangSmith resources
7such as runs (~trace spans), datasets, examples (~records),
8feedback (~metrics), projects (tracer sessions/groups), etc.
10For detailed API documentation, visit the [LangSmith docs](https://docs.langchain.com/langsmith/home).
11"""
13from __future__ import annotations
15import atexit
16import collections
17import concurrent.futures as cf
18import contextlib
19import datetime
20import functools
21import importlib
22import importlib.metadata
23import io
24import itertools
25import json
26import logging
27import os
28import random
29import threading
30import time
31import traceback
32import typing
33import uuid
34import warnings
35import weakref
36from collections.abc import AsyncIterable, Iterable, Iterator, Mapping, Sequence
37from inspect import signature
38from pathlib import Path
39from queue import PriorityQueue
40from typing import (
41 TYPE_CHECKING,
42 Annotated,
43 Any,
44 Callable,
45 Literal,
46 Optional,
47 Union,
48 cast,
49)
50from urllib import parse as urllib_parse
52import requests
53from pydantic import Field
54from requests import adapters as requests_adapters
55from requests_toolbelt import ( # type: ignore[import-untyped]
56 multipart as rqtb_multipart,
57)
58from typing_extensions import TypeGuard, overload
59from urllib3.poolmanager import PoolKey # type: ignore[attr-defined, import-untyped]
60from urllib3.util import Retry # type: ignore[import-untyped]
62import langsmith
63from langsmith import env as ls_env
64from langsmith import schemas as ls_schemas
65from langsmith import utils as ls_utils
66from langsmith._internal import _orjson
67from langsmith._internal._background_thread import (
68 TracingQueueItem,
69)
70from langsmith._internal._background_thread import (
71 tracing_control_thread_func as _tracing_control_thread_func,
72)
73from langsmith._internal._beta_decorator import warn_beta
74from langsmith._internal._compressed_traces import CompressedTraces
75from langsmith._internal._constants import (
76 _AUTO_SCALE_UP_NTHREADS_LIMIT,
77 _BLOCKSIZE_BYTES,
78 _BOUNDARY,
79 _SIZE_LIMIT_BYTES,
80)
81from langsmith._internal._multipart import (
82 MultipartPart,
83 MultipartPartsAndContext,
84 join_multipart_parts_and_context,
85)
86from langsmith._internal._operations import (
87 SerializedFeedbackOperation,
88 SerializedRunOperation,
89 combine_serialized_queue_operations,
90 compress_multipart_parts_and_context,
91 serialize_feedback_dict,
92 serialize_run_dict,
93 serialized_feedback_operation_to_multipart_parts_and_context,
94 serialized_run_operation_to_multipart_parts_and_context,
95)
96from langsmith._internal._serde import dumps_json as _dumps_json
97from langsmith._internal._uuid import uuid7
98from langsmith.schemas import AttachmentInfo, ExampleWithRuns
100_OPENAI_API_KEY = "OPENAI_API_KEY"
101_ANTHROPIC_API_KEY = "ANTHROPIC_API_KEY"
104def _check_otel_enabled() -> bool:
105 """Check if OTEL is enabled and imports are available."""
106 return ls_utils.is_env_var_truish("OTEL_ENABLED")
109def _import_otel():
110 """Dynamically import OTEL modules when needed."""
111 try:
112 from opentelemetry import trace as otel_trace # type: ignore[import]
113 from opentelemetry.trace import set_span_in_context # type: ignore[import]
115 from langsmith._internal.otel._otel_client import (
116 get_otlp_tracer_provider,
117 )
118 from langsmith._internal.otel._otel_exporter import OTELExporter
120 return otel_trace, set_span_in_context, get_otlp_tracer_provider, OTELExporter
121 except ImportError:
122 raise ImportError(
123 "To use OTEL tracing, you must install it with `pip install langsmith[otel]`"
124 )
127try:
128 from zoneinfo import ZoneInfo # type: ignore[import-not-found]
129except ImportError:
131 class ZoneInfo: # type: ignore[no-redef]
132 """Introduced in python 3.9."""
135try:
136 from opentelemetry.sdk.trace import TracerProvider # type: ignore[import-not-found]
137except ImportError:
139 class TracerProvider: # type: ignore[no-redef]
140 """Used for optional OTEL tracing."""
143if TYPE_CHECKING:
144 import pandas as pd # type: ignore
145 from langchain_core.runnables import Runnable
147 from langsmith import schemas
149 # OTEL imports for type hints
150 try:
151 from opentelemetry import trace as otel_trace # type: ignore[import]
153 from langsmith._internal.otel._otel_exporter import OTELExporter
154 except ImportError:
155 otel_trace = Any # type: ignore[assignment, misc]
156 OTELExporter = Any # type: ignore[assignment, misc]
157 from langsmith.evaluation import evaluator as ls_evaluator
158 from langsmith.evaluation._arunner import (
159 AEVALUATOR_T,
160 ATARGET_T,
161 AsyncExperimentResults,
162 )
163 from langsmith.evaluation._runner import (
164 COMPARATIVE_EVALUATOR_T,
165 DATA_T,
166 EVALUATOR_T,
167 EXPERIMENT_T,
168 SUMMARY_EVALUATOR_T,
169 TARGET_T,
170 ComparativeExperimentResults,
171 ExperimentResults,
172 )
175logger = logging.getLogger(__name__)
176_urllib3_logger = logging.getLogger("urllib3.connectionpool")
178X_API_KEY = "x-api-key"
179EMPTY_SEQ: tuple[dict, ...] = ()
180URLLIB3_SUPPORTS_BLOCKSIZE = "key_blocksize" in signature(PoolKey).parameters
181DEFAULT_INSTRUCTIONS = "How are people using my agent? What are they asking about?"
184def _parse_token_or_url(
185 url_or_token: Union[str, uuid.UUID],
186 api_url: str,
187 num_parts: int = 2,
188 kind: str = "dataset",
189) -> tuple[str, str]:
190 """Parse a public dataset URL or share token."""
191 try:
192 if isinstance(url_or_token, uuid.UUID) or uuid.UUID(url_or_token):
193 return api_url, str(url_or_token)
194 except ValueError:
195 pass
197 # Then it's a URL
198 parsed_url = urllib_parse.urlparse(str(url_or_token))
199 # Extract the UUID from the path
200 path_parts = parsed_url.path.split("/")
201 if len(path_parts) >= num_parts:
202 token_uuid = path_parts[-num_parts]
203 _as_uuid(token_uuid, var="token parts")
204 else:
205 raise ls_utils.LangSmithUserError(f"Invalid public {kind} URL: {url_or_token}")
206 if parsed_url.netloc == "smith.langchain.com":
207 api_url = "https://api.smith.langchain.com"
208 elif parsed_url.netloc == "beta.smith.langchain.com":
209 api_url = "https://beta.api.smith.langchain.com"
210 return api_url, token_uuid
213def _is_langchain_hosted(url: str) -> bool:
214 """Check if the URL is langchain hosted.
216 Args:
217 url (str): The URL to check.
219 Returns:
220 bool: True if the URL is langchain hosted, False otherwise.
221 """
222 try:
223 netloc = urllib_parse.urlsplit(url).netloc.split(":")[0]
224 return netloc == "langchain.com" or netloc.endswith(".langchain.com")
225 except Exception:
226 return False
229ID_TYPE = Union[uuid.UUID, str]
230RUN_TYPE_T = Literal[
231 "tool", "chain", "llm", "retriever", "embedding", "prompt", "parser"
232]
235@functools.lru_cache(maxsize=1)
236def _default_retry_config() -> Retry:
237 """Get the default retry configuration.
239 If urllib3 version is 1.26 or greater, retry on all methods.
241 Returns:
242 Retry: The default retry configuration.
243 """
244 retry_params = dict(
245 total=3,
246 status_forcelist=[502, 503, 504, 408, 425],
247 backoff_factor=0.5,
248 # Sadly urllib3 1.x doesn't support backoff_jitter
249 raise_on_redirect=False,
250 raise_on_status=False,
251 respect_retry_after_header=True,
252 )
254 # the `allowed_methods` keyword is not available in urllib3 < 1.26
256 # check to see if urllib3 version is 1.26 or greater
257 urllib3_version = importlib.metadata.version("urllib3")
258 use_allowed_methods = tuple(map(int, urllib3_version.split("."))) >= (1, 26)
260 if use_allowed_methods:
261 # Retry on all methods
262 retry_params["allowed_methods"] = None
264 return ls_utils.LangSmithRetry(**retry_params) # type: ignore
267def close_session(session: requests.Session) -> None:
268 """Close the session.
270 Args:
271 session (requests.Session): The session to close.
272 """
273 logger.debug("Closing Client.session")
274 session.close()
277def _validate_api_key_if_hosted(api_url: str, api_key: Optional[str]) -> None:
278 """Verify API key is provided if url not localhost.
280 Args:
281 api_url (str): The API URL.
282 api_key (Optional[str]): The API key.
284 Returns:
285 None
287 Raises:
288 LangSmithUserError: If the API key is not provided when using the hosted service.
289 """
290 # If the domain is langchain.com, raise error if no api_key
291 if not api_key:
292 if (
293 _is_langchain_hosted(api_url)
294 and not ls_utils.is_env_var_truish("OTEL_ENABLED")
295 and ls_utils.tracing_is_enabled()
296 ):
297 warnings.warn(
298 "API key must be provided when using hosted LangSmith API",
299 ls_utils.LangSmithMissingAPIKeyWarning,
300 )
303def _format_feedback_score(score: Union[float, int, bool, None]):
304 """Format a feedback score by truncating numerical values to 4 decimal places.
306 Args:
307 score: The score to format, can be a number or any other type
309 Returns:
310 The formatted score
311 """
312 if isinstance(score, float):
313 # Truncate at 4 decimal places
314 return round(score, 4)
315 return score
318def _get_tracing_sampling_rate(
319 tracing_sampling_rate: Optional[float] = None,
320) -> float | None:
321 """Get the tracing sampling rate.
323 Returns:
324 Optional[float]: The tracing sampling rate.
325 """
326 if tracing_sampling_rate is None:
327 sampling_rate_str = ls_utils.get_env_var("TRACING_SAMPLING_RATE")
328 if not sampling_rate_str:
329 return None
330 else:
331 sampling_rate_str = str(tracing_sampling_rate)
332 sampling_rate = float(sampling_rate_str)
333 if sampling_rate < 0 or sampling_rate > 1:
334 raise ls_utils.LangSmithUserError(
335 "LANGSMITH_TRACING_SAMPLING_RATE must be between 0 and 1 if set."
336 f" Got: {sampling_rate}"
337 )
338 return sampling_rate
341def _get_write_api_urls(_write_api_urls: Optional[dict[str, str]]) -> dict[str, str]:
342 # Note: LANGSMITH_RUNS_ENDPOINTS is now handled via replicas, not _write_api_urls
343 _write_api_urls = _write_api_urls or {}
344 processed_write_api_urls = {}
345 for url, api_key in _write_api_urls.items():
346 processed_url = url.strip()
347 if not processed_url:
348 raise ls_utils.LangSmithUserError("LangSmith runs API URL cannot be empty")
349 processed_url = processed_url.strip().strip('"').strip("'").rstrip("/")
350 processed_api_key = api_key.strip().strip('"').strip("'")
351 _validate_api_key_if_hosted(processed_url, processed_api_key)
352 processed_write_api_urls[processed_url] = processed_api_key
354 return processed_write_api_urls
357def _as_uuid(value: ID_TYPE, var: Optional[str] = None) -> uuid.UUID:
358 try:
359 return uuid.UUID(value) if not isinstance(value, uuid.UUID) else value
360 except ValueError as e:
361 var = var or "value"
362 raise ls_utils.LangSmithUserError(
363 f"{var} must be a valid UUID or UUID string. Got {value}"
364 ) from e
367@typing.overload
368def _ensure_uuid(value: Optional[Union[str, uuid.UUID]]) -> uuid.UUID: ...
371@typing.overload
372def _ensure_uuid(
373 value: Optional[Union[str, uuid.UUID]], *, accept_null: bool = True
374) -> Optional[uuid.UUID]: ...
377def _ensure_uuid(value: Optional[Union[str, uuid.UUID]], *, accept_null: bool = False):
378 if value is None:
379 if accept_null:
380 return None
381 return uuid7()
382 return _as_uuid(value)
385@functools.lru_cache(maxsize=1)
386def _parse_url(url):
387 parsed_url = urllib_parse.urlparse(url)
388 host = parsed_url.netloc.split(":")[0]
389 return host
392class _LangSmithHttpAdapter(requests_adapters.HTTPAdapter):
393 __attrs__ = [
394 "max_retries",
395 "config",
396 "_pool_connections",
397 "_pool_maxsize",
398 "_pool_block",
399 "_blocksize",
400 ]
402 def __init__(
403 self,
404 pool_connections: int = requests_adapters.DEFAULT_POOLSIZE,
405 pool_maxsize: int = requests_adapters.DEFAULT_POOLSIZE,
406 max_retries: Union[Retry, int, None] = requests_adapters.DEFAULT_RETRIES,
407 pool_block: bool = requests_adapters.DEFAULT_POOLBLOCK,
408 blocksize: int = 16384, # default from urllib3.BaseHTTPSConnection
409 ) -> None:
410 self._blocksize = blocksize
411 super().__init__(pool_connections, pool_maxsize, max_retries, pool_block)
413 def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs):
414 if URLLIB3_SUPPORTS_BLOCKSIZE:
415 # urllib3 before 2.0 doesn't support blocksize
416 pool_kwargs["blocksize"] = self._blocksize
417 return super().init_poolmanager(connections, maxsize, block, **pool_kwargs)
420class Client:
421 """Client for interacting with the LangSmith API."""
423 __slots__ = [
424 "__weakref__",
425 "api_url",
426 "_api_key",
427 "_workspace_id",
428 "_headers",
429 "_custom_headers",
430 "retry_config",
431 "timeout_ms",
432 "_timeout",
433 "session",
434 "_get_data_type_cached",
435 "_web_url",
436 "_tenant_id",
437 "tracing_sample_rate",
438 "_filtered_post_uuids",
439 "tracing_queue",
440 "_anonymizer",
441 "_hide_inputs",
442 "_hide_outputs",
443 "_hide_metadata",
444 "_omit_traced_runtime_info",
445 "_process_buffered_run_ops",
446 "_run_ops_buffer_size",
447 "_run_ops_buffer_timeout_ms",
448 "_run_ops_buffer_last_flush_time",
449 "_info",
450 "_write_api_urls",
451 "_settings",
452 "_manual_cleanup",
453 "_pyo3_client",
454 "compressed_traces",
455 "_data_available_event",
456 "_futures",
457 "_run_ops_buffer",
458 "_run_ops_buffer_lock",
459 "otel_exporter",
460 "_otel_trace",
461 "_set_span_in_context",
462 "_max_batch_size_bytes",
463 "_tracing_error_callback",
464 ]
466 _api_key: Optional[str]
467 _headers: dict[str, str]
468 _custom_headers: dict[str, str]
469 _timeout: tuple[float, float]
470 _manual_cleanup: bool
472 def __init__(
473 self,
474 api_url: Optional[str] = None,
475 *,
476 api_key: Optional[str] = None,
477 retry_config: Optional[Retry] = None,
478 timeout_ms: Optional[Union[int, tuple[int, int]]] = None,
479 web_url: Optional[str] = None,
480 session: Optional[requests.Session] = None,
481 auto_batch_tracing: bool = True,
482 anonymizer: Optional[Callable[[dict], dict]] = None,
483 hide_inputs: Optional[Union[Callable[[dict], dict], bool]] = None,
484 hide_outputs: Optional[Union[Callable[[dict], dict], bool]] = None,
485 hide_metadata: Optional[Union[Callable[[dict], dict], bool]] = None,
486 omit_traced_runtime_info: bool = False,
487 process_buffered_run_ops: Optional[
488 Callable[[Sequence[dict]], Sequence[dict]]
489 ] = None,
490 run_ops_buffer_size: Optional[int] = None,
491 run_ops_buffer_timeout_ms: Optional[float] = None,
492 info: Optional[Union[dict, ls_schemas.LangSmithInfo]] = None,
493 api_urls: Optional[dict[str, str]] = None,
494 otel_tracer_provider: Optional[TracerProvider] = None,
495 otel_enabled: Optional[bool] = None,
496 tracing_sampling_rate: Optional[float] = None,
497 workspace_id: Optional[str] = None,
498 max_batch_size_bytes: Optional[int] = None,
499 headers: Optional[dict[str, str]] = None,
500 tracing_error_callback: Optional[Callable[[Exception], None]] = None,
501 ) -> None:
502 """Initialize a `Client` instance.
504 Args:
505 api_url (Optional[str]): URL for the LangSmith API. Defaults to the `LANGCHAIN_ENDPOINT`
506 environment variable or `https://api.smith.langchain.com` if not set.
507 api_key (Optional[str]): API key for the LangSmith API. Defaults to the `LANGCHAIN_API_KEY`
508 environment variable.
509 retry_config (Optional[Retry]): Retry configuration for the `HTTPAdapter`.
510 timeout_ms (Optional[Union[int, Tuple[int, int]]]): Timeout for the `HTTPAdapter`.
512 Can also be a 2-tuple of `(connect timeout, read timeout)` to set them separately.
513 web_url (Optional[str]): URL for the LangSmith web app. Default is auto-inferred from
514 the `ENDPOINT`.
515 session (Optional[requests.Session]): The session to use for requests.
517 If `None`, a new session will be created.
518 auto_batch_tracing (bool, default=True): Whether to automatically batch tracing.
519 anonymizer (Optional[Callable[[dict], dict]]): A function applied for masking serialized run inputs and outputs,
520 before sending to the API.
521 hide_inputs (Optional[Union[Callable[[dict], dict], bool]]): Whether to hide run inputs when tracing with this client.
523 If `True`, hides the entire inputs.
525 If a function, applied to all run inputs when creating runs.
526 hide_outputs (Optional[Union[Callable[[dict], dict], bool]]): Whether to hide run outputs when tracing with this client.
528 If `True`, hides the entire outputs.
530 If a function, applied to all run outputs when creating runs.
531 hide_metadata (Optional[Union[Callable[[dict], dict], bool]]): Whether to hide run metadata when tracing with this client.
533 If `True`, hides the entire metadata.
535 If a function, applied to all run metadata when creating runs.
536 omit_traced_runtime_info (bool): Whether to omit runtime information from traced runs.
538 If `True`, runtime information (SDK version, platform, Python version, etc.)
539 will not be stored in the `extra.runtime` field of runs.
541 Defaults to `False`.
542 process_buffered_run_ops (Optional[Callable[[Sequence[dict]], Sequence[dict]]]): A function applied to buffered run operations
543 that allows for modification of the raw run dicts before they are converted to multipart and compressed.
545 Useful specifically for high throughput tracing where you need to apply a rate-limited API or other
546 costly process to the runs before they are sent to the API.
548 Note that the buffer will only flush automatically when `run_ops_buffer_size` is reached or a new run is added to the
549 buffer after `run_ops_buffer_timeout_ms` has elapsed - it will not flush outside of these conditions unless you manually
550 call `client.flush()`, so be sure to do this before your code exits.
551 run_ops_buffer_size (Optional[int]): Maximum number of run operations to collect in the buffer before applying
552 `process_buffered_run_ops` and sending to the API.
554 Required when `process_buffered_run_ops` is provided.
555 run_ops_buffer_timeout_ms (Optional[int]): Maximum time in milliseconds to wait before flushing the run ops buffer
556 when new runs are added.
558 Defaults to `5000`.
560 Only used when `process_buffered_run_ops` is provided.
561 info: The information about the LangSmith API.
563 If not provided, it will be fetched from the API.
564 api_urls (Optional[Dict[str, str]]): A dictionary of write API URLs and their corresponding API keys.
566 Useful for multi-tenant setups. Data is only read from the first
567 URL in the dictionary. However, ONLY Runs are written (`POST` and `PATCH`)
568 to all URLs in the dictionary. Feedback, sessions, datasets, examples,
569 annotation queues and evaluation results are only written to the first.
570 otel_tracer_provider (Optional[TracerProvider]): Optional tracer provider for OpenTelemetry integration.
572 If not provided, a LangSmith-specific tracer provider will be used.
573 tracing_sampling_rate (Optional[float]): The sampling rate for tracing.
575 If provided, overrides the `LANGCHAIN_TRACING_SAMPLING_RATE` environment variable.
577 Should be a float between `0` and `1`, where `1` means trace everything
578 and `0` means trace nothing.
579 workspace_id (Optional[str]): The workspace ID.
581 Required for org-scoped API keys.
582 max_batch_size_bytes (Optional[int]): The maximum size of a batch of runs in bytes.
584 If not provided, the default is set by the server.
585 headers (Optional[Dict[str, str]]): Additional HTTP headers to include in all requests.
586 These headers will be merged with the default headers (User-Agent, Accept, x-api-key, etc.).
587 Custom headers will not override the default required headers.
588 tracing_error_callback (Optional[Callable[[Exception], None]]): Optional callback function to handle errors.
590 Called when exceptions occur during tracing operations.
592 Raises:
593 LangSmithUserError: If the API key is not provided when using the hosted service.
594 LangSmithUserError: If both `api_url` and `api_urls` are provided.
595 """
596 if api_url and api_urls:
597 raise ls_utils.LangSmithUserError(
598 "You cannot provide both api_url and api_urls."
599 )
601 if (
602 os.getenv("LANGSMITH_ENDPOINT") or os.getenv("LANGCHAIN_ENDPOINT")
603 ) and os.getenv("LANGSMITH_RUNS_ENDPOINTS"):
604 raise ls_utils.LangSmithUserError(
605 "You cannot provide both LANGSMITH_ENDPOINT / LANGCHAIN_ENDPOINT "
606 "and LANGSMITH_RUNS_ENDPOINTS."
607 )
609 self.tracing_sample_rate = _get_tracing_sampling_rate(tracing_sampling_rate)
610 self._filtered_post_uuids: set[uuid.UUID] = set()
611 self._write_api_urls: Mapping[str, Optional[str]] = _get_write_api_urls(
612 api_urls
613 )
614 # Initialize workspace attribute first
615 self._workspace_id = ls_utils.get_workspace_id(workspace_id)
616 # Store custom headers
617 self._custom_headers = headers or {}
619 if self._write_api_urls:
620 self.api_url = next(iter(self._write_api_urls))
621 self.api_key = self._write_api_urls[self.api_url]
622 else:
623 self.api_url = ls_utils.get_api_url(api_url)
624 self.api_key = ls_utils.get_api_key(api_key)
625 _validate_api_key_if_hosted(self.api_url, self.api_key)
626 self._write_api_urls = {self.api_url: self.api_key}
627 self.retry_config = retry_config or _default_retry_config()
628 self.timeout_ms = (
629 (timeout_ms, timeout_ms)
630 if isinstance(timeout_ms, int)
631 else (timeout_ms or (10_000, 90_001))
632 )
633 self._timeout = (self.timeout_ms[0] / 1000, self.timeout_ms[1] / 1000)
634 self._web_url = web_url
635 self._tenant_id: Optional[uuid.UUID] = None
636 # Create a session and register a finalizer to close it
637 session_ = session if session else requests.Session()
638 self.session = session_
639 self._info = (
640 info
641 if info is None or isinstance(info, ls_schemas.LangSmithInfo)
642 else ls_schemas.LangSmithInfo(**info)
643 )
644 weakref.finalize(self, close_session, self.session)
645 atexit.register(close_session, session_)
646 self.compressed_traces: Optional[CompressedTraces] = None
647 self._data_available_event: Optional[threading.Event] = None
648 self._futures: Optional[weakref.WeakSet[cf.Future]] = None
649 self._run_ops_buffer: list[tuple[str, dict]] = []
650 self._run_ops_buffer_lock = threading.Lock()
651 self.otel_exporter: Optional[OTELExporter] = None
652 self._max_batch_size_bytes = max_batch_size_bytes
654 # Initialize auto batching
655 if auto_batch_tracing:
656 self.tracing_queue: Optional[PriorityQueue] = PriorityQueue()
658 threading.Thread(
659 target=_tracing_control_thread_func,
660 # arg must be a weakref to self to avoid the Thread object
661 # preventing garbage collection of the Client object
662 args=(weakref.ref(self),),
663 ).start()
664 else:
665 self.tracing_queue = None
667 # Mount the HTTPAdapter with the retry configuration.
668 adapter = _LangSmithHttpAdapter(
669 max_retries=self.retry_config,
670 blocksize=_BLOCKSIZE_BYTES,
671 # We need to set the pool_maxsize to a value greater than the
672 # number of threads used for batch tracing, plus 1 for other
673 # requests.
674 pool_maxsize=_AUTO_SCALE_UP_NTHREADS_LIMIT + 1,
675 )
676 self.session.mount("http://", adapter)
677 self.session.mount("https://", adapter)
678 self._get_data_type_cached = functools.lru_cache(maxsize=10)(
679 self._get_data_type
680 )
681 self._anonymizer = anonymizer
682 self._hide_inputs = (
683 hide_inputs
684 if hide_inputs is not None
685 else ls_utils.get_env_var("HIDE_INPUTS") == "true"
686 )
687 self._hide_outputs = (
688 hide_outputs
689 if hide_outputs is not None
690 else ls_utils.get_env_var("HIDE_OUTPUTS") == "true"
691 )
692 self._hide_metadata = (
693 hide_metadata
694 if hide_metadata is not None
695 else ls_utils.get_env_var("HIDE_METADATA") == "true"
696 )
697 self._omit_traced_runtime_info = omit_traced_runtime_info
698 self._process_buffered_run_ops = process_buffered_run_ops
699 self._run_ops_buffer_size = run_ops_buffer_size
700 self._run_ops_buffer_timeout_ms = run_ops_buffer_timeout_ms or 5000
701 self._run_ops_buffer_last_flush_time = time.time()
703 # Validate that run_ops_buffer_size is provided when process_buffered_run_ops is used
704 if process_buffered_run_ops is not None and run_ops_buffer_size is None:
705 raise ValueError(
706 "run_ops_buffer_size must be provided when process_buffered_run_ops is specified"
707 )
708 if process_buffered_run_ops is None and run_ops_buffer_size is not None:
709 raise ValueError(
710 "process_buffered_run_ops must be provided when run_ops_buffer_size is specified"
711 )
713 # To trigger this code, set the `LANGSMITH_USE_PYO3_CLIENT` env var to any value.
714 self._pyo3_client = None
715 if ls_utils.get_env_var("USE_PYO3_CLIENT") is not None:
716 langsmith_pyo3 = None
717 try:
718 import langsmith_pyo3 # type: ignore[import-not-found, no-redef]
719 except ImportError as e:
720 logger.warning(
721 "Failed to import `langsmith_pyo3` when PyO3 client was requested, "
722 "falling back to Python impl: %s",
723 repr(e),
724 )
726 if langsmith_pyo3:
727 # TODO: tweak these constants as needed
728 queue_capacity = 1_000_000
729 batch_size = 100
730 batch_timeout_millis = 1000
731 worker_threads = 1
733 try:
734 self._pyo3_client = langsmith_pyo3.BlockingTracingClient(
735 self.api_url,
736 self.api_key,
737 queue_capacity,
738 batch_size,
739 batch_timeout_millis,
740 worker_threads,
741 )
742 except Exception as e:
743 logger.warning(
744 "Failed to instantiate `langsmith_pyo3.BlockingTracingClient` "
745 "when PyO3 client was requested, falling back to Python impl: %s",
746 repr(e),
747 )
749 self._settings: Union[ls_schemas.LangSmithSettings, None] = None
751 self._manual_cleanup = False
753 if _check_otel_enabled() or otel_enabled:
754 try:
755 (
756 otel_trace,
757 set_span_in_context,
758 get_otlp_tracer_provider,
759 OTELExporter,
760 ) = _import_otel()
762 existing_provider = otel_trace.get_tracer_provider()
763 tracer = existing_provider.get_tracer(__name__)
764 if otel_tracer_provider is None:
765 # Use existing global provider if available
766 if not (
767 isinstance(existing_provider, otel_trace.ProxyTracerProvider)
768 and hasattr(tracer, "_tracer")
769 and isinstance(
770 cast(
771 otel_trace.ProxyTracer, # type: ignore[attr-defined, name-defined]
772 tracer,
773 )._tracer,
774 otel_trace.NoOpTracer,
775 )
776 ):
777 otel_tracer_provider = cast(TracerProvider, existing_provider)
778 else:
779 otel_tracer_provider = get_otlp_tracer_provider()
780 otel_trace.set_tracer_provider(otel_tracer_provider)
782 self.otel_exporter = OTELExporter(tracer_provider=otel_tracer_provider)
784 # Store imports for later use
785 self._otel_trace = otel_trace
786 self._set_span_in_context = set_span_in_context
788 except ImportError:
789 warnings.warn(
790 "LANGSMITH_OTEL_ENABLED is set but OpenTelemetry packages are not installed: Install with `pip install langsmith[otel]"
791 )
792 self.otel_exporter = None
793 else:
794 self.otel_exporter = None
796 self._tracing_error_callback = tracing_error_callback
798 def _repr_html_(self) -> str:
799 """Return an HTML representation of the instance with a link to the URL.
801 Returns:
802 str: The HTML representation of the instance.
803 """
804 link = self._host_url
805 return f'<a href="{link}", target="_blank" rel="noopener">LangSmith Client</a>'
807 def _invoke_tracing_error_callback(self, error: Exception) -> None:
808 """Invoke the background tracing error callback if configured.
810 Args:
811 error: The exception that occurred during background tracing.
812 """
813 if self._tracing_error_callback:
814 try:
815 self._tracing_error_callback(error)
816 except Exception:
817 logger.error(
818 "Error in tracing_error_callback:\n",
819 exc_info=True,
820 )
822 def __repr__(self) -> str:
823 """Return a string representation of the instance with a link to the URL.
825 Returns:
826 str: The string representation of the instance.
827 """
828 return f"Client (API URL: {self.api_url})"
830 @property
831 def _host(self) -> str:
832 return _parse_url(self.api_url)
834 @property
835 def _host_url(self) -> str:
836 """The web host url."""
837 return ls_utils.get_host_url(self._web_url, self.api_url)
839 def _compute_headers(self) -> dict[str, str]:
840 headers = {
841 "User-Agent": f"langsmith-py/{langsmith.__version__}",
842 "Accept": "application/json",
843 }
844 # Merge custom headers first so they don't override required headers
845 headers.update(self._custom_headers)
846 # Required headers that should not be overridden
847 if self.api_key:
848 headers[X_API_KEY] = self.api_key
849 if self._workspace_id:
850 headers["X-Tenant-Id"] = self._workspace_id
851 return headers
853 def _set_header_affecting_attr(self, attr_name: str, value: Any) -> None:
854 """Set attributes that affect headers and recalculate them."""
855 object.__setattr__(self, attr_name, value)
856 object.__setattr__(self, "_headers", self._compute_headers())
858 @property
859 def api_key(self) -> Optional[str]:
860 """Return the API key used for authentication."""
861 return self._api_key
863 @api_key.setter
864 def api_key(self, value: Optional[str]) -> None:
865 self._set_header_affecting_attr("_api_key", value)
867 @property
868 def workspace_id(self) -> Optional[str]:
869 """Return the workspace ID used for API requests."""
870 return self._workspace_id
872 @workspace_id.setter
873 def workspace_id(self, value: Optional[str]) -> None:
874 self._set_header_affecting_attr("_workspace_id", value)
876 @property
877 def info(self) -> ls_schemas.LangSmithInfo:
878 """Get the information about the LangSmith API.
880 Returns:
881 The information about the LangSmith API, or `None` if the API is not available.
882 """
883 if self._info is not None:
884 return self._info
886 # Skip API call when using OTEL-only mode
887 otel_only_mode = ls_utils.is_env_var_truish(
888 "OTEL_ENABLED"
889 ) and ls_utils.is_env_var_truish("OTEL_ONLY")
891 if otel_only_mode:
892 self._info = ls_schemas.LangSmithInfo()
893 return self._info
895 # Fetch info from API
896 try:
897 response = self.request_with_retries(
898 "GET",
899 "/info",
900 headers={"Accept": "application/json"},
901 timeout=self._timeout,
902 )
903 ls_utils.raise_for_status_with_text(response)
904 self._info = ls_schemas.LangSmithInfo(**response.json())
905 except BaseException as e:
906 logger.warning(
907 f"Failed to get info from {self.api_url}: {repr(e)}",
908 )
909 self._info = ls_schemas.LangSmithInfo()
911 return self._info
913 def _get_settings(self) -> ls_schemas.LangSmithSettings:
914 """Get the settings for the current tenant.
916 Returns:
917 dict: The settings for the current tenant.
918 """
919 if self._settings is None:
920 response = self.request_with_retries("GET", "/settings")
921 ls_utils.raise_for_status_with_text(response)
922 self._settings = ls_schemas.LangSmithSettings(**response.json())
924 return self._settings
926 def _content_above_size(self, content_length: Optional[int]) -> Optional[str]:
927 if content_length is None or self._info is None:
928 return None
929 info = cast(ls_schemas.LangSmithInfo, self._info)
930 bic = info.batch_ingest_config
931 if not bic:
932 return None
933 size_limit = self._max_batch_size_bytes or bic.get("size_limit_bytes")
934 if size_limit is None:
935 return None
936 if content_length > size_limit:
937 return (
938 f"The content length of {content_length} bytes exceeds the "
939 f"maximum size limit of {size_limit} bytes."
940 )
941 return None
943 def request_with_retries(
944 self,
945 /,
946 method: Literal["GET", "POST", "PUT", "PATCH", "DELETE"],
947 pathname: str,
948 *,
949 request_kwargs: Optional[Mapping] = None,
950 stop_after_attempt: int = 1,
951 retry_on: Optional[Sequence[type[BaseException]]] = None,
952 to_ignore: Optional[Sequence[type[BaseException]]] = None,
953 handle_response: Optional[Callable[[requests.Response, int], Any]] = None,
954 _context: str = "",
955 **kwargs: Any,
956 ) -> requests.Response:
957 """Send a request with retries.
959 Args:
960 method (str): The HTTP request method.
961 pathname (str): The pathname of the request URL. Will be appended to the API URL.
962 request_kwargs (Mapping): Additional request parameters.
963 stop_after_attempt (int, default=1): The number of attempts to make.
964 retry_on (Optional[Sequence[Type[BaseException]]]): The exceptions to retry on.
966 In addition to: `[LangSmithConnectionError, LangSmithAPIError]`.
967 to_ignore (Optional[Sequence[Type[BaseException]]]): The exceptions to ignore / pass on.
968 handle_response (Optional[Callable[[requests.Response, int], Any]]): A function to handle the response and return whether to continue retrying.
969 _context (str, default=""): The context of the request.
970 **kwargs (Any): Additional keyword arguments to pass to the request.
972 Returns:
973 The response object.
975 Raises:
976 LangSmithAPIError: If a server error occurs.
977 LangSmithUserError: If the request fails.
978 LangSmithConnectionError: If a connection error occurs.
979 LangSmithError: If the request fails.
980 """
981 request_kwargs = request_kwargs or {}
982 request_kwargs = {
983 "timeout": self._timeout,
984 **request_kwargs,
985 **kwargs,
986 "headers": {
987 **self._headers,
988 **request_kwargs.get("headers", {}),
989 **kwargs.get("headers", {}),
990 },
991 }
992 if (
993 method != "GET"
994 and "data" in request_kwargs
995 and "files" not in request_kwargs
996 and not request_kwargs["headers"].get("Content-Type")
997 ):
998 request_kwargs["headers"]["Content-Type"] = "application/json"
999 logging_filters = [
1000 ls_utils.FilterLangSmithRetry(),
1001 ls_utils.FilterPoolFullWarning(host=str(self._host)),
1002 ]
1003 retry_on_: tuple[type[BaseException], ...] = (
1004 *(retry_on or ()),
1005 *(
1006 ls_utils.LangSmithConnectionError,
1007 ls_utils.LangSmithRequestTimeout, # 408
1008 ls_utils.LangSmithAPIError, # 500
1009 ),
1010 )
1011 to_ignore_: tuple[type[BaseException], ...] = (*(to_ignore or ()),)
1012 response = None
1013 for idx in range(stop_after_attempt):
1014 try:
1015 try:
1016 with ls_utils.filter_logs(_urllib3_logger, logging_filters):
1017 response = self.session.request(
1018 method,
1019 _construct_url(self.api_url, pathname),
1020 stream=False,
1021 **request_kwargs,
1022 )
1023 ls_utils.raise_for_status_with_text(response)
1024 return response
1025 except requests.exceptions.ReadTimeout as e:
1026 logger.debug("Passing on exception %s", e)
1027 if idx + 1 == stop_after_attempt:
1028 raise
1029 sleep_time = 2**idx + (random.random() * 0.5)
1030 time.sleep(sleep_time)
1031 continue
1033 except requests.HTTPError as e:
1034 if response is not None:
1035 if handle_response is not None:
1036 if idx + 1 < stop_after_attempt:
1037 should_continue = handle_response(response, idx + 1)
1038 if should_continue:
1039 continue
1040 if response.status_code == 500:
1041 raise ls_utils.LangSmithAPIError(
1042 f"Server error caused failure to {method}"
1043 f" {pathname} in"
1044 f" LangSmith API. {repr(e)}"
1045 f"{_context}"
1046 )
1047 elif response.status_code == 408:
1048 raise ls_utils.LangSmithRequestTimeout(
1049 f"Client took too long to send request to {method}"
1050 f"{pathname} {_context}"
1051 )
1052 elif response.status_code == 429:
1053 raise ls_utils.LangSmithRateLimitError(
1054 f"Rate limit exceeded for {pathname}. {repr(e)}"
1055 f"{_context}"
1056 )
1057 elif response.status_code == 401:
1058 raise ls_utils.LangSmithAuthError(
1059 f"Authentication failed for {pathname}. {repr(e)}"
1060 f"{_context}"
1061 )
1062 elif response.status_code == 404:
1063 raise ls_utils.LangSmithNotFoundError(
1064 f"Resource not found for {pathname}. {repr(e)}"
1065 f"{_context}"
1066 )
1067 elif response.status_code == 409:
1068 raise ls_utils.LangSmithConflictError(
1069 f"Conflict for {pathname}. {repr(e)}{_context}"
1070 )
1071 elif response.status_code == 403:
1072 try:
1073 error_data = response.json()
1074 error_code = error_data.get("error", "")
1075 if error_code == "org_scoped_key_requires_workspace":
1076 raise ls_utils.LangSmithUserError(
1077 "This API key is org-scoped and requires workspace specification. "
1078 "Please provide 'workspace_id' parameter, "
1079 "or set LANGSMITH_WORKSPACE_ID environment variable."
1080 )
1081 except (ValueError, KeyError):
1082 pass
1083 raise ls_utils.LangSmithError(
1084 f"Failed to {method} {pathname} in LangSmith"
1085 f" API. {repr(e)}"
1086 )
1087 else:
1088 raise ls_utils.LangSmithError(
1089 f"Failed to {method} {pathname} in LangSmith"
1090 f" API. {repr(e)}"
1091 )
1093 else:
1094 raise ls_utils.LangSmithUserError(
1095 f"Failed to {method} {pathname} in LangSmith API. {repr(e)}"
1096 )
1097 except requests.ConnectionError as e:
1098 recommendation = (
1099 "Please confirm your LANGCHAIN_ENDPOINT."
1100 if self.api_url != "https://api.smith.langchain.com"
1101 else "Please confirm your internet connection."
1102 )
1103 try:
1104 content_length = int(
1105 str(e.request.headers.get("Content-Length"))
1106 if e.request
1107 else ""
1108 )
1109 size_rec = self._content_above_size(content_length)
1110 if size_rec:
1111 recommendation = size_rec
1112 except ValueError:
1113 content_length = None
1115 api_key = (
1116 e.request.headers.get("x-api-key") or "" if e.request else ""
1117 )
1118 prefix, suffix = api_key[:5], api_key[-2:]
1119 filler = "*" * (max(0, len(api_key) - 7))
1120 masked_api_key = f"{prefix}{filler}{suffix}"
1122 raise ls_utils.LangSmithConnectionError(
1123 f"Connection error caused failure to {method} {pathname}"
1124 f" in LangSmith API. {recommendation}"
1125 f" {repr(e)}"
1126 f"\nContent-Length: {content_length}"
1127 f"\nAPI Key: {masked_api_key}"
1128 f"{_context}"
1129 ) from e
1130 except Exception as e:
1131 args = list(e.args)
1132 msg = args[1] if len(args) > 1 else ""
1133 msg = msg.replace("session", "session (project)")
1134 if args:
1135 emsg = "\n".join(
1136 [str(args[0])]
1137 + [msg]
1138 + [str(arg) for arg in (args[2:] if len(args) > 2 else [])]
1139 )
1140 else:
1141 emsg = msg
1142 raise ls_utils.LangSmithError(
1143 f"Failed to {method} {pathname} in LangSmith API. {emsg}"
1144 f"{_context}"
1145 ) from e
1146 except to_ignore_ as e:
1147 if response is not None:
1148 logger.debug("Passing on exception %s", e)
1149 return response
1150 except ls_utils.LangSmithRateLimitError:
1151 if idx + 1 == stop_after_attempt:
1152 raise
1153 if response is not None:
1154 try:
1155 retry_after = float(response.headers.get("retry-after", "30"))
1156 except Exception as e:
1157 logger.warning(
1158 "Invalid retry-after header: %s",
1159 repr(e),
1160 )
1161 retry_after = 30
1162 # Add exponential backoff
1163 retry_after = retry_after * 2**idx + random.random()
1164 time.sleep(retry_after)
1165 except retry_on_:
1166 # Handle other exceptions more immediately
1167 if idx + 1 == stop_after_attempt:
1168 raise
1169 sleep_time = 2**idx + (random.random() * 0.5)
1170 time.sleep(sleep_time)
1171 continue
1172 # Else we still raise an error
1174 raise ls_utils.LangSmithError(
1175 f"Failed to {method} {pathname} in LangSmith API."
1176 )
1178 def _get_paginated_list(
1179 self, path: str, *, params: Optional[dict] = None
1180 ) -> Iterator[dict]:
1181 """Get a paginated list of items.
1183 Args:
1184 path (str): The path of the request URL.
1185 params (Optional[dict]): The query parameters.
1187 Yields:
1188 The items in the paginated list.
1189 """
1190 params_ = params.copy() if params else {}
1191 offset = params_.get("offset", 0)
1192 params_["limit"] = params_.get("limit", 100)
1193 while True:
1194 params_["offset"] = offset
1195 response = self.request_with_retries(
1196 "GET",
1197 path,
1198 params=params_,
1199 )
1200 items = response.json()
1201 if not items:
1202 break
1203 yield from items
1204 if len(items) < params_["limit"]:
1205 # offset and limit isn't respected if we're
1206 # querying for specific values
1207 break
1208 offset += len(items)
1210 def _get_cursor_paginated_list(
1211 self,
1212 path: str,
1213 *,
1214 body: Optional[dict] = None,
1215 request_method: Literal["GET", "POST"] = "POST",
1216 data_key: str = "runs",
1217 ) -> Iterator[dict]:
1218 """Get a cursor paginated list of items.
1220 Args:
1221 path (str): The path of the request URL.
1222 body (Optional[dict]): The query body.
1223 request_method (Literal["GET", "POST"], default="POST"): The HTTP request method.
1224 data_key (str, default="runs"): The key in the response body that contains the items.
1226 Yields:
1227 The items in the paginated list.
1228 """
1229 params_ = body.copy() if body else {}
1230 while True:
1231 response = self.request_with_retries(
1232 request_method,
1233 path,
1234 request_kwargs={
1235 "data": _dumps_json(params_),
1236 },
1237 )
1238 response_body = response.json()
1239 if not response_body:
1240 break
1241 if not response_body.get(data_key):
1242 break
1243 yield from response_body[data_key]
1244 cursors = response_body.get("cursors")
1245 if not cursors:
1246 break
1247 if not cursors.get("next"):
1248 break
1249 params_["cursor"] = cursors["next"]
1251 def upload_dataframe(
1252 self,
1253 df: pd.DataFrame,
1254 name: str,
1255 input_keys: Sequence[str],
1256 output_keys: Sequence[str],
1257 *,
1258 description: Optional[str] = None,
1259 data_type: Optional[ls_schemas.DataType] = ls_schemas.DataType.kv,
1260 ) -> ls_schemas.Dataset:
1261 """Upload a dataframe as individual examples to the LangSmith API.
1263 Args:
1264 df (pd.DataFrame): The dataframe to upload.
1265 name (str): The name of the dataset.
1266 input_keys (Sequence[str]): The input keys.
1267 output_keys (Sequence[str]): The output keys.
1268 description (Optional[str]): The description of the dataset.
1269 data_type (Optional[DataType]): The data type of the dataset.
1271 Returns:
1272 Dataset: The uploaded dataset.
1274 Raises:
1275 ValueError: If the `csv_file` is not a `str` or `tuple`.
1277 Example:
1278 ```python
1279 from langsmith import Client
1280 import os
1281 import pandas as pd
1283 client = Client()
1285 df = pd.read_parquet("path/to/your/myfile.parquet")
1286 input_keys = ["column1", "column2"] # replace with your input column names
1287 output_keys = ["output1", "output2"] # replace with your output column names
1289 dataset = client.upload_dataframe(
1290 df=df,
1291 input_keys=input_keys,
1292 output_keys=output_keys,
1293 name="My Parquet Dataset",
1294 description="Dataset created from a parquet file",
1295 data_type="kv", # The default
1296 )
1297 ```
1298 """
1299 csv_file = io.BytesIO()
1300 df.to_csv(csv_file, index=False)
1301 csv_file.seek(0)
1302 return self.upload_csv(
1303 ("data.csv", csv_file),
1304 input_keys=input_keys,
1305 output_keys=output_keys,
1306 description=description,
1307 name=name,
1308 data_type=data_type,
1309 )
1311 def upload_csv(
1312 self,
1313 csv_file: Union[str, tuple[str, io.BytesIO]],
1314 input_keys: Sequence[str],
1315 output_keys: Sequence[str],
1316 *,
1317 name: Optional[str] = None,
1318 description: Optional[str] = None,
1319 data_type: Optional[ls_schemas.DataType] = ls_schemas.DataType.kv,
1320 ) -> ls_schemas.Dataset:
1321 """Upload a CSV file to the LangSmith API.
1323 Args:
1324 csv_file (Union[str, Tuple[str, io.BytesIO]]): The CSV file to upload.
1326 If a string, it should be the path.
1328 If a tuple, it should be a tuple containing the filename
1329 and a `BytesIO` object.
1330 input_keys (Sequence[str]): The input keys.
1331 output_keys (Sequence[str]): The output keys.
1332 name (Optional[str]): The name of the dataset.
1333 description (Optional[str]): The description of the dataset.
1334 data_type (Optional[ls_schemas.DataType]): The data type of the dataset.
1336 Returns:
1337 Dataset: The uploaded dataset.
1339 Raises:
1340 ValueError: If the `csv_file` is not a string or tuple.
1342 Example:
1343 ```python
1344 from langsmith import Client
1345 import os
1347 client = Client()
1349 csv_file = "path/to/your/myfile.csv"
1350 input_keys = ["column1", "column2"] # replace with your input column names
1351 output_keys = ["output1", "output2"] # replace with your output column names
1353 dataset = client.upload_csv(
1354 csv_file=csv_file,
1355 input_keys=input_keys,
1356 output_keys=output_keys,
1357 name="My CSV Dataset",
1358 description="Dataset created from a CSV file",
1359 data_type="kv", # The default
1360 )
1361 ```
1362 """
1363 data = {
1364 "input_keys": input_keys,
1365 "output_keys": output_keys,
1366 }
1367 if name:
1368 data["name"] = name
1369 if description:
1370 data["description"] = description
1371 if data_type:
1372 data["data_type"] = ls_utils.get_enum_value(data_type)
1373 data["id"] = str(uuid.uuid4())
1374 if isinstance(csv_file, str):
1375 with open(csv_file, "rb") as f:
1376 file_ = {"file": f}
1377 response = self.request_with_retries(
1378 "POST",
1379 "/datasets/upload",
1380 data=data,
1381 files=file_,
1382 )
1383 elif isinstance(csv_file, tuple):
1384 response = self.request_with_retries(
1385 "POST",
1386 "/datasets/upload",
1387 data=data,
1388 files={"file": csv_file},
1389 )
1390 else:
1391 raise ValueError("csv_file must be a string or tuple")
1392 ls_utils.raise_for_status_with_text(response)
1393 result = response.json()
1394 # TODO: Make this more robust server-side
1395 if "detail" in result and "already exists" in result["detail"]:
1396 file_name = csv_file if isinstance(csv_file, str) else csv_file[0]
1397 file_name = file_name.split("/")[-1]
1398 raise ValueError(f"Dataset {file_name} already exists")
1399 return ls_schemas.Dataset(
1400 **result,
1401 _host_url=self._host_url,
1402 _tenant_id=self._get_optional_tenant_id(),
1403 )
1405 def _run_transform(
1406 self,
1407 run: Union[ls_schemas.Run, dict, ls_schemas.RunLikeDict],
1408 update: bool = False,
1409 copy: bool = False,
1410 ) -> dict:
1411 """Transform the given run object into a dictionary representation.
1413 Args:
1414 run (Union[ls_schemas.Run, dict]): The run object to transform.
1415 update (Optional[bool]): Whether the payload is for an "update" event.
1416 copy (Optional[bool]): Whether to deepcopy run inputs/outputs.
1418 Returns:
1419 dict: The transformed run object as a dictionary.
1420 """
1421 if hasattr(run, "dict") and callable(getattr(run, "dict")):
1422 run_create: dict = run.dict() # type: ignore
1423 else:
1424 run_create = cast(dict, run)
1425 if "id" not in run_create:
1426 run_create["id"] = uuid.uuid4()
1427 elif isinstance(run_create["id"], str):
1428 run_create["id"] = uuid.UUID(run_create["id"])
1429 if "inputs" in run_create and run_create["inputs"] is not None:
1430 if copy:
1431 run_create["inputs"] = ls_utils.deepish_copy(run_create["inputs"])
1432 run_create["inputs"] = self._hide_run_inputs(run_create["inputs"])
1433 if "outputs" in run_create and run_create["outputs"] is not None:
1434 if copy:
1435 run_create["outputs"] = ls_utils.deepish_copy(run_create["outputs"])
1436 run_create["outputs"] = self._hide_run_outputs(run_create["outputs"])
1437 # Hide metadata in extra if present
1438 if "extra" in run_create and isinstance(run_create["extra"], dict):
1439 extra = run_create["extra"]
1440 if "metadata" in extra and extra["metadata"] is not None:
1441 if copy:
1442 extra["metadata"] = ls_utils.deepish_copy(extra["metadata"])
1443 extra["metadata"] = self._hide_run_metadata(extra["metadata"])
1444 if not update and not run_create.get("start_time"):
1445 run_create["start_time"] = datetime.datetime.now(datetime.timezone.utc)
1447 # Only retain LLM & Prompt manifests
1448 if "serialized" in run_create:
1449 if run_create.get("run_type") not in ("llm", "prompt"):
1450 # Drop completely
1451 run_create.pop("serialized", None)
1452 elif run_create.get("serialized"):
1453 # Drop graph
1454 run_create["serialized"].pop("graph", None)
1456 return run_create
1458 def _insert_runtime_env(self, runs: Sequence[dict]) -> None:
1459 if self._omit_traced_runtime_info:
1460 return
1461 runtime_env = ls_env.get_runtime_environment()
1462 for run_create in runs:
1463 run_extra = cast(dict, run_create.setdefault("extra", {}))
1464 # update runtime
1465 runtime: dict = run_extra.setdefault("runtime", {})
1466 run_extra["runtime"] = {**runtime_env, **runtime}
1467 # update metadata
1468 metadata: dict = run_extra.setdefault("metadata", {})
1469 langchain_metadata = ls_env.get_langchain_env_var_metadata()
1470 metadata.update(
1471 {k: v for k, v in langchain_metadata.items() if k not in metadata}
1472 )
1474 def _should_sample(self) -> bool:
1475 if self.tracing_sample_rate is None:
1476 return True
1477 return random.random() < self.tracing_sample_rate
1479 def _filter_for_sampling(
1480 self, runs: Iterable[dict], *, patch: bool = False
1481 ) -> list[dict]:
1482 if self.tracing_sample_rate is None:
1483 return list(runs)
1485 if patch:
1486 sampled = []
1487 for run in runs:
1488 trace_id = _as_uuid(run["trace_id"])
1489 if trace_id not in self._filtered_post_uuids:
1490 sampled.append(run)
1491 elif run["id"] == trace_id:
1492 self._filtered_post_uuids.remove(trace_id)
1493 return sampled
1494 else:
1495 sampled = []
1496 for run in runs:
1497 trace_id = run.get("trace_id") or run["id"]
1499 # If we've already made a decision about this trace, follow it
1500 if trace_id in self._filtered_post_uuids:
1501 continue
1503 # For new traces, apply sampling
1504 if run["id"] == trace_id:
1505 if self._should_sample():
1506 sampled.append(run)
1507 else:
1508 self._filtered_post_uuids.add(trace_id)
1509 else:
1510 # Child runs follow their trace's sampling decision
1511 sampled.append(run)
1512 return sampled
1514 def create_run(
1515 self,
1516 name: str,
1517 inputs: dict[str, Any],
1518 run_type: RUN_TYPE_T,
1519 *,
1520 project_name: Optional[str] = None,
1521 revision_id: Optional[str] = None,
1522 dangerously_allow_filesystem: bool = False,
1523 api_key: Optional[str] = None,
1524 api_url: Optional[str] = None,
1525 **kwargs: Any,
1526 ) -> None:
1527 """Persist a run to the LangSmith API.
1529 Args:
1530 name (str): The name of the run.
1531 inputs (Dict[str, Any]): The input values for the run.
1532 run_type (str): The type of the run, such as tool, chain, llm, retriever,
1533 embedding, prompt, or parser.
1534 project_name (Optional[str]): The project name of the run.
1535 revision_id (Optional[Union[UUID, str]]): The revision ID of the run.
1536 api_key (Optional[str]): The API key to use for this specific run.
1537 api_url (Optional[str]): The API URL to use for this specific run.
1538 **kwargs (Any): Additional keyword arguments.
1540 Returns:
1541 None
1543 Raises:
1544 LangSmithUserError: If the API key is not provided when using the hosted service.
1546 Example:
1547 ```python
1548 from langsmith import Client
1549 import datetime
1550 from uuid import uuid4
1552 client = Client()
1554 run_id = uuid4()
1555 client.create_run(
1556 id=run_id,
1557 project_name=project_name,
1558 name="test_run",
1559 run_type="llm",
1560 inputs={"prompt": "hello world"},
1561 outputs={"generation": "hi there"},
1562 start_time=datetime.datetime.now(datetime.timezone.utc),
1563 end_time=datetime.datetime.now(datetime.timezone.utc),
1564 hide_inputs=True,
1565 hide_outputs=True,
1566 )
1567 ```
1568 """
1569 project_name = project_name or kwargs.pop(
1570 "session_name",
1571 # if the project is not provided, use the environment's project
1572 ls_utils.get_tracer_project(),
1573 )
1574 run_create = {
1575 **kwargs,
1576 "session_name": project_name,
1577 "name": name,
1578 "inputs": inputs,
1579 "run_type": run_type,
1580 }
1581 if not self._filter_for_sampling([run_create]):
1582 return
1583 if revision_id is not None:
1584 run_create["extra"]["metadata"]["revision_id"] = revision_id
1585 run_create = self._run_transform(run_create, copy=False)
1586 self._insert_runtime_env([run_create])
1587 if run_create.get("attachments") is not None:
1588 for attachment in run_create["attachments"].values():
1589 if (
1590 isinstance(attachment, tuple)
1591 and isinstance(attachment[1], Path)
1592 and not dangerously_allow_filesystem
1593 ):
1594 raise ValueError(
1595 "Must set dangerously_allow_filesystem=True to allow passing in Paths for attachments."
1596 )
1597 # If process_buffered_run_ops is enabled, collect run ops in batches
1598 # before batching
1599 if self._process_buffered_run_ops and not kwargs.get("is_run_ops_buffer_flush"):
1600 with self._run_ops_buffer_lock:
1601 self._run_ops_buffer.append(("post", run_create))
1602 # Process batch when we have enough runs or enough time has passed
1603 if self._should_flush_run_ops_buffer():
1604 self._flush_run_ops_buffer()
1605 return
1606 else:
1607 self._create_run(run_create, api_key=api_key, api_url=api_url)
1609 def _create_run(
1610 self,
1611 run_create: dict,
1612 *,
1613 api_key: Optional[str] = None,
1614 api_url: Optional[str] = None,
1615 ) -> None:
1616 if (
1617 # batch ingest requires trace_id and dotted_order to be set
1618 run_create.get("trace_id") is not None
1619 and run_create.get("dotted_order") is not None
1620 ):
1621 if self._pyo3_client is not None:
1622 self._pyo3_client.create_run(run_create)
1623 elif (
1624 self.compressed_traces is not None
1625 and api_key is None
1626 and api_url is None
1627 ):
1628 if self._data_available_event is None:
1629 raise ValueError(
1630 "Run compression is enabled but threading event is not configured"
1631 )
1632 serialized_op = serialize_run_dict("post", run_create)
1633 (
1634 multipart_form,
1635 opened_files,
1636 ) = serialized_run_operation_to_multipart_parts_and_context(
1637 serialized_op
1638 )
1639 logger.log(
1640 5,
1641 "Adding compressed multipart to queue with context: %s",
1642 multipart_form.context,
1643 )
1644 with self.compressed_traces.lock:
1645 enqueued = compress_multipart_parts_and_context(
1646 multipart_form,
1647 self.compressed_traces,
1648 _BOUNDARY,
1649 )
1650 if enqueued:
1651 self.compressed_traces.trace_count += 1
1652 self._data_available_event.set()
1654 _close_files(list(opened_files.values()))
1655 elif self.tracing_queue is not None:
1656 serialized_op = serialize_run_dict("post", run_create)
1657 logger.log(
1658 5,
1659 "Adding to tracing queue: trace_id=%s, run_id=%s",
1660 serialized_op.trace_id,
1661 serialized_op.id,
1662 )
1663 if self.otel_exporter is not None:
1664 self.tracing_queue.put(
1665 TracingQueueItem(
1666 run_create["dotted_order"],
1667 serialized_op,
1668 api_key=api_key,
1669 api_url=api_url,
1670 otel_context=self._set_span_in_context(
1671 self._otel_trace.get_current_span()
1672 ),
1673 )
1674 )
1675 else:
1676 self.tracing_queue.put(
1677 TracingQueueItem(
1678 run_create["dotted_order"],
1679 serialized_op,
1680 api_key=api_key,
1681 api_url=api_url,
1682 )
1683 )
1684 else:
1685 # Neither Rust nor Python batch ingestion is configured,
1686 # fall back to the non-batch approach.
1687 self._create_run_non_batch(run_create, api_key=api_key, api_url=api_url)
1688 else:
1689 self._create_run_non_batch(run_create, api_key=api_key, api_url=api_url)
1691 def _create_run_non_batch(
1692 self,
1693 run_create: dict,
1694 *,
1695 api_key: Optional[str] = None,
1696 api_url: Optional[str] = None,
1697 ):
1698 errors = []
1699 # If specific api_key/api_url provided, use those; otherwise use all configured endpoints
1700 if api_key is not None or api_url is not None:
1701 target_api_url = api_url or self.api_url
1702 target_api_key = api_key or self.api_key
1703 headers = {**self._headers, X_API_KEY: target_api_key}
1704 try:
1705 self.request_with_retries(
1706 "POST",
1707 f"{target_api_url}/runs",
1708 request_kwargs={
1709 "data": _dumps_json(run_create),
1710 "headers": headers,
1711 },
1712 to_ignore=(ls_utils.LangSmithConflictError,),
1713 )
1714 except Exception as e:
1715 errors.append(e)
1716 else:
1717 # Use all configured write API URLs
1718 for write_api_url, write_api_key in self._write_api_urls.items():
1719 headers = {**self._headers, X_API_KEY: write_api_key}
1720 try:
1721 self.request_with_retries(
1722 "POST",
1723 f"{write_api_url}/runs",
1724 request_kwargs={
1725 "data": _dumps_json(run_create),
1726 "headers": headers,
1727 },
1728 to_ignore=(ls_utils.LangSmithConflictError,),
1729 )
1730 except Exception as e:
1731 errors.append(e)
1732 if errors:
1733 # Invoke callback for the errors
1734 if len(errors) > 1:
1735 exception_group = ls_utils.LangSmithExceptionGroup(exceptions=errors)
1736 self._invoke_tracing_error_callback(exception_group)
1737 raise exception_group
1738 else:
1739 self._invoke_tracing_error_callback(errors[0])
1740 raise errors[0]
1742 def _hide_run_inputs(self, inputs: dict):
1743 if self._hide_inputs is True:
1744 return {}
1745 if self._anonymizer:
1746 json_inputs = _orjson.loads(_dumps_json(inputs))
1747 return self._anonymizer(json_inputs)
1748 if self._hide_inputs is False:
1749 return inputs
1750 return self._hide_inputs(inputs)
1752 def _hide_run_outputs(self, outputs: dict):
1753 if self._hide_outputs is True:
1754 return {}
1755 if self._anonymizer:
1756 json_outputs = _orjson.loads(_dumps_json(outputs))
1757 return self._anonymizer(json_outputs)
1758 if self._hide_outputs is False:
1759 return outputs
1760 return self._hide_outputs(outputs)
1762 def _hide_run_metadata(self, metadata: dict) -> dict:
1763 if self._hide_metadata is True:
1764 return {}
1765 if self._hide_metadata is False:
1766 return metadata
1767 return self._hide_metadata(metadata)
1769 def _should_flush_run_ops_buffer(self) -> bool:
1770 """Check if the run ops buffer should be flushed based on size or time."""
1771 if not self._run_ops_buffer:
1772 return False
1774 # Check size-based flushing
1775 if (
1776 self._run_ops_buffer_size is not None
1777 and len(self._run_ops_buffer) >= self._run_ops_buffer_size
1778 ):
1779 return True
1781 # Check time-based flushing
1782 if self._run_ops_buffer_timeout_ms is not None:
1783 time_since_last_flush = time.time() - self._run_ops_buffer_last_flush_time
1784 if time_since_last_flush >= (self._run_ops_buffer_timeout_ms / 1000):
1785 return True
1787 return False
1789 def _flush_run_ops_buffer(self) -> None:
1790 """Process and flush run ops buffer in a background thread."""
1791 if not self._run_ops_buffer:
1792 return
1794 # Copy the buffer contents and clear it immediately to avoid blocking
1795 batch_to_process = list(self._run_ops_buffer)
1796 self._run_ops_buffer.clear()
1797 self._run_ops_buffer_last_flush_time = time.time()
1799 # Submit the processing to processing thread pool
1800 from langsmith._internal._background_thread import (
1801 LANGSMITH_CLIENT_THREAD_POOL,
1802 _process_buffered_run_ops_batch,
1803 )
1805 try:
1806 future = LANGSMITH_CLIENT_THREAD_POOL.submit(
1807 _process_buffered_run_ops_batch, self, batch_to_process
1808 )
1809 # Track the future if we have a futures set
1810 if self._futures is not None:
1811 self._futures.add(future)
1812 except RuntimeError:
1813 # Thread pool is shut down, process synchronously as fallback
1814 _process_buffered_run_ops_batch(self, batch_to_process)
1816 def _batch_ingest_run_ops(
1817 self,
1818 ops: list[SerializedRunOperation],
1819 *,
1820 api_url: Optional[str] = None,
1821 api_key: Optional[str] = None,
1822 ) -> None:
1823 ids_and_partial_body: dict[
1824 Literal["post", "patch"], list[tuple[str, bytes]]
1825 ] = {
1826 "post": [],
1827 "patch": [],
1828 }
1830 # form the partial body and ids
1831 for op in ops:
1832 if isinstance(op, SerializedRunOperation):
1833 curr_dict = _orjson.loads(op._none)
1834 if op.inputs:
1835 curr_dict["inputs"] = _orjson.Fragment(op.inputs)
1836 if op.outputs:
1837 curr_dict["outputs"] = _orjson.Fragment(op.outputs)
1838 if op.events:
1839 curr_dict["events"] = _orjson.Fragment(op.events)
1840 if op.extra:
1841 curr_dict["extra"] = _orjson.Fragment(op.extra)
1842 if op.error:
1843 curr_dict["error"] = _orjson.Fragment(op.error)
1844 if op.serialized:
1845 curr_dict["serialized"] = _orjson.Fragment(op.serialized)
1846 if op.attachments:
1847 logger.warning(
1848 "Attachments are not supported when use_multipart_endpoint "
1849 "is False"
1850 )
1851 ids_and_partial_body[op.operation].append(
1852 (f"trace={op.trace_id},id={op.id}", _orjson.dumps(curr_dict))
1853 )
1854 elif isinstance(op, SerializedFeedbackOperation):
1855 logger.warning(
1856 "Feedback operations are not supported in non-multipart mode"
1857 )
1858 else:
1859 logger.error("Unknown item type in tracing queue: %s", type(op))
1861 # send the requests in batches
1862 info = self.info
1863 size_limit_bytes = (
1864 self._max_batch_size_bytes
1865 or (info.batch_ingest_config or {}).get("size_limit_bytes")
1866 or _SIZE_LIMIT_BYTES
1867 )
1869 body_chunks: collections.defaultdict[str, list] = collections.defaultdict(list)
1870 context_ids: collections.defaultdict[str, list] = collections.defaultdict(list)
1871 body_size = 0
1872 for key in cast(list[Literal["post", "patch"]], ["post", "patch"]):
1873 body_deque = collections.deque(ids_and_partial_body[key])
1874 while body_deque:
1875 if (
1876 body_size > 0
1877 and body_size + len(body_deque[0][1]) > size_limit_bytes
1878 ):
1879 self._post_batch_ingest_runs(
1880 _orjson.dumps(body_chunks),
1881 _context=f"\n{key}: {'; '.join(context_ids[key])}",
1882 api_url=api_url,
1883 api_key=api_key,
1884 )
1885 body_size = 0
1886 body_chunks.clear()
1887 context_ids.clear()
1888 curr_id, curr_body = body_deque.popleft()
1889 body_size += len(curr_body)
1890 body_chunks[key].append(_orjson.Fragment(curr_body))
1891 context_ids[key].append(curr_id)
1892 if body_size:
1893 context = "; ".join(f"{k}: {'; '.join(v)}" for k, v in context_ids.items())
1894 self._post_batch_ingest_runs(
1895 _orjson.dumps(body_chunks),
1896 _context="\n" + context,
1897 api_url=api_url,
1898 api_key=api_key,
1899 )
1901 def batch_ingest_runs(
1902 self,
1903 create: Optional[
1904 Sequence[Union[ls_schemas.Run, ls_schemas.RunLikeDict, dict]]
1905 ] = None,
1906 update: Optional[
1907 Sequence[Union[ls_schemas.Run, ls_schemas.RunLikeDict, dict]]
1908 ] = None,
1909 *,
1910 pre_sampled: bool = False,
1911 ) -> None:
1912 """Batch ingest/upsert multiple runs in the Langsmith system.
1914 Args:
1915 create (Optional[Sequence[Union[Run, RunLikeDict]]]):
1916 A sequence of `Run` objects or equivalent dictionaries representing
1917 runs to be created / posted.
1918 update (Optional[Sequence[Union[Run, RunLikeDict]]]):
1919 A sequence of `Run` objects or equivalent dictionaries representing
1920 runs that have already been created and should be updated / patched.
1921 pre_sampled (bool, default=False): Whether the runs have already been subject
1922 to sampling, and therefore should not be sampled again.
1924 Raises:
1925 LangsmithAPIError: If there is an error in the API request.
1927 Returns:
1928 None
1930 !!! note
1932 The run objects MUST contain the `dotted_order` and `trace_id` fields
1933 to be accepted by the API.
1935 Example:
1936 ```python
1937 from langsmith import Client
1938 import datetime
1939 from uuid import uuid4
1941 client = Client()
1942 _session = "__test_batch_ingest_runs"
1943 trace_id = uuid4()
1944 trace_id_2 = uuid4()
1945 run_id_2 = uuid4()
1946 current_time = datetime.datetime.now(datetime.timezone.utc).strftime(
1947 "%Y%m%dT%H%M%S%fZ"
1948 )
1949 later_time = (
1950 datetime.datetime.now(datetime.timezone.utc) + timedelta(seconds=1)
1951 ).strftime("%Y%m%dT%H%M%S%fZ")
1953 runs_to_create = [
1954 {
1955 "id": str(trace_id),
1956 "session_name": _session,
1957 "name": "run 1",
1958 "run_type": "chain",
1959 "dotted_order": f"{current_time}{str(trace_id)}",
1960 "trace_id": str(trace_id),
1961 "inputs": {"input1": 1, "input2": 2},
1962 "outputs": {"output1": 3, "output2": 4},
1963 },
1964 {
1965 "id": str(trace_id_2),
1966 "session_name": _session,
1967 "name": "run 3",
1968 "run_type": "chain",
1969 "dotted_order": f"{current_time}{str(trace_id_2)}",
1970 "trace_id": str(trace_id_2),
1971 "inputs": {"input1": 1, "input2": 2},
1972 "error": "error",
1973 },
1974 {
1975 "id": str(run_id_2),
1976 "session_name": _session,
1977 "name": "run 2",
1978 "run_type": "chain",
1979 "dotted_order": f"{current_time}{str(trace_id)}."
1980 f"{later_time}{str(run_id_2)}",
1981 "trace_id": str(trace_id),
1982 "parent_run_id": str(trace_id),
1983 "inputs": {"input1": 5, "input2": 6},
1984 },
1985 ]
1986 runs_to_update = [
1987 {
1988 "id": str(run_id_2),
1989 "dotted_order": f"{current_time}{str(trace_id)}."
1990 f"{later_time}{str(run_id_2)}",
1991 "trace_id": str(trace_id),
1992 "parent_run_id": str(trace_id),
1993 "outputs": {"output1": 4, "output2": 5},
1994 },
1995 ]
1997 client.batch_ingest_runs(create=runs_to_create, update=runs_to_update)
1998 ```
1999 """
2000 if not create and not update:
2001 return
2002 # transform and convert to dicts
2003 create_dicts = [
2004 self._run_transform(run, copy=False) for run in create or EMPTY_SEQ
2005 ]
2006 update_dicts = [
2007 self._run_transform(run, update=True, copy=False)
2008 for run in update or EMPTY_SEQ
2009 ]
2010 for run in create_dicts:
2011 if not run.get("trace_id") or not run.get("dotted_order"):
2012 raise ls_utils.LangSmithUserError(
2013 "Batch ingest requires trace_id and dotted_order to be set."
2014 )
2015 for run in update_dicts:
2016 if not run.get("trace_id") or not run.get("dotted_order"):
2017 raise ls_utils.LangSmithUserError(
2018 "Batch ingest requires trace_id and dotted_order to be set."
2019 )
2020 # filter out runs that are not sampled
2021 if not pre_sampled:
2022 create_dicts = self._filter_for_sampling(create_dicts)
2023 update_dicts = self._filter_for_sampling(update_dicts, patch=True)
2025 if not create_dicts and not update_dicts:
2026 return
2028 # Apply process_buffered_run_ops function if provided
2029 if self._process_buffered_run_ops:
2030 if create_dicts:
2031 create_dicts = list(self._process_buffered_run_ops(create_dicts))
2032 if update_dicts:
2033 update_dicts = list(self._process_buffered_run_ops(update_dicts))
2035 self._insert_runtime_env(create_dicts + update_dicts)
2037 # convert to serialized ops
2038 serialized_ops = cast(
2039 list[SerializedRunOperation],
2040 combine_serialized_queue_operations(
2041 list(
2042 itertools.chain(
2043 (serialize_run_dict("post", run) for run in create_dicts),
2044 (serialize_run_dict("patch", run) for run in update_dicts),
2045 )
2046 )
2047 ),
2048 )
2050 self._batch_ingest_run_ops(serialized_ops)
2052 def _post_batch_ingest_runs(
2053 self,
2054 body: bytes,
2055 *,
2056 _context: str,
2057 api_url: Optional[str] = None,
2058 api_key: Optional[str] = None,
2059 ):
2060 # Use provided endpoint or fall back to all configured endpoints
2061 endpoints: Mapping[str, Optional[str]]
2062 if api_url is not None and api_key is not None:
2063 endpoints = {api_url: api_key}
2064 else:
2065 endpoints = self._write_api_urls
2067 for target_api_url, target_api_key in endpoints.items():
2068 try:
2069 logger.debug(
2070 f"Sending batch ingest request to {target_api_url} with context: {_context}"
2071 )
2072 self.request_with_retries(
2073 "POST",
2074 f"{target_api_url}/runs/batch",
2075 request_kwargs={
2076 "data": body,
2077 "headers": {
2078 **self._headers,
2079 X_API_KEY: target_api_key,
2080 },
2081 },
2082 to_ignore=(ls_utils.LangSmithConflictError,),
2083 stop_after_attempt=3,
2084 _context=_context,
2085 )
2086 except Exception as e:
2087 try:
2088 exc_desc_lines = traceback.format_exception_only(type(e), e)
2089 exc_desc = "".join(exc_desc_lines).rstrip()
2090 logger.warning(f"Failed to batch ingest runs: {exc_desc}")
2091 except Exception:
2092 logger.warning(f"Failed to batch ingest runs: {repr(e)}")
2093 self._invoke_tracing_error_callback(e)
2095 def _multipart_ingest_ops(
2096 self,
2097 ops: list[Union[SerializedRunOperation, SerializedFeedbackOperation]],
2098 *,
2099 api_url: Optional[str] = None,
2100 api_key: Optional[str] = None,
2101 ) -> None:
2102 parts: list[MultipartPartsAndContext] = []
2103 opened_files_dict: dict[str, io.BufferedReader] = {}
2104 for op in ops:
2105 if isinstance(op, SerializedRunOperation):
2106 (
2107 part,
2108 opened_files,
2109 ) = serialized_run_operation_to_multipart_parts_and_context(op)
2110 parts.append(part)
2111 opened_files_dict.update(opened_files)
2112 elif isinstance(op, SerializedFeedbackOperation):
2113 parts.append(
2114 serialized_feedback_operation_to_multipart_parts_and_context(op)
2115 )
2116 else:
2117 logger.error("Unknown operation type in tracing queue: %s", type(op))
2118 acc_multipart = join_multipart_parts_and_context(parts)
2119 if acc_multipart:
2120 try:
2121 self._send_multipart_req(
2122 acc_multipart, api_url=api_url, api_key=api_key
2123 )
2124 finally:
2125 _close_files(list(opened_files_dict.values()))
2127 def multipart_ingest(
2128 self,
2129 create: Optional[
2130 Sequence[Union[ls_schemas.Run, ls_schemas.RunLikeDict, dict]]
2131 ] = None,
2132 update: Optional[
2133 Sequence[Union[ls_schemas.Run, ls_schemas.RunLikeDict, dict]]
2134 ] = None,
2135 *,
2136 pre_sampled: bool = False,
2137 dangerously_allow_filesystem: bool = False,
2138 ) -> None:
2139 """Batch ingest/upsert multiple runs in the Langsmith system.
2141 Args:
2142 create (Optional[Sequence[Union[ls_schemas.Run, RunLikeDict]]]):
2143 A sequence of `Run` objects or equivalent dictionaries representing
2144 runs to be created / posted.
2145 update (Optional[Sequence[Union[ls_schemas.Run, RunLikeDict]]]):
2146 A sequence of `Run` objects or equivalent dictionaries representing
2147 runs that have already been created and should be updated / patched.
2148 pre_sampled (bool, default=False): Whether the runs have already been subject
2149 to sampling, and therefore should not be sampled again.
2151 Raises:
2152 LangsmithAPIError: If there is an error in the API request.
2154 !!! note
2156 The run objects MUST contain the `dotted_order` and `trace_id` fields
2157 to be accepted by the API.
2159 Example:
2160 ```python
2161 from langsmith import Client
2162 import datetime
2163 from uuid import uuid4
2165 client = Client()
2166 _session = "__test_batch_ingest_runs"
2167 trace_id = uuid4()
2168 trace_id_2 = uuid4()
2169 run_id_2 = uuid4()
2170 current_time = datetime.datetime.now(datetime.timezone.utc).strftime(
2171 "%Y%m%dT%H%M%S%fZ"
2172 )
2173 later_time = (
2174 datetime.datetime.now(datetime.timezone.utc) + timedelta(seconds=1)
2175 ).strftime("%Y%m%dT%H%M%S%fZ")
2177 runs_to_create = [
2178 {
2179 "id": str(trace_id),
2180 "session_name": _session,
2181 "name": "run 1",
2182 "run_type": "chain",
2183 "dotted_order": f"{current_time}{str(trace_id)}",
2184 "trace_id": str(trace_id),
2185 "inputs": {"input1": 1, "input2": 2},
2186 "outputs": {"output1": 3, "output2": 4},
2187 },
2188 {
2189 "id": str(trace_id_2),
2190 "session_name": _session,
2191 "name": "run 3",
2192 "run_type": "chain",
2193 "dotted_order": f"{current_time}{str(trace_id_2)}",
2194 "trace_id": str(trace_id_2),
2195 "inputs": {"input1": 1, "input2": 2},
2196 "error": "error",
2197 },
2198 {
2199 "id": str(run_id_2),
2200 "session_name": _session,
2201 "name": "run 2",
2202 "run_type": "chain",
2203 "dotted_order": f"{current_time}{str(trace_id)}."
2204 f"{later_time}{str(run_id_2)}",
2205 "trace_id": str(trace_id),
2206 "parent_run_id": str(trace_id),
2207 "inputs": {"input1": 5, "input2": 6},
2208 },
2209 ]
2210 runs_to_update = [
2211 {
2212 "id": str(run_id_2),
2213 "dotted_order": f"{current_time}{str(trace_id)}."
2214 f"{later_time}{str(run_id_2)}",
2215 "trace_id": str(trace_id),
2216 "parent_run_id": str(trace_id),
2217 "outputs": {"output1": 4, "output2": 5},
2218 },
2219 ]
2221 client.multipart_ingest(create=runs_to_create, update=runs_to_update)
2222 ```
2223 """
2224 if not (create or update):
2225 return
2226 # transform and convert to dicts
2227 create_dicts = [self._run_transform(run) for run in create or EMPTY_SEQ]
2228 update_dicts = [
2229 self._run_transform(run, update=True) for run in update or EMPTY_SEQ
2230 ]
2231 # require trace_id and dotted_order
2232 if create_dicts:
2233 for run in create_dicts:
2234 if not run.get("trace_id") or not run.get("dotted_order"):
2235 raise ls_utils.LangSmithUserError(
2236 "Multipart ingest requires trace_id and dotted_order"
2237 " to be set in create dicts."
2238 )
2239 else:
2240 del run
2241 if update_dicts:
2242 for run in update_dicts:
2243 if not run.get("trace_id") or not run.get("dotted_order"):
2244 raise ls_utils.LangSmithUserError(
2245 "Multipart ingest requires trace_id and dotted_order"
2246 " to be set in update dicts."
2247 )
2248 else:
2249 del run
2250 # combine post and patch dicts where possible
2251 if update_dicts and create_dicts:
2252 create_by_id = {run["id"]: run for run in create_dicts}
2253 standalone_updates: list[dict] = []
2254 for run in update_dicts:
2255 if run["id"] in create_by_id:
2256 for k, v in run.items():
2257 if v is not None:
2258 create_by_id[run["id"]][k] = v
2259 else:
2260 standalone_updates.append(run)
2261 else:
2262 del run
2263 update_dicts = standalone_updates
2264 # filter out runs that are not sampled
2265 if not pre_sampled:
2266 create_dicts = self._filter_for_sampling(create_dicts)
2267 update_dicts = self._filter_for_sampling(update_dicts, patch=True)
2268 if not create_dicts and not update_dicts:
2269 return
2270 # insert runtime environment
2271 self._insert_runtime_env(create_dicts)
2272 self._insert_runtime_env(update_dicts)
2274 # format as serialized operations
2275 serialized_ops = combine_serialized_queue_operations(
2276 list(
2277 itertools.chain(
2278 (serialize_run_dict("post", run) for run in create_dicts),
2279 (serialize_run_dict("patch", run) for run in update_dicts),
2280 )
2281 )
2282 )
2284 for op in serialized_ops:
2285 if isinstance(op, SerializedRunOperation) and op.attachments:
2286 for attachment in op.attachments.values():
2287 if (
2288 isinstance(attachment, tuple)
2289 and isinstance(attachment[1], Path)
2290 and not dangerously_allow_filesystem
2291 ):
2292 raise ValueError(
2293 "Must set dangerously_allow_filesystem=True to allow passing in Paths for attachments."
2294 )
2296 # sent the runs in multipart requests
2297 self._multipart_ingest_ops(serialized_ops)
2299 def _send_multipart_req(
2300 self,
2301 acc: MultipartPartsAndContext,
2302 *,
2303 attempts: int = 3,
2304 api_url: Optional[str] = None,
2305 api_key: Optional[str] = None,
2306 ):
2307 parts = acc.parts
2308 _context = acc.context
2310 # Use provided endpoint or fall back to all configured endpoints
2311 if api_url is not None and api_key is not None:
2312 endpoints: Mapping[str, str | None] = {api_url: api_key}
2313 else:
2314 endpoints = self._write_api_urls
2316 for target_api_url, target_api_key in endpoints.items():
2317 for idx in range(1, attempts + 1):
2318 try:
2319 encoder = rqtb_multipart.MultipartEncoder(parts, boundary=_BOUNDARY)
2320 if encoder.len <= 20_000_000: # ~20 MB
2321 data = encoder.to_string()
2322 else:
2323 data = encoder
2324 logger.debug(
2325 f"Sending multipart request to {target_api_url} with context: {_context}"
2326 )
2327 self.request_with_retries(
2328 "POST",
2329 f"{target_api_url}/runs/multipart",
2330 request_kwargs={
2331 "data": data,
2332 "headers": {
2333 **self._headers,
2334 X_API_KEY: target_api_key,
2335 "Content-Type": encoder.content_type,
2336 },
2337 },
2338 stop_after_attempt=1,
2339 _context=_context,
2340 )
2341 break
2342 except ls_utils.LangSmithConflictError:
2343 break
2344 except (
2345 ls_utils.LangSmithConnectionError,
2346 ls_utils.LangSmithRequestTimeout,
2347 ls_utils.LangSmithAPIError,
2348 ) as exc:
2349 if idx == attempts:
2350 logger.warning(f"Failed to multipart ingest runs: {exc}")
2351 self._invoke_tracing_error_callback(exc)
2352 else:
2353 continue
2354 except Exception as e:
2355 try:
2356 exc_desc_lines = traceback.format_exception_only(type(e), e)
2357 exc_desc = "".join(exc_desc_lines).rstrip()
2358 logger.warning(f"Failed to multipart ingest runs: {exc_desc}")
2359 except Exception:
2360 logger.warning(f"Failed to multipart ingest runs: {repr(e)}")
2361 self._invoke_tracing_error_callback(e)
2362 # do not retry by default
2363 break
2365 def _send_compressed_multipart_req(
2366 self,
2367 data_stream: io.BytesIO,
2368 compressed_traces_info: Optional[tuple[int, int]],
2369 *,
2370 attempts: int = 3,
2371 ):
2372 """Send a zstd-compressed multipart form data stream to the backend."""
2373 _context: str = "; ".join(getattr(data_stream, "context", []))
2375 for api_url, api_key in self._write_api_urls.items():
2376 data_stream.seek(0)
2378 for idx in range(1, attempts + 1):
2379 try:
2380 headers = {
2381 **self._headers,
2382 "X-API-KEY": api_key,
2383 "Content-Type": f"multipart/form-data; boundary={_BOUNDARY}",
2384 "Content-Encoding": "zstd",
2385 "X-Pre-Compressed-Size": (
2386 str(compressed_traces_info[0])
2387 if compressed_traces_info
2388 else ""
2389 ),
2390 "X-Post-Compressed-Size": (
2391 str(compressed_traces_info[1])
2392 if compressed_traces_info
2393 else ""
2394 ),
2395 }
2396 logger.debug(
2397 f"Sending compressed multipart request with context: {_context}"
2398 )
2399 self.request_with_retries(
2400 "POST",
2401 f"{api_url}/runs/multipart",
2402 request_kwargs={
2403 "data": data_stream,
2404 "headers": headers,
2405 },
2406 stop_after_attempt=1,
2407 _context=_context,
2408 )
2409 break
2410 except ls_utils.LangSmithConflictError:
2411 break
2412 except (
2413 ls_utils.LangSmithConnectionError,
2414 ls_utils.LangSmithRequestTimeout,
2415 ls_utils.LangSmithAPIError,
2416 ) as exc:
2417 if idx == attempts:
2418 logger.warning(
2419 f"Failed to send compressed multipart ingest: {exc}"
2420 )
2421 self._invoke_tracing_error_callback(exc)
2422 else:
2423 continue
2424 except Exception as e:
2425 try:
2426 exc_desc_lines = traceback.format_exception_only(type(e), e)
2427 exc_desc = "".join(exc_desc_lines).rstrip()
2428 logger.warning(
2429 f"Failed to send compressed multipart ingest: {exc_desc}"
2430 )
2431 except Exception:
2432 logger.warning(
2433 f"Failed to send compressed multipart ingest: {repr(e)}"
2434 )
2435 self._invoke_tracing_error_callback(e)
2436 # Do not retry by default after unknown exceptions
2437 break
2439 def update_run(
2440 self,
2441 run_id: ID_TYPE,
2442 *,
2443 name: Optional[str] = None,
2444 run_type: Optional[RUN_TYPE_T] = None,
2445 start_time: Optional[datetime.datetime] = None,
2446 end_time: Optional[datetime.datetime] = None,
2447 error: Optional[str] = None,
2448 inputs: Optional[dict] = None,
2449 outputs: Optional[dict] = None,
2450 events: Optional[Sequence[dict]] = None,
2451 extra: Optional[dict] = None,
2452 tags: Optional[list[str]] = None,
2453 attachments: Optional[ls_schemas.Attachments] = None,
2454 dangerously_allow_filesystem: bool = False,
2455 reference_example_id: str | uuid.UUID | None = None,
2456 api_key: Optional[str] = None,
2457 api_url: Optional[str] = None,
2458 **kwargs: Any,
2459 ) -> None:
2460 """Update a run in the LangSmith API.
2462 Args:
2463 run_id (Union[UUID, str]): The ID of the run to update.
2464 name (Optional[str]): The name of the run.
2465 run_type (Optional[str]): The type of the run (e.g., llm, chain, tool).
2466 start_time (Optional[datetime.datetime]): The start time of the run.
2467 end_time (Optional[datetime.datetime]): The end time of the run.
2468 error (Optional[str]): The error message of the run.
2469 inputs (Optional[Dict]): The input values for the run.
2470 outputs (Optional[Dict]): The output values for the run.
2471 events (Optional[Sequence[dict]]): The events for the run.
2472 extra (Optional[Dict]): The extra information for the run.
2473 tags (Optional[List[str]]): The tags for the run.
2474 attachments (Optional[Dict[str, Attachment]]): A dictionary of attachments to add to the run. The keys are the attachment names,
2475 and the values are Attachment objects containing the data and mime type.
2476 reference_example_id (Optional[Union[str, uuid.UUID]]): ID of the example
2477 that was the source of the run inputs. Used for runs that were part of
2478 an experiment.
2479 api_key (Optional[str]): The API key to use for this specific run.
2480 api_url (Optional[str]): The API URL to use for this specific run.
2481 **kwargs (Any): Kwargs are ignored.
2483 Returns:
2484 None
2486 Examples:
2487 ```python
2488 from langsmith import Client
2489 import datetime
2490 from uuid import uuid4
2492 client = Client()
2493 project_name = "__test_update_run"
2495 start_time = datetime.datetime.now()
2496 revision_id = uuid4()
2497 run: dict = dict(
2498 id=uuid4(),
2499 name="test_run",
2500 run_type="llm",
2501 inputs={"text": "hello world"},
2502 project_name=project_name,
2503 api_url=os.getenv("LANGCHAIN_ENDPOINT"),
2504 start_time=start_time,
2505 extra={"extra": "extra"},
2506 revision_id=revision_id,
2507 )
2508 # Create the run
2509 client.create_run(**run)
2510 run["outputs"] = {"output": ["Hi"]}
2511 run["extra"]["foo"] = "bar"
2512 run["name"] = "test_run_updated"
2513 # Update the run
2514 client.update_run(run["id"], **run)
2515 ```
2516 """
2517 data: dict[str, Any] = {
2518 "id": _as_uuid(run_id, "run_id"),
2519 "name": name,
2520 "run_type": run_type,
2521 "trace_id": kwargs.pop("trace_id", None),
2522 "parent_run_id": kwargs.pop("parent_run_id", None),
2523 "dotted_order": kwargs.pop("dotted_order", None),
2524 "tags": tags,
2525 "extra": extra,
2526 "session_id": kwargs.pop("session_id", None),
2527 "session_name": kwargs.pop("session_name", None),
2528 }
2529 if start_time is not None:
2530 data["start_time"] = start_time.isoformat()
2531 if attachments:
2532 for _, attachment in attachments.items():
2533 if (
2534 isinstance(attachment, tuple)
2535 and isinstance(attachment[1], Path)
2536 and not dangerously_allow_filesystem
2537 ):
2538 raise ValueError(
2539 "Must set dangerously_allow_filesystem=True to allow passing in Paths for attachments."
2540 )
2541 data["attachments"] = attachments
2542 use_multipart = (
2543 (self.tracing_queue is not None or self.compressed_traces is not None)
2544 # batch ingest requires trace_id and dotted_order to be set
2545 and data["trace_id"] is not None
2546 and data["dotted_order"] is not None
2547 )
2548 if not self._filter_for_sampling([data], patch=True):
2549 return
2550 if end_time is not None:
2551 data["end_time"] = end_time.isoformat()
2552 else:
2553 data["end_time"] = datetime.datetime.now(datetime.timezone.utc).isoformat()
2554 if error is not None:
2555 data["error"] = error
2556 if inputs is not None:
2557 data["inputs"] = self._hide_run_inputs(inputs)
2558 if outputs is not None:
2559 if not use_multipart:
2560 outputs = ls_utils.deepish_copy(outputs)
2561 data["outputs"] = self._hide_run_outputs(outputs)
2562 if events is not None:
2563 data["events"] = events
2564 if data["extra"]:
2565 self._insert_runtime_env([data])
2566 if metadata := data["extra"].get("metadata"):
2567 data["extra"]["metadata"] = self._hide_run_metadata(metadata)
2568 if reference_example_id is not None:
2569 data["reference_example_id"] = reference_example_id
2571 # If process_buffered_run_ops is enabled, collect runs in batches
2572 if self._process_buffered_run_ops and not kwargs.get("is_run_ops_buffer_flush"):
2573 with self._run_ops_buffer_lock:
2574 self._run_ops_buffer.append(("patch", data))
2575 # Process batch when we have enough runs or enough time has passed
2576 if self._should_flush_run_ops_buffer():
2577 self._flush_run_ops_buffer()
2578 return
2579 else:
2580 self._update_run(data, api_key=api_key, api_url=api_url)
2582 def _update_run(
2583 self,
2584 run_update: dict,
2585 *,
2586 api_key: Optional[str] = None,
2587 api_url: Optional[str] = None,
2588 ):
2589 use_multipart = (
2590 (self.tracing_queue is not None or self.compressed_traces is not None)
2591 # batch ingest requires trace_id and dotted_order to be set
2592 and run_update["trace_id"] is not None
2593 and run_update["dotted_order"] is not None
2594 )
2595 if self._pyo3_client is not None:
2596 self._pyo3_client.update_run(run_update)
2597 elif use_multipart:
2598 serialized_op = serialize_run_dict(operation="patch", payload=run_update)
2599 if (
2600 self.compressed_traces is not None
2601 and api_key is None
2602 and api_url is None
2603 ):
2604 (
2605 multipart_form,
2606 opened_files,
2607 ) = serialized_run_operation_to_multipart_parts_and_context(
2608 serialized_op
2609 )
2610 logger.log(
2611 5,
2612 "Adding compressed multipart to queue with context: %s",
2613 multipart_form.context,
2614 )
2615 with self.compressed_traces.lock:
2616 if self._data_available_event is None:
2617 raise ValueError(
2618 "Run compression is enabled but threading event is not configured"
2619 )
2620 enqueued = compress_multipart_parts_and_context(
2621 multipart_form,
2622 self.compressed_traces,
2623 _BOUNDARY,
2624 )
2625 if enqueued:
2626 self.compressed_traces.trace_count += 1
2627 self._data_available_event.set()
2628 _close_files(list(opened_files.values()))
2629 elif self.tracing_queue is not None:
2630 logger.log(
2631 5,
2632 "Adding to tracing queue: trace_id=%s, run_id=%s",
2633 serialized_op.trace_id,
2634 serialized_op.id,
2635 )
2636 if self.otel_exporter is not None:
2637 self.tracing_queue.put(
2638 TracingQueueItem(
2639 run_update["dotted_order"],
2640 serialized_op,
2641 api_key=api_key,
2642 api_url=api_url,
2643 otel_context=self._set_span_in_context(
2644 self._otel_trace.get_current_span()
2645 ),
2646 )
2647 )
2648 else:
2649 self.tracing_queue.put(
2650 TracingQueueItem(
2651 run_update["dotted_order"],
2652 serialized_op,
2653 api_key=api_key,
2654 api_url=api_url,
2655 )
2656 )
2657 else:
2658 self._update_run_non_batch(run_update, api_key=api_key, api_url=api_url)
2660 def _update_run_non_batch(
2661 self,
2662 run_update: dict,
2663 *,
2664 api_key: Optional[str] = None,
2665 api_url: Optional[str] = None,
2666 ) -> None:
2667 # If specific api_key/api_url provided, use those; otherwise use all configured endpoints
2668 if api_key is not None or api_url is not None:
2669 target_api_url = api_url or self.api_url
2670 target_api_key = api_key or self.api_key
2671 headers = {
2672 **self._headers,
2673 X_API_KEY: target_api_key,
2674 }
2676 self.request_with_retries(
2677 "PATCH",
2678 f"{target_api_url}/runs/{run_update['id']}",
2679 request_kwargs={
2680 "data": _dumps_json(run_update),
2681 "headers": headers,
2682 },
2683 )
2684 else:
2685 # Use all configured write API URLs
2686 for write_api_url, write_api_key in self._write_api_urls.items():
2687 headers = {
2688 **self._headers,
2689 X_API_KEY: write_api_key,
2690 }
2692 self.request_with_retries(
2693 "PATCH",
2694 f"{write_api_url}/runs/{run_update['id']}",
2695 request_kwargs={
2696 "data": _dumps_json(run_update),
2697 "headers": headers,
2698 },
2699 )
2701 def flush_compressed_traces(self, attempts: int = 3) -> None:
2702 """Force flush the currently buffered compressed runs."""
2703 if self.compressed_traces is None:
2704 return
2706 if self._futures is None:
2707 raise ValueError(
2708 "Run compression is enabled but request pool futures is not set"
2709 )
2711 # Attempt to drain and send any remaining data
2712 from langsmith._internal._background_thread import (
2713 LANGSMITH_CLIENT_THREAD_POOL,
2714 _tracing_thread_drain_compressed_buffer,
2715 )
2717 (
2718 final_data_stream,
2719 compressed_traces_info,
2720 ) = _tracing_thread_drain_compressed_buffer(
2721 self, size_limit=1, size_limit_bytes=1
2722 )
2724 if final_data_stream is not None:
2725 # We have data to send
2726 future = None
2727 try:
2728 future = LANGSMITH_CLIENT_THREAD_POOL.submit(
2729 self._send_compressed_multipart_req,
2730 final_data_stream,
2731 compressed_traces_info,
2732 attempts=attempts,
2733 )
2734 self._futures.add(future)
2735 except RuntimeError:
2736 # In case the ThreadPoolExecutor is already shutdown
2737 self._send_compressed_multipart_req(
2738 final_data_stream, compressed_traces_info, attempts=attempts
2739 )
2741 # If we got a future, wait for it to complete
2742 if self._futures:
2743 futures = list(self._futures)
2744 done, _ = cf.wait(futures)
2745 # Remove completed futures
2746 self._futures.difference_update(done)
2748 def flush(self) -> None:
2749 """Flush either queue or compressed buffer, depending on mode."""
2750 # Flush any remaining batch items first
2751 if self._process_buffered_run_ops:
2752 with self._run_ops_buffer_lock:
2753 if self._run_ops_buffer:
2754 self._flush_run_ops_buffer()
2755 if self.compressed_traces is not None:
2756 self.flush_compressed_traces()
2757 elif self.tracing_queue is not None:
2758 self.tracing_queue.join()
2760 def _load_child_runs(self, run: ls_schemas.Run) -> ls_schemas.Run:
2761 """Load child runs for a given run.
2763 Args:
2764 run (Run): The run to load child runs for.
2766 Returns:
2767 Run: The run with loaded child runs.
2769 Raises:
2770 LangSmithError: If a child run has no parent.
2771 """
2772 child_runs = self.list_runs(
2773 is_root=False, session_id=run.session_id, trace_id=run.trace_id
2774 )
2775 treemap: collections.defaultdict[uuid.UUID, list[ls_schemas.Run]] = (
2776 collections.defaultdict(list)
2777 )
2778 runs: dict[uuid.UUID, ls_schemas.Run] = {}
2779 run_id_str = str(run.id)
2781 for child_run in sorted(
2782 child_runs,
2783 key=lambda r: r.dotted_order,
2784 ):
2785 if child_run.parent_run_id is None:
2786 raise ls_utils.LangSmithError(f"Child run {child_run.id} has no parent")
2788 # Only track downstream children
2789 ancestor_ids = {
2790 seg.split("Z", 1)[1]
2791 for seg in child_run.dotted_order.split(".")
2792 if "Z" in seg
2793 }
2794 if run_id_str in ancestor_ids and child_run.id != run.id:
2795 treemap[child_run.parent_run_id].append(child_run)
2796 runs[child_run.id] = child_run
2797 run.child_runs = treemap.pop(run.id, [])
2798 for run_id, children in treemap.items():
2799 runs[run_id].child_runs = children
2800 return run
2802 def read_run(
2803 self, run_id: ID_TYPE, load_child_runs: bool = False
2804 ) -> ls_schemas.Run:
2805 """Read a run from the LangSmith API.
2807 Args:
2808 run_id (Union[UUID, str]):
2809 The ID of the run to read.
2810 load_child_runs (bool, default=False):
2811 Whether to load nested child runs.
2813 Returns:
2814 Run: The run read from the LangSmith API.
2816 Examples:
2817 ```python
2818 from langsmith import Client
2820 # Existing run
2821 run_id = "your-run-id"
2823 client = Client()
2824 stored_run = client.read_run(run_id)
2825 ```
2826 """
2827 response = self.request_with_retries(
2828 "GET", f"/runs/{_as_uuid(run_id, 'run_id')}"
2829 )
2830 attachments = _convert_stored_attachments_to_attachments_dict(
2831 response.json(), attachments_key="s3_urls", api_url=self.api_url
2832 )
2833 run = ls_schemas.Run(
2834 attachments=attachments, **response.json(), _host_url=self._host_url
2835 )
2837 if load_child_runs:
2838 run = self._load_child_runs(run)
2839 return run
2841 def list_runs(
2842 self,
2843 *,
2844 project_id: Optional[Union[ID_TYPE, Sequence[ID_TYPE]]] = None,
2845 project_name: Optional[Union[str, Sequence[str]]] = None,
2846 run_type: Optional[str] = None,
2847 trace_id: Optional[ID_TYPE] = None,
2848 reference_example_id: Optional[ID_TYPE] = None,
2849 query: Optional[str] = None,
2850 filter: Optional[str] = None,
2851 trace_filter: Optional[str] = None,
2852 tree_filter: Optional[str] = None,
2853 is_root: Optional[bool] = None,
2854 parent_run_id: Optional[ID_TYPE] = None,
2855 start_time: Optional[datetime.datetime] = None,
2856 error: Optional[bool] = None,
2857 run_ids: Optional[Sequence[ID_TYPE]] = None,
2858 select: Optional[Sequence[str]] = None,
2859 limit: Optional[int] = None,
2860 **kwargs: Any,
2861 ) -> Iterator[ls_schemas.Run]:
2862 """List runs from the LangSmith API.
2864 Args:
2865 project_id (Optional[Union[UUID, str], Sequence[Union[UUID, str]]]):
2866 The ID(s) of the project to filter by.
2867 project_name (Optional[Union[str, Sequence[str]]]): The name(s) of the project to filter by.
2868 run_type (Optional[str]): The type of the runs to filter by.
2869 trace_id (Optional[Union[UUID, str]]): The ID of the trace to filter by.
2870 reference_example_id (Optional[Union[UUID, str]]): The ID of the reference example to filter by.
2871 query (Optional[str]): The query string to filter by.
2872 filter (Optional[str]): The filter string to filter by.
2873 trace_filter (Optional[str]): Filter to apply to the ROOT run in the trace tree. This is meant to
2874 be used in conjunction with the regular `filter` parameter to let you
2875 filter runs by attributes of the root run within a trace.
2876 tree_filter (Optional[str]): Filter to apply to OTHER runs in the trace tree, including
2877 sibling and child runs. This is meant to be used in conjunction with
2878 the regular `filter` parameter to let you filter runs by attributes
2879 of any run within a trace.
2880 is_root (Optional[bool]): Whether to filter by root runs.
2881 parent_run_id (Optional[Union[UUID, str]]):
2882 The ID of the parent run to filter by.
2883 start_time (Optional[datetime.datetime]):
2884 The start time to filter by.
2885 error (Optional[bool]): Whether to filter by error status.
2886 run_ids (Optional[Sequence[Union[UUID, str]]]):
2887 The IDs of the runs to filter by.
2888 select (Optional[Sequence[str]]): The fields to select.
2889 limit (Optional[int]): The maximum number of runs to return.
2890 **kwargs (Any): Additional keyword arguments.
2892 Yields:
2893 The runs.
2895 Examples:
2896 ```python
2897 # List all runs in a project
2898 project_runs = client.list_runs(project_name="<your_project>")
2900 # List LLM and Chat runs in the last 24 hours
2901 todays_llm_runs = client.list_runs(
2902 project_name="<your_project>",
2903 start_time=datetime.now() - timedelta(days=1),
2904 run_type="llm",
2905 )
2907 # List root traces in a project
2908 root_runs = client.list_runs(project_name="<your_project>", is_root=1)
2910 # List runs without errors
2911 correct_runs = client.list_runs(project_name="<your_project>", error=False)
2913 # List runs and only return their inputs/outputs (to speed up the query)
2914 input_output_runs = client.list_runs(
2915 project_name="<your_project>", select=["inputs", "outputs"]
2916 )
2918 # List runs by run ID
2919 run_ids = [
2920 "a36092d2-4ad5-4fb4-9c0d-0dba9a2ed836",
2921 "9398e6be-964f-4aa4-8ae9-ad78cd4b7074",
2922 ]
2923 selected_runs = client.list_runs(id=run_ids)
2925 # List all "chain" type runs that took more than 10 seconds and had
2926 # `total_tokens` greater than 5000
2927 chain_runs = client.list_runs(
2928 project_name="<your_project>",
2929 filter='and(eq(run_type, "chain"), gt(latency, 10), gt(total_tokens, 5000))',
2930 )
2932 # List all runs called "extractor" whose root of the trace was assigned feedback "user_score" score of 1
2933 good_extractor_runs = client.list_runs(
2934 project_name="<your_project>",
2935 filter='eq(name, "extractor")',
2936 trace_filter='and(eq(feedback_key, "user_score"), eq(feedback_score, 1))',
2937 )
2939 # List all runs that started after a specific timestamp and either have "error" not equal to null or a "Correctness" feedback score equal to 0
2940 complex_runs = client.list_runs(
2941 project_name="<your_project>",
2942 filter='and(gt(start_time, "2023-07-15T12:34:56Z"), or(neq(error, null), and(eq(feedback_key, "Correctness"), eq(feedback_score, 0.0))))',
2943 )
2945 # List all runs where `tags` include "experimental" or "beta" and `latency` is greater than 2 seconds
2946 tagged_runs = client.list_runs(
2947 project_name="<your_project>",
2948 filter='and(or(has(tags, "experimental"), has(tags, "beta")), gt(latency, 2))',
2949 )
2950 ```
2951 """ # noqa: E501
2952 project_ids = []
2953 if isinstance(project_id, (uuid.UUID, str)):
2954 project_ids.append(project_id)
2955 elif isinstance(project_id, list):
2956 project_ids.extend(project_id)
2957 if project_name is not None:
2958 if isinstance(project_name, str):
2959 project_name = [project_name]
2960 project_ids.extend(
2961 [self.read_project(project_name=name).id for name in project_name]
2962 )
2963 default_select = [
2964 "app_path",
2965 "completion_cost",
2966 "completion_tokens",
2967 "dotted_order",
2968 "end_time",
2969 "error",
2970 "events",
2971 "extra",
2972 "feedback_stats",
2973 "first_token_time",
2974 "id",
2975 "inputs",
2976 "name",
2977 "outputs",
2978 "parent_run_id",
2979 "parent_run_ids",
2980 "prompt_cost",
2981 "prompt_tokens",
2982 "reference_example_id",
2983 "run_type",
2984 "session_id",
2985 "start_time",
2986 "status",
2987 "tags",
2988 "total_cost",
2989 "total_tokens",
2990 "trace_id",
2991 ]
2993 select = select or default_select
2995 if "child_run_ids" in select:
2996 warnings.warn(
2997 "The child_run_ids field is deprecated and will be removed in following versions",
2998 DeprecationWarning,
2999 )
3001 body_query: dict[str, Any] = {
3002 "session": project_ids if project_ids else None,
3003 "run_type": run_type,
3004 "reference_example": (
3005 [reference_example_id] if reference_example_id else None
3006 ),
3007 "query": query,
3008 "filter": filter,
3009 "trace_filter": trace_filter,
3010 "tree_filter": tree_filter,
3011 "is_root": is_root,
3012 "parent_run": parent_run_id,
3013 "start_time": start_time.isoformat() if start_time else None,
3014 "error": error,
3015 "id": run_ids,
3016 "trace": trace_id,
3017 "select": select,
3018 "limit": limit,
3019 **kwargs,
3020 }
3021 body_query = {k: v for k, v in body_query.items() if v is not None}
3022 for i, run in enumerate(
3023 self._get_cursor_paginated_list("/runs/query", body=body_query)
3024 ):
3025 # Should this be behind a flag?
3026 attachments = _convert_stored_attachments_to_attachments_dict(
3027 run, attachments_key="s3_urls", api_url=self.api_url
3028 )
3029 yield ls_schemas.Run(
3030 attachments=attachments, **run, _host_url=self._host_url
3031 )
3032 if limit is not None and i + 1 >= limit:
3033 break
3035 def get_run_stats(
3036 self,
3037 *,
3038 id: Optional[list[ID_TYPE]] = None,
3039 trace: Optional[ID_TYPE] = None,
3040 parent_run: Optional[ID_TYPE] = None,
3041 run_type: Optional[str] = None,
3042 project_names: Optional[list[str]] = None,
3043 project_ids: Optional[list[ID_TYPE]] = None,
3044 reference_example_ids: Optional[list[ID_TYPE]] = None,
3045 start_time: Optional[str] = None,
3046 end_time: Optional[str] = None,
3047 error: Optional[bool] = None,
3048 query: Optional[str] = None,
3049 filter: Optional[str] = None,
3050 trace_filter: Optional[str] = None,
3051 tree_filter: Optional[str] = None,
3052 is_root: Optional[bool] = None,
3053 data_source_type: Optional[str] = None,
3054 ) -> dict[str, Any]:
3055 """Get aggregate statistics over queried runs.
3057 Takes in similar query parameters to `list_runs` and returns statistics
3058 based on the runs that match the query.
3060 Args:
3061 id (Optional[List[Union[UUID, str]]]): List of run IDs to filter by.
3062 trace (Optional[Union[UUID, str]]): Trace ID to filter by.
3063 parent_run (Optional[Union[UUID, str]]): Parent run ID to filter by.
3064 run_type (Optional[str]): Run type to filter by.
3065 project_names (Optional[List[str]]): List of project names to filter by.
3066 project_ids (Optional[List[Union[UUID, str]]]): List of project IDs to filter by.
3067 reference_example_ids (Optional[List[Union[UUID, str]]]): List of reference example IDs to filter by.
3068 start_time (Optional[str]): Start time to filter by.
3069 end_time (Optional[str]): End time to filter by.
3070 error (Optional[bool]): Filter by error status.
3071 query (Optional[str]): Query string to filter by.
3072 filter (Optional[str]): Filter string to apply.
3073 trace_filter (Optional[str]): Trace filter string to apply.
3074 tree_filter (Optional[str]): Tree filter string to apply.
3075 is_root (Optional[bool]): Filter by root run status.
3076 data_source_type (Optional[str]): Data source type to filter by.
3078 Returns:
3079 Dict[str, Any]: A dictionary containing the run statistics.
3080 """ # noqa: E501
3081 from concurrent.futures import ThreadPoolExecutor, as_completed # type: ignore
3083 project_ids = project_ids or []
3084 if project_names:
3085 with ThreadPoolExecutor() as executor:
3086 futures = [
3087 executor.submit(self.read_project, project_name=name)
3088 for name in project_names
3089 ]
3090 for future in as_completed(futures):
3091 project_ids.append(future.result().id)
3092 payload = {
3093 "id": id,
3094 "trace": trace,
3095 "parent_run": parent_run,
3096 "run_type": run_type,
3097 "session": project_ids,
3098 "reference_example": reference_example_ids,
3099 "start_time": start_time,
3100 "end_time": end_time,
3101 "error": error,
3102 "query": query,
3103 "filter": filter,
3104 "trace_filter": trace_filter,
3105 "tree_filter": tree_filter,
3106 "is_root": is_root,
3107 "data_source_type": data_source_type,
3108 }
3110 # Remove None values from the payload
3111 payload = {k: v for k, v in payload.items() if v is not None}
3113 response = self.request_with_retries(
3114 "POST",
3115 "/runs/stats",
3116 request_kwargs={
3117 "data": _dumps_json(payload),
3118 },
3119 )
3120 ls_utils.raise_for_status_with_text(response)
3121 return response.json()
3123 def get_run_url(
3124 self,
3125 *,
3126 run: ls_schemas.RunBase,
3127 project_name: Optional[str] = None,
3128 project_id: Optional[ID_TYPE] = None,
3129 ) -> str:
3130 """Get the URL for a run.
3132 Not recommended for use within your agent runtime.
3133 More for use interacting with runs after the fact
3134 for data analysis or ETL workloads.
3136 Args:
3137 run (RunBase): The run.
3138 project_name (Optional[str]): The name of the project.
3139 project_id (Optional[Union[UUID, str]]): The ID of the project.
3141 Returns:
3142 str: The URL for the run.
3143 """
3144 if session_id := getattr(run, "session_id", None):
3145 pass
3146 elif session_name := getattr(run, "session_name", None):
3147 session_id = self.read_project(project_name=session_name).id
3148 elif project_id is not None:
3149 session_id = project_id
3150 elif project_name is not None:
3151 session_id = self.read_project(project_name=project_name).id
3152 else:
3153 project_name = ls_utils.get_tracer_project()
3154 session_id = self.read_project(project_name=project_name).id
3155 session_id_ = _as_uuid(session_id, "session_id")
3156 return (
3157 f"{self._host_url}/o/{self._get_tenant_id()}/projects/p/{session_id_}/"
3158 f"r/{run.id}?poll=true"
3159 )
3161 def share_run(self, run_id: ID_TYPE, *, share_id: Optional[ID_TYPE] = None) -> str:
3162 """Get a share link for a run.
3164 Args:
3165 run_id (Union[UUID, str]): The ID of the run to share.
3166 share_id (Optional[Union[UUID, str]]): Custom share ID.
3167 If not provided, a random UUID will be generated.
3169 Returns:
3170 str: The URL of the shared run.
3171 """
3172 run_id_ = _as_uuid(run_id, "run_id")
3173 data = {
3174 "run_id": str(run_id_),
3175 "share_token": share_id or str(uuid.uuid4()),
3176 }
3177 response = self.request_with_retries(
3178 "PUT",
3179 f"/runs/{run_id_}/share",
3180 headers=self._headers,
3181 json=data,
3182 )
3183 ls_utils.raise_for_status_with_text(response)
3184 share_token = response.json()["share_token"]
3185 return f"{self._host_url}/public/{share_token}/r"
3187 def unshare_run(self, run_id: ID_TYPE) -> None:
3188 """Delete share link for a run.
3190 Args:
3191 run_id (Union[UUID, str]): The ID of the run to unshare.
3193 Returns:
3194 None
3195 """
3196 response = self.request_with_retries(
3197 "DELETE",
3198 f"/runs/{_as_uuid(run_id, 'run_id')}/share",
3199 headers=self._headers,
3200 )
3201 ls_utils.raise_for_status_with_text(response)
3203 def read_run_shared_link(self, run_id: ID_TYPE) -> Optional[str]:
3204 """Retrieve the shared link for a specific run.
3206 Args:
3207 run_id (Union[UUID, str]): The ID of the run.
3209 Returns:
3210 Optional[str]: The shared link for the run, or None if the link is not
3211 available.
3212 """
3213 response = self.request_with_retries(
3214 "GET",
3215 f"/runs/{_as_uuid(run_id, 'run_id')}/share",
3216 headers=self._headers,
3217 )
3218 ls_utils.raise_for_status_with_text(response)
3219 result = response.json()
3220 if result is None or "share_token" not in result:
3221 return None
3222 return f"{self._host_url}/public/{result['share_token']}/r"
3224 def run_is_shared(self, run_id: ID_TYPE) -> bool:
3225 """Get share state for a run.
3227 Args:
3228 run_id (Union[UUID, str]): The ID of the run.
3230 Returns:
3231 bool: True if the run is shared, False otherwise.
3232 """
3233 link = self.read_run_shared_link(_as_uuid(run_id, "run_id"))
3234 return link is not None
3236 def read_shared_run(
3237 self, share_token: Union[ID_TYPE, str], run_id: Optional[ID_TYPE] = None
3238 ) -> ls_schemas.Run:
3239 """Get shared runs.
3241 Args:
3242 share_token (Union[UUID, str]): The share token or URL of the shared run.
3243 run_id (Optional[Union[UUID, str]]): The ID of the specific run to retrieve.
3244 If not provided, the full shared run will be returned.
3246 Returns:
3247 Run: The shared run.
3248 """
3249 _, token_uuid = _parse_token_or_url(share_token, "", kind="run")
3250 path = f"/public/{token_uuid}/run"
3251 if run_id is not None:
3252 path += f"/{_as_uuid(run_id, 'run_id')}"
3253 response = self.request_with_retries(
3254 "GET",
3255 path,
3256 headers=self._headers,
3257 )
3258 ls_utils.raise_for_status_with_text(response)
3259 return ls_schemas.Run(**response.json(), _host_url=self._host_url)
3261 def list_shared_runs(
3262 self, share_token: Union[ID_TYPE, str], run_ids: Optional[list[str]] = None
3263 ) -> Iterator[ls_schemas.Run]:
3264 """Get shared runs.
3266 Args:
3267 share_token (Union[UUID, str]): The share token or URL of the shared run.
3268 run_ids (Optional[List[str]]): A list of run IDs to filter the results by.
3270 Yields:
3271 A shared run.
3272 """
3273 body = {"id": run_ids} if run_ids else {}
3274 _, token_uuid = _parse_token_or_url(share_token, "", kind="run")
3275 for run in self._get_cursor_paginated_list(
3276 f"/public/{token_uuid}/runs/query", body=body
3277 ):
3278 yield ls_schemas.Run(**run, _host_url=self._host_url)
3280 def read_dataset_shared_schema(
3281 self,
3282 dataset_id: Optional[ID_TYPE] = None,
3283 *,
3284 dataset_name: Optional[str] = None,
3285 ) -> ls_schemas.DatasetShareSchema:
3286 """Retrieve the shared schema of a dataset.
3288 Args:
3289 dataset_id (Optional[Union[UUID, str]]): The ID of the dataset.
3290 Either `dataset_id` or `dataset_name` must be given.
3291 dataset_name (Optional[str]): The name of the dataset.
3292 Either `dataset_id` or `dataset_name` must be given.
3294 Returns:
3295 ls_schemas.DatasetShareSchema: The shared schema of the dataset.
3297 Raises:
3298 ValueError: If neither `dataset_id` nor `dataset_name` is given.
3299 """
3300 if dataset_id is None and dataset_name is None:
3301 raise ValueError("Either dataset_id or dataset_name must be given")
3302 if dataset_id is None:
3303 dataset_id = self.read_dataset(dataset_name=dataset_name).id
3304 response = self.request_with_retries(
3305 "GET",
3306 f"/datasets/{_as_uuid(dataset_id, 'dataset_id')}/share",
3307 headers=self._headers,
3308 )
3309 ls_utils.raise_for_status_with_text(response)
3310 d = response.json()
3311 return cast(
3312 ls_schemas.DatasetShareSchema,
3313 {
3314 **d,
3315 "url": f"{self._host_url}/public/"
3316 f"{_as_uuid(d['share_token'], 'response.share_token')}/d",
3317 },
3318 )
3320 def share_dataset(
3321 self,
3322 dataset_id: Optional[ID_TYPE] = None,
3323 *,
3324 dataset_name: Optional[str] = None,
3325 ) -> ls_schemas.DatasetShareSchema:
3326 """Get a share link for a dataset.
3328 Args:
3329 dataset_id (Optional[Union[UUID, str]]): The ID of the dataset.
3330 Either `dataset_id` or `dataset_name` must be given.
3331 dataset_name (Optional[str]): The name of the dataset.
3332 Either `dataset_id` or `dataset_name` must be given.
3334 Returns:
3335 ls_schemas.DatasetShareSchema: The shared schema of the dataset.
3337 Raises:
3338 ValueError: If neither `dataset_id` nor `dataset_name` is given.
3339 """
3340 if dataset_id is None and dataset_name is None:
3341 raise ValueError("Either dataset_id or dataset_name must be given")
3342 if dataset_id is None:
3343 dataset_id = self.read_dataset(dataset_name=dataset_name).id
3344 data = {
3345 "dataset_id": str(dataset_id),
3346 }
3347 response = self.request_with_retries(
3348 "PUT",
3349 f"/datasets/{_as_uuid(dataset_id, 'dataset_id')}/share",
3350 headers=self._headers,
3351 json=data,
3352 )
3353 ls_utils.raise_for_status_with_text(response)
3354 d: dict = response.json()
3355 return cast(
3356 ls_schemas.DatasetShareSchema,
3357 {**d, "url": f"{self._host_url}/public/{d['share_token']}/d"},
3358 )
3360 def unshare_dataset(self, dataset_id: ID_TYPE) -> None:
3361 """Delete share link for a dataset.
3363 Args:
3364 dataset_id (Union[UUID, str]): The ID of the dataset to unshare.
3366 Returns:
3367 None
3368 """
3369 response = self.request_with_retries(
3370 "DELETE",
3371 f"/datasets/{_as_uuid(dataset_id, 'dataset_id')}/share",
3372 headers=self._headers,
3373 )
3374 ls_utils.raise_for_status_with_text(response)
3376 def read_shared_dataset(
3377 self,
3378 share_token: str,
3379 ) -> ls_schemas.Dataset:
3380 """Get shared datasets.
3382 Args:
3383 share_token (Union[UUID, str]): The share token or URL of the shared dataset.
3385 Returns:
3386 Dataset: The shared dataset.
3387 """
3388 _, token_uuid = _parse_token_or_url(share_token, self.api_url)
3389 response = self.request_with_retries(
3390 "GET",
3391 f"/public/{token_uuid}/datasets",
3392 headers=self._headers,
3393 )
3394 ls_utils.raise_for_status_with_text(response)
3395 return ls_schemas.Dataset(
3396 **response.json(),
3397 _host_url=self._host_url,
3398 _public_path=f"/public/{share_token}/d",
3399 )
3401 def list_shared_examples(
3402 self,
3403 share_token: str,
3404 *,
3405 example_ids: Optional[list[ID_TYPE]] = None,
3406 limit: Optional[int] = None,
3407 ) -> Iterator[ls_schemas.Example]:
3408 """Get shared examples.
3410 Args:
3411 share_token (Union[UUID, str]): The share token or URL of the shared dataset.
3412 example_ids (Optional[List[UUID, str]], optional): The IDs of the examples to filter by.
3413 limit (Optional[int]): Maximum number of examples to return, by default None.
3415 Returns:
3416 List[ls_schemas.Example]: The list of shared examples.
3417 """
3418 params = {}
3419 if example_ids is not None:
3420 params["id"] = [str(id) for id in example_ids]
3421 for i, example in enumerate(
3422 self._get_paginated_list(
3423 f"/public/{_as_uuid(share_token, 'share_token')}/examples",
3424 params=params,
3425 )
3426 ):
3427 yield ls_schemas.Example(**example, _host_url=self._host_url)
3428 if limit is not None and i + 1 >= limit:
3429 break
3431 def list_shared_projects(
3432 self,
3433 *,
3434 dataset_share_token: str,
3435 project_ids: Optional[list[ID_TYPE]] = None,
3436 name: Optional[str] = None,
3437 name_contains: Optional[str] = None,
3438 limit: Optional[int] = None,
3439 ) -> Iterator[ls_schemas.TracerSessionResult]:
3440 """List shared projects.
3442 Args:
3443 dataset_share_token (str): The share token of the dataset.
3444 project_ids (Optional[List[Union[UUID, str]]]): List of project IDs to filter the results, by default None.
3445 name (Optional[str]): Name of the project to filter the results, by default None.
3446 name_contains (Optional[str]): Substring to search for in project names, by default None.
3447 limit (Optional[int]): Maximum number of projects to return, by default None.
3449 Yields:
3450 The shared projects.
3451 """
3452 params = {"id": project_ids, "name": name, "name_contains": name_contains}
3453 share_token = _as_uuid(dataset_share_token, "dataset_share_token")
3454 for i, project in enumerate(
3455 self._get_paginated_list(
3456 f"/public/{share_token}/datasets/sessions",
3457 params=params,
3458 )
3459 ):
3460 yield ls_schemas.TracerSessionResult(**project, _host_url=self._host_url)
3461 if limit is not None and i + 1 >= limit:
3462 break
3464 def create_project(
3465 self,
3466 project_name: str,
3467 *,
3468 description: Optional[str] = None,
3469 metadata: Optional[dict] = None,
3470 upsert: bool = False,
3471 project_extra: Optional[dict] = None,
3472 reference_dataset_id: Optional[ID_TYPE] = None,
3473 ) -> ls_schemas.TracerSession:
3474 """Create a project on the LangSmith API.
3476 Args:
3477 project_name (str): The name of the project.
3478 project_extra (Optional[dict]): Additional project information.
3479 metadata (Optional[dict]): Additional metadata to associate with the project.
3480 description (Optional[str]): The description of the project.
3481 upsert (bool, default=False): Whether to update the project if it already exists.
3482 reference_dataset_id (Optional[Union[UUID, str]): The ID of the reference dataset to associate with the project.
3484 Returns:
3485 TracerSession: The created project.
3486 """
3487 endpoint = f"{self.api_url}/sessions"
3488 extra = project_extra
3489 if metadata:
3490 extra = {**(extra or {}), "metadata": metadata}
3491 body: dict[str, Any] = {
3492 "name": project_name,
3493 "extra": extra,
3494 "description": description,
3495 "id": str(uuid.uuid4()),
3496 }
3497 params = {}
3498 if upsert:
3499 params["upsert"] = True
3500 if reference_dataset_id is not None:
3501 body["reference_dataset_id"] = reference_dataset_id
3502 response = self.request_with_retries(
3503 "POST",
3504 endpoint,
3505 headers={**self._headers, "Content-Type": "application/json"},
3506 data=_dumps_json(body),
3507 )
3508 ls_utils.raise_for_status_with_text(response)
3509 return ls_schemas.TracerSession(**response.json(), _host_url=self._host_url)
3511 def update_project(
3512 self,
3513 project_id: ID_TYPE,
3514 *,
3515 name: Optional[str] = None,
3516 description: Optional[str] = None,
3517 metadata: Optional[dict] = None,
3518 project_extra: Optional[dict] = None,
3519 end_time: Optional[datetime.datetime] = None,
3520 ) -> ls_schemas.TracerSession:
3521 """Update a LangSmith project.
3523 Args:
3524 project_id (Union[UUID, str]):
3525 The ID of the project to update.
3526 name (Optional[str]):
3527 The new name to give the project. This is only valid if the project
3528 has been assigned an end_time, meaning it has been completed/closed.
3529 description (Optional[str]):
3530 The new description to give the project.
3531 metadata (Optional[dict]):
3532 Additional metadata to associate with the project.
3533 project_extra (Optional[dict]):
3534 Additional project information.
3535 end_time (Optional[datetime.datetime]):
3536 The time the project was completed.
3538 Returns:
3539 TracerSession: The updated project.
3540 """
3541 endpoint = f"{self.api_url}/sessions/{_as_uuid(project_id, 'project_id')}"
3542 extra = project_extra
3543 if metadata:
3544 extra = {**(extra or {}), "metadata": metadata}
3545 body: dict[str, Any] = {
3546 "name": name,
3547 "extra": extra,
3548 "description": description,
3549 "end_time": end_time.isoformat() if end_time else None,
3550 }
3551 response = self.request_with_retries(
3552 "PATCH",
3553 endpoint,
3554 headers={**self._headers, "Content-Type": "application/json"},
3555 data=_dumps_json(body),
3556 )
3557 ls_utils.raise_for_status_with_text(response)
3558 return ls_schemas.TracerSession(**response.json(), _host_url=self._host_url)
3560 def _get_optional_tenant_id(self) -> Optional[uuid.UUID]:
3561 if self._tenant_id is not None:
3562 return self._tenant_id
3563 try:
3564 response = self.request_with_retries(
3565 "GET", "/sessions", params={"limit": 1}
3566 )
3567 result = response.json()
3568 if isinstance(result, list) and len(result) > 0:
3569 tracer_session = ls_schemas.TracerSessionResult(
3570 **result[0], _host_url=self._host_url
3571 )
3572 self._tenant_id = tracer_session.tenant_id
3573 return self._tenant_id
3574 except Exception as e:
3575 logger.debug(
3576 "Failed to get tenant ID from LangSmith: %s", repr(e), exc_info=True
3577 )
3578 return None
3580 def _get_tenant_id(self) -> uuid.UUID:
3581 tenant_id = self._get_optional_tenant_id()
3582 if tenant_id is None:
3583 raise ls_utils.LangSmithError("No tenant ID found")
3584 return tenant_id
3586 @ls_utils.xor_args(("project_id", "project_name"))
3587 def read_project(
3588 self,
3589 *,
3590 project_id: Optional[str] = None,
3591 project_name: Optional[str] = None,
3592 include_stats: bool = False,
3593 ) -> ls_schemas.TracerSessionResult:
3594 """Read a project from the LangSmith API.
3596 Args:
3597 project_id (Optional[str]):
3598 The ID of the project to read.
3599 project_name (Optional[str]): The name of the project to read.
3600 Only one of project_id or project_name may be given.
3601 include_stats (bool, default=False):
3602 Whether to include a project's aggregate statistics in the response.
3604 Returns:
3605 TracerSessionResult: The project.
3606 """
3607 path = "/sessions"
3608 params: dict[str, Any] = {"limit": 1}
3609 if project_id is not None:
3610 path += f"/{_as_uuid(project_id, 'project_id')}"
3611 elif project_name is not None:
3612 params["name"] = project_name
3613 else:
3614 raise ValueError("Must provide project_name or project_id")
3615 params["include_stats"] = include_stats
3616 response = self.request_with_retries("GET", path, params=params)
3617 result = response.json()
3618 if isinstance(result, list):
3619 if len(result) == 0:
3620 raise ls_utils.LangSmithNotFoundError(
3621 f"Project {project_name} not found"
3622 )
3623 return ls_schemas.TracerSessionResult(**result[0], _host_url=self._host_url)
3624 return ls_schemas.TracerSessionResult(
3625 **response.json(), _host_url=self._host_url
3626 )
3628 def has_project(
3629 self, project_name: str, *, project_id: Optional[str] = None
3630 ) -> bool:
3631 """Check if a project exists.
3633 Args:
3634 project_name (str):
3635 The name of the project to check for.
3636 project_id (Optional[str]):
3637 The ID of the project to check for.
3639 Returns:
3640 bool: Whether the project exists.
3641 """
3642 try:
3643 self.read_project(project_name=project_name)
3644 except ls_utils.LangSmithNotFoundError:
3645 return False
3646 return True
3648 def get_test_results(
3649 self,
3650 *,
3651 project_id: Optional[ID_TYPE] = None,
3652 project_name: Optional[str] = None,
3653 ) -> pd.DataFrame:
3654 """Read the record-level information from an experiment into a Pandas DF.
3656 !!! note
3658 This will fetch whatever data exists in the DB. Results are not
3659 immediately available in the DB upon evaluation run completion.
3661 Feedback score values will be returned as an average across all runs for
3662 the experiment. Non-numeric feedback scores will be omitted.
3664 Args:
3665 project_id (Optional[Union[UUID, str]]): The ID of the project.
3666 project_name (Optional[str]): The name of the project.
3668 Returns:
3669 pd.DataFrame: A dataframe containing the test results.
3670 """
3671 warnings.warn(
3672 "Function get_test_results is in beta.", UserWarning, stacklevel=2
3673 )
3674 from concurrent.futures import ThreadPoolExecutor, as_completed # type: ignore
3676 import pandas as pd # type: ignore
3678 runs = self.list_runs(
3679 project_id=project_id,
3680 project_name=project_name,
3681 is_root=True,
3682 select=[
3683 "id",
3684 "reference_example_id",
3685 "inputs",
3686 "outputs",
3687 "error",
3688 "feedback_stats",
3689 "start_time",
3690 "end_time",
3691 ],
3692 )
3693 results: list[dict] = []
3694 example_ids = []
3696 def fetch_examples(batch):
3697 examples = self.list_examples(example_ids=batch)
3698 return [
3699 {
3700 "example_id": example.id,
3701 **{f"reference.{k}": v for k, v in (example.outputs or {}).items()},
3702 }
3703 for example in examples
3704 ]
3706 batch_size = 50
3707 cursor = 0
3708 with ThreadPoolExecutor() as executor:
3709 futures = []
3710 for r in runs:
3711 row = {
3712 "example_id": r.reference_example_id,
3713 **{f"input.{k}": v for k, v in r.inputs.items()},
3714 **{f"outputs.{k}": v for k, v in (r.outputs or {}).items()},
3715 "execution_time": (
3716 (r.end_time - r.start_time).total_seconds()
3717 if r.end_time
3718 else None
3719 ),
3720 "error": r.error,
3721 "id": r.id,
3722 }
3723 if r.feedback_stats:
3724 row.update(
3725 {
3726 f"feedback.{k}": v.get("avg")
3727 for k, v in r.feedback_stats.items()
3728 if not (k == "note" and v.get("comments"))
3729 }
3730 )
3731 if r.feedback_stats.get("note") and (
3732 comments := r.feedback_stats["note"].get("comments")
3733 ):
3734 row["notes"] = comments
3735 if r.reference_example_id:
3736 example_ids.append(r.reference_example_id)
3737 else:
3738 logger.warning(f"Run {r.id} has no reference example ID.")
3739 if len(example_ids) % batch_size == 0:
3740 # Ensure not empty
3741 if batch := example_ids[cursor : cursor + batch_size]:
3742 futures.append(executor.submit(fetch_examples, batch))
3743 cursor += batch_size
3744 results.append(row)
3746 # Handle any remaining examples
3747 if example_ids[cursor:]:
3748 futures.append(executor.submit(fetch_examples, example_ids[cursor:]))
3749 result_df = pd.DataFrame(results).set_index("example_id")
3750 example_outputs = [
3751 output for future in as_completed(futures) for output in future.result()
3752 ]
3753 if example_outputs:
3754 example_df = pd.DataFrame(example_outputs).set_index("example_id")
3755 result_df = example_df.merge(result_df, left_index=True, right_index=True)
3757 # Flatten dict columns into dot syntax for easier access
3758 return pd.json_normalize(result_df.to_dict(orient="records"))
3760 def list_projects(
3761 self,
3762 project_ids: Optional[list[ID_TYPE]] = None,
3763 name: Optional[str] = None,
3764 name_contains: Optional[str] = None,
3765 reference_dataset_id: Optional[ID_TYPE] = None,
3766 reference_dataset_name: Optional[str] = None,
3767 reference_free: Optional[bool] = None,
3768 include_stats: Optional[bool] = None,
3769 dataset_version: Optional[str] = None,
3770 limit: Optional[int] = None,
3771 metadata: Optional[dict[str, Any]] = None,
3772 ) -> Iterator[ls_schemas.TracerSessionResult]:
3773 """List projects from the LangSmith API.
3775 Args:
3776 project_ids (Optional[List[Union[UUID, str]]]):
3777 A list of project IDs to filter by, by default None
3778 name (Optional[str]):
3779 The name of the project to filter by, by default None
3780 name_contains (Optional[str]):
3781 A string to search for in the project name, by default None
3782 reference_dataset_id (Optional[List[Union[UUID, str]]]):
3783 A dataset ID to filter by, by default None
3784 reference_dataset_name (Optional[str]):
3785 The name of the reference dataset to filter by, by default None
3786 reference_free (Optional[bool]):
3787 Whether to filter for only projects not associated with a dataset.
3788 limit (Optional[int]):
3789 The maximum number of projects to return, by default None
3790 metadata (Optional[Dict[str, Any]]):
3791 Metadata to filter by.
3793 Yields:
3794 The projects.
3796 Raises:
3797 ValueError: If both reference_dataset_id and reference_dataset_name are given.
3798 """
3799 params: dict[str, Any] = {
3800 "limit": min(limit, 100) if limit is not None else 100
3801 }
3802 if project_ids is not None:
3803 params["id"] = project_ids
3804 if name is not None:
3805 params["name"] = name
3806 if name_contains is not None:
3807 params["name_contains"] = name_contains
3808 if reference_dataset_id is not None:
3809 if reference_dataset_name is not None:
3810 raise ValueError(
3811 "Only one of reference_dataset_id or"
3812 " reference_dataset_name may be given"
3813 )
3814 params["reference_dataset"] = reference_dataset_id
3815 elif reference_dataset_name is not None:
3816 reference_dataset_id = self.read_dataset(
3817 dataset_name=reference_dataset_name
3818 ).id
3819 params["reference_dataset"] = reference_dataset_id
3820 if reference_free is not None:
3821 params["reference_free"] = reference_free
3822 if include_stats is not None:
3823 params["include_stats"] = include_stats
3824 if dataset_version is not None:
3825 params["dataset_version"] = dataset_version
3826 if metadata is not None:
3827 params["metadata"] = json.dumps(metadata)
3828 for i, project in enumerate(
3829 self._get_paginated_list("/sessions", params=params)
3830 ):
3831 yield ls_schemas.TracerSessionResult(**project, _host_url=self._host_url)
3832 if limit is not None and i + 1 >= limit:
3833 break
3835 @ls_utils.xor_args(("project_name", "project_id"))
3836 def delete_project(
3837 self, *, project_name: Optional[str] = None, project_id: Optional[str] = None
3838 ) -> None:
3839 """Delete a project from LangSmith.
3841 Args:
3842 project_name (Optional[str]):
3843 The name of the project to delete.
3844 project_id (Optional[str]):
3845 The ID of the project to delete.
3847 Returns:
3848 None
3850 Raises:
3851 ValueError: If neither project_name or project_id is provided.
3852 """
3853 if project_name is not None:
3854 project_id = str(self.read_project(project_name=project_name).id)
3855 elif project_id is None:
3856 raise ValueError("Must provide project_name or project_id")
3857 response = self.request_with_retries(
3858 "DELETE",
3859 f"/sessions/{_as_uuid(project_id, 'project_id')}",
3860 headers=self._headers,
3861 )
3862 ls_utils.raise_for_status_with_text(response)
3864 def create_dataset(
3865 self,
3866 dataset_name: str,
3867 *,
3868 description: Optional[str] = None,
3869 data_type: ls_schemas.DataType = ls_schemas.DataType.kv,
3870 inputs_schema: Optional[dict[str, Any]] = None,
3871 outputs_schema: Optional[dict[str, Any]] = None,
3872 transformations: Optional[list[ls_schemas.DatasetTransformation]] = None,
3873 metadata: Optional[dict] = None,
3874 ) -> ls_schemas.Dataset:
3875 """Create a dataset in the LangSmith API.
3877 Args:
3878 dataset_name (str):
3879 The name of the dataset.
3880 description (Optional[str]):
3881 The description of the dataset.
3882 data_type (DataType, default=DataType.kv):
3883 The data type of the dataset.
3884 inputs_schema (Optional[Dict[str, Any]]):
3885 The schema definition for the inputs of the dataset.
3886 outputs_schema (Optional[Dict[str, Any]]):
3887 The schema definition for the outputs of the dataset.
3888 transformations (Optional[List[DatasetTransformation]]):
3889 A list of transformations to apply to the dataset.
3890 metadata (Optional[dict]):
3891 Additional metadata to associate with the dataset.
3893 Returns:
3894 Dataset: The created dataset.
3896 Raises:
3897 requests.HTTPError: If the request to create the dataset fails.
3898 """
3899 metadata = {"runtime": ls_env.get_runtime_environment(), **(metadata or {})}
3900 dataset: dict[str, Any] = {
3901 "name": dataset_name,
3902 "data_type": data_type.value,
3903 "transformations": transformations,
3904 "extra": {
3905 "metadata": {
3906 "runtime": ls_env.get_runtime_environment(),
3907 **(metadata or {}),
3908 }
3909 },
3910 }
3911 if description is not None:
3912 dataset["description"] = description
3914 if inputs_schema is not None:
3915 dataset["inputs_schema_definition"] = inputs_schema
3917 if outputs_schema is not None:
3918 dataset["outputs_schema_definition"] = outputs_schema
3920 response = self.request_with_retries(
3921 "POST",
3922 "/datasets",
3923 headers={**self._headers, "Content-Type": "application/json"},
3924 data=_orjson.dumps(dataset),
3925 )
3926 ls_utils.raise_for_status_with_text(response)
3928 json_response = response.json()
3929 json_response["metadata"] = json_response.get("metadata") or metadata
3930 return ls_schemas.Dataset(
3931 **json_response,
3932 _host_url=self._host_url,
3933 _tenant_id=self._get_optional_tenant_id(),
3934 )
3936 def has_dataset(
3937 self,
3938 *,
3939 dataset_name: Optional[str] = None,
3940 dataset_id: Optional[ID_TYPE] = None,
3941 ) -> bool:
3942 """Check whether a dataset exists in your tenant.
3944 Args:
3945 dataset_name (Optional[str]):
3946 The name of the dataset to check.
3947 dataset_id (Optional[Union[UUID, str]]):
3948 The ID of the dataset to check.
3950 Returns:
3951 bool: Whether the dataset exists.
3952 """
3953 try:
3954 self.read_dataset(dataset_name=dataset_name, dataset_id=dataset_id)
3955 return True
3956 except ls_utils.LangSmithNotFoundError:
3957 return False
3959 @ls_utils.xor_args(("dataset_name", "dataset_id"))
3960 def read_dataset(
3961 self,
3962 *,
3963 dataset_name: Optional[str] = None,
3964 dataset_id: Optional[ID_TYPE] = None,
3965 ) -> ls_schemas.Dataset:
3966 """Read a dataset from the LangSmith API.
3968 Args:
3969 dataset_name (Optional[str]):
3970 The name of the dataset to read.
3971 dataset_id (Optional[Union[UUID, str]]):
3972 The ID of the dataset to read.
3974 Returns:
3975 Dataset: The dataset.
3976 """
3977 path = "/datasets"
3978 params: dict[str, Any] = {"limit": 1}
3979 if dataset_id is not None:
3980 path += f"/{_as_uuid(dataset_id, 'dataset_id')}"
3981 elif dataset_name is not None:
3982 params["name"] = dataset_name
3983 else:
3984 raise ValueError("Must provide dataset_name or dataset_id")
3985 response = self.request_with_retries(
3986 "GET",
3987 path,
3988 params=params,
3989 )
3990 result = response.json()
3991 if isinstance(result, list):
3992 if len(result) == 0:
3993 raise ls_utils.LangSmithNotFoundError(
3994 f"Dataset {dataset_name} not found"
3995 )
3996 return ls_schemas.Dataset(
3997 **result[0],
3998 _host_url=self._host_url,
3999 _tenant_id=self._get_optional_tenant_id(),
4000 )
4001 return ls_schemas.Dataset(
4002 **result,
4003 _host_url=self._host_url,
4004 _tenant_id=self._get_optional_tenant_id(),
4005 )
4007 def diff_dataset_versions(
4008 self,
4009 dataset_id: Optional[ID_TYPE] = None,
4010 *,
4011 dataset_name: Optional[str] = None,
4012 from_version: Union[str, datetime.datetime],
4013 to_version: Union[str, datetime.datetime],
4014 ) -> ls_schemas.DatasetDiffInfo:
4015 """Get the difference between two versions of a dataset.
4017 Args:
4018 dataset_id (Optional[Union[UUID, str]]):
4019 The ID of the dataset.
4020 dataset_name (Optional[str]):
4021 The name of the dataset.
4022 from_version (Union[str, datetime.datetime]):
4023 The starting version for the diff.
4024 to_version (Union[str, datetime.datetime]):
4025 The ending version for the diff.
4027 Returns:
4028 DatasetDiffInfo: The difference between the two versions of the dataset.
4030 Examples:
4031 ```python
4032 # Get the difference between two tagged versions of a dataset
4033 from_version = "prod"
4034 to_version = "dev"
4035 diff = client.diff_dataset_versions(
4036 dataset_name="my-dataset",
4037 from_version=from_version,
4038 to_version=to_version,
4039 )
4041 # Get the difference between two timestamped versions of a dataset
4042 from_version = datetime.datetime(2024, 1, 1)
4043 to_version = datetime.datetime(2024, 2, 1)
4044 diff = client.diff_dataset_versions(
4045 dataset_name="my-dataset",
4046 from_version=from_version,
4047 to_version=to_version,
4048 )
4049 ```
4050 """
4051 if dataset_id is None:
4052 if dataset_name is None:
4053 raise ValueError("Must provide either dataset name or ID")
4054 dataset_id = self.read_dataset(dataset_name=dataset_name).id
4055 dsid = _as_uuid(dataset_id, "dataset_id")
4056 response = self.request_with_retries(
4057 "GET",
4058 f"/datasets/{dsid}/versions/diff",
4059 headers=self._headers,
4060 params={
4061 "from_version": (
4062 from_version.isoformat()
4063 if isinstance(from_version, datetime.datetime)
4064 else from_version
4065 ),
4066 "to_version": (
4067 to_version.isoformat()
4068 if isinstance(to_version, datetime.datetime)
4069 else to_version
4070 ),
4071 },
4072 )
4073 ls_utils.raise_for_status_with_text(response)
4074 return ls_schemas.DatasetDiffInfo(**response.json())
4076 def read_dataset_openai_finetuning(
4077 self,
4078 dataset_id: Optional[ID_TYPE] = None,
4079 *,
4080 dataset_name: Optional[str] = None,
4081 ) -> list:
4082 """Download a dataset in OpenAI Jsonl format and load it as a list of dicts.
4084 Args:
4085 dataset_id (Optional[Union[UUID, str]]):
4086 The ID of the dataset to download.
4087 dataset_name (Optional[str]):
4088 The name of the dataset to download.
4090 Returns:
4091 list[dict]: The dataset loaded as a list of dicts.
4093 Raises:
4094 ValueError: If neither dataset_id nor dataset_name is provided.
4095 """
4096 path = "/datasets"
4097 if dataset_id is not None:
4098 pass
4099 elif dataset_name is not None:
4100 dataset_id = self.read_dataset(dataset_name=dataset_name).id
4101 else:
4102 raise ValueError("Must provide dataset_name or dataset_id")
4103 response = self.request_with_retries(
4104 "GET",
4105 f"{path}/{_as_uuid(dataset_id, 'dataset_id')}/openai_ft",
4106 )
4107 dataset = [json.loads(line) for line in response.text.strip().split("\n")]
4108 return dataset
4110 def list_datasets(
4111 self,
4112 *,
4113 dataset_ids: Optional[list[ID_TYPE]] = None,
4114 data_type: Optional[str] = None,
4115 dataset_name: Optional[str] = None,
4116 dataset_name_contains: Optional[str] = None,
4117 metadata: Optional[dict[str, Any]] = None,
4118 limit: Optional[int] = None,
4119 ) -> Iterator[ls_schemas.Dataset]:
4120 """List the datasets on the LangSmith API.
4122 Args:
4123 dataset_ids (Optional[List[Union[UUID, str]]]):
4124 A list of dataset IDs to filter the results by.
4125 data_type (Optional[str]):
4126 The data type of the datasets to filter the results by.
4127 dataset_name (Optional[str]):
4128 The name of the dataset to filter the results by.
4129 dataset_name_contains (Optional[str]):
4130 A substring to search for in the dataset names.
4131 metadata (Optional[Dict[str, Any]]):
4132 A dictionary of metadata to filter the results by.
4133 limit (Optional[int]):
4134 The maximum number of datasets to return.
4136 Yields:
4137 The datasets.
4138 """
4139 params: dict[str, Any] = {
4140 "limit": min(limit, 100) if limit is not None else 100
4141 }
4142 if dataset_ids is not None:
4143 params["id"] = dataset_ids
4144 if data_type is not None:
4145 params["data_type"] = data_type
4146 if dataset_name is not None:
4147 params["name"] = dataset_name
4148 if dataset_name_contains is not None:
4149 params["name_contains"] = dataset_name_contains
4150 if metadata is not None:
4151 params["metadata"] = json.dumps(metadata)
4152 for i, dataset in enumerate(
4153 self._get_paginated_list("/datasets", params=params)
4154 ):
4155 yield ls_schemas.Dataset(
4156 **dataset,
4157 _host_url=self._host_url,
4158 _tenant_id=self._get_optional_tenant_id(),
4159 )
4160 if limit is not None and i + 1 >= limit:
4161 break
4163 @ls_utils.xor_args(("dataset_id", "dataset_name"))
4164 def delete_dataset(
4165 self,
4166 *,
4167 dataset_id: Optional[ID_TYPE] = None,
4168 dataset_name: Optional[str] = None,
4169 ) -> None:
4170 """Delete a dataset from the LangSmith API.
4172 Args:
4173 dataset_id (Optional[Union[UUID, str]]):
4174 The ID of the dataset to delete.
4175 dataset_name (Optional[str]):
4176 The name of the dataset to delete.
4178 Returns:
4179 None
4180 """
4181 if dataset_name is not None:
4182 dataset_id = self.read_dataset(dataset_name=dataset_name).id
4183 if dataset_id is None:
4184 raise ValueError("Must provide either dataset name or ID")
4185 response = self.request_with_retries(
4186 "DELETE",
4187 f"/datasets/{_as_uuid(dataset_id, 'dataset_id')}",
4188 headers=self._headers,
4189 )
4190 ls_utils.raise_for_status_with_text(response)
4192 def update_dataset_tag(
4193 self,
4194 *,
4195 dataset_id: Optional[ID_TYPE] = None,
4196 dataset_name: Optional[str] = None,
4197 as_of: datetime.datetime,
4198 tag: str,
4199 ) -> None:
4200 """Update the tags of a dataset.
4202 If the tag is already assigned to a different version of this dataset,
4203 the tag will be moved to the new version. The as_of parameter is used to
4204 determine which version of the dataset to apply the new tags to.
4205 It must be an exact version of the dataset to succeed. You can
4206 use the read_dataset_version method to find the exact version
4207 to apply the tags to.
4209 Args:
4210 dataset_id (Optional[Union[UUID, str]]):
4211 The ID of the dataset to update.
4212 dataset_name (Optional[str]):
4213 The name of the dataset to update.
4214 as_of (datetime.datetime):
4215 The timestamp of the dataset to apply the new tags to.
4216 tag (str):
4217 The new tag to apply to the dataset.
4219 Returns:
4220 None
4222 Examples:
4223 ```python
4224 dataset_name = "my-dataset"
4225 # Get the version of a dataset <= a given timestamp
4226 dataset_version = client.read_dataset_version(
4227 dataset_name=dataset_name, as_of=datetime.datetime(2024, 1, 1)
4228 )
4229 # Assign that version a new tag
4230 client.update_dataset_tags(
4231 dataset_name="my-dataset",
4232 as_of=dataset_version.as_of,
4233 tag="prod",
4234 )
4235 ```
4236 """
4237 if dataset_name is not None:
4238 dataset_id = self.read_dataset(dataset_name=dataset_name).id
4239 if dataset_id is None:
4240 raise ValueError("Must provide either dataset name or ID")
4241 response = self.request_with_retries(
4242 "PUT",
4243 f"/datasets/{_as_uuid(dataset_id, 'dataset_id')}/tags",
4244 headers=self._headers,
4245 json={
4246 "as_of": as_of.isoformat(),
4247 "tag": tag,
4248 },
4249 )
4250 ls_utils.raise_for_status_with_text(response)
4252 def list_dataset_versions(
4253 self,
4254 *,
4255 dataset_id: Optional[ID_TYPE] = None,
4256 dataset_name: Optional[str] = None,
4257 search: Optional[str] = None,
4258 limit: Optional[int] = None,
4259 ) -> Iterator[ls_schemas.DatasetVersion]:
4260 """List dataset versions.
4262 Args:
4263 dataset_id (Optional[Union[UUID, str]]): The ID of the dataset.
4264 dataset_name (Optional[str]): The name of the dataset.
4265 search (Optional[str]): The search query.
4266 limit (Optional[int]): The maximum number of versions to return.
4268 Yields:
4269 The dataset versions.
4270 """
4271 if dataset_id is None:
4272 dataset_id = self.read_dataset(dataset_name=dataset_name).id
4273 params = {
4274 "search": search,
4275 "limit": min(limit, 100) if limit is not None else 100,
4276 }
4277 for i, version in enumerate(
4278 self._get_paginated_list(
4279 f"/datasets/{_as_uuid(dataset_id, 'dataset_id')}/versions",
4280 params=params,
4281 )
4282 ):
4283 yield ls_schemas.DatasetVersion(**version)
4284 if limit is not None and i + 1 >= limit:
4285 break
4287 def read_dataset_version(
4288 self,
4289 *,
4290 dataset_id: Optional[ID_TYPE] = None,
4291 dataset_name: Optional[str] = None,
4292 as_of: Optional[datetime.datetime] = None,
4293 tag: Optional[str] = None,
4294 ) -> ls_schemas.DatasetVersion:
4295 """Get dataset version by `as_of` or exact tag.
4297 Ues this to resolve the nearest version to a given timestamp or for a given tag.
4299 Args:
4300 dataset_id (Optional[ID_TYPE]): The ID of the dataset.
4301 dataset_name (Optional[str]): The name of the dataset.
4302 as_of (Optional[datetime.datetime]): The timestamp of the dataset
4303 to retrieve.
4304 tag (Optional[str]): The tag of the dataset to retrieve.
4306 Returns:
4307 DatasetVersion: The dataset version.
4309 Examples:
4310 ```python
4311 # Get the latest version of a dataset
4312 client.read_dataset_version(dataset_name="my-dataset", tag="latest")
4314 # Get the version of a dataset <= a given timestamp
4315 client.read_dataset_version(
4316 dataset_name="my-dataset",
4317 as_of=datetime.datetime(2024, 1, 1),
4318 )
4321 # Get the version of a dataset with a specific tag
4322 client.read_dataset_version(dataset_name="my-dataset", tag="prod")
4323 ```
4324 """
4325 if dataset_id is None:
4326 dataset_id = self.read_dataset(dataset_name=dataset_name).id
4327 if (as_of and tag) or (as_of is None and tag is None):
4328 raise ValueError("Exactly one of as_of and tag must be specified.")
4329 response = self.request_with_retries(
4330 "GET",
4331 f"/datasets/{_as_uuid(dataset_id, 'dataset_id')}/version",
4332 params={"as_of": as_of, "tag": tag},
4333 )
4334 return ls_schemas.DatasetVersion(**response.json())
4336 def clone_public_dataset(
4337 self,
4338 token_or_url: str,
4339 *,
4340 source_api_url: Optional[str] = None,
4341 dataset_name: Optional[str] = None,
4342 ) -> ls_schemas.Dataset:
4343 """Clone a public dataset to your own langsmith tenant.
4345 This operation is idempotent. If you already have a dataset with the given name,
4346 this function will do nothing.
4348 Args:
4349 token_or_url (str): The token of the public dataset to clone.
4350 source_api_url (Optional[str]): The URL of the langsmith server where the data is hosted.
4351 Defaults to the API URL of your current client.
4352 dataset_name (Optional[str]): The name of the dataset to create in your tenant.
4353 Defaults to the name of the public dataset.
4355 Returns:
4356 Dataset: The cloned dataset.
4357 """
4358 source_api_url = source_api_url or self.api_url
4359 source_api_url, token_uuid = _parse_token_or_url(token_or_url, source_api_url)
4360 source_client = Client(
4361 # Placeholder API key not needed anymore in most cases, but
4362 # some private deployments may have API key-based rate limiting
4363 # that would cause this to fail if we provide no value.
4364 api_url=source_api_url,
4365 api_key="placeholder",
4366 )
4367 ds = source_client.read_shared_dataset(token_uuid)
4368 dataset_name = dataset_name or ds.name
4369 try:
4370 ds = self.read_dataset(dataset_name=dataset_name)
4371 logger.info(
4372 f"Dataset {dataset_name} already exists in your tenant. Skipping."
4373 )
4374 return ds
4375 except ls_utils.LangSmithNotFoundError:
4376 pass
4378 try:
4379 # Fetch examples first
4380 examples = list(source_client.list_shared_examples(token_uuid))
4381 dataset = self.create_dataset(
4382 dataset_name=dataset_name,
4383 description=ds.description,
4384 data_type=ds.data_type or ls_schemas.DataType.kv,
4385 inputs_schema=ds.inputs_schema,
4386 outputs_schema=ds.outputs_schema,
4387 transformations=ds.transformations,
4388 )
4389 try:
4390 self.create_examples(
4391 inputs=[e.inputs for e in examples],
4392 outputs=[e.outputs for e in examples],
4393 dataset_id=dataset.id,
4394 )
4395 except BaseException as e:
4396 # Let's not do automatic clean up for now in case there might be
4397 # some other reasons why create_examples fails (i.e., not network issue
4398 # or keyboard interrupt).
4399 # The risk is that this is an existing dataset that has valid examples
4400 # populated from another source so we don't want to delete it.
4401 logger.error(
4402 f"An error occurred while creating dataset {dataset_name}. "
4403 "You should delete it manually."
4404 )
4405 raise e
4406 finally:
4407 del source_client
4408 return dataset
4410 def _get_data_type(self, dataset_id: ID_TYPE) -> ls_schemas.DataType:
4411 dataset = self.read_dataset(dataset_id=dataset_id)
4412 return dataset.data_type
4414 @ls_utils.xor_args(("dataset_id", "dataset_name"))
4415 def create_llm_example(
4416 self,
4417 prompt: str,
4418 generation: Optional[str] = None,
4419 dataset_id: Optional[ID_TYPE] = None,
4420 dataset_name: Optional[str] = None,
4421 created_at: Optional[datetime.datetime] = None,
4422 ) -> ls_schemas.Example:
4423 """Add an example (row) to an LLM-type dataset.
4425 Args:
4426 prompt (str):
4427 The input prompt for the example.
4428 generation (Optional[str]):
4429 The output generation for the example.
4430 dataset_id (Optional[Union[UUID, str]]):
4431 The ID of the dataset.
4432 dataset_name (Optional[str]):
4433 The name of the dataset.
4434 created_at (Optional[datetime.datetime]):
4435 The creation timestamp of the example.
4437 Returns:
4438 Example: The created example
4439 """
4440 return self.create_example(
4441 inputs={"input": prompt},
4442 outputs={"output": generation},
4443 dataset_id=dataset_id,
4444 dataset_name=dataset_name,
4445 created_at=created_at,
4446 )
4448 @ls_utils.xor_args(("dataset_id", "dataset_name"))
4449 def create_chat_example(
4450 self,
4451 messages: list[Union[Mapping[str, Any], ls_schemas.BaseMessageLike]],
4452 generations: Optional[
4453 Union[Mapping[str, Any], ls_schemas.BaseMessageLike]
4454 ] = None,
4455 dataset_id: Optional[ID_TYPE] = None,
4456 dataset_name: Optional[str] = None,
4457 created_at: Optional[datetime.datetime] = None,
4458 ) -> ls_schemas.Example:
4459 """Add an example (row) to a Chat-type dataset.
4461 Args:
4462 messages (List[Union[Mapping[str, Any], BaseMessageLike]]):
4463 The input messages for the example.
4464 generations (Optional[Union[Mapping[str, Any], BaseMessageLike]]):
4465 The output messages for the example.
4466 dataset_id (Optional[Union[UUID, str]]):
4467 The ID of the dataset.
4468 dataset_name (Optional[str]):
4469 The name of the dataset.
4470 created_at (Optional[datetime.datetime]):
4471 The creation timestamp of the example.
4473 Returns:
4474 Example: The created example
4475 """
4476 final_input = []
4477 for message in messages:
4478 if ls_utils.is_base_message_like(message):
4479 final_input.append(
4480 ls_utils.convert_langchain_message(
4481 cast(ls_schemas.BaseMessageLike, message)
4482 )
4483 )
4484 else:
4485 final_input.append(cast(dict, message))
4486 final_generations = None
4487 if generations is not None:
4488 if ls_utils.is_base_message_like(generations):
4489 final_generations = ls_utils.convert_langchain_message(
4490 cast(ls_schemas.BaseMessageLike, generations)
4491 )
4492 else:
4493 final_generations = cast(dict, generations)
4494 return self.create_example(
4495 inputs={"input": final_input},
4496 outputs=(
4497 {"output": final_generations} if final_generations is not None else None
4498 ),
4499 dataset_id=dataset_id,
4500 dataset_name=dataset_name,
4501 created_at=created_at,
4502 )
4504 def create_example_from_run(
4505 self,
4506 run: ls_schemas.Run,
4507 dataset_id: Optional[ID_TYPE] = None,
4508 dataset_name: Optional[str] = None,
4509 created_at: Optional[datetime.datetime] = None,
4510 ) -> ls_schemas.Example:
4511 """Add an example (row) to a dataset from a run.
4513 Args:
4514 run (Run): The run to create an example from.
4515 dataset_id (Optional[Union[UUID, str]]): The ID of the dataset.
4516 dataset_name (Optional[str]): The name of the dataset.
4517 created_at (Optional[datetime.datetime]): The creation timestamp of the example.
4519 Returns:
4520 Example: The created example
4521 """
4522 if dataset_id is None:
4523 dataset_id = self.read_dataset(dataset_name=dataset_name).id
4524 dataset_name = None # Nested call expects only 1 defined
4525 dataset_type = self._get_data_type_cached(dataset_id)
4526 if dataset_type == ls_schemas.DataType.llm:
4527 if run.run_type != "llm":
4528 raise ValueError(
4529 f"Run type {run.run_type} is not supported"
4530 " for dataset of type 'LLM'"
4531 )
4532 try:
4533 prompt = ls_utils.get_prompt_from_inputs(run.inputs)
4534 except ValueError:
4535 raise ValueError(
4536 "Error converting LLM run inputs to prompt for run"
4537 f" {run.id} with inputs {run.inputs}"
4538 )
4539 inputs: dict[str, Any] = {"input": prompt}
4540 if not run.outputs:
4541 outputs: Optional[dict[str, Any]] = None
4542 else:
4543 try:
4544 generation = ls_utils.get_llm_generation_from_outputs(run.outputs)
4545 except ValueError:
4546 raise ValueError(
4547 "Error converting LLM run outputs to generation for run"
4548 f" {run.id} with outputs {run.outputs}"
4549 )
4550 outputs = {"output": generation}
4551 elif dataset_type == ls_schemas.DataType.chat:
4552 if run.run_type != "llm":
4553 raise ValueError(
4554 f"Run type {run.run_type} is not supported"
4555 " for dataset of type 'chat'"
4556 )
4557 try:
4558 inputs = {"input": ls_utils.get_messages_from_inputs(run.inputs)}
4559 except ValueError:
4560 raise ValueError(
4561 "Error converting LLM run inputs to chat messages for run"
4562 f" {run.id} with inputs {run.inputs}"
4563 )
4564 if not run.outputs:
4565 outputs = None
4566 else:
4567 try:
4568 outputs = {
4569 "output": ls_utils.get_message_generation_from_outputs(
4570 run.outputs
4571 )
4572 }
4573 except ValueError:
4574 raise ValueError(
4575 "Error converting LLM run outputs to chat generations"
4576 f" for run {run.id} with outputs {run.outputs}"
4577 )
4578 elif dataset_type == ls_schemas.DataType.kv:
4579 # Anything goes
4580 inputs = run.inputs
4581 outputs = run.outputs
4583 else:
4584 raise ValueError(f"Dataset type {dataset_type} not recognized.")
4585 return self.create_example(
4586 inputs=inputs,
4587 outputs=outputs,
4588 dataset_id=dataset_id,
4589 dataset_name=dataset_name,
4590 created_at=created_at,
4591 )
4593 def _prepare_multipart_data(
4594 self,
4595 examples: Union[
4596 list[ls_schemas.ExampleCreate]
4597 | list[ls_schemas.ExampleUpsertWithAttachments]
4598 | list[ls_schemas.ExampleUpdate],
4599 ],
4600 include_dataset_id: bool = False,
4601 dangerously_allow_filesystem: bool = False,
4602 ) -> tuple[Any, bytes, dict[str, io.BufferedReader]]:
4603 parts: list[MultipartPart] = []
4604 opened_files_dict: dict[str, io.BufferedReader] = {}
4605 if include_dataset_id:
4606 if not isinstance(examples[0], ls_schemas.ExampleUpsertWithAttachments):
4607 raise ValueError(
4608 "The examples must be of type ExampleUpsertWithAttachments"
4609 " if include_dataset_id is True"
4610 )
4611 dataset_id = examples[0].dataset_id
4613 for example in examples:
4614 if (
4615 not isinstance(example, ls_schemas.ExampleCreate)
4616 and not isinstance(example, ls_schemas.ExampleUpsertWithAttachments)
4617 and not isinstance(example, ls_schemas.ExampleUpdate)
4618 ):
4619 raise ValueError(
4620 "The examples must be of type ExampleCreate"
4621 " or ExampleUpsertWithAttachments"
4622 " or ExampleUpdate"
4623 )
4624 if example.id is not None:
4625 example_id = str(example.id)
4626 else:
4627 example_id = str(uuid.uuid4())
4629 if isinstance(example, ls_schemas.ExampleUpdate):
4630 created_at = None
4631 else:
4632 created_at = example.created_at
4634 if isinstance(example, ls_schemas.ExampleCreate):
4635 use_source_run_io = example.use_source_run_io
4636 use_source_run_attachments = example.use_source_run_attachments
4637 source_run_id = example.source_run_id
4638 else:
4639 use_source_run_io, use_source_run_attachments, source_run_id = (
4640 None,
4641 None,
4642 None,
4643 )
4645 example_body = {
4646 **({"dataset_id": dataset_id} if include_dataset_id else {}),
4647 **({"created_at": created_at} if created_at is not None else {}),
4648 **(
4649 {"use_source_run_io": use_source_run_io}
4650 if use_source_run_io
4651 else {}
4652 ),
4653 **(
4654 {"use_source_run_attachments": use_source_run_attachments}
4655 if use_source_run_attachments
4656 else {}
4657 ),
4658 **({"source_run_id": source_run_id} if source_run_id else {}),
4659 }
4660 if example.metadata is not None:
4661 example_body["metadata"] = example.metadata
4662 if example.split is not None:
4663 example_body["split"] = example.split
4664 valb = _dumps_json(example_body)
4666 parts.append(
4667 (
4668 f"{example_id}",
4669 (
4670 None,
4671 valb,
4672 "application/json",
4673 {},
4674 ),
4675 )
4676 )
4678 if example.inputs is not None:
4679 inputsb = _dumps_json(example.inputs)
4680 parts.append(
4681 (
4682 f"{example_id}.inputs",
4683 (
4684 None,
4685 inputsb,
4686 "application/json",
4687 {},
4688 ),
4689 )
4690 )
4692 if example.outputs is not None:
4693 outputsb = _dumps_json(example.outputs)
4694 parts.append(
4695 (
4696 f"{example_id}.outputs",
4697 (
4698 None,
4699 outputsb,
4700 "application/json",
4701 {},
4702 ),
4703 )
4704 )
4706 if example.attachments:
4707 for name, attachment in example.attachments.items():
4708 if isinstance(attachment, dict):
4709 mime_type = attachment["mime_type"]
4710 attachment_data = attachment["data"]
4711 else:
4712 mime_type, attachment_data = attachment
4713 if isinstance(attachment_data, Path):
4714 if dangerously_allow_filesystem:
4715 try:
4716 file_size = os.path.getsize(attachment_data)
4717 file = open(attachment_data, "rb")
4718 except FileNotFoundError:
4719 logger.warning(
4720 "Attachment file not found for example %s: %s",
4721 example_id,
4722 attachment_data,
4723 )
4724 continue
4725 opened_files_dict[
4726 str(attachment_data) + str(uuid.uuid4())
4727 ] = file
4729 parts.append(
4730 (
4731 f"{example_id}.attachment.{name}",
4732 (
4733 None,
4734 file, # type: ignore[arg-type]
4735 f"{mime_type}; length={file_size}",
4736 {},
4737 ),
4738 )
4739 )
4740 else:
4741 raise ValueError(
4742 "dangerously_allow_filesystem must be True to upload files from the filesystem"
4743 )
4744 else:
4745 parts.append(
4746 (
4747 f"{example_id}.attachment.{name}",
4748 (
4749 None,
4750 attachment_data,
4751 f"{mime_type}; length={len(attachment_data)}",
4752 {},
4753 ),
4754 )
4755 )
4757 if (
4758 isinstance(example, ls_schemas.ExampleUpdate)
4759 and example.attachments_operations
4760 ):
4761 attachments_operationsb = _dumps_json(example.attachments_operations)
4762 parts.append(
4763 (
4764 f"{example_id}.attachments_operations",
4765 (
4766 None,
4767 attachments_operationsb,
4768 "application/json",
4769 {},
4770 ),
4771 )
4772 )
4774 encoder = rqtb_multipart.MultipartEncoder(parts, boundary=_BOUNDARY)
4775 if encoder.len <= 20_000_000: # ~20 MB
4776 data = encoder.to_string()
4777 else:
4778 data = encoder
4780 return encoder, data, opened_files_dict
4782 def update_examples_multipart(
4783 self,
4784 *,
4785 dataset_id: ID_TYPE,
4786 updates: Optional[list[ls_schemas.ExampleUpdate]] = None,
4787 dangerously_allow_filesystem: bool = False,
4788 ) -> ls_schemas.UpsertExamplesResponse:
4789 """Update examples using multipart.
4791 .. deprecated:: 0.3.9
4793 Use Client.update_examples instead. Will be removed in 0.4.0.
4794 """
4795 return self._update_examples_multipart(
4796 dataset_id=dataset_id,
4797 updates=updates,
4798 dangerously_allow_filesystem=dangerously_allow_filesystem,
4799 )
4801 def _update_examples_multipart(
4802 self,
4803 *,
4804 dataset_id: ID_TYPE,
4805 updates: Optional[list[ls_schemas.ExampleUpdate]] = None,
4806 dangerously_allow_filesystem: bool = False,
4807 ) -> ls_schemas.UpsertExamplesResponse:
4808 """Update examples using multipart.
4810 Args:
4811 dataset_id (Union[UUID, str]): The ID of the dataset to update.
4812 updates (Optional[List[ExampleUpdate]]): The updates to apply to the examples.
4814 Raises:
4815 ValueError: If the multipart examples endpoint is not enabled.
4816 """
4817 if not (self.info.instance_flags or {}).get(
4818 "dataset_examples_multipart_enabled", False
4819 ):
4820 raise ValueError(
4821 "Your LangSmith deployment does not allow using the latest examples "
4822 "endpoints, please upgrade your deployment to the latest version or downgrade your SDK "
4823 "to langsmith<0.3.9."
4824 )
4825 if updates is None:
4826 updates = []
4828 encoder, data, opened_files_dict = self._prepare_multipart_data(
4829 updates,
4830 include_dataset_id=False,
4831 dangerously_allow_filesystem=dangerously_allow_filesystem,
4832 )
4834 try:
4835 response = self.request_with_retries(
4836 "PATCH",
4837 _dataset_examples_path(self.api_url, dataset_id),
4838 request_kwargs={
4839 "data": data,
4840 "headers": {
4841 **self._headers,
4842 "Content-Type": encoder.content_type,
4843 },
4844 },
4845 )
4846 ls_utils.raise_for_status_with_text(response)
4847 finally:
4848 _close_files(list(opened_files_dict.values()))
4849 return response.json()
4851 def upload_examples_multipart(
4852 self,
4853 *,
4854 dataset_id: ID_TYPE,
4855 uploads: Optional[list[ls_schemas.ExampleCreate]] = None,
4856 dangerously_allow_filesystem: bool = False,
4857 ) -> ls_schemas.UpsertExamplesResponse:
4858 """Upload examples using multipart.
4860 .. deprecated:: 0.3.9
4862 Use Client.create_examples instead. Will be removed in 0.4.0.
4863 """
4864 return self._upload_examples_multipart(
4865 dataset_id=dataset_id,
4866 uploads=uploads,
4867 dangerously_allow_filesystem=dangerously_allow_filesystem,
4868 )
4870 def _estimate_example_size(self, example: ls_schemas.ExampleCreate) -> int:
4871 """Estimate the size of an example in bytes for batching purposes."""
4872 size = 1000 # Base overhead for JSON structure and boundaries
4874 if example.inputs:
4875 size += len(_dumps_json(example.inputs))
4876 if example.outputs:
4877 size += len(_dumps_json(example.outputs))
4878 if example.metadata:
4879 size += len(_dumps_json(example.metadata))
4881 # Estimate attachments
4882 if example.attachments:
4883 for _, attachment in example.attachments.items():
4884 if isinstance(attachment, dict):
4885 attachment_data = attachment["data"]
4886 else:
4887 _, attachment_data = attachment
4889 if isinstance(attachment_data, Path):
4890 try:
4891 size += os.path.getsize(attachment_data)
4892 except (FileNotFoundError, OSError):
4893 size += 1_000_000 # 1MB fallback estimate
4894 else:
4895 size += len(attachment_data)
4896 size += 200 # Multipart headers overhead per attachment
4898 return size
4900 def _batch_examples_by_size(
4901 self,
4902 examples: list[ls_schemas.ExampleCreate],
4903 max_batch_size_bytes: int = 20_000_000, # 20MB limit per batch
4904 ) -> list[list[ls_schemas.ExampleCreate]]:
4905 """Batch examples by size limits."""
4906 batches = []
4907 current_batch: list[ls_schemas.ExampleCreate] = []
4908 current_size = 0
4910 for example in examples:
4911 example_size = self._estimate_example_size(example)
4913 # Handle oversized single examples
4914 if example_size > max_batch_size_bytes:
4915 # Flush current batch first
4916 if current_batch:
4917 batches.append(current_batch)
4918 current_batch = []
4919 current_size = 0
4920 # oversized example
4921 batches.append([example])
4922 continue
4924 size_exceeded = current_size + example_size > max_batch_size_bytes
4926 # new batch
4927 if current_batch and size_exceeded:
4928 batches.append(current_batch)
4929 current_batch = [example]
4930 current_size = example_size
4931 else:
4932 current_batch.append(example)
4933 current_size += example_size
4935 # final batch
4936 if current_batch:
4937 batches.append(current_batch)
4939 return batches
4941 def _upload_examples_multipart(
4942 self,
4943 *,
4944 dataset_id: ID_TYPE,
4945 uploads: Optional[list[ls_schemas.ExampleCreate]] = None,
4946 dangerously_allow_filesystem: bool = False,
4947 ) -> ls_schemas.UpsertExamplesResponse:
4948 """Upload examples using multipart.
4950 Args:
4951 dataset_id (Union[UUID, str]): The ID of the dataset to upload to.
4952 uploads (Optional[List[ExampleCreate]]): The examples to upload.
4953 dangerously_allow_filesystem (bool): Whether to allow uploading files from the filesystem.
4955 Returns:
4956 ls_schemas.UpsertExamplesResponse: The count and ids of the successfully uploaded examples
4958 Raises:
4959 ValueError: If the multipart examples endpoint is not enabled.
4960 """
4961 if not (self.info.instance_flags or {}).get(
4962 "dataset_examples_multipart_enabled", False
4963 ):
4964 raise ValueError(
4965 "Your LangSmith deployment does not allow using the multipart examples endpoint, please upgrade your deployment to the latest version."
4966 )
4967 if uploads is None:
4968 uploads = []
4969 encoder, data, opened_files_dict = self._prepare_multipart_data(
4970 uploads,
4971 include_dataset_id=False,
4972 dangerously_allow_filesystem=dangerously_allow_filesystem,
4973 )
4975 try:
4976 response = self.request_with_retries(
4977 "POST",
4978 _dataset_examples_path(self.api_url, dataset_id),
4979 request_kwargs={
4980 "data": data,
4981 "headers": {
4982 **self._headers,
4983 "Content-Type": encoder.content_type,
4984 },
4985 },
4986 )
4987 ls_utils.raise_for_status_with_text(response)
4988 finally:
4989 _close_files(list(opened_files_dict.values()))
4990 return response.json()
4992 def upsert_examples_multipart(
4993 self,
4994 *,
4995 upserts: Optional[list[ls_schemas.ExampleUpsertWithAttachments]] = None,
4996 dangerously_allow_filesystem: bool = False,
4997 ) -> ls_schemas.UpsertExamplesResponse:
4998 """Upsert examples.
5000 .. deprecated:: 0.3.9
5002 Use Client.create_examples and Client.update_examples instead. Will be
5003 removed in 0.4.0.
5004 """
5005 if not (self.info.instance_flags or {}).get(
5006 "examples_multipart_enabled", False
5007 ):
5008 raise ValueError(
5009 "Your LangSmith deployment does not allow using the multipart examples endpoint, please upgrade your deployment to the latest version."
5010 )
5011 if upserts is None:
5012 upserts = []
5014 encoder, data, opened_files_dict = self._prepare_multipart_data(
5015 upserts,
5016 include_dataset_id=True,
5017 dangerously_allow_filesystem=dangerously_allow_filesystem,
5018 )
5020 try:
5021 response = self.request_with_retries(
5022 "POST",
5023 (
5024 "/v1/platform/examples/multipart"
5025 if self.api_url[-3:] != "/v1" and self.api_url[-4:] != "/v1/"
5026 else "/platform/examples/multipart"
5027 ),
5028 request_kwargs={
5029 "data": data,
5030 "headers": {
5031 **self._headers,
5032 "Content-Type": encoder.content_type,
5033 },
5034 },
5035 )
5036 ls_utils.raise_for_status_with_text(response)
5037 finally:
5038 _close_files(list(opened_files_dict.values()))
5039 return response.json()
5041 @ls_utils.xor_args(("dataset_id", "dataset_name"))
5042 def create_examples(
5043 self,
5044 *,
5045 dataset_name: Optional[str] = None,
5046 dataset_id: Optional[ID_TYPE] = None,
5047 examples: Optional[Sequence[ls_schemas.ExampleCreate | dict]] = None,
5048 dangerously_allow_filesystem: bool = False,
5049 max_concurrency: Annotated[int, Field(ge=1, le=3)] = 1,
5050 **kwargs: Any,
5051 ) -> ls_schemas.UpsertExamplesResponse | dict[str, Any]:
5052 """Create examples in a dataset.
5054 Args:
5055 dataset_name (str | None):
5056 The name of the dataset to create the examples in. Must specify exactly
5057 one of dataset_name or dataset_id.
5058 dataset_id (UUID | str | None):
5059 The ID of the dataset to create the examples in. Must specify exactly
5060 one of dataset_name or dataset_id
5061 examples (Sequence[ExampleCreate | dict]):
5062 The examples to create.
5063 dangerously_allow_filesystem (bool):
5064 Whether to allow uploading files from the filesystem.
5065 **kwargs (Any): Legacy keyword args. Should not be specified if 'examples' is specified.
5067 - inputs (Sequence[Mapping[str, Any]]): The input values for the examples.
5068 - outputs (Optional[Sequence[Optional[Mapping[str, Any]]]]): The output values for the examples.
5069 - metadata (Optional[Sequence[Optional[Mapping[str, Any]]]]): The metadata for the examples.
5070 - splits (Optional[Sequence[Optional[str | List[str]]]]): The splits for the examples, which are divisions of your dataset such as 'train', 'test', or 'validation'.
5071 - source_run_ids (Optional[Sequence[Optional[Union[UUID, str]]]]): The IDs of the source runs associated with the examples.
5072 - ids (Optional[Sequence[Union[UUID, str]]]): The IDs of the examples.
5074 Raises:
5075 ValueError: If 'examples' and legacy args are both provided.
5077 Returns:
5078 The LangSmith JSON response. Includes 'count' and 'example_ids'.
5080 !!! warning "Behavior changed in `langsmith` 0.3.11"
5082 Updated to take argument 'examples', a single list where each
5083 element is the full example to create. This should be used instead of the
5084 legacy 'inputs', 'outputs', etc. arguments which split each examples
5085 attributes across arguments.
5087 Updated to support creating examples with attachments.
5089 Example:
5090 ```python
5091 from langsmith import Client
5093 client = Client()
5095 dataset = client.create_dataset("agent-qa")
5097 examples = [
5098 {
5099 "inputs": {"question": "what's an agent"},
5100 "outputs": {"answer": "an agent is..."},
5101 "metadata": {"difficulty": "easy"},
5102 },
5103 {
5104 "inputs": {
5105 "question": "can you explain the agent architecture in this diagram?"
5106 },
5107 "outputs": {"answer": "this diagram shows..."},
5108 "attachments": {"diagram": {"mime_type": "image/png", "data": b"..."}},
5109 "metadata": {"difficulty": "medium"},
5110 },
5111 # more examples...
5112 ]
5114 response = client.create_examples(dataset_name="agent-qa", examples=examples)
5115 # -> {"example_ids": [...
5116 ```
5117 """ # noqa: E501
5118 if not 1 <= max_concurrency <= 3:
5119 raise ValueError("max_concurrency must be between 1 and 3")
5121 if kwargs and examples:
5122 kwarg_keys = ", ".join([f"'{k}'" for k in kwargs])
5123 raise ValueError(
5124 f"Cannot specify {kwarg_keys} when 'examples' is specified."
5125 )
5127 supported_kwargs = {
5128 "inputs",
5129 "outputs",
5130 "metadata",
5131 "splits",
5132 "ids",
5133 "source_run_ids",
5134 }
5135 if kwargs and (unsupported := set(kwargs).difference(supported_kwargs)):
5136 raise ValueError(
5137 f"Received unsupported keyword arguments: {tuple(unsupported)}."
5138 )
5140 if not (dataset_id or dataset_name):
5141 raise ValueError("Either dataset_id or dataset_name must be provided.")
5142 elif not dataset_id:
5143 dataset_id = self.read_dataset(dataset_name=dataset_name).id
5145 if examples:
5146 uploads = [
5147 ls_schemas.ExampleCreate(**x) if isinstance(x, dict) else x
5148 for x in examples
5149 ]
5151 # For backwards compatibility
5152 else:
5153 inputs = kwargs.get("inputs")
5154 if not inputs:
5155 raise ValueError("Must specify either 'examples' or 'inputs.'")
5156 # Since inputs are required, we will check against them
5157 input_len = len(inputs)
5158 for arg_name, arg_value in kwargs.items():
5159 if arg_value is not None and len(arg_value) != input_len:
5160 raise ValueError(
5161 f"Length of {arg_name} ({len(arg_value)}) does not match"
5162 f" length of inputs ({input_len})"
5163 )
5164 uploads = [
5165 ls_schemas.ExampleCreate(
5166 **{
5167 "inputs": in_,
5168 "outputs": out_,
5169 "metadata": metadata_,
5170 "split": split_,
5171 "id": id_ or str(uuid.uuid4()),
5172 "source_run_id": source_run_id_,
5173 }
5174 )
5175 for in_, out_, metadata_, split_, id_, source_run_id_ in zip(
5176 inputs,
5177 kwargs.get("outputs") or (None for _ in range(input_len)),
5178 kwargs.get("metadata") or (None for _ in range(input_len)),
5179 kwargs.get("splits") or (None for _ in range(input_len)),
5180 kwargs.get("ids") or (None for _ in range(input_len)),
5181 kwargs.get("source_run_ids") or (None for _ in range(input_len)),
5182 )
5183 ]
5185 if not uploads:
5186 return ls_schemas.UpsertExamplesResponse(example_ids=[], count=0)
5188 # Use size-aware batching to prevent payload limit errors
5189 batches = self._batch_examples_by_size(uploads)
5191 return self._upload_examples_batches_parallel(
5192 batches, dataset_id, dangerously_allow_filesystem, max_concurrency
5193 )
5195 def _upload_examples_batches_parallel(
5196 self, batches, dataset_id, dangerously_allow_filesystem, max_concurrency
5197 ):
5198 all_examples_ids = []
5199 total_count = 0
5200 from langsmith.utils import ContextThreadPoolExecutor
5202 with ContextThreadPoolExecutor(max_workers=max_concurrency) as executor:
5203 # submit all batch uploads to thread pool
5204 futures = [
5205 executor.submit(
5206 self._upload_single_batch,
5207 batch,
5208 dataset_id,
5209 dangerously_allow_filesystem,
5210 )
5211 for batch in batches
5212 ]
5213 # collect results as they complete
5214 for future in cf.as_completed(futures):
5215 response = future.result()
5216 all_examples_ids.extend(response.get("example_ids", []))
5217 total_count += response.get("count", 0)
5219 return ls_schemas.UpsertExamplesResponse(
5220 example_ids=all_examples_ids, count=total_count
5221 )
5223 def _upload_single_batch(self, batch, dataset_id, dangerously_allow_filesystem):
5224 """Upload a single batch of examples (used by both sequential and parallel)."""
5225 if (self.info.instance_flags or {}).get(
5226 "dataset_examples_multipart_enabled", False
5227 ):
5228 response = self._upload_examples_multipart(
5229 dataset_id=cast(uuid.UUID, dataset_id),
5230 uploads=batch, # batch is a list of ExampleCreate objects
5231 dangerously_allow_filesystem=dangerously_allow_filesystem,
5232 )
5233 return {
5234 "example_ids": response.get("example_ids", []),
5235 "count": response.get("count", 0),
5236 }
5237 else:
5238 # Strip attachments for legacy endpoint
5239 for upload in batch:
5240 if getattr(upload, "attachments") is not None:
5241 upload.attachments = None
5242 warnings.warn(
5243 "Must upgrade your LangSmith version to use attachments."
5244 )
5246 response = self.request_with_retries(
5247 "POST",
5248 "/examples/bulk",
5249 headers={**self._headers, "Content-Type": "application/json"},
5250 data=_dumps_json(
5251 [
5252 {
5253 **dump_model(upload, exclude_none=True),
5254 "dataset_id": str(dataset_id),
5255 }
5256 for upload in batch
5257 ]
5258 ),
5259 )
5260 ls_utils.raise_for_status_with_text(response)
5261 response_data = response.json()
5262 return {
5263 "example_ids": [data["id"] for data in response_data],
5264 "count": len(response_data),
5265 }
5267 @ls_utils.xor_args(("dataset_id", "dataset_name"))
5268 def create_example(
5269 self,
5270 inputs: Optional[Mapping[str, Any]] = None,
5271 dataset_id: Optional[ID_TYPE] = None,
5272 dataset_name: Optional[str] = None,
5273 created_at: Optional[datetime.datetime] = None,
5274 outputs: Optional[Mapping[str, Any]] = None,
5275 metadata: Optional[Mapping[str, Any]] = None,
5276 split: Optional[str | list[str]] = None,
5277 example_id: Optional[ID_TYPE] = None,
5278 source_run_id: Optional[ID_TYPE] = None,
5279 use_source_run_io: bool = False,
5280 use_source_run_attachments: Optional[list[str]] = None,
5281 attachments: Optional[ls_schemas.Attachments] = None,
5282 ) -> ls_schemas.Example:
5283 """Create a dataset example in the LangSmith API.
5285 Examples are rows in a dataset, containing the inputs
5286 and expected outputs (or other reference information)
5287 for a model or chain.
5289 Args:
5290 inputs (Mapping[str, Any]):
5291 The input values for the example.
5292 dataset_id (Optional[Union[UUID, str]]):
5293 The ID of the dataset to create the example in.
5294 dataset_name (Optional[str]):
5295 The name of the dataset to create the example in.
5296 created_at (Optional[datetime.datetime]):
5297 The creation timestamp of the example.
5298 outputs (Optional[Mapping[str, Any]]):
5299 The output values for the example.
5300 metadata (Optional[Mapping[str, Any]]):
5301 The metadata for the example.
5302 split (Optional[str | List[str]]):
5303 The splits for the example, which are divisions
5304 of your dataset such as 'train', 'test', or 'validation'.
5305 example_id (Optional[Union[UUID, str]]):
5306 The ID of the example to create. If not provided, a new
5307 example will be created.
5308 source_run_id (Optional[Union[UUID, str]]):
5309 The ID of the source run associated with this example.
5310 use_source_run_io (bool):
5311 Whether to use the inputs, outputs, and attachments from the source run.
5312 use_source_run_attachments (Optional[List[str]]):
5313 Which attachments to use from the source run. If use_source_run_io
5314 is True, all attachments will be used regardless of this param.
5315 attachments (Optional[Attachments]):
5316 The attachments for the example.
5318 Returns:
5319 Example: The created example.
5320 """
5321 if inputs is None and not use_source_run_io:
5322 raise ValueError("Must provide either inputs or use_source_run_io")
5324 if dataset_id is None:
5325 dataset_id = self.read_dataset(dataset_name=dataset_name).id
5327 data = ls_schemas.ExampleCreate(
5328 **{
5329 "inputs": inputs,
5330 "outputs": outputs,
5331 "metadata": metadata,
5332 "split": split,
5333 "source_run_id": source_run_id,
5334 "use_source_run_io": use_source_run_io,
5335 "use_source_run_attachments": use_source_run_attachments,
5336 "attachments": attachments,
5337 }
5338 )
5339 if created_at:
5340 data.created_at = created_at
5341 data.id = (
5342 (uuid.UUID(example_id) if isinstance(example_id, str) else example_id)
5343 if example_id
5344 else uuid.uuid4()
5345 )
5347 if (self.info.instance_flags or {}).get(
5348 "dataset_examples_multipart_enabled", False
5349 ):
5350 self._upload_examples_multipart(dataset_id=dataset_id, uploads=[data])
5351 return self.read_example(example_id=data.id)
5352 else:
5353 # fallback to old method
5354 if getattr(data, "attachments") is not None:
5355 data.attachments = None
5356 warnings.warn("Must upgrade your LangSmith version to use attachments")
5357 response = self.request_with_retries(
5358 "POST",
5359 "/examples",
5360 headers={**self._headers, "Content-Type": "application/json"},
5361 data=_dumps_json(
5362 {
5363 **{k: v for k, v in dump_model(data).items() if v is not None},
5364 "dataset_id": str(dataset_id),
5365 }
5366 ),
5367 )
5368 ls_utils.raise_for_status_with_text(response)
5369 result = response.json()
5370 return ls_schemas.Example(
5371 **result,
5372 _host_url=self._host_url,
5373 _tenant_id=self._get_optional_tenant_id(),
5374 )
5376 def read_example(
5377 self, example_id: ID_TYPE, *, as_of: Optional[datetime.datetime] = None
5378 ) -> ls_schemas.Example:
5379 """Read an example from the LangSmith API.
5381 Args:
5382 example_id (Union[UUID, str]): The ID of the example to read.
5383 as_of (Optional[datetime.datetime]): The dataset version tag OR
5384 timestamp to retrieve the example as of.
5385 Response examples will only be those that were present at the time
5386 of the tagged (or timestamped) version.
5388 Returns:
5389 Example: The example.
5390 """
5391 response = self.request_with_retries(
5392 "GET",
5393 f"/examples/{_as_uuid(example_id, 'example_id')}",
5394 params={
5395 "as_of": as_of.isoformat() if as_of else None,
5396 },
5397 )
5399 example = response.json()
5400 attachments = _convert_stored_attachments_to_attachments_dict(
5401 example, attachments_key="attachment_urls", api_url=self.api_url
5402 )
5404 return ls_schemas.Example(
5405 **{k: v for k, v in example.items() if k != "attachment_urls"},
5406 attachments=attachments,
5407 _host_url=self._host_url,
5408 _tenant_id=self._get_optional_tenant_id(),
5409 )
5411 def list_examples(
5412 self,
5413 dataset_id: Optional[ID_TYPE] = None,
5414 dataset_name: Optional[str] = None,
5415 example_ids: Optional[Sequence[ID_TYPE]] = None,
5416 as_of: Optional[Union[datetime.datetime, str]] = None,
5417 splits: Optional[Sequence[str]] = None,
5418 inline_s3_urls: bool = True,
5419 *,
5420 offset: int = 0,
5421 limit: Optional[int] = None,
5422 metadata: Optional[dict] = None,
5423 filter: Optional[str] = None,
5424 include_attachments: bool = False,
5425 **kwargs: Any,
5426 ) -> Iterator[ls_schemas.Example]:
5427 r"""Retrieve the example rows of the specified dataset.
5429 Args:
5430 dataset_id (Optional[Union[UUID, str]]): The ID of the dataset to filter by.
5431 dataset_name (Optional[str]): The name of the dataset to filter by.
5432 example_ids (Optional[Sequence[Union[UUID, str]]): The IDs of the examples to filter by.
5433 as_of (Optional[Union[datetime.datetime, str]]): The dataset version tag OR
5434 timestamp to retrieve the examples as of.
5435 Response examples will only be those that were present at the time
5436 of the tagged (or timestamped) version.
5437 splits (Optional[Sequence[str]]): A list of dataset splits, which are
5438 divisions of your dataset such as 'train', 'test', or 'validation'.
5439 Returns examples only from the specified splits.
5440 inline_s3_urls (bool, default=True): Whether to inline S3 URLs.
5441 offset (int, default=0): The offset to start from. Defaults to 0.
5442 limit (Optional[int]): The maximum number of examples to return.
5443 metadata (Optional[dict]): A dictionary of metadata to filter by.
5444 filter (Optional[str]): A structured filter string to apply to
5445 the examples.
5446 include_attachments (bool, default=False): Whether to include the
5447 attachments in the response.
5448 **kwargs (Any): Additional keyword arguments are ignored.
5450 Yields:
5451 The examples.
5453 Examples:
5454 List all examples for a dataset:
5456 ```python
5457 from langsmith import Client
5459 client = Client()
5461 # By Dataset ID
5462 examples = client.list_examples(
5463 dataset_id="c9ace0d8-a82c-4b6c-13d2-83401d68e9ab"
5464 )
5465 # By Dataset Name
5466 examples = client.list_examples(dataset_name="My Test Dataset")
5467 ```
5469 List examples by id
5471 ```python
5472 example_ids = [
5473 "734fc6a0-c187-4266-9721-90b7a025751a",
5474 "d6b4c1b9-6160-4d63-9b61-b034c585074f",
5475 "4d31df4e-f9c3-4a6e-8b6c-65701c2fed13",
5476 ]
5477 examples = client.list_examples(example_ids=example_ids)
5478 ```
5480 List examples by metadata
5482 ```python
5483 examples = client.list_examples(
5484 dataset_name=dataset_name, metadata={"foo": "bar"}
5485 )
5486 ```
5488 List examples by structured filter
5490 ```python
5491 examples = client.list_examples(
5492 dataset_name=dataset_name,
5493 filter='and(not(has(metadata, \'{"foo": "bar"}\')), exists(metadata, "tenant_id"))',
5494 )
5495 ```
5496 """
5497 params: dict[str, Any] = {
5498 **kwargs,
5499 "offset": offset,
5500 "id": example_ids,
5501 "as_of": (
5502 as_of.isoformat() if isinstance(as_of, datetime.datetime) else as_of
5503 ),
5504 "splits": splits,
5505 "inline_s3_urls": inline_s3_urls,
5506 "limit": min(limit, 100) if limit is not None else 100,
5507 "filter": filter,
5508 }
5509 if metadata is not None:
5510 params["metadata"] = _dumps_json(metadata)
5511 if dataset_id is not None:
5512 params["dataset"] = dataset_id
5513 elif dataset_name is not None:
5514 dataset_id = self.read_dataset(dataset_name=dataset_name).id
5515 params["dataset"] = dataset_id
5516 else:
5517 pass
5518 if include_attachments:
5519 params["select"] = ["attachment_urls", "outputs", "metadata"]
5520 for i, example in enumerate(
5521 self._get_paginated_list("/examples", params=params)
5522 ):
5523 attachments = _convert_stored_attachments_to_attachments_dict(
5524 example, attachments_key="attachment_urls", api_url=self.api_url
5525 )
5527 yield ls_schemas.Example(
5528 **{k: v for k, v in example.items() if k != "attachment_urls"},
5529 attachments=attachments,
5530 _host_url=self._host_url,
5531 _tenant_id=self._get_optional_tenant_id(),
5532 )
5533 if limit is not None and i + 1 >= limit:
5534 break
5536 @warn_beta
5537 def index_dataset(
5538 self,
5539 *,
5540 dataset_id: ID_TYPE,
5541 tag: str = "latest",
5542 **kwargs: Any,
5543 ) -> None:
5544 """Enable dataset indexing. Examples are indexed by their inputs.
5546 This enables searching for similar examples by inputs with
5547 ``client.similar_examples()``.
5549 Args:
5550 dataset_id (Union[UUID, str]): The ID of the dataset to index.
5551 tag (Optional[str]): The version of the dataset to index. If 'latest'
5552 then any updates to the dataset (additions, updates, deletions of
5553 examples) will be reflected in the index.
5554 **kwargs (Any): Additional keyword arguments to pass as part of request body.
5556 Returns:
5557 None
5558 """ # noqa: E501
5559 dataset_id = _as_uuid(dataset_id, "dataset_id")
5560 resp = self.request_with_retries(
5561 "POST",
5562 f"/datasets/{dataset_id}/index",
5563 headers=self._headers,
5564 data=json.dumps({"tag": tag, **kwargs}),
5565 )
5566 ls_utils.raise_for_status_with_text(resp)
5568 @warn_beta
5569 def sync_indexed_dataset(
5570 self,
5571 *,
5572 dataset_id: ID_TYPE,
5573 **kwargs: Any,
5574 ) -> None:
5575 """Sync dataset index.
5577 This already happens automatically every 5 minutes, but you can call this to
5578 force a sync.
5580 Args:
5581 dataset_id (Union[UUID, str]): The ID of the dataset to sync.
5583 Returns:
5584 None
5585 """ # noqa: E501
5586 dataset_id = _as_uuid(dataset_id, "dataset_id")
5587 resp = self.request_with_retries(
5588 "POST",
5589 f"/datasets/{dataset_id}/index/sync",
5590 headers=self._headers,
5591 data=json.dumps({**kwargs}),
5592 )
5593 ls_utils.raise_for_status_with_text(resp)
5595 # NOTE: dataset_name arg explicitly not supported to avoid extra API calls.
5596 @warn_beta
5597 def similar_examples(
5598 self,
5599 inputs: dict,
5600 /,
5601 *,
5602 limit: int,
5603 dataset_id: ID_TYPE,
5604 filter: Optional[str] = None,
5605 **kwargs: Any,
5606 ) -> list[ls_schemas.ExampleSearch]:
5607 r"""Retrieve the dataset examples whose inputs best match the current inputs.
5609 !!! note
5611 Must have few-shot indexing enabled for the dataset. See `client.index_dataset()`.
5613 Args:
5614 inputs (dict): The inputs to use as a search query. Must match the dataset
5615 input schema. Must be JSON serializable.
5616 limit (int): The maximum number of examples to return.
5617 dataset_id (Union[UUID, str]): The ID of the dataset to search over.
5618 filter (Optional[str]): A filter string to apply to the search results. Uses
5619 the same syntax as the `filter` parameter in `list_runs()`. Only a subset
5620 of operations are supported.
5622 For example, you can use ``and(eq(metadata.some_tag, 'some_value'), neq(metadata.env, 'dev'))``
5623 to filter only examples where some_tag has some_value, and the environment is not dev.
5624 **kwargs: Additional keyword arguments to pass as part of request body.
5626 Returns:
5627 list[ExampleSearch]: List of ExampleSearch objects.
5629 Examples:
5630 ```python
5631 from langsmith import Client
5633 client = Client()
5634 client.similar_examples(
5635 {"question": "When would i use the runnable generator"},
5636 limit=3,
5637 dataset_id="...",
5638 )
5639 ```
5641 ```python
5642 [
5643 ExampleSearch(
5644 inputs={
5645 "question": "How do I cache a Chat model? What caches can I use?"
5646 },
5647 outputs={
5648 "answer": "You can use LangChain's caching layer for Chat Models. This can save you money by reducing the number of API calls you make to the LLM provider, if you're often requesting the same completion multiple times, and speed up your application.\n\nfrom langchain.cache import InMemoryCache\nlangchain.llm_cache = InMemoryCache()\n\n# The first time, it is not yet in cache, so it should take longer\nllm.predict('Tell me a joke')\n\nYou can also use SQLite Cache which uses a SQLite database:\n\nrm .langchain.db\n\nfrom langchain.cache import SQLiteCache\nlangchain.llm_cache = SQLiteCache(database_path=\".langchain.db\")\n\n# The first time, it is not yet in cache, so it should take longer\nllm.predict('Tell me a joke') \n"
5649 },
5650 metadata=None,
5651 id=UUID("b2ddd1c4-dff6-49ae-8544-f48e39053398"),
5652 dataset_id=UUID("01b6ce0f-bfb6-4f48-bbb8-f19272135d40"),
5653 ),
5654 ExampleSearch(
5655 inputs={"question": "What's a runnable lambda?"},
5656 outputs={
5657 "answer": "A runnable lambda is an object that implements LangChain's `Runnable` interface and runs a callbale (i.e., a function). Note the function must accept a single argument."
5658 },
5659 metadata=None,
5660 id=UUID("f94104a7-2434-4ba7-8293-6a283f4860b4"),
5661 dataset_id=UUID("01b6ce0f-bfb6-4f48-bbb8-f19272135d40"),
5662 ),
5663 ExampleSearch(
5664 inputs={"question": "Show me how to use RecursiveURLLoader"},
5665 outputs={
5666 "answer": 'The RecursiveURLLoader comes from the langchain.document_loaders.recursive_url_loader module. Here\'s an example of how to use it:\n\nfrom langchain.document_loaders.recursive_url_loader import RecursiveUrlLoader\n\n# Create an instance of RecursiveUrlLoader with the URL you want to load\nloader = RecursiveUrlLoader(url="https://example.com")\n\n# Load all child links from the URL page\nchild_links = loader.load()\n\n# Print the child links\nfor link in child_links:\n print(link)\n\nMake sure to replace "https://example.com" with the actual URL you want to load. The load() method returns a list of child links found on the URL page. You can iterate over this list to access each child link.'
5667 },
5668 metadata=None,
5669 id=UUID("0308ea70-a803-4181-a37d-39e95f138f8c"),
5670 dataset_id=UUID("01b6ce0f-bfb6-4f48-bbb8-f19272135d40"),
5671 ),
5672 ]
5673 ```
5674 """
5675 dataset_id = _as_uuid(dataset_id, "dataset_id")
5676 req = {
5677 "inputs": inputs,
5678 "limit": limit,
5679 **kwargs,
5680 }
5681 if filter is not None:
5682 req["filter"] = filter
5684 resp = self.request_with_retries(
5685 "POST",
5686 f"/datasets/{dataset_id}/search",
5687 headers=self._headers,
5688 data=json.dumps(req),
5689 )
5690 ls_utils.raise_for_status_with_text(resp)
5691 examples = []
5692 for ex in resp.json()["examples"]:
5693 examples.append(ls_schemas.ExampleSearch(**ex, dataset_id=dataset_id))
5694 return examples
5696 def update_example(
5697 self,
5698 example_id: ID_TYPE,
5699 *,
5700 inputs: Optional[dict[str, Any]] = None,
5701 outputs: Optional[Mapping[str, Any]] = None,
5702 metadata: Optional[dict] = None,
5703 split: Optional[str | list[str]] = None,
5704 dataset_id: Optional[ID_TYPE] = None,
5705 attachments_operations: Optional[ls_schemas.AttachmentsOperations] = None,
5706 attachments: Optional[ls_schemas.Attachments] = None,
5707 ) -> dict[str, Any]:
5708 """Update a specific example.
5710 Args:
5711 example_id (Union[UUID, str]):
5712 The ID of the example to update.
5713 inputs (Optional[Dict[str, Any]]):
5714 The input values to update.
5715 outputs (Optional[Mapping[str, Any]]):
5716 The output values to update.
5717 metadata (Optional[Dict]):
5718 The metadata to update.
5719 split (Optional[str | List[str]]):
5720 The dataset split to update, such as
5721 'train', 'test', or 'validation'.
5722 dataset_id (Optional[Union[UUID, str]]):
5723 The ID of the dataset to update.
5724 attachments_operations (Optional[AttachmentsOperations]):
5725 The attachments operations to perform.
5726 attachments (Optional[Attachments]):
5727 The attachments to add to the example.
5729 Returns:
5730 Dict[str, Any]: The updated example.
5731 """
5732 if attachments_operations is not None:
5733 if not (self.info.instance_flags or {}).get(
5734 "dataset_examples_multipart_enabled", False
5735 ):
5736 raise ValueError(
5737 "Your LangSmith deployment does not allow using the attachment operations, please upgrade your deployment to the latest version."
5738 )
5739 example_dict = dict(
5740 inputs=inputs,
5741 outputs=outputs,
5742 id=example_id,
5743 metadata=metadata,
5744 split=split,
5745 attachments_operations=attachments_operations,
5746 attachments=attachments,
5747 )
5748 example = ls_schemas.ExampleUpdate(
5749 **{k: v for k, v in example_dict.items() if v is not None}
5750 )
5752 if dataset_id is None:
5753 dataset_id = self.read_example(example_id).dataset_id
5755 if (self.info.instance_flags or {}).get(
5756 "dataset_examples_multipart_enabled", False
5757 ):
5758 return dict(
5759 self._update_examples_multipart(
5760 dataset_id=dataset_id, updates=[example]
5761 )
5762 )
5763 else:
5764 # fallback to old method
5765 response = self.request_with_retries(
5766 "PATCH",
5767 f"/examples/{_as_uuid(example_id, 'example_id')}",
5768 headers={**self._headers, "Content-Type": "application/json"},
5769 data=_dumps_json(
5770 {
5771 **{
5772 k: v
5773 for k, v in dump_model(example).items()
5774 if v is not None
5775 },
5776 "dataset_id": str(dataset_id),
5777 }
5778 ),
5779 )
5780 ls_utils.raise_for_status_with_text(response)
5781 return response.json()
5783 def update_examples(
5784 self,
5785 *,
5786 dataset_name: str | None = None,
5787 dataset_id: ID_TYPE | None = None,
5788 updates: Optional[Sequence[ls_schemas.ExampleUpdate | dict]] = None,
5789 dangerously_allow_filesystem: bool = False,
5790 **kwargs: Any,
5791 ) -> dict[str, Any]:
5792 """Update multiple examples.
5794 Examples are expected to all be part of the same dataset.
5796 Args:
5797 dataset_name (str | None):
5798 The name of the dataset to update. Should specify exactly one of
5799 'dataset_name' or 'dataset_id'.
5800 dataset_id (UUID | str | None):
5801 The ID of the dataset to update. Should specify exactly one of
5802 'dataset_name' or 'dataset_id'.
5803 updates (Sequence[ExampleUpdate | dict] | None):
5804 The example updates. Overwrites any specified fields and does not
5805 update any unspecified fields.
5806 dangerously_allow_filesystem (bool):
5807 Whether to allow using filesystem paths as attachments.
5808 **kwargs (Any):
5809 Legacy keyword args. Should not be specified if 'updates' is specified.
5811 - example_ids (Sequence[UUID | str]): The IDs of the examples to update.
5812 - inputs (Sequence[dict | None] | None): The input values for the examples.
5813 - outputs (Sequence[dict | None] | None): The output values for the examples.
5814 - metadata (Sequence[dict | None] | None): The metadata for the examples.
5815 - splits (Sequence[str | list[str] | None] | None): The splits for the examples, which are divisions of your dataset such as 'train', 'test', or 'validation'.
5816 - attachments_operations (Sequence[AttachmentsOperations | None] | None): The operations to perform on the attachments.
5817 - dataset_ids (Sequence[UUID | str] | None): The IDs of the datasets to move the examples to.
5819 Returns:
5820 The LangSmith JSON response. Includes 'message', 'count', and 'example_ids'.
5822 !!! warning "Behavior changed in `langsmith` 0.3.9"
5824 Updated to ...
5826 Example:
5827 ```python
5828 from langsmith import Client
5830 client = Client()
5832 dataset = client.create_dataset("agent-qa")
5834 examples = [
5835 {
5836 "inputs": {"question": "what's an agent"},
5837 "outputs": {"answer": "an agent is..."},
5838 "metadata": {"difficulty": "easy"},
5839 },
5840 {
5841 "inputs": {
5842 "question": "can you explain the agent architecture in this diagram?"
5843 },
5844 "outputs": {"answer": "this diagram shows..."},
5845 "attachments": {"diagram": {"mime_type": "image/png", "data": b"..."}},
5846 "metadata": {"difficulty": "medium"},
5847 },
5848 # more examples...
5849 ]
5851 response = client.create_examples(dataset_name="agent-qa", examples=examples)
5852 example_ids = response["example_ids"]
5854 updates = [
5855 {
5856 "id": example_ids[0],
5857 "inputs": {"question": "what isn't an agent"},
5858 "outputs": {"answer": "an agent is not..."},
5859 },
5860 {
5861 "id": example_ids[1],
5862 "attachments_operations": [
5863 {"rename": {"diagram": "agent_diagram"}, "retain": []}
5864 ],
5865 },
5866 ]
5867 response = client.update_examples(dataset_name="agent-qa", updates=updates)
5868 # -> {"example_ids": [...
5869 ```
5870 """ # noqa: E501
5871 if kwargs and updates:
5872 raise ValueError(
5873 f"Must pass in either 'updates' or args {tuple(kwargs)}, not both."
5874 )
5875 if not (kwargs or updates):
5876 raise ValueError("Please pass in a non-empty sequence for arg 'updates'.")
5878 if dataset_name and dataset_id:
5879 raise ValueError(
5880 "Must pass in exactly one of 'dataset_name' or 'dataset_id'."
5881 )
5882 elif dataset_name:
5883 dataset_id = self.read_dataset(dataset_name=dataset_name).id
5885 if updates:
5886 updates_obj = [
5887 ls_schemas.ExampleUpdate(**x) if isinstance(x, dict) else x
5888 for x in updates
5889 ]
5891 if not dataset_id:
5892 if updates_obj[0].dataset_id:
5893 dataset_id = updates_obj[0].dataset_id
5894 else:
5895 raise ValueError(
5896 "Must pass in (exactly) one of 'dataset_name' or 'dataset_id'."
5897 )
5899 # For backwards compatibility
5900 else:
5901 example_ids = kwargs.get("example_ids", None)
5902 if not example_ids:
5903 raise ValueError(
5904 "Must pass in (exactly) one of 'updates' or 'example_ids'."
5905 )
5906 if not dataset_id:
5907 if "dataset_ids" not in kwargs:
5908 # Assume all examples belong to same dataset
5909 dataset_id = self.read_example(example_ids[0]).dataset_id
5910 elif len(set(kwargs["dataset_ids"])) > 1:
5911 raise ValueError("Dataset IDs must be the same for all examples")
5912 elif not kwargs["dataset_ids"][0]:
5913 raise ValueError("If specified, dataset_ids must be non-null.")
5914 else:
5915 dataset_id = kwargs["dataset_ids"][0]
5917 multipart_enabled = (self.info.instance_flags or {}).get(
5918 "dataset_examples_multipart_enabled"
5919 )
5920 if (
5921 not multipart_enabled
5922 and (kwargs.get("attachments_operations") or kwargs.get("attachments"))
5923 is not None
5924 ):
5925 raise ValueError(
5926 "Your LangSmith deployment does not allow using the attachment "
5927 "operations, please upgrade your deployment to the latest version."
5928 )
5929 # Since ids are required, we will check against them
5930 examples_len = len(example_ids)
5931 for arg_name, arg_value in kwargs.items():
5932 if arg_value is not None and len(arg_value) != examples_len:
5933 raise ValueError(
5934 f"Length of {arg_name} ({len(arg_value)}) does not match"
5935 f" length of examples ({examples_len})"
5936 )
5937 updates_obj = [
5938 ls_schemas.ExampleUpdate(
5939 **{
5940 "id": id_,
5941 "inputs": in_,
5942 "outputs": out_,
5943 "dataset_id": dataset_id_,
5944 "metadata": metadata_,
5945 "split": split_,
5946 "attachments": attachments_,
5947 "attachments_operations": attachments_operations_,
5948 }
5949 )
5950 for id_, in_, out_, metadata_, split_, dataset_id_, attachments_, attachments_operations_ in zip(
5951 example_ids,
5952 kwargs.get("inputs", (None for _ in range(examples_len))),
5953 kwargs.get("outputs", (None for _ in range(examples_len))),
5954 kwargs.get("metadata", (None for _ in range(examples_len))),
5955 kwargs.get("splits", (None for _ in range(examples_len))),
5956 kwargs.get("dataset_ids", (None for _ in range(examples_len))),
5957 kwargs.get("attachments", (None for _ in range(examples_len))),
5958 kwargs.get(
5959 "attachments_operations", (None for _ in range(examples_len))
5960 ),
5961 )
5962 ]
5964 response: Any = None
5965 if (self.info.instance_flags or {}).get(
5966 "dataset_examples_multipart_enabled", False
5967 ):
5968 response = self._update_examples_multipart(
5969 dataset_id=cast(uuid.UUID, dataset_id),
5970 updates=updates_obj,
5971 dangerously_allow_filesystem=dangerously_allow_filesystem,
5972 )
5974 return {
5975 "message": f"{response.get('count', 0)} examples updated",
5976 **response,
5977 }
5978 else:
5979 # fallback to old method
5980 response = self.request_with_retries(
5981 "PATCH",
5982 "/examples/bulk",
5983 headers={**self._headers, "Content-Type": "application/json"},
5984 data=(
5985 _dumps_json(
5986 [
5987 {
5988 k: v
5989 for k, v in dump_model(example).items()
5990 if v is not None
5991 }
5992 for example in updates_obj
5993 ]
5994 )
5995 ),
5996 )
5997 ls_utils.raise_for_status_with_text(response)
5998 return response.json()
6000 def delete_example(self, example_id: ID_TYPE) -> None:
6001 """Delete an example by ID.
6003 Args:
6004 example_id (Union[UUID, str]):
6005 The ID of the example to delete.
6007 Returns:
6008 None
6009 """
6010 response = self.request_with_retries(
6011 "DELETE",
6012 f"/examples/{_as_uuid(example_id, 'example_id')}",
6013 headers=self._headers,
6014 )
6015 ls_utils.raise_for_status_with_text(response)
6017 def delete_examples(
6018 self, example_ids: Sequence[ID_TYPE], *, hard_delete: bool = False
6019 ) -> None:
6020 """Delete multiple examples by ID.
6022 Parameters
6023 ----------
6024 example_ids : Sequence[ID_TYPE]
6025 The IDs of the examples to delete.
6026 hard_delete : bool, default=False
6027 If True, permanently delete the examples. If False, soft delete them.
6028 """
6029 if hard_delete:
6030 # Hard delete uses POST to a different endpoint
6031 # The platform endpoint is at /v1/platform/... instead of /api/v1/...
6032 # So we need to use a different base URL
6033 body = {
6034 "example_ids": [
6035 str(_as_uuid(id_, f"example_ids[{i}]"))
6036 for i, id_ in enumerate(example_ids)
6037 ],
6038 "hard_delete": True,
6039 }
6040 # Use platform path helper for consistent URL construction
6041 path = _platform_path(self.api_url, "datasets/examples/delete")
6042 full_url = _construct_url(self.api_url, path)
6043 response = self.session.request(
6044 "POST",
6045 full_url,
6046 headers={**self._headers, "Content-Type": "application/json"},
6047 data=_dumps_json(body),
6048 timeout=self._timeout,
6049 )
6050 else:
6051 # Soft delete uses DELETE with query params
6052 params: dict[str, Any] = {
6053 "example_ids": [
6054 str(_as_uuid(id_, f"example_ids[{i}]"))
6055 for i, id_ in enumerate(example_ids)
6056 ]
6057 }
6058 response = self.request_with_retries(
6059 "DELETE",
6060 "/examples",
6061 headers={**self._headers, "Content-Type": "application/json"},
6062 params=params,
6063 )
6064 ls_utils.raise_for_status_with_text(response)
6066 def list_dataset_splits(
6067 self,
6068 *,
6069 dataset_id: Optional[ID_TYPE] = None,
6070 dataset_name: Optional[str] = None,
6071 as_of: Optional[Union[str, datetime.datetime]] = None,
6072 ) -> list[str]:
6073 """Get the splits for a dataset.
6075 Args:
6076 dataset_id (Optional[Union[UUID, str]]): The ID of the dataset.
6077 dataset_name (Optional[str]): The name of the dataset.
6078 as_of (Optional[Union[str, datetime.datetime]]): The version
6079 of the dataset to retrieve splits for. Can be a timestamp or a
6080 string tag. Defaults to "latest".
6082 Returns:
6083 List[str]: The names of this dataset's splits.
6084 """
6085 if dataset_id is None:
6086 if dataset_name is None:
6087 raise ValueError("Must provide dataset name or ID")
6088 dataset_id = self.read_dataset(dataset_name=dataset_name).id
6089 params = {}
6090 if as_of is not None:
6091 params["as_of"] = (
6092 as_of.isoformat() if isinstance(as_of, datetime.datetime) else as_of
6093 )
6095 response = self.request_with_retries(
6096 "GET",
6097 f"/datasets/{_as_uuid(dataset_id, 'dataset_id')}/splits",
6098 params=params,
6099 )
6100 ls_utils.raise_for_status_with_text(response)
6101 return response.json()
6103 def update_dataset_splits(
6104 self,
6105 *,
6106 dataset_id: Optional[ID_TYPE] = None,
6107 dataset_name: Optional[str] = None,
6108 split_name: str,
6109 example_ids: list[ID_TYPE],
6110 remove: bool = False,
6111 ) -> None:
6112 """Update the splits for a dataset.
6114 Args:
6115 dataset_id (Optional[Union[UUID, str]]): The ID of the dataset to update.
6116 dataset_name (Optional[str]): The name of the dataset to update.
6117 split_name (str): The name of the split to update.
6118 example_ids (List[Union[UUID, str]]): The IDs of the examples to add to or
6119 remove from the split.
6120 remove (Optional[bool]): If True, remove the examples from the split.
6121 If False, add the examples to the split.
6123 Returns:
6124 None
6125 """
6126 if dataset_id is None:
6127 if dataset_name is None:
6128 raise ValueError("Must provide dataset name or ID")
6129 dataset_id = self.read_dataset(dataset_name=dataset_name).id
6130 data = {
6131 "split_name": split_name,
6132 "examples": [
6133 str(_as_uuid(id_, f"example_ids[{i}]"))
6134 for i, id_ in enumerate(example_ids)
6135 ],
6136 "remove": remove,
6137 }
6139 response = self.request_with_retries(
6140 "PUT", f"/datasets/{_as_uuid(dataset_id, 'dataset_id')}/splits", json=data
6141 )
6142 ls_utils.raise_for_status_with_text(response)
6144 def _resolve_run_id(
6145 self,
6146 run: Union[ls_schemas.Run, ls_schemas.RunBase, str, uuid.UUID],
6147 load_child_runs: bool,
6148 ) -> ls_schemas.Run:
6149 """Resolve the run ID.
6151 Args:
6152 run (Union[Run, RunBase, str, UUID]):
6153 The run to resolve.
6154 load_child_runs (bool):
6155 Whether to load child runs.
6157 Returns:
6158 Run: The resolved run.
6160 Raises:
6161 TypeError: If the run type is invalid.
6162 """
6163 if isinstance(run, (str, uuid.UUID)):
6164 run_ = self.read_run(run, load_child_runs=load_child_runs)
6165 else:
6166 run_ = cast(ls_schemas.Run, run)
6167 return run_
6169 def _resolve_example_id(
6170 self,
6171 example: Union[ls_schemas.Example, str, uuid.UUID, dict, None],
6172 run: ls_schemas.Run,
6173 ) -> Optional[ls_schemas.Example]:
6174 """Resolve the example ID.
6176 Args:
6177 example (Optional[Union[Example, str, UUID, dict]]):
6178 The example to resolve.
6179 run (Run):
6180 The run associated with the example.
6182 Returns:
6183 Optional[Example]: The resolved example.
6184 """
6185 if isinstance(example, (str, uuid.UUID)):
6186 reference_example_ = self.read_example(example)
6187 elif isinstance(example, ls_schemas.Example):
6188 reference_example_ = example
6189 elif isinstance(example, dict):
6190 reference_example_ = ls_schemas.Example(
6191 **example,
6192 _host_url=self._host_url,
6193 _tenant_id=self._get_optional_tenant_id(),
6194 )
6195 elif run.reference_example_id is not None:
6196 reference_example_ = self.read_example(run.reference_example_id)
6197 else:
6198 reference_example_ = None
6199 return reference_example_
6201 def _select_eval_results(
6202 self,
6203 results: Union[
6204 ls_evaluator.EvaluationResult, ls_evaluator.EvaluationResults, dict
6205 ],
6206 *,
6207 fn_name: Optional[str] = None,
6208 ) -> list[ls_evaluator.EvaluationResult]:
6209 from langsmith.evaluation import evaluator as ls_evaluator # noqa: F811
6211 def _cast_result(
6212 single_result: Union[ls_evaluator.EvaluationResult, dict],
6213 ) -> ls_evaluator.EvaluationResult:
6214 if isinstance(single_result, dict):
6215 return ls_evaluator.EvaluationResult(
6216 **{
6217 "key": fn_name,
6218 "comment": single_result.get("reasoning"),
6219 **single_result,
6220 }
6221 )
6222 return single_result
6224 def _is_eval_results(results: Any) -> TypeGuard[ls_evaluator.EvaluationResults]:
6225 return isinstance(results, dict) and "results" in results
6227 if isinstance(results, ls_evaluator.EvaluationResult):
6228 results_ = [results]
6229 elif _is_eval_results(results):
6230 results_ = [_cast_result(r) for r in results["results"]]
6231 elif isinstance(results, dict):
6232 results_ = [_cast_result(cast(dict, results))]
6233 else:
6234 raise ValueError(
6235 f"Invalid evaluation results type: {type(results)}."
6236 " Must be EvaluationResult, EvaluationResults."
6237 )
6238 return results_
6240 def evaluate_run(
6241 self,
6242 run: Union[ls_schemas.Run, ls_schemas.RunBase, str, uuid.UUID],
6243 evaluator: ls_evaluator.RunEvaluator,
6244 *,
6245 source_info: Optional[dict[str, Any]] = None,
6246 reference_example: Optional[
6247 Union[ls_schemas.Example, str, dict, uuid.UUID]
6248 ] = None,
6249 load_child_runs: bool = False,
6250 ) -> ls_evaluator.EvaluationResult:
6251 """Evaluate a run.
6253 Args:
6254 run (Union[Run, RunBase, str, UUID]):
6255 The run to evaluate.
6256 evaluator (RunEvaluator):
6257 The evaluator to use.
6258 source_info (Optional[Dict[str, Any]]):
6259 Additional information about the source of the evaluation to log
6260 as feedback metadata.
6261 reference_example (Optional[Union[Example, str, dict, UUID]]):
6262 The example to use as a reference for the evaluation.
6263 If not provided, the run's reference example will be used.
6264 load_child_runs (bool, default=False):
6265 Whether to load child runs when resolving the run ID.
6267 Returns:
6268 Feedback: The feedback object created by the evaluation.
6269 """
6270 run_ = self._resolve_run_id(run, load_child_runs=load_child_runs)
6271 reference_example_ = self._resolve_example_id(reference_example, run_)
6272 evaluator_response = evaluator.evaluate_run(
6273 run_,
6274 example=reference_example_,
6275 )
6276 results = self._log_evaluation_feedback(
6277 evaluator_response,
6278 run_,
6279 source_info=source_info,
6280 )
6281 # TODO: Return all results
6282 return results[0]
6284 def _log_evaluation_feedback(
6285 self,
6286 evaluator_response: Union[
6287 ls_evaluator.EvaluationResult, ls_evaluator.EvaluationResults, dict
6288 ],
6289 run: Optional[ls_schemas.Run] = None,
6290 source_info: Optional[dict[str, Any]] = None,
6291 project_id: Optional[ID_TYPE] = None,
6292 *,
6293 _executor: Optional[cf.ThreadPoolExecutor] = None,
6294 ) -> list[ls_evaluator.EvaluationResult]:
6295 results = self._select_eval_results(evaluator_response)
6297 def _submit_feedback(**kwargs):
6298 if _executor:
6299 _executor.submit(self.create_feedback, **kwargs)
6300 else:
6301 self.create_feedback(**kwargs)
6303 for res in results:
6304 source_info_ = source_info or {}
6305 if res.evaluator_info:
6306 source_info_ = {**res.evaluator_info, **source_info_}
6307 run_id_ = None
6308 if res.target_run_id:
6309 run_id_ = res.target_run_id
6310 elif run is not None:
6311 run_id_ = run.id
6312 error = res.extra.get("error", None) if res.extra is not None else None
6314 _submit_feedback(
6315 run_id=run_id_,
6316 key=res.key,
6317 score=res.score,
6318 value=res.value,
6319 comment=res.comment,
6320 correction=res.correction,
6321 source_info=source_info_,
6322 source_run_id=res.source_run_id,
6323 feedback_config=cast(
6324 Optional[ls_schemas.FeedbackConfig], res.feedback_config
6325 ),
6326 feedback_source_type=ls_schemas.FeedbackSourceType.MODEL,
6327 project_id=project_id,
6328 extra=res.extra,
6329 trace_id=run.trace_id if run else None,
6330 error=error,
6331 )
6332 return results
6334 async def aevaluate_run(
6335 self,
6336 run: Union[ls_schemas.Run, str, uuid.UUID],
6337 evaluator: ls_evaluator.RunEvaluator,
6338 *,
6339 source_info: Optional[dict[str, Any]] = None,
6340 reference_example: Optional[
6341 Union[ls_schemas.Example, str, dict, uuid.UUID]
6342 ] = None,
6343 load_child_runs: bool = False,
6344 ) -> ls_evaluator.EvaluationResult:
6345 """Evaluate a run asynchronously.
6347 Args:
6348 run (Union[Run, str, UUID]):
6349 The run to evaluate.
6350 evaluator (RunEvaluator):
6351 The evaluator to use.
6352 source_info (Optional[Dict[str, Any]]):
6353 Additional information about the source of the evaluation to log
6354 as feedback metadata.
6355 reference_example (Optional[Union[Example, str, dict, UUID]]):
6356 The example to use as a reference for the evaluation.
6357 If not provided, the run's reference example will be used.
6358 load_child_runs (bool, default=False):
6359 Whether to load child runs when resolving the run ID.
6361 Returns:
6362 EvaluationResult: The evaluation result object created by the evaluation.
6363 """
6364 run_ = self._resolve_run_id(run, load_child_runs=load_child_runs)
6365 reference_example_ = self._resolve_example_id(reference_example, run_)
6366 evaluator_response = await evaluator.aevaluate_run(
6367 run_,
6368 example=reference_example_,
6369 )
6370 # TODO: Return all results and use async API
6371 results = self._log_evaluation_feedback(
6372 evaluator_response,
6373 run_,
6374 source_info=source_info,
6375 )
6376 return results[0]
6378 def create_feedback(
6379 self,
6380 # TODO: make run_id a kwarg and drop default value for 'key' in breaking release.
6381 run_id: Optional[ID_TYPE] = None,
6382 key: str = "unnamed",
6383 *,
6384 score: Union[float, int, bool, None] = None,
6385 value: Union[str, dict, None] = None,
6386 trace_id: Optional[ID_TYPE] = None,
6387 correction: Union[dict, None] = None,
6388 comment: Union[str, None] = None,
6389 source_info: Optional[dict[str, Any]] = None,
6390 feedback_source_type: Union[
6391 ls_schemas.FeedbackSourceType, str
6392 ] = ls_schemas.FeedbackSourceType.API,
6393 source_run_id: Optional[ID_TYPE] = None,
6394 feedback_id: Optional[ID_TYPE] = None,
6395 feedback_config: Optional[ls_schemas.FeedbackConfig] = None,
6396 stop_after_attempt: int = 10,
6397 project_id: Optional[ID_TYPE] = None,
6398 comparative_experiment_id: Optional[ID_TYPE] = None,
6399 feedback_group_id: Optional[ID_TYPE] = None,
6400 extra: Optional[dict] = None,
6401 error: Optional[bool] = None,
6402 **kwargs: Any,
6403 ) -> ls_schemas.Feedback:
6404 """Create feedback for a run.
6406 !!! note
6408 To enable feedback to be batch uploaded in the background you must
6409 specify `trace_id`. *We highly encourage this for latency-sensitive environments.*
6411 Args:
6412 key (str):
6413 The name of the feedback metric.
6414 score (Optional[Union[float, int, bool]]):
6415 The score to rate this run on the metric or aspect.
6416 value (Optional[Union[float, int, bool, str, dict]]):
6417 The display value or non-numeric value for this feedback.
6418 run_id (Optional[Union[UUID, str]]):
6419 The ID of the run to provide feedback for. At least one of run_id,
6420 trace_id, or project_id must be specified.
6421 trace_id (Optional[Union[UUID, str]]):
6422 The ID of the trace (i.e. root parent run) of the run to provide
6423 feedback for (specified by run_id). If run_id and trace_id are the
6424 same, only trace_id needs to be specified. **NOTE**: trace_id is
6425 required feedback ingestion to be batched and backgrounded.
6426 correction (Optional[dict]):
6427 The proper ground truth for this run.
6428 comment (Optional[str]):
6429 A comment about this feedback, such as a justification for the score or
6430 chain-of-thought trajectory for an LLM judge.
6431 source_info (Optional[Dict[str, Any]]):
6432 Information about the source of this feedback.
6433 feedback_source_type (Union[FeedbackSourceType, str]):
6434 The type of feedback source, such as model (for model-generated feedback)
6435 or API.
6436 source_run_id (Optional[Union[UUID, str]]):
6437 The ID of the run that generated this feedback, if a "model" type.
6438 feedback_id (Optional[Union[UUID, str]]):
6439 The ID of the feedback to create. If not provided, a random UUID will be
6440 generated.
6441 feedback_config (Optional[FeedbackConfig]):
6442 The configuration specifying how to interpret feedback with this key.
6443 Examples include continuous (with min/max bounds), categorical,
6444 or freeform.
6445 stop_after_attempt (int, default=10):
6446 The number of times to retry the request before giving up.
6447 project_id (Optional[Union[UUID, str]]):
6448 The ID of the project (or experiment) to provide feedback on. This is
6449 used for creating summary metrics for experiments. Cannot specify
6450 run_id or trace_id if project_id is specified, and vice versa.
6451 comparative_experiment_id (Optional[Union[UUID, str]]):
6452 If this feedback was logged as a part of a comparative experiment, this
6453 associates the feedback with that experiment.
6454 feedback_group_id (Optional[Union[UUID, str]]):
6455 When logging preferences, ranking runs, or other comparative feedback,
6456 this is used to group feedback together.
6457 extra (Optional[Dict]):
6458 Metadata for the feedback.
6459 **kwargs (Any):
6460 Additional keyword arguments.
6462 Returns:
6463 Feedback: The created feedback object.
6465 Example:
6466 ```python
6467 from langsmith import trace, traceable, Client
6470 @traceable
6471 def foo(x):
6472 return {"y": x * 2}
6475 @traceable
6476 def bar(y):
6477 return {"z": y - 1}
6480 client = Client()
6482 inputs = {"x": 1}
6483 with trace(name="foobar", inputs=inputs) as root_run:
6484 result = foo(**inputs)
6485 result = bar(**result)
6486 root_run.outputs = result
6487 trace_id = root_run.id
6488 child_runs = root_run.child_runs
6490 # Provide feedback for a trace (a.k.a. a root run)
6491 client.create_feedback(
6492 key="user_feedback",
6493 score=1,
6494 trace_id=trace_id,
6495 )
6497 # Provide feedback for a child run
6498 foo_run_id = [run for run in child_runs if run.name == "foo"][0].id
6499 client.create_feedback(
6500 key="correctness",
6501 score=0,
6502 run_id=foo_run_id,
6503 # trace_id= is optional but recommended to enable batched and backgrounded
6504 # feedback ingestion.
6505 trace_id=trace_id,
6506 )
6507 ```
6508 """
6509 run_id = run_id or trace_id
6510 if run_id is None and project_id is None:
6511 raise ValueError("One of run_id, trace_id, or project_id must be provided")
6512 if run_id is not None and project_id is not None:
6513 raise ValueError(
6514 "project_id cannot be provided if run_id or trace_id is provided"
6515 )
6516 if kwargs:
6517 warnings.warn(
6518 "The following arguments are no longer used in the create_feedback"
6519 f" endpoint: {sorted(kwargs)}",
6520 DeprecationWarning,
6521 )
6522 try:
6523 if not isinstance(feedback_source_type, ls_schemas.FeedbackSourceType):
6524 feedback_source_type = ls_schemas.FeedbackSourceType(
6525 feedback_source_type
6526 )
6527 if feedback_source_type == ls_schemas.FeedbackSourceType.API:
6528 feedback_source: ls_schemas.FeedbackSourceBase = (
6529 ls_schemas.APIFeedbackSource(metadata=source_info)
6530 )
6531 elif feedback_source_type == ls_schemas.FeedbackSourceType.MODEL:
6532 feedback_source = ls_schemas.ModelFeedbackSource(metadata=source_info)
6533 else:
6534 raise ValueError(f"Unknown feedback source type {feedback_source_type}")
6535 feedback_source.metadata = (
6536 feedback_source.metadata if feedback_source.metadata is not None else {}
6537 )
6538 if source_run_id is not None and "__run" not in feedback_source.metadata:
6539 feedback_source.metadata["__run"] = {"run_id": str(source_run_id)}
6540 if feedback_source.metadata and "__run" in feedback_source.metadata:
6541 # Validate that the linked run ID is a valid UUID
6542 # Run info may be a base model or dict.
6543 _run_meta: Union[dict, Any] = feedback_source.metadata["__run"]
6544 if hasattr(_run_meta, "dict") and callable(_run_meta):
6545 _run_meta = _run_meta.dict()
6546 if "run_id" in _run_meta:
6547 _run_meta["run_id"] = str(
6548 _as_uuid(
6549 feedback_source.metadata["__run"]["run_id"],
6550 "feedback_source.metadata['__run']['run_id']",
6551 )
6552 )
6553 feedback_source.metadata["__run"] = _run_meta
6554 feedback = ls_schemas.FeedbackCreate(
6555 id=_ensure_uuid(feedback_id),
6556 # If run_id is None, this is interpreted as session-level
6557 # feedback.
6558 run_id=_ensure_uuid(run_id, accept_null=True),
6559 trace_id=_ensure_uuid(trace_id, accept_null=True),
6560 key=key,
6561 score=_format_feedback_score(score),
6562 value=value,
6563 correction=correction,
6564 comment=comment,
6565 feedback_source=feedback_source,
6566 created_at=datetime.datetime.now(datetime.timezone.utc),
6567 modified_at=datetime.datetime.now(datetime.timezone.utc),
6568 feedback_config=feedback_config,
6569 session_id=_ensure_uuid(project_id, accept_null=True),
6570 comparative_experiment_id=_ensure_uuid(
6571 comparative_experiment_id, accept_null=True
6572 ),
6573 feedback_group_id=_ensure_uuid(feedback_group_id, accept_null=True),
6574 extra=extra,
6575 error=error,
6576 )
6578 use_multipart = (self.info.batch_ingest_config or {}).get(
6579 "use_multipart_endpoint", False
6580 )
6582 if (
6583 use_multipart
6584 and self.info.version # TODO: Remove version check once versions have updated
6585 and ls_utils.is_version_greater_or_equal(self.info.version, "0.8.10")
6586 and (
6587 self.tracing_queue is not None or self.compressed_traces is not None
6588 )
6589 and feedback.trace_id is not None
6590 and self.otel_exporter is None
6591 ):
6592 serialized_op = serialize_feedback_dict(feedback)
6593 if self.compressed_traces is not None:
6594 multipart_form = (
6595 serialized_feedback_operation_to_multipart_parts_and_context(
6596 serialized_op
6597 )
6598 )
6599 with self.compressed_traces.lock:
6600 enqueued = compress_multipart_parts_and_context(
6601 multipart_form,
6602 self.compressed_traces,
6603 _BOUNDARY,
6604 )
6605 if enqueued:
6606 self.compressed_traces.trace_count += 1
6607 if self._data_available_event:
6608 self._data_available_event.set()
6609 elif self.tracing_queue is not None:
6610 self.tracing_queue.put(
6611 TracingQueueItem(str(feedback.id), serialized_op)
6612 )
6613 else:
6614 feedback_block = _dumps_json(feedback.dict(exclude_none=True))
6615 self.request_with_retries(
6616 "POST",
6617 "/feedback",
6618 request_kwargs={
6619 "data": feedback_block,
6620 },
6621 stop_after_attempt=stop_after_attempt,
6622 retry_on=(ls_utils.LangSmithNotFoundError,),
6623 )
6624 return ls_schemas.Feedback(**feedback.dict())
6625 except Exception as e:
6626 logger.error("Error creating feedback", exc_info=True)
6627 raise e
6629 def update_feedback(
6630 self,
6631 feedback_id: ID_TYPE,
6632 *,
6633 score: Union[float, int, bool, None] = None,
6634 value: Union[float, int, bool, str, dict, None] = None,
6635 correction: Union[dict, None] = None,
6636 comment: Union[str, None] = None,
6637 ) -> None:
6638 """Update a feedback in the LangSmith API.
6640 Args:
6641 feedback_id (Union[UUID, str]):
6642 The ID of the feedback to update.
6643 score (Optional[Union[float, int, bool]]):
6644 The score to update the feedback with.
6645 value (Optional[Union[float, int, bool, str, dict]]):
6646 The value to update the feedback with.
6647 correction (Optional[dict]):
6648 The correction to update the feedback with.
6649 comment (Optional[str]):
6650 The comment to update the feedback with.
6652 Returns:
6653 None
6654 """
6655 feedback_update: dict[str, Any] = {}
6656 if score is not None:
6657 feedback_update["score"] = _format_feedback_score(score)
6658 if value is not None:
6659 feedback_update["value"] = value
6660 if correction is not None:
6661 feedback_update["correction"] = correction
6662 if comment is not None:
6663 feedback_update["comment"] = comment
6664 response = self.request_with_retries(
6665 "PATCH",
6666 f"/feedback/{_as_uuid(feedback_id, 'feedback_id')}",
6667 headers={**self._headers, "Content-Type": "application/json"},
6668 data=_dumps_json(feedback_update),
6669 )
6670 ls_utils.raise_for_status_with_text(response)
6672 def read_feedback(self, feedback_id: ID_TYPE) -> ls_schemas.Feedback:
6673 """Read a feedback from the LangSmith API.
6675 Args:
6676 feedback_id (Union[UUID, str]):
6677 The ID of the feedback to read.
6679 Returns:
6680 Feedback: The feedback.
6681 """
6682 response = self.request_with_retries(
6683 "GET",
6684 f"/feedback/{_as_uuid(feedback_id, 'feedback_id')}",
6685 )
6686 return ls_schemas.Feedback(**response.json())
6688 def list_feedback(
6689 self,
6690 *,
6691 run_ids: Optional[Sequence[ID_TYPE]] = None,
6692 feedback_key: Optional[Sequence[str]] = None,
6693 feedback_source_type: Optional[Sequence[ls_schemas.FeedbackSourceType]] = None,
6694 limit: Optional[int] = None,
6695 **kwargs: Any,
6696 ) -> Iterator[ls_schemas.Feedback]:
6697 """List the feedback objects on the LangSmith API.
6699 Args:
6700 run_ids (Optional[Sequence[Union[UUID, str]]]):
6701 The IDs of the runs to filter by.
6702 feedback_key (Optional[Sequence[str]]):
6703 The feedback key(s) to filter by. Examples: 'correctness'
6704 The query performs a union of all feedback keys.
6705 feedback_source_type (Optional[Sequence[FeedbackSourceType]]):
6706 The type of feedback source, such as model or API.
6707 limit (Optional[int]):
6708 The maximum number of feedback to return.
6709 **kwargs (Any):
6710 Additional keyword arguments.
6712 Yields:
6713 The feedback objects.
6714 """
6715 params: dict = {
6716 "run": run_ids,
6717 "limit": min(limit, 100) if limit is not None else 100,
6718 **kwargs,
6719 }
6720 if feedback_key is not None:
6721 params["key"] = feedback_key
6722 if feedback_source_type is not None:
6723 params["source"] = feedback_source_type
6724 for i, feedback in enumerate(
6725 self._get_paginated_list("/feedback", params=params)
6726 ):
6727 yield ls_schemas.Feedback(**feedback)
6728 if limit is not None and i + 1 >= limit:
6729 break
6731 def delete_feedback(self, feedback_id: ID_TYPE) -> None:
6732 """Delete a feedback by ID.
6734 Args:
6735 feedback_id (Union[UUID, str]):
6736 The ID of the feedback to delete.
6738 Returns:
6739 None
6740 """
6741 response = self.request_with_retries(
6742 "DELETE",
6743 f"/feedback/{_as_uuid(feedback_id, 'feedback_id')}",
6744 headers=self._headers,
6745 )
6746 ls_utils.raise_for_status_with_text(response)
6748 def create_feedback_from_token(
6749 self,
6750 token_or_url: Union[str, uuid.UUID],
6751 score: Union[float, int, bool, None] = None,
6752 *,
6753 value: Union[float, int, bool, str, dict, None] = None,
6754 correction: Union[dict, None] = None,
6755 comment: Union[str, None] = None,
6756 metadata: Optional[dict] = None,
6757 ) -> None:
6758 """Create feedback from a presigned token or URL.
6760 Args:
6761 token_or_url (Union[str, uuid.UUID]): The token or URL from which to create
6762 feedback.
6763 score (Optional[Union[float, int, bool]]): The score of the feedback.
6764 value (Optional[Union[float, int, bool, str, dict]]): The value of the
6765 feedback.
6766 correction (Optional[dict]): The correction of the feedback.
6767 comment (Optional[str]): The comment of the feedback.
6768 metadata (Optional[dict]): Additional metadata for the feedback.
6770 Raises:
6771 ValueError: If the source API URL is invalid.
6773 Returns:
6774 None
6775 """
6776 source_api_url, token_uuid = _parse_token_or_url(
6777 token_or_url, self.api_url, num_parts=1
6778 )
6779 if source_api_url != self.api_url:
6780 raise ValueError(f"Invalid source API URL. {source_api_url}")
6781 response = self.request_with_retries(
6782 "POST",
6783 f"/feedback/tokens/{_as_uuid(token_uuid)}",
6784 data=_dumps_json(
6785 {
6786 "score": score,
6787 "value": value,
6788 "correction": correction,
6789 "comment": comment,
6790 "metadata": metadata,
6791 # TODO: Add ID once the API supports it.
6792 }
6793 ),
6794 headers=self._headers,
6795 )
6796 ls_utils.raise_for_status_with_text(response)
6798 def create_presigned_feedback_token(
6799 self,
6800 run_id: ID_TYPE,
6801 feedback_key: str,
6802 *,
6803 expiration: Optional[datetime.datetime | datetime.timedelta] = None,
6804 feedback_config: Optional[ls_schemas.FeedbackConfig] = None,
6805 feedback_id: Optional[ID_TYPE] = None,
6806 ) -> ls_schemas.FeedbackIngestToken:
6807 """Create a pre-signed URL to send feedback data to.
6809 This is useful for giving browser-based clients a way to upload
6810 feedback data directly to LangSmith without accessing the
6811 API key.
6813 Args:
6814 run_id (Union[UUID, str]):
6815 The ID of the run.
6816 feedback_key (str):
6817 The key of the feedback to create.
6818 expiration (Optional[datetime.datetime | datetime.timedelta]): The expiration time of the pre-signed URL.
6819 Either a datetime or a timedelta offset from now.
6820 Default to 3 hours.
6821 feedback_config (Optional[FeedbackConfig]):
6822 If creating a feedback_key for the first time,
6823 this defines how the metric should be interpreted,
6824 such as a continuous score (w/ optional bounds),
6825 or distribution over categorical values.
6826 feedback_id (Optional[Union[UUID, str]): The ID of the feedback to create. If not provided, a new
6827 feedback will be created.
6829 Returns:
6830 FeedbackIngestToken: The pre-signed URL for uploading feedback data.
6831 """
6832 body: dict[str, Any] = {
6833 "run_id": run_id,
6834 "feedback_key": feedback_key,
6835 "feedback_config": feedback_config,
6836 "id": feedback_id or str(uuid.uuid4()),
6837 }
6838 if expiration is None:
6839 body["expires_in"] = ls_schemas.TimeDeltaInput(
6840 days=0,
6841 hours=3,
6842 minutes=0,
6843 )
6844 elif isinstance(expiration, datetime.datetime):
6845 body["expires_at"] = expiration.isoformat()
6846 elif isinstance(expiration, datetime.timedelta):
6847 body["expires_in"] = ls_schemas.TimeDeltaInput(
6848 days=expiration.days,
6849 hours=expiration.seconds // 3600,
6850 minutes=(expiration.seconds // 60) % 60,
6851 )
6852 else:
6853 raise ValueError(f"Unknown expiration type: {type(expiration)}")
6855 response = self.request_with_retries(
6856 "POST",
6857 "/feedback/tokens",
6858 data=_dumps_json(body),
6859 )
6860 ls_utils.raise_for_status_with_text(response)
6861 return ls_schemas.FeedbackIngestToken(**response.json())
6863 def create_presigned_feedback_tokens(
6864 self,
6865 run_id: ID_TYPE,
6866 feedback_keys: Sequence[str],
6867 *,
6868 expiration: Optional[datetime.datetime | datetime.timedelta] = None,
6869 feedback_configs: Optional[
6870 Sequence[Optional[ls_schemas.FeedbackConfig]]
6871 ] = None,
6872 ) -> Sequence[ls_schemas.FeedbackIngestToken]:
6873 """Create a pre-signed URL to send feedback data to.
6875 This is useful for giving browser-based clients a way to upload
6876 feedback data directly to LangSmith without accessing the
6877 API key.
6879 Args:
6880 run_id (Union[UUID, str]):
6881 The ID of the run.
6882 feedback_keys (Sequence[str]):
6883 The key of the feedback to create.
6884 expiration (Optional[datetime.datetime | datetime.timedelta]): The expiration time of the pre-signed URL.
6885 Either a datetime or a timedelta offset from now.
6886 Default to 3 hours.
6887 feedback_configs (Optional[Sequence[Optional[FeedbackConfig]]]):
6888 If creating a feedback_key for the first time,
6889 this defines how the metric should be interpreted,
6890 such as a continuous score (w/ optional bounds),
6891 or distribution over categorical values.
6893 Returns:
6894 Sequence[FeedbackIngestToken]: The pre-signed URL for uploading feedback data.
6895 """
6896 # validate
6897 if feedback_configs is not None and len(feedback_keys) != len(feedback_configs):
6898 raise ValueError(
6899 "The length of feedback_keys and feedback_configs must be the same."
6900 )
6901 if not feedback_configs:
6902 feedback_configs = [None] * len(feedback_keys)
6903 # build expiry option
6904 expires_in, expires_at = None, None
6905 if expiration is None:
6906 expires_in = ls_schemas.TimeDeltaInput(
6907 days=0,
6908 hours=3,
6909 minutes=0,
6910 )
6911 elif isinstance(expiration, datetime.datetime):
6912 expires_at = expiration.isoformat()
6913 elif isinstance(expiration, datetime.timedelta):
6914 expires_in = ls_schemas.TimeDeltaInput(
6915 days=expiration.days,
6916 hours=expiration.seconds // 3600,
6917 minutes=(expiration.seconds // 60) % 60,
6918 )
6919 else:
6920 raise ValueError(f"Unknown expiration type: {type(expiration)}")
6921 # assemble body, one entry per key
6922 body = _dumps_json(
6923 [
6924 {
6925 "run_id": run_id,
6926 "feedback_key": feedback_key,
6927 "feedback_config": feedback_config,
6928 "expires_in": expires_in,
6929 "expires_at": expires_at,
6930 }
6931 for feedback_key, feedback_config in zip(
6932 feedback_keys, feedback_configs
6933 )
6934 ]
6935 )
6937 def req(api_url: str, api_key: Optional[str]) -> list:
6938 response = self.request_with_retries(
6939 "POST",
6940 f"{api_url}/feedback/tokens",
6941 request_kwargs={
6942 "data": body,
6943 "headers": {
6944 **self._headers,
6945 X_API_KEY: api_key or self.api_key,
6946 },
6947 },
6948 )
6949 ls_utils.raise_for_status_with_text(response)
6950 return response.json()
6952 tokens = []
6953 with cf.ThreadPoolExecutor(max_workers=len(self._write_api_urls)) as executor:
6954 futs = [
6955 executor.submit(req, api_url, api_key)
6956 for api_url, api_key in self._write_api_urls.items()
6957 ]
6958 for fut in cf.as_completed(futs):
6959 response = fut.result()
6960 tokens.extend(
6961 [ls_schemas.FeedbackIngestToken(**part) for part in response]
6962 )
6963 return tokens
6965 def list_presigned_feedback_tokens(
6966 self,
6967 run_id: ID_TYPE,
6968 *,
6969 limit: Optional[int] = None,
6970 ) -> Iterator[ls_schemas.FeedbackIngestToken]:
6971 """List the feedback ingest tokens for a run.
6973 Args:
6974 run_id (Union[UUID, str]): The ID of the run to filter by.
6975 limit (Optional[int]): The maximum number of tokens to return.
6977 Yields:
6978 The feedback ingest tokens.
6979 """
6980 params = {
6981 "run_id": _as_uuid(run_id, "run_id"),
6982 "limit": min(limit, 100) if limit is not None else 100,
6983 }
6984 for i, token in enumerate(
6985 self._get_paginated_list("/feedback/tokens", params=params)
6986 ):
6987 yield ls_schemas.FeedbackIngestToken(**token)
6988 if limit is not None and i + 1 >= limit:
6989 break
6991 def list_feedback_formulas(
6992 self,
6993 *,
6994 dataset_id: Optional[ID_TYPE] = None,
6995 session_id: Optional[ID_TYPE] = None,
6996 limit: Optional[int] = None,
6997 offset: int = 0,
6998 ) -> Iterator[ls_schemas.FeedbackFormula]:
6999 """List feedback formulas.
7001 Args:
7002 dataset_id (Optional[Union[UUID, str]]):
7003 The ID of the dataset to filter by.
7004 session_id (Optional[Union[UUID, str]]):
7005 The ID of the session to filter by.
7006 limit (Optional[int]):
7007 The maximum number of feedback formulas to return.
7008 offset (int):
7009 The starting offset for pagination.
7011 Yields:
7012 The feedback formulas.
7013 """
7014 params: dict[str, Any] = {
7015 "dataset_id": (
7016 _as_uuid(dataset_id, "dataset_id") if dataset_id is not None else None
7017 ),
7018 "session_id": (
7019 _as_uuid(session_id, "session_id") if session_id is not None else None
7020 ),
7021 "limit": min(limit, 100) if limit is not None else 100,
7022 "offset": offset,
7023 }
7024 for i, feedback_formula in enumerate(
7025 self._get_paginated_list("/feedback/formulas", params=params)
7026 ):
7027 yield ls_schemas.FeedbackFormula(**feedback_formula)
7028 if limit is not None and i + 1 >= limit:
7029 break
7031 def get_feedback_formula_by_id(
7032 self, feedback_formula_id: ID_TYPE
7033 ) -> ls_schemas.FeedbackFormula:
7034 """Get a feedback formula by ID.
7036 Args:
7037 feedback_formula_id (Union[UUID, str]):
7038 The ID of the feedback formula to retrieve.
7040 Returns:
7041 The requested feedback formula.
7042 """
7043 response = self.request_with_retries(
7044 "GET",
7045 f"/feedback/formulas/{_as_uuid(feedback_formula_id, 'feedback_formula_id')}",
7046 )
7047 ls_utils.raise_for_status_with_text(response)
7048 return ls_schemas.FeedbackFormula(**response.json())
7050 def create_feedback_formula(
7051 self,
7052 *,
7053 feedback_key: str,
7054 aggregation_type: Literal["sum", "avg"],
7055 formula_parts: Sequence[
7056 Union[ls_schemas.FeedbackFormulaWeightedVariable, dict]
7057 ],
7058 dataset_id: Optional[ID_TYPE] = None,
7059 session_id: Optional[ID_TYPE] = None,
7060 ) -> ls_schemas.FeedbackFormula:
7061 """Create a feedback formula.
7063 Args:
7064 feedback_key (str):
7065 The feedback key for the formula.
7066 aggregation_type (Literal["sum", "avg"]):
7067 The aggregation type to use when combining parts.
7068 formula_parts (Sequence[FeedbackFormulaWeightedVariable | dict]):
7069 The weighted feedback keys included in the formula.
7070 dataset_id (Optional[Union[UUID, str]]):
7071 The dataset to scope the formula to.
7072 session_id (Optional[Union[UUID, str]]):
7073 The session to scope the formula to.
7075 Returns:
7076 The created feedback formula.
7077 """
7078 typed_parts: list[ls_schemas.FeedbackFormulaWeightedVariable] = [
7079 part
7080 if isinstance(part, ls_schemas.FeedbackFormulaWeightedVariable)
7081 else ls_schemas.FeedbackFormulaWeightedVariable(**part)
7082 for part in formula_parts
7083 ]
7084 payload = ls_schemas.FeedbackFormulaCreate(
7085 feedback_key=feedback_key,
7086 aggregation_type=aggregation_type,
7087 formula_parts=typed_parts,
7088 dataset_id=(
7089 _as_uuid(dataset_id, "dataset_id") if dataset_id is not None else None
7090 ),
7091 session_id=(
7092 _as_uuid(session_id, "session_id") if session_id is not None else None
7093 ),
7094 )
7095 response = self.request_with_retries(
7096 "POST",
7097 "/feedback/formulas",
7098 request_kwargs={
7099 "data": _dumps_json(payload.dict(exclude_none=True)),
7100 },
7101 )
7102 ls_utils.raise_for_status_with_text(response)
7103 return ls_schemas.FeedbackFormula(**response.json())
7105 def update_feedback_formula(
7106 self,
7107 feedback_formula_id: ID_TYPE,
7108 *,
7109 feedback_key: str,
7110 aggregation_type: Literal["sum", "avg"],
7111 formula_parts: Sequence[
7112 Union[ls_schemas.FeedbackFormulaWeightedVariable, dict]
7113 ],
7114 ) -> ls_schemas.FeedbackFormula:
7115 """Update a feedback formula.
7117 Args:
7118 feedback_formula_id (Union[UUID, str]):
7119 The ID of the feedback formula to update.
7120 feedback_key (str):
7121 The feedback key for the formula.
7122 aggregation_type (Literal["sum", "avg"]):
7123 The aggregation type to use when combining parts.
7124 formula_parts (Sequence[FeedbackFormulaWeightedVariable | dict]):
7125 The weighted feedback keys included in the formula.
7127 Returns:
7128 The updated feedback formula.
7129 """
7130 typed_parts: list[ls_schemas.FeedbackFormulaWeightedVariable] = [
7131 part
7132 if isinstance(part, ls_schemas.FeedbackFormulaWeightedVariable)
7133 else ls_schemas.FeedbackFormulaWeightedVariable(**part)
7134 for part in formula_parts
7135 ]
7136 payload = ls_schemas.FeedbackFormulaUpdate(
7137 feedback_key=feedback_key,
7138 aggregation_type=aggregation_type,
7139 formula_parts=typed_parts,
7140 )
7141 response = self.request_with_retries(
7142 "PUT",
7143 f"/feedback/formulas/{_as_uuid(feedback_formula_id, 'feedback_formula_id')}",
7144 request_kwargs={
7145 "data": _dumps_json(payload.dict(exclude_none=True)),
7146 },
7147 )
7148 ls_utils.raise_for_status_with_text(response)
7149 return ls_schemas.FeedbackFormula(**response.json())
7151 def delete_feedback_formula(self, feedback_formula_id: ID_TYPE) -> None:
7152 """Delete a feedback formula by ID.
7154 Args:
7155 feedback_formula_id (Union[UUID, str]):
7156 The ID of the feedback formula to delete.
7157 """
7158 response = self.request_with_retries(
7159 "DELETE",
7160 f"/feedback/formulas/{_as_uuid(feedback_formula_id, 'feedback_formula_id')}",
7161 )
7162 ls_utils.raise_for_status_with_text(response)
7164 # Annotation Queue API
7166 def list_annotation_queues(
7167 self,
7168 *,
7169 queue_ids: Optional[list[ID_TYPE]] = None,
7170 name: Optional[str] = None,
7171 name_contains: Optional[str] = None,
7172 limit: Optional[int] = None,
7173 ) -> Iterator[ls_schemas.AnnotationQueue]:
7174 """List the annotation queues on the LangSmith API.
7176 Args:
7177 queue_ids (Optional[List[Union[UUID, str]]]):
7178 The IDs of the queues to filter by.
7179 name (Optional[str]):
7180 The name of the queue to filter by.
7181 name_contains (Optional[str]):
7182 The substring that the queue name should contain.
7183 limit (Optional[int]):
7184 The maximum number of queues to return.
7186 Yields:
7187 The annotation queues.
7188 """
7189 params: dict = {
7190 "ids": (
7191 [_as_uuid(id_, f"queue_ids[{i}]") for i, id_ in enumerate(queue_ids)]
7192 if queue_ids is not None
7193 else None
7194 ),
7195 "name": name,
7196 "name_contains": name_contains,
7197 "limit": min(limit, 100) if limit is not None else 100,
7198 }
7199 for i, queue in enumerate(
7200 self._get_paginated_list("/annotation-queues", params=params)
7201 ):
7202 yield ls_schemas.AnnotationQueue(
7203 **queue,
7204 )
7205 if limit is not None and i + 1 >= limit:
7206 break
7208 def create_annotation_queue(
7209 self,
7210 *,
7211 name: str,
7212 description: Optional[str] = None,
7213 queue_id: Optional[ID_TYPE] = None,
7214 rubric_instructions: Optional[str] = None,
7215 ) -> ls_schemas.AnnotationQueueWithDetails:
7216 """Create an annotation queue on the LangSmith API.
7218 Args:
7219 name (str):
7220 The name of the annotation queue.
7221 description (Optional[str]):
7222 The description of the annotation queue.
7223 queue_id (Optional[Union[UUID, str]]):
7224 The ID of the annotation queue.
7225 rubric_instructions (Optional[str]):
7226 The rubric instructions for the annotation queue.
7228 Returns:
7229 AnnotationQueue: The created annotation queue object.
7230 """
7231 body = {
7232 "name": name,
7233 "description": description,
7234 "id": str(queue_id) if queue_id is not None else str(uuid.uuid4()),
7235 "rubric_instructions": rubric_instructions,
7236 }
7237 response = self.request_with_retries(
7238 "POST",
7239 "/annotation-queues",
7240 json={k: v for k, v in body.items() if v is not None},
7241 )
7242 ls_utils.raise_for_status_with_text(response)
7243 return ls_schemas.AnnotationQueueWithDetails(
7244 **response.json(),
7245 )
7247 def read_annotation_queue(self, queue_id: ID_TYPE) -> ls_schemas.AnnotationQueue:
7248 """Read an annotation queue with the specified `queue_id`.
7250 Args:
7251 queue_id (Union[UUID, str]): The ID of the annotation queue to read.
7253 Returns:
7254 AnnotationQueue: The annotation queue object.
7255 """
7256 base_url = f"/annotation-queues/{_as_uuid(queue_id, 'queue_id')}"
7257 response = self.request_with_retries(
7258 "GET",
7259 f"{base_url}",
7260 headers=self._headers,
7261 )
7262 ls_utils.raise_for_status_with_text(response)
7263 return ls_schemas.AnnotationQueueWithDetails(**response.json())
7265 def update_annotation_queue(
7266 self,
7267 queue_id: ID_TYPE,
7268 *,
7269 name: str,
7270 description: Optional[str] = None,
7271 rubric_instructions: Optional[str] = None,
7272 ) -> None:
7273 """Update an annotation queue with the specified `queue_id`.
7275 Args:
7276 queue_id (Union[UUID, str]): The ID of the annotation queue to update.
7277 name (str): The new name for the annotation queue.
7278 description (Optional[str]): The new description for the
7279 annotation queue.
7280 rubric_instructions (Optional[str]): The new rubric instructions for the
7281 annotation queue.
7283 Returns:
7284 None
7285 """
7286 response = self.request_with_retries(
7287 "PATCH",
7288 f"/annotation-queues/{_as_uuid(queue_id, 'queue_id')}",
7289 json={
7290 "name": name,
7291 "description": description,
7292 "rubric_instructions": rubric_instructions,
7293 },
7294 )
7295 ls_utils.raise_for_status_with_text(response)
7297 def delete_annotation_queue(self, queue_id: ID_TYPE) -> None:
7298 """Delete an annotation queue with the specified `queue_id`.
7300 Args:
7301 queue_id (Union[UUID, str]): The ID of the annotation queue to delete.
7303 Returns:
7304 None
7305 """
7306 response = self.request_with_retries(
7307 "DELETE",
7308 f"/annotation-queues/{_as_uuid(queue_id, 'queue_id')}",
7309 headers={"Accept": "application/json", **self._headers},
7310 )
7311 ls_utils.raise_for_status_with_text(response)
7313 def add_runs_to_annotation_queue(
7314 self, queue_id: ID_TYPE, *, run_ids: list[ID_TYPE]
7315 ) -> None:
7316 """Add runs to an annotation queue with the specified `queue_id`.
7318 Args:
7319 queue_id (Union[UUID, str]): The ID of the annotation queue.
7320 run_ids (List[Union[UUID, str]]): The IDs of the runs to be added to the annotation
7321 queue.
7323 Returns:
7324 None
7325 """
7326 response = self.request_with_retries(
7327 "POST",
7328 f"/annotation-queues/{_as_uuid(queue_id, 'queue_id')}/runs",
7329 json=[str(_as_uuid(id_, f"run_ids[{i}]")) for i, id_ in enumerate(run_ids)],
7330 )
7331 ls_utils.raise_for_status_with_text(response)
7333 def delete_run_from_annotation_queue(
7334 self, queue_id: ID_TYPE, *, run_id: ID_TYPE
7335 ) -> None:
7336 """Delete a run from an annotation queue with the specified `queue_id` and `run_id`.
7338 Args:
7339 queue_id (Union[UUID, str]): The ID of the annotation queue.
7340 run_id (Union[UUID, str]): The ID of the run to be added to the annotation
7341 queue.
7343 Returns:
7344 None
7345 """
7346 response = self.request_with_retries(
7347 "DELETE",
7348 f"/annotation-queues/{_as_uuid(queue_id, 'queue_id')}/runs/{_as_uuid(run_id, 'run_id')}",
7349 )
7350 ls_utils.raise_for_status_with_text(response)
7352 def get_run_from_annotation_queue(
7353 self, queue_id: ID_TYPE, *, index: int
7354 ) -> ls_schemas.RunWithAnnotationQueueInfo:
7355 """Get a run from an annotation queue at the specified index.
7357 Args:
7358 queue_id (Union[UUID, str]): The ID of the annotation queue.
7359 index (int): The index of the run to retrieve.
7361 Returns:
7362 RunWithAnnotationQueueInfo: The run at the specified index.
7364 Raises:
7365 LangSmithNotFoundError: If the run is not found at the given index.
7366 LangSmithError: For other API-related errors.
7367 """
7368 base_url = f"/annotation-queues/{_as_uuid(queue_id, 'queue_id')}/run"
7369 response = self.request_with_retries(
7370 "GET",
7371 f"{base_url}/{index}",
7372 headers=self._headers,
7373 )
7374 ls_utils.raise_for_status_with_text(response)
7375 return ls_schemas.RunWithAnnotationQueueInfo(**response.json())
7377 def create_comparative_experiment(
7378 self,
7379 name: str,
7380 experiments: Sequence[ID_TYPE],
7381 *,
7382 reference_dataset: Optional[ID_TYPE] = None,
7383 description: Optional[str] = None,
7384 created_at: Optional[datetime.datetime] = None,
7385 metadata: Optional[dict[str, Any]] = None,
7386 id: Optional[ID_TYPE] = None,
7387 ) -> ls_schemas.ComparativeExperiment:
7388 """Create a comparative experiment on the LangSmith API.
7390 These experiments compare 2 or more experiment results over a shared dataset.
7392 Args:
7393 name (str): The name of the comparative experiment.
7394 experiments (Sequence[Union[UUID, str]]): The IDs of the experiments to compare.
7395 reference_dataset (Optional[Union[UUID, str]]): The ID of the dataset these experiments are compared on.
7396 description (Optional[str]): The description of the comparative experiment.
7397 created_at (Optional[datetime.datetime]): The creation time of the comparative experiment.
7398 metadata (Optional[Dict[str, Any]]): Additional metadata for the comparative experiment.
7399 id (Optional[Union[UUID, str]]): The ID of the comparative experiment.
7401 Returns:
7402 ComparativeExperiment: The created comparative experiment object.
7403 """
7404 if not experiments:
7405 raise ValueError("At least one experiment is required.")
7406 if reference_dataset is None:
7407 # Get one of the experiments' reference dataset
7408 reference_dataset = self.read_project(
7409 project_id=experiments[0]
7410 ).reference_dataset_id
7411 if not reference_dataset:
7412 raise ValueError("A reference dataset is required.")
7413 body: dict[str, Any] = {
7414 "id": id or str(uuid.uuid4()),
7415 "name": name,
7416 "experiment_ids": experiments,
7417 "reference_dataset_id": reference_dataset,
7418 "description": description,
7419 "created_at": created_at or datetime.datetime.now(datetime.timezone.utc),
7420 "extra": {},
7421 }
7422 if metadata is not None:
7423 body["extra"]["metadata"] = metadata
7424 ser = _dumps_json({k: v for k, v in body.items()}) # if v is not None})
7425 response = self.request_with_retries(
7426 "POST",
7427 "/datasets/comparative",
7428 request_kwargs={
7429 "data": ser,
7430 },
7431 )
7432 ls_utils.raise_for_status_with_text(response)
7433 response_d = response.json()
7434 return ls_schemas.ComparativeExperiment(**response_d)
7436 async def arun_on_dataset(
7437 self,
7438 dataset_name: str,
7439 llm_or_chain_factory: Any,
7440 *,
7441 evaluation: Optional[Any] = None,
7442 concurrency_level: int = 5,
7443 project_name: Optional[str] = None,
7444 project_metadata: Optional[dict[str, Any]] = None,
7445 dataset_version: Optional[Union[datetime.datetime, str]] = None,
7446 verbose: bool = False,
7447 input_mapper: Optional[Callable[[dict], Any]] = None,
7448 revision_id: Optional[str] = None,
7449 **kwargs: Any,
7450 ) -> dict[str, Any]:
7451 """Asynchronously run the Chain or language model on a dataset.
7453 .. deprecated:: 0.1.0
7455 This method is deprecated. Use :func:`langsmith.aevaluate` instead.
7456 """ # noqa: E501
7457 warnings.warn(
7458 "The `arun_on_dataset` method is deprecated and"
7459 " will be removed in a future version."
7460 "Please use the `aevaluate` method instead.",
7461 DeprecationWarning,
7462 )
7463 try:
7464 from langchain.smith import ( # type: ignore[import-not-found]
7465 arun_on_dataset as _arun_on_dataset,
7466 )
7467 except ImportError:
7468 raise ImportError(
7469 "The client.arun_on_dataset function requires the langchain"
7470 "package to run.\nInstall with pip install langchain"
7471 )
7472 return await _arun_on_dataset(
7473 dataset_name=dataset_name,
7474 llm_or_chain_factory=llm_or_chain_factory,
7475 client=self,
7476 evaluation=evaluation,
7477 concurrency_level=concurrency_level,
7478 project_name=project_name,
7479 project_metadata=project_metadata,
7480 verbose=verbose,
7481 input_mapper=input_mapper,
7482 revision_id=revision_id,
7483 dataset_version=dataset_version,
7484 **kwargs,
7485 )
7487 def run_on_dataset(
7488 self,
7489 dataset_name: str,
7490 llm_or_chain_factory: Any,
7491 *,
7492 evaluation: Optional[Any] = None,
7493 concurrency_level: int = 5,
7494 project_name: Optional[str] = None,
7495 project_metadata: Optional[dict[str, Any]] = None,
7496 dataset_version: Optional[Union[datetime.datetime, str]] = None,
7497 verbose: bool = False,
7498 input_mapper: Optional[Callable[[dict], Any]] = None,
7499 revision_id: Optional[str] = None,
7500 **kwargs: Any,
7501 ) -> dict[str, Any]:
7502 """Run the Chain or language model on a dataset.
7504 .. deprecated:: 0.1.0
7506 This method is deprecated. Use :func:`langsmith.aevaluate` instead.
7507 """ # noqa: E501 # noqa: E501
7508 warnings.warn(
7509 "The `run_on_dataset` method is deprecated and"
7510 " will be removed in a future version."
7511 "Please use the `evaluate` method instead.",
7512 DeprecationWarning,
7513 )
7514 try:
7515 from langchain.smith import (
7516 run_on_dataset as _run_on_dataset, # type: ignore
7517 )
7518 except ImportError:
7519 raise ImportError(
7520 "The client.run_on_dataset function requires the langchain"
7521 "package to run.\nInstall with pip install langchain"
7522 )
7523 return _run_on_dataset(
7524 dataset_name=dataset_name,
7525 llm_or_chain_factory=llm_or_chain_factory,
7526 concurrency_level=concurrency_level,
7527 client=self,
7528 evaluation=evaluation,
7529 project_name=project_name,
7530 project_metadata=project_metadata,
7531 verbose=verbose,
7532 input_mapper=input_mapper,
7533 revision_id=revision_id,
7534 dataset_version=dataset_version,
7535 **kwargs,
7536 )
7538 def _current_tenant_is_owner(self, owner: str) -> bool:
7539 """Check if the current workspace has the same handle as owner.
7541 Args:
7542 owner (str): The owner to check against.
7544 Returns:
7545 bool: True if the current tenant is the owner, False otherwise.
7546 """
7547 settings = self._get_settings()
7548 return owner == "-" or settings.tenant_handle == owner
7550 def _owner_conflict_error(
7551 self, action: str, owner: str
7552 ) -> ls_utils.LangSmithUserError:
7553 return ls_utils.LangSmithUserError(
7554 f"Cannot {action} for another tenant.\n"
7555 f"Current tenant: {self._get_settings().tenant_handle},\n"
7556 f"Requested tenant: {owner}"
7557 )
7559 def _get_latest_commit_hash(
7560 self, prompt_owner_and_name: str, limit: int = 1, offset: int = 0
7561 ) -> Optional[str]:
7562 """Get the latest commit hash for a prompt.
7564 Args:
7565 prompt_owner_and_name (str): The owner and name of the prompt.
7566 limit (int, default=1): The maximum number of commits to fetch. Defaults to 1.
7567 offset (int, default=0): The number of commits to skip. Defaults to 0.
7569 Returns:
7570 Optional[str]: The latest commit hash, or None if no commits are found.
7571 """
7572 response = self.request_with_retries(
7573 "GET",
7574 f"/commits/{prompt_owner_and_name}/",
7575 params={"limit": limit, "offset": offset},
7576 )
7577 commits = response.json()["commits"]
7578 return commits[0]["commit_hash"] if commits else None
7580 def _like_or_unlike_prompt(
7581 self, prompt_identifier: str, like: bool
7582 ) -> dict[str, int]:
7583 """Like or unlike a prompt.
7585 Args:
7586 prompt_identifier (str): The identifier of the prompt.
7587 like (bool): True to like the prompt, False to unlike it.
7589 Returns:
7590 A dictionary with the key 'likes' and the count of likes as the value.
7592 Raises:
7593 requests.exceptions.HTTPError: If the prompt is not found or
7594 another error occurs.
7595 """
7596 owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier)
7597 response = self.request_with_retries(
7598 "POST", f"/likes/{owner}/{prompt_name}", json={"like": like}
7599 )
7600 response.raise_for_status()
7601 return response.json()
7603 def _get_prompt_url(self, prompt_identifier: str) -> str:
7604 """Get a URL for a prompt.
7606 Args:
7607 prompt_identifier (str): The identifier of the prompt.
7609 Returns:
7610 str: The URL for the prompt.
7612 """
7613 owner, prompt_name, commit_hash = ls_utils.parse_prompt_identifier(
7614 prompt_identifier
7615 )
7617 if not self._current_tenant_is_owner(owner):
7618 return f"{self._host_url}/hub/{owner}/{prompt_name}:{commit_hash[:8]}"
7620 settings = self._get_settings()
7621 return (
7622 f"{self._host_url}/prompts/{prompt_name}/{commit_hash[:8]}"
7623 f"?organizationId={settings.id}"
7624 )
7626 def _prompt_exists(self, prompt_identifier: str) -> bool:
7627 """Check if a prompt exists.
7629 Args:
7630 prompt_identifier (str): The identifier of the prompt.
7632 Returns:
7633 bool: True if the prompt exists, False otherwise.
7634 """
7635 prompt = self.get_prompt(prompt_identifier)
7636 return True if prompt else False
7638 def like_prompt(self, prompt_identifier: str) -> dict[str, int]:
7639 """Like a prompt.
7641 Args:
7642 prompt_identifier (str): The identifier of the prompt.
7644 Returns:
7645 Dict[str, int]: A dictionary with the key 'likes' and the count of likes as the value.
7647 """
7648 return self._like_or_unlike_prompt(prompt_identifier, like=True)
7650 def unlike_prompt(self, prompt_identifier: str) -> dict[str, int]:
7651 """Unlike a prompt.
7653 Args:
7654 prompt_identifier (str): The identifier of the prompt.
7656 Returns:
7657 Dict[str, int]: A dictionary with the key 'likes' and the count of likes as the value.
7659 """
7660 return self._like_or_unlike_prompt(prompt_identifier, like=False)
7662 def list_prompts(
7663 self,
7664 *,
7665 limit: int = 100,
7666 offset: int = 0,
7667 is_public: Optional[bool] = None,
7668 is_archived: Optional[bool] = False,
7669 sort_field: ls_schemas.PromptSortField = ls_schemas.PromptSortField.updated_at,
7670 sort_direction: Literal["desc", "asc"] = "desc",
7671 query: Optional[str] = None,
7672 ) -> ls_schemas.ListPromptsResponse:
7673 """List prompts with pagination.
7675 Args:
7676 limit (int, default=100): The maximum number of prompts to return. Defaults to 100.
7677 offset (int, default=0): The number of prompts to skip. Defaults to 0.
7678 is_public (Optional[bool]): Filter prompts by if they are public.
7679 is_archived (Optional[bool]): Filter prompts by if they are archived.
7680 sort_field (PromptSortField): The field to sort by.
7681 Defaults to "updated_at".
7682 sort_direction (Literal["desc", "asc"], default="desc"): The order to sort by.
7683 Defaults to "desc".
7684 query (Optional[str]): Filter prompts by a search query.
7686 Returns:
7687 ListPromptsResponse: A response object containing
7688 the list of prompts.
7689 """
7690 params = {
7691 "limit": limit,
7692 "offset": offset,
7693 "is_public": (
7694 "true" if is_public else "false" if is_public is not None else None
7695 ),
7696 "is_archived": "true" if is_archived else "false",
7697 "sort_field": sort_field,
7698 "sort_direction": sort_direction,
7699 "query": query,
7700 "match_prefix": "true" if query else None,
7701 }
7703 response = self.request_with_retries("GET", "/repos/", params=params)
7704 return ls_schemas.ListPromptsResponse(**response.json())
7706 def get_prompt(self, prompt_identifier: str) -> Optional[ls_schemas.Prompt]:
7707 """Get a specific prompt by its identifier.
7709 Args:
7710 prompt_identifier (str): The identifier of the prompt.
7711 The identifier should be in the format "prompt_name" or "owner/prompt_name".
7713 Returns:
7714 Optional[Prompt]: The prompt object.
7716 Raises:
7717 requests.exceptions.HTTPError: If the prompt is not found or
7718 another error occurs.
7719 """
7720 owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier)
7721 try:
7722 response = self.request_with_retries("GET", f"/repos/{owner}/{prompt_name}")
7723 return ls_schemas.Prompt(**response.json()["repo"])
7724 except ls_utils.LangSmithNotFoundError:
7725 return None
7727 def create_prompt(
7728 self,
7729 prompt_identifier: str,
7730 *,
7731 description: Optional[str] = None,
7732 readme: Optional[str] = None,
7733 tags: Optional[Sequence[str]] = None,
7734 is_public: bool = False,
7735 ) -> ls_schemas.Prompt:
7736 """Create a new prompt.
7738 Does not attach prompt object, just creates an empty prompt.
7740 Args:
7741 prompt_identifier (str): The identifier of the prompt.
7742 The identifier should be in the formatof owner/name:hash, name:hash, owner/name, or name
7743 description (Optional[str]): A description of the prompt.
7744 readme (Optional[str]): A readme for the prompt.
7745 tags (Optional[Sequence[str]]): A list of tags for the prompt.
7746 is_public (bool): Whether the prompt should be public.
7748 Returns:
7749 Prompt: The created prompt object.
7751 Raises:
7752 ValueError: If the current tenant is not the owner.
7753 HTTPError: If the server request fails.
7754 """
7755 settings = self._get_settings()
7756 if is_public and not settings.tenant_handle:
7757 raise ls_utils.LangSmithUserError(
7758 "Cannot create a public prompt without first\n"
7759 "creating a LangChain Hub handle. "
7760 "You can add a handle by creating a public prompt at:\n"
7761 "https://smith.langchain.com/prompts"
7762 )
7764 owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier)
7765 if not self._current_tenant_is_owner(owner=owner):
7766 raise self._owner_conflict_error("create a prompt", owner)
7768 json: dict[str, Union[str, bool, Sequence[str]]] = {
7769 "repo_handle": prompt_name,
7770 "description": description or "",
7771 "readme": readme or "",
7772 "tags": tags or [],
7773 "is_public": is_public,
7774 }
7776 response = self.request_with_retries("POST", "/repos/", json=json)
7777 response.raise_for_status()
7778 return ls_schemas.Prompt(**response.json()["repo"])
7780 def create_commit(
7781 self,
7782 prompt_identifier: str,
7783 object: Any,
7784 *,
7785 parent_commit_hash: Optional[str] = None,
7786 ) -> str:
7787 """Create a commit for an existing prompt.
7789 Args:
7790 prompt_identifier (str): The identifier of the prompt.
7791 object (Any): The LangChain object to commit.
7792 parent_commit_hash (Optional[str]): The hash of the parent commit.
7793 Defaults to latest commit.
7795 Returns:
7796 str: The url of the prompt commit.
7798 Raises:
7799 HTTPError: If the server request fails.
7800 ValueError: If the prompt does not exist.
7801 """
7802 if not self._prompt_exists(prompt_identifier):
7803 raise ls_utils.LangSmithNotFoundError(
7804 "Prompt does not exist, you must create it first."
7805 )
7807 try:
7808 from langchain_core.load import dumps
7809 except ImportError:
7810 raise ImportError(
7811 "The client.create_commit function requires the langchain-core"
7812 "package to run.\nInstall with `pip install langchain-core`"
7813 )
7815 json_object = dumps(prep_obj_for_push(object))
7816 manifest_dict = json.loads(json_object)
7818 owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier)
7819 prompt_owner_and_name = f"{owner}/{prompt_name}"
7821 if parent_commit_hash == "latest" or parent_commit_hash is None:
7822 parent_commit_hash = self._get_latest_commit_hash(prompt_owner_and_name)
7824 request_dict = {"parent_commit": parent_commit_hash, "manifest": manifest_dict}
7825 response = self.request_with_retries(
7826 "POST", f"/commits/{prompt_owner_and_name}", json=request_dict
7827 )
7829 commit_hash = response.json()["commit"]["commit_hash"]
7830 return self._get_prompt_url(f"{prompt_owner_and_name}:{commit_hash}")
7832 def update_prompt(
7833 self,
7834 prompt_identifier: str,
7835 *,
7836 description: Optional[str] = None,
7837 readme: Optional[str] = None,
7838 tags: Optional[Sequence[str]] = None,
7839 is_public: Optional[bool] = None,
7840 is_archived: Optional[bool] = None,
7841 ) -> dict[str, Any]:
7842 """Update a prompt's metadata.
7844 To update the content of a prompt, use push_prompt or create_commit instead.
7846 Args:
7847 prompt_identifier (str): The identifier of the prompt to update.
7848 description (Optional[str]): New description for the prompt.
7849 readme (Optional[str]): New readme for the prompt.
7850 tags (Optional[Sequence[str]]): New list of tags for the prompt.
7851 is_public (Optional[bool]): New public status for the prompt.
7852 is_archived (Optional[bool]): New archived status for the prompt.
7854 Returns:
7855 Dict[str, Any]: The updated prompt data as returned by the server.
7857 Raises:
7858 ValueError: If the prompt_identifier is empty.
7859 HTTPError: If the server request fails.
7860 """
7861 settings = self._get_settings()
7862 if is_public and not settings.tenant_handle:
7863 raise ValueError(
7864 "Cannot create a public prompt without first\n"
7865 "creating a LangChain Hub handle. "
7866 "You can add a handle by creating a public prompt at:\n"
7867 "https://smith.langchain.com/prompts"
7868 )
7870 json: dict[str, Union[str, bool, Sequence[str]]] = {}
7872 if description is not None:
7873 json["description"] = description
7874 if readme is not None:
7875 json["readme"] = readme
7876 if is_public is not None:
7877 json["is_public"] = is_public
7878 if is_archived is not None:
7879 json["is_archived"] = is_archived
7880 if tags is not None:
7881 json["tags"] = tags
7883 owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier)
7884 response = self.request_with_retries(
7885 "PATCH", f"/repos/{owner}/{prompt_name}", json=json
7886 )
7887 response.raise_for_status()
7888 return response.json()
7890 def delete_prompt(self, prompt_identifier: str) -> None:
7891 """Delete a prompt.
7893 Args:
7894 prompt_identifier (str): The identifier of the prompt to delete.
7896 Returns:
7897 bool: True if the prompt was successfully deleted, False otherwise.
7899 Raises:
7900 ValueError: If the current tenant is not the owner of the prompt.
7901 """
7902 owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier)
7903 if not self._current_tenant_is_owner(owner):
7904 raise self._owner_conflict_error("delete a prompt", owner)
7906 response = self.request_with_retries("DELETE", f"/repos/{owner}/{prompt_name}")
7907 response.raise_for_status()
7909 def pull_prompt_commit(
7910 self,
7911 prompt_identifier: str,
7912 *,
7913 include_model: Optional[bool] = False,
7914 ) -> ls_schemas.PromptCommit:
7915 """Pull a prompt object from the LangSmith API.
7917 Args:
7918 prompt_identifier (str): The identifier of the prompt.
7920 Returns:
7921 PromptCommit: The prompt object.
7923 Raises:
7924 ValueError: If no commits are found for the prompt.
7925 """
7926 owner, prompt_name, commit_hash = ls_utils.parse_prompt_identifier(
7927 prompt_identifier
7928 )
7929 response = self.request_with_retries(
7930 "GET",
7931 (
7932 f"/commits/{owner}/{prompt_name}/{commit_hash}"
7933 f"{'?include_model=true' if include_model else ''}"
7934 ),
7935 )
7936 return ls_schemas.PromptCommit(
7937 **{"owner": owner, "repo": prompt_name, **response.json()}
7938 )
7940 def list_prompt_commits(
7941 self,
7942 prompt_identifier: str,
7943 *,
7944 limit: Optional[int] = None,
7945 offset: int = 0,
7946 include_model: bool = False,
7947 ) -> Iterator[ls_schemas.ListedPromptCommit]:
7948 """List commits for a given prompt.
7950 Args:
7951 prompt_identifier (str): The identifier of the prompt in the format 'owner/repo_name'.
7952 limit (Optional[int]): The maximum number of commits to return. If None, returns all commits.
7953 offset (int, default=0): The number of commits to skip before starting to return results.
7954 include_model (bool, default=False): Whether to include the model information in the commit data.
7956 Yields:
7957 A ListedPromptCommit object for each commit.
7959 !!! note
7961 This method uses pagination to retrieve commits. It will make multiple API calls if necessary to retrieve all commits
7962 or up to the specified limit.
7963 """
7964 owner, prompt_name, _ = ls_utils.parse_prompt_identifier(prompt_identifier)
7966 params = {
7967 "limit": min(100, limit) if limit is not None else limit,
7968 "offset": offset,
7969 "include_model": include_model,
7970 }
7971 i = 0
7972 while True:
7973 params["offset"] = offset
7974 response = self.request_with_retries(
7975 "GET",
7976 f"/commits/{owner}/{prompt_name}/",
7977 params=params,
7978 )
7979 val = response.json()
7980 items = val["commits"]
7981 total = val["total"]
7983 if not items:
7984 break
7985 for it in items:
7986 if limit is not None and i >= limit:
7987 return # Stop iteration if we've reached the limit
7988 yield ls_schemas.ListedPromptCommit(
7989 **{"owner": owner, "repo": prompt_name, **it}
7990 )
7991 i += 1
7993 offset += len(items)
7994 if offset >= total:
7995 break
7997 def pull_prompt(
7998 self, prompt_identifier: str, *, include_model: Optional[bool] = False
7999 ) -> Any:
8000 """Pull a prompt and return it as a LangChain `PromptTemplate`.
8002 This method requires [`langchain-core`](https://pypi.org/project/langchain-core).
8004 Args:
8005 prompt_identifier: The identifier of the prompt.
8006 include_model: Whether to include the model information in the prompt data.
8008 Returns:
8009 Any: The prompt object in the specified format.
8010 """
8011 try:
8012 from langchain_core.language_models.base import BaseLanguageModel
8013 from langchain_core.load.load import loads
8014 from langchain_core.output_parsers import BaseOutputParser
8015 from langchain_core.prompts import BasePromptTemplate
8016 from langchain_core.prompts.structured import StructuredPrompt
8017 from langchain_core.runnables.base import RunnableBinding, RunnableSequence
8018 except ImportError:
8019 raise ImportError(
8020 "The client.pull_prompt function requires the langchain-core"
8021 "package to run.\nInstall with `pip install langchain-core`"
8022 )
8023 try:
8024 from langchain_core._api import suppress_langchain_beta_warning
8025 except ImportError:
8027 @contextlib.contextmanager
8028 def suppress_langchain_beta_warning():
8029 yield
8031 prompt_object = self.pull_prompt_commit(
8032 prompt_identifier, include_model=include_model
8033 )
8034 with suppress_langchain_beta_warning():
8035 prompt = loads(json.dumps(prompt_object.manifest))
8037 if (
8038 isinstance(prompt, BasePromptTemplate)
8039 or isinstance(prompt, RunnableSequence)
8040 and isinstance(prompt.first, BasePromptTemplate)
8041 ):
8042 prompt_template = (
8043 prompt
8044 if isinstance(prompt, BasePromptTemplate)
8045 else (
8046 prompt.first
8047 if isinstance(prompt, RunnableSequence)
8048 and isinstance(prompt.first, BasePromptTemplate)
8049 else None
8050 )
8051 )
8052 if prompt_template is None:
8053 raise ls_utils.LangSmithError(
8054 "Prompt object is not a valid prompt template."
8055 )
8057 if prompt_template.metadata is None:
8058 prompt_template.metadata = {}
8059 prompt_template.metadata.update(
8060 {
8061 "lc_hub_owner": prompt_object.owner,
8062 "lc_hub_repo": prompt_object.repo,
8063 "lc_hub_commit_hash": prompt_object.commit_hash,
8064 }
8065 )
8067 # Transform 2-step RunnableSequence to 3-step for structured prompts
8068 # See create_commit for the reverse transformation
8069 if (
8070 include_model
8071 and isinstance(prompt, RunnableSequence)
8072 and isinstance(prompt.first, StructuredPrompt)
8073 # Make forward-compatible in case we let update the response type
8074 and (
8075 len(prompt.steps) == 2 and not isinstance(prompt.last, BaseOutputParser)
8076 )
8077 ):
8078 if isinstance(prompt.last, RunnableBinding) and isinstance(
8079 prompt.last.bound, BaseLanguageModel
8080 ):
8081 seq = cast(RunnableSequence, prompt.first | prompt.last.bound)
8082 if len(seq.steps) == 3: # prompt | bound llm | output parser
8083 rebound_llm = seq.steps[1]
8084 prompt = RunnableSequence(
8085 prompt.first,
8086 rebound_llm.bind(**{**prompt.last.kwargs}),
8087 seq.last,
8088 )
8089 else:
8090 prompt = seq # Not sure
8092 elif isinstance(prompt.last, BaseLanguageModel):
8093 prompt: RunnableSequence = prompt.first | prompt.last # type: ignore[no-redef, assignment]
8094 else:
8095 pass
8097 return prompt
8099 def push_prompt(
8100 self,
8101 prompt_identifier: str,
8102 *,
8103 object: Optional[Any] = None,
8104 parent_commit_hash: str = "latest",
8105 is_public: Optional[bool] = None,
8106 description: Optional[str] = None,
8107 readme: Optional[str] = None,
8108 tags: Optional[Sequence[str]] = None,
8109 ) -> str:
8110 """Push a prompt to the LangSmith API.
8112 Can be used to update prompt metadata or prompt content.
8114 If the prompt does not exist, it will be created.
8115 If the prompt exists, it will be updated.
8117 Args:
8118 prompt_identifier (str): The identifier of the prompt.
8119 object (Optional[Any]): The LangChain object to push.
8120 parent_commit_hash (str): The parent commit hash.
8121 Defaults to "latest".
8122 is_public (Optional[bool]): Whether the prompt should be public.
8123 If None (default), the current visibility status is maintained for existing prompts.
8124 For new prompts, None defaults to private.
8125 Set to True to make public, or False to make private.
8126 description (Optional[str]): A description of the prompt.
8127 Defaults to an empty string.
8128 readme (Optional[str]): A readme for the prompt.
8129 Defaults to an empty string.
8130 tags (Optional[Sequence[str]]): A list of tags for the prompt.
8131 Defaults to an empty list.
8133 Returns:
8134 str: The URL of the prompt.
8135 """
8136 # Create or update prompt metadata
8137 if self._prompt_exists(prompt_identifier):
8138 if any(
8139 param is not None for param in [is_public, description, readme, tags]
8140 ):
8141 self.update_prompt(
8142 prompt_identifier,
8143 description=description,
8144 readme=readme,
8145 tags=tags,
8146 is_public=is_public,
8147 )
8148 else:
8149 self.create_prompt(
8150 prompt_identifier,
8151 is_public=is_public if is_public is not None else False,
8152 description=description,
8153 readme=readme,
8154 tags=tags,
8155 )
8157 if object is None:
8158 return self._get_prompt_url(prompt_identifier=prompt_identifier)
8160 # Create a commit with the new manifest
8161 url = self.create_commit(
8162 prompt_identifier,
8163 object,
8164 parent_commit_hash=parent_commit_hash,
8165 )
8166 return url
8168 def cleanup(self) -> None:
8169 """Manually trigger cleanup of the background thread."""
8170 self._manual_cleanup = True
8172 @overload
8173 def evaluate(
8174 self,
8175 target: Union[TARGET_T, Runnable, EXPERIMENT_T],
8176 /,
8177 data: Optional[DATA_T] = None,
8178 evaluators: Optional[Sequence[EVALUATOR_T]] = None,
8179 summary_evaluators: Optional[Sequence[SUMMARY_EVALUATOR_T]] = None,
8180 metadata: Optional[dict] = None,
8181 experiment_prefix: Optional[str] = None,
8182 description: Optional[str] = None,
8183 max_concurrency: Optional[int] = 0,
8184 num_repetitions: int = 1,
8185 blocking: bool = True,
8186 experiment: Optional[EXPERIMENT_T] = None,
8187 upload_results: bool = True,
8188 **kwargs: Any,
8189 ) -> ExperimentResults: ...
8191 @overload
8192 def evaluate(
8193 self,
8194 target: Union[tuple[EXPERIMENT_T, EXPERIMENT_T]],
8195 /,
8196 data: Optional[DATA_T] = None,
8197 evaluators: Optional[Sequence[COMPARATIVE_EVALUATOR_T]] = None,
8198 summary_evaluators: Optional[Sequence[SUMMARY_EVALUATOR_T]] = None,
8199 metadata: Optional[dict] = None,
8200 experiment_prefix: Optional[str] = None,
8201 description: Optional[str] = None,
8202 max_concurrency: Optional[int] = 0,
8203 num_repetitions: int = 1,
8204 blocking: bool = True,
8205 experiment: Optional[EXPERIMENT_T] = None,
8206 upload_results: bool = True,
8207 **kwargs: Any,
8208 ) -> ComparativeExperimentResults: ...
8210 def evaluate(
8211 self,
8212 target: Union[
8213 TARGET_T, Runnable, EXPERIMENT_T, tuple[EXPERIMENT_T, EXPERIMENT_T]
8214 ],
8215 /,
8216 data: Optional[DATA_T] = None,
8217 evaluators: Optional[
8218 Union[Sequence[EVALUATOR_T], Sequence[COMPARATIVE_EVALUATOR_T]]
8219 ] = None,
8220 summary_evaluators: Optional[Sequence[SUMMARY_EVALUATOR_T]] = None,
8221 metadata: Optional[dict] = None,
8222 experiment_prefix: Optional[str] = None,
8223 description: Optional[str] = None,
8224 max_concurrency: Optional[int] = 0,
8225 num_repetitions: int = 1,
8226 blocking: bool = True,
8227 experiment: Optional[EXPERIMENT_T] = None,
8228 upload_results: bool = True,
8229 error_handling: Literal["log", "ignore"] = "log",
8230 **kwargs: Any,
8231 ) -> Union[ExperimentResults, ComparativeExperimentResults]:
8232 r"""Evaluate a target system on a given dataset.
8234 Args:
8235 target (Union[TARGET_T, Runnable, EXPERIMENT_T, Tuple[EXPERIMENT_T, EXPERIMENT_T]]):
8236 The target system or experiment(s) to evaluate.
8238 Can be a function that takes a `dict` and returns a `dict`, a langchain `Runnable`, an
8239 existing experiment ID, or a two-tuple of experiment IDs.
8240 data (DATA_T): The dataset to evaluate on.
8242 Can be a dataset name, a list of examples, or a generator of examples.
8243 evaluators (Optional[Union[Sequence[EVALUATOR_T], Sequence[COMPARATIVE_EVALUATOR_T]]]):
8244 A list of evaluators to run on each example. The evaluator signature
8245 depends on the target type. Default to None.
8246 summary_evaluators (Optional[Sequence[SUMMARY_EVALUATOR_T]]): A list of summary
8247 evaluators to run on the entire dataset. Should not be specified if
8248 comparing two existing experiments.
8249 metadata (Optional[dict]): Metadata to attach to the experiment.
8250 experiment_prefix (Optional[str]): A prefix to provide for your experiment name.
8251 description (Optional[str]): A free-form text description for the experiment.
8252 max_concurrency (Optional[int], default=0): The maximum number of concurrent
8253 evaluations to run.
8255 If `None` then no limit is set. If `0` then no concurrency.
8256 blocking (bool, default=True): Whether to block until the evaluation is complete.
8257 num_repetitions (int, default=1): The number of times to run the evaluation.
8258 Each item in the dataset will be run and evaluated this many times.
8259 Defaults to 1.
8260 experiment (Optional[EXPERIMENT_T]): An existing experiment to
8261 extend.
8263 If provided, `experiment_prefix` is ignored.
8265 For advanced usage only. Should not be specified if target is an existing experiment or
8266 two-tuple fo experiments.
8267 upload_results (bool, default=True): Whether to upload the results to LangSmith.
8268 error_handling (str, default="log"): How to handle individual run errors.
8270 `'log'` will trace the runs with the error message as part of the
8271 experiment, `'ignore'` will not count the run as part of the experiment at
8272 all.
8273 **kwargs (Any): Additional keyword arguments to pass to the evaluator.
8275 Returns:
8276 ExperimentResults: If target is a function, Runnable, or existing experiment.
8277 ComparativeExperimentResults: If target is a two-tuple of existing experiments.
8279 Examples:
8280 Prepare the dataset:
8282 ```python
8283 from langsmith import Client
8285 client = Client()
8286 dataset = client.clone_public_dataset(
8287 "https://smith.langchain.com/public/419dcab2-1d66-4b94-8901-0357ead390df/d"
8288 )
8289 dataset_name = "Evaluate Examples"
8290 ```
8292 Basic usage:
8294 ```python
8295 def accuracy(outputs: dict, reference_outputs: dict) -> dict:
8296 # Row-level evaluator for accuracy.
8297 pred = outputs["response"]
8298 expected = reference_outputs["answer"]
8299 return {"score": expected.lower() == pred.lower()}
8300 ```
8302 ```python
8303 def precision(outputs: list[dict], reference_outputs: list[dict]) -> dict:
8304 # Experiment-level evaluator for precision.
8305 # TP / (TP + FP)
8306 predictions = [out["response"].lower() for out in outputs]
8307 expected = [ref["answer"].lower() for ref in reference_outputs]
8308 # yes and no are the only possible answers
8309 tp = sum([p == e for p, e in zip(predictions, expected) if p == "yes"])
8310 fp = sum([p == "yes" and e == "no" for p, e in zip(predictions, expected)])
8311 return {"score": tp / (tp + fp)}
8314 def predict(inputs: dict) -> dict:
8315 # This can be any function or just an API call to your app.
8316 return {"response": "Yes"}
8319 results = client.evaluate(
8320 predict,
8321 data=dataset_name,
8322 evaluators=[accuracy],
8323 summary_evaluators=[precision],
8324 experiment_prefix="My Experiment",
8325 description="Evaluating the accuracy of a simple prediction model.",
8326 metadata={
8327 "my-prompt-version": "abcd-1234",
8328 },
8329 )
8330 ```
8332 Evaluating over only a subset of the examples
8334 ```python
8335 experiment_name = results.experiment_name
8336 examples = client.list_examples(dataset_name=dataset_name, limit=5)
8337 results = client.evaluate(
8338 predict,
8339 data=examples,
8340 evaluators=[accuracy],
8341 summary_evaluators=[precision],
8342 experiment_prefix="My Experiment",
8343 description="Just testing a subset synchronously.",
8344 )
8345 ```
8347 Streaming each prediction to more easily + eagerly debug.
8349 ```python
8350 results = client.evaluate(
8351 predict,
8352 data=dataset_name,
8353 evaluators=[accuracy],
8354 summary_evaluators=[precision],
8355 description="I don't even have to block!",
8356 blocking=False,
8357 )
8358 for i, result in enumerate(results): # doctest: +ELLIPSIS
8359 pass
8360 ```
8362 Using the `evaluate` API with an off-the-shelf LangChain evaluator:
8364 ```python
8365 from langsmith.evaluation import LangChainStringEvaluator
8366 from langchain.chat_models import init_chat_model
8369 def prepare_criteria_data(run: Run, example: Example):
8370 return {
8371 "prediction": run.outputs["output"],
8372 "reference": example.outputs["answer"],
8373 "input": str(example.inputs),
8374 }
8377 results = client.evaluate(
8378 predict,
8379 data=dataset_name,
8380 evaluators=[
8381 accuracy,
8382 LangChainStringEvaluator("embedding_distance"),
8383 LangChainStringEvaluator(
8384 "labeled_criteria",
8385 config={
8386 "criteria": {
8387 "usefulness": "The prediction is useful if it is correct"
8388 " and/or asks a useful followup question."
8389 },
8390 "llm": init_chat_model("gpt-4o"),
8391 },
8392 prepare_data=prepare_criteria_data,
8393 ),
8394 ],
8395 description="Evaluating with off-the-shelf LangChain evaluators.",
8396 summary_evaluators=[precision],
8397 )
8398 ```
8400 View the evaluation results for experiment:...
8401 Evaluating a LangChain object:
8403 ```python
8404 from langchain_core.runnables import chain as as_runnable
8407 @as_runnable
8408 def nested_predict(inputs):
8409 return {"response": "Yes"}
8412 @as_runnable
8413 def lc_predict(inputs):
8414 return nested_predict.invoke(inputs)
8417 results = client.evaluate(
8418 lc_predict,
8419 data=dataset_name,
8420 evaluators=[accuracy],
8421 description="This time we're evaluating a LangChain object.",
8422 summary_evaluators=[precision],
8423 )
8424 ```
8426 Comparative evaluation:
8428 ```python
8429 results = client.evaluate(
8430 # The target is a tuple of the experiment IDs to compare
8431 target=(
8432 "12345678-1234-1234-1234-123456789012",
8433 "98765432-1234-1234-1234-123456789012",
8434 ),
8435 evaluators=[accuracy],
8436 summary_evaluators=[precision],
8437 )
8438 ```
8440 Evaluate an existing experiment:
8442 ```python
8443 results = client.evaluate(
8444 # The target is the ID of the experiment we are evaluating
8445 target="12345678-1234-1234-1234-123456789012",
8446 evaluators=[accuracy],
8447 summary_evaluators=[precision],
8448 )
8449 ```
8451 !!! version-added "Added in `langsmith` 0.2.0"
8452 """ # noqa: E501
8453 from langsmith.evaluation._runner import evaluate as evaluate_
8455 # Need to ignore because it fails when there are too many union types +
8456 # overloads.
8457 return evaluate_( # type: ignore[misc]
8458 target, # type: ignore[arg-type]
8459 data=data,
8460 evaluators=evaluators, # type: ignore[arg-type]
8461 summary_evaluators=summary_evaluators,
8462 metadata=metadata,
8463 experiment_prefix=experiment_prefix,
8464 description=description,
8465 max_concurrency=max_concurrency,
8466 num_repetitions=num_repetitions,
8467 client=self,
8468 blocking=blocking,
8469 experiment=experiment,
8470 upload_results=upload_results,
8471 error_handling=error_handling,
8472 **kwargs,
8473 )
8475 async def aevaluate(
8476 self,
8477 target: Union[
8478 ATARGET_T,
8479 AsyncIterable[dict],
8480 Runnable,
8481 str,
8482 uuid.UUID,
8483 schemas.TracerSession,
8484 ],
8485 /,
8486 data: Union[
8487 DATA_T, AsyncIterable[schemas.Example], Iterable[schemas.Example], None
8488 ] = None,
8489 evaluators: Optional[Sequence[Union[EVALUATOR_T, AEVALUATOR_T]]] = None,
8490 summary_evaluators: Optional[Sequence[SUMMARY_EVALUATOR_T]] = None,
8491 metadata: Optional[dict] = None,
8492 experiment_prefix: Optional[str] = None,
8493 description: Optional[str] = None,
8494 max_concurrency: Optional[int] = 0,
8495 num_repetitions: int = 1,
8496 blocking: bool = True,
8497 experiment: Optional[Union[schemas.TracerSession, str, uuid.UUID]] = None,
8498 upload_results: bool = True,
8499 error_handling: Literal["log", "ignore"] = "log",
8500 **kwargs: Any,
8501 ) -> AsyncExperimentResults:
8502 r"""Evaluate an async target system on a given dataset.
8504 Args:
8505 target (Union[ATARGET_T, AsyncIterable[dict], Runnable, str, uuid.UUID, TracerSession]):
8506 The target system or experiment(s) to evaluate.
8508 Can be an async function that takes a `dict` and returns a `dict`, a langchain `Runnable`, an
8509 existing experiment ID, or a two-tuple of experiment IDs.
8510 data (Union[DATA_T, AsyncIterable[Example]]): The dataset to evaluate on.
8512 Can be a dataset name, a list of examples, an async generator of examples, or an async iterable of examples.
8513 evaluators (Optional[Sequence[EVALUATOR_T]]): A list of evaluators to run
8514 on each example.
8515 summary_evaluators (Optional[Sequence[SUMMARY_EVALUATOR_T]]): A list of summary
8516 evaluators to run on the entire dataset.
8517 metadata (Optional[dict]): Metadata to attach to the experiment.
8518 experiment_prefix (Optional[str]): A prefix to provide for your experiment name.
8519 description (Optional[str]): A description of the experiment.
8520 max_concurrency (Optional[int], default=0): The maximum number of concurrent
8521 evaluations to run.
8523 If `None` then no limit is set. If `0` then no concurrency.
8524 num_repetitions (int, default=1): The number of times to run the evaluation.
8525 Each item in the dataset will be run and evaluated this many times.
8526 Defaults to 1.
8527 blocking (bool, default=True): Whether to block until the evaluation is complete.
8528 experiment (Optional[TracerSession]): An existing experiment to
8529 extend.
8531 If provided, `experiment_prefix` is ignored.
8533 For advanced usage only.
8534 upload_results (bool, default=True): Whether to upload the results to LangSmith.
8535 error_handling (str, default="log"): How to handle individual run errors.
8537 `'log'` will trace the runs with the error message as part of the
8538 experiment, `'ignore'` will not count the run as part of the experiment at
8539 all.
8540 **kwargs (Any): Additional keyword arguments to pass to the evaluator.
8542 Returns:
8543 An async iterator over the experiment results.
8545 Environment:
8546 - `LANGSMITH_TEST_CACHE`: If set, API calls will be cached to disk to save time and
8547 cost during testing.
8549 Recommended to commit the cache files to your repository for faster CI/CD runs.
8551 Requires the `'langsmith[vcr]'` package to be installed.
8553 Examples:
8554 Prepare the dataset:
8556 ```python
8557 import asyncio
8558 from langsmith import Client
8560 client = Client()
8561 dataset = client.clone_public_dataset(
8562 "https://smith.langchain.com/public/419dcab2-1d66-4b94-8901-0357ead390df/d"
8563 )
8564 dataset_name = "Evaluate Examples"
8565 ```
8567 Basic usage:
8569 ```python
8570 def accuracy(outputs: dict, reference_outputs: dict) -> dict:
8571 # Row-level evaluator for accuracy.
8572 pred = outputs["resposen"]
8573 expected = reference_outputs["answer"]
8574 return {"score": expected.lower() == pred.lower()}
8577 def precision(outputs: list[dict], reference_outputs: list[dict]) -> dict:
8578 # Experiment-level evaluator for precision.
8579 # TP / (TP + FP)
8580 predictions = [out["response"].lower() for out in outputs]
8581 expected = [ref["answer"].lower() for ref in reference_outputs]
8582 # yes and no are the only possible answers
8583 tp = sum([p == e for p, e in zip(predictions, expected) if p == "yes"])
8584 fp = sum([p == "yes" and e == "no" for p, e in zip(predictions, expected)])
8585 return {"score": tp / (tp + fp)}
8588 async def apredict(inputs: dict) -> dict:
8589 # This can be any async function or just an API call to your app.
8590 await asyncio.sleep(0.1)
8591 return {"response": "Yes"}
8594 results = asyncio.run(
8595 client.aevaluate(
8596 apredict,
8597 data=dataset_name,
8598 evaluators=[accuracy],
8599 summary_evaluators=[precision],
8600 experiment_prefix="My Experiment",
8601 description="Evaluate the accuracy of the model asynchronously.",
8602 metadata={
8603 "my-prompt-version": "abcd-1234",
8604 },
8605 )
8606 )
8607 ```
8609 Evaluating over only a subset of the examples using an async generator:
8611 ```python
8612 async def example_generator():
8613 examples = client.list_examples(dataset_name=dataset_name, limit=5)
8614 for example in examples:
8615 yield example
8618 results = asyncio.run(
8619 client.aevaluate(
8620 apredict,
8621 data=example_generator(),
8622 evaluators=[accuracy],
8623 summary_evaluators=[precision],
8624 experiment_prefix="My Subset Experiment",
8625 description="Evaluate a subset of examples asynchronously.",
8626 )
8627 )
8628 ```
8630 Streaming each prediction to more easily + eagerly debug.
8632 ```python
8633 results = asyncio.run(
8634 client.aevaluate(
8635 apredict,
8636 data=dataset_name,
8637 evaluators=[accuracy],
8638 summary_evaluators=[precision],
8639 experiment_prefix="My Streaming Experiment",
8640 description="Streaming predictions for debugging.",
8641 blocking=False,
8642 )
8643 )
8646 async def aenumerate(iterable):
8647 async for elem in iterable:
8648 print(elem)
8651 asyncio.run(aenumerate(results))
8652 ```
8654 Running without concurrency:
8656 ```python
8657 results = asyncio.run(
8658 client.aevaluate(
8659 apredict,
8660 data=dataset_name,
8661 evaluators=[accuracy],
8662 summary_evaluators=[precision],
8663 experiment_prefix="My Experiment Without Concurrency",
8664 description="This was run without concurrency.",
8665 max_concurrency=0,
8666 )
8667 )
8668 ```
8670 Using Async evaluators:
8672 ```python
8673 async def helpfulness(outputs: dict) -> dict:
8674 # Row-level evaluator for helpfulness.
8675 await asyncio.sleep(5) # Replace with your LLM API call
8676 return {"score": outputs["output"] == "Yes"}
8679 results = asyncio.run(
8680 client.aevaluate(
8681 apredict,
8682 data=dataset_name,
8683 evaluators=[helpfulness],
8684 summary_evaluators=[precision],
8685 experiment_prefix="My Helpful Experiment",
8686 description="Applying async evaluators example.",
8687 )
8688 )
8689 ```
8691 Evaluate an existing experiment:
8693 ```python
8694 results = asyncio.run(
8695 client.aevaluate(
8696 # The target is the ID of the experiment we are evaluating
8697 target="419dcab2-1d66-4b94-8901-0357ead390df",
8698 evaluators=[accuracy, helpfulness],
8699 summary_evaluators=[precision],
8700 )
8701 )
8702 ```
8704 !!! version-added "Added in `langsmith` 0.2.0"
8705 """ # noqa: E501
8706 from langsmith.evaluation._arunner import aevaluate as aevaluate_
8708 return await aevaluate_(
8709 target,
8710 data=data,
8711 evaluators=evaluators,
8712 summary_evaluators=summary_evaluators,
8713 metadata=metadata,
8714 experiment_prefix=experiment_prefix,
8715 description=description,
8716 max_concurrency=max_concurrency,
8717 num_repetitions=num_repetitions,
8718 client=self,
8719 blocking=blocking,
8720 experiment=experiment,
8721 upload_results=upload_results,
8722 error_handling=error_handling,
8723 **kwargs,
8724 )
8726 def _paginate_examples_with_runs(
8727 self,
8728 dataset_id: ID_TYPE,
8729 session_id: uuid.UUID,
8730 preview: bool = False,
8731 comparative_experiment_id: Optional[uuid.UUID] = None,
8732 filters: dict[uuid.UUID, list[str]] | None = None,
8733 limit: Optional[int] = None,
8734 ) -> Iterator[list[ExampleWithRuns]]:
8735 """Paginate through examples with runs and yield batches.
8737 Args:
8738 dataset_id: Dataset UUID to fetch examples with runs
8739 session_id: Session UUID to filter runs by, same as project_id
8740 preview: Whether to return preview data only
8741 comparative_experiment_id: Optional comparative experiment UUID
8742 filters: Optional filters to apply
8743 limit: Maximum total number of results to return
8745 Yields:
8746 Batches of run results as lists of ExampleWithRuns instances
8747 """
8748 offset = 0
8749 results_count = 0
8751 while True:
8752 remaining = (limit - results_count) if limit else None
8753 batch_limit = min(100, remaining) if remaining else 100
8755 body = {
8756 "session_ids": [session_id],
8757 "offset": offset,
8758 "limit": batch_limit,
8759 "preview": preview,
8760 "comparative_experiment_id": comparative_experiment_id,
8761 "filters": filters,
8762 }
8764 response = self.request_with_retries(
8765 "POST",
8766 f"/datasets/{dataset_id}/runs",
8767 request_kwargs={"data": _dumps_json(body)},
8768 )
8770 batch = response.json()
8771 if not batch:
8772 break
8774 # Transform raw dictionaries to ExampleWithRuns instances
8775 examples_batch = [ls_schemas.ExampleWithRuns(**result) for result in batch]
8776 yield examples_batch
8777 results_count += len(batch)
8779 if len(batch) < batch_limit or (limit and results_count >= limit):
8780 break
8782 offset += len(batch)
8784 def get_experiment_results(
8785 self,
8786 name: Optional[str] = None,
8787 project_id: Optional[uuid.UUID] = None,
8788 preview: bool = False,
8789 comparative_experiment_id: Optional[uuid.UUID] = None,
8790 filters: dict[uuid.UUID, list[str]] | None = None,
8791 limit: Optional[int] = None,
8792 ) -> ls_schemas.ExperimentResults:
8793 """Get results for an experiment, including experiment session aggregated stats and experiment runs for each dataset example.
8795 Experiment results may not be available immediately after the experiment is created.
8797 Args:
8798 name: The experiment name.
8799 project_id: Experiment's tracing project id, also called session_id, can be found in the url of the LS experiment page
8800 preview: Whether to return lightweight preview data only. When True,
8801 fetches inputs_preview/outputs_preview summaries instead of full inputs/outputs from S3 storage.
8802 Faster and less bandwidth.
8803 comparative_experiment_id: Optional comparative experiment UUID for pairwise comparison experiment results.
8804 filters: Optional filters to apply to results
8805 limit: Maximum number of results to return
8807 Returns:
8808 ExperimentResults with:
8809 - feedback_stats: Combined feedback statistics including session-level feedback
8810 - run_stats: Aggregated run statistics (latency, tokens, cost, etc.)
8811 - examples_with_runs: Iterator of ExampleWithRuns
8813 Raises:
8814 ValueError: If project not found for the given session_id
8816 Example:
8817 ```python
8818 client = Client()
8819 results = client.get_experiment_results(
8820 project_id="037ae90f-f297-4926-b93c-37d8abf6899f",
8821 )
8822 for example_with_runs in results["examples_with_runs"]:
8823 print(example_with_runs.dict())
8825 # Access aggregated experiment statistics
8826 print(f"Total runs: {results['run_stats']['run_count']}")
8827 print(f"Total cost: {results['run_stats']['total_cost']}")
8828 print(f"P50 latency: {results['run_stats']['latency_p50']}")
8830 # Access feedback statistics
8831 print(f"Feedback stats: {results['feedback_stats']}")
8832 ```
8833 """
8834 project = self.read_project(
8835 project_name=name, project_id=project_id, include_stats=True
8836 )
8838 if not project:
8839 raise ValueError(f"No experiment found with project_id: '{project_id}'")
8841 def _get_examples_with_runs_iterator():
8842 """Yield examples with corresponding experiment runs."""
8843 for batch in self._paginate_examples_with_runs(
8844 dataset_id=project.reference_dataset_id,
8845 session_id=project.id,
8846 preview=preview,
8847 comparative_experiment_id=comparative_experiment_id,
8848 filters=filters,
8849 limit=limit,
8850 ):
8851 yield from batch
8853 run_stats: ls_schemas.ExperimentRunStats = {
8854 "run_count": project.run_count,
8855 "latency_p50": project.latency_p50,
8856 "latency_p99": project.latency_p99,
8857 "total_tokens": project.total_tokens,
8858 "prompt_tokens": project.prompt_tokens,
8859 "completion_tokens": project.completion_tokens,
8860 "last_run_start_time": project.last_run_start_time,
8861 "run_facets": project.run_facets,
8862 "total_cost": project.total_cost,
8863 "prompt_cost": project.prompt_cost,
8864 "completion_cost": project.completion_cost,
8865 "first_token_p50": project.first_token_p50,
8866 "first_token_p99": project.first_token_p99,
8867 "error_rate": project.error_rate,
8868 }
8869 feedback_stats = {
8870 **(project.feedback_stats or {}),
8871 **(project.session_feedback_stats or {}),
8872 }
8873 return ls_schemas.ExperimentResults(
8874 feedback_stats=feedback_stats,
8875 run_stats=run_stats,
8876 examples_with_runs=_get_examples_with_runs_iterator(),
8877 )
8879 @warn_beta
8880 def generate_insights(
8881 self,
8882 *,
8883 chat_histories: list[list[dict]],
8884 instructions: str = DEFAULT_INSTRUCTIONS,
8885 name: str | None = None,
8886 model: Literal["openai", "anthropic"] | None = None,
8887 openai_api_key: str | None = None,
8888 anthropic_api_key: str | None = None,
8889 ) -> ls_schemas.InsightsReport:
8890 """Generate Insights over your agent chat histories.
8892 !!! note
8894 - Only available to Plus and higher tier LangSmith users.
8895 - Insights Agent uses user's model API key. The cost of the report
8896 grows linearly with the number of chat histories you upload and the
8897 size of each history. For more see [insights](https://docs.langchain.com/langsmith/insights).
8898 - This method will upload your chat histories as traces to LangSmith.
8899 - If you pass in a model API key this will be set as a workspace secret
8900 meaning it will be usedin for evaluators and the playground.
8902 Args:
8903 chat_histories: A list of chat histories. Each chat history should be a
8904 list of messages. We recommend formatting these as OpenAI messages with
8905 a "role" and "content" key. Max length 1000 items.
8906 instructions: Instructions for the Insights agent. Should focus on what
8907 your agent does and what types of insights you
8908 want to generate.
8909 name: Name for the generated Insights report.
8910 model: Whether to use OpenAI or Anthropic models. This will impact the
8911 cost of generating the Insights Report.
8912 openai_api_key: OpenAI API key to use. Only needed if you have not already
8913 stored this in LangSmith as a workspace secret.
8914 anthropic_api_key: Anthropic API key to use. Only needed if you have not
8915 already stored this in LangSmith as a workspace secret.
8917 Example:
8918 ```python
8919 import os
8920 from langsmith import Client
8922 client = client()
8924 chat_histories = [
8925 [
8926 {"role": "user", "content": "how are you"},
8927 {"role": "assistant", "content": "good!"},
8928 ],
8929 [
8930 {"role": "user", "content": "do you like art"},
8931 {"role": "assistant", "content": "only Tarkovsky"},
8932 ],
8933 ]
8935 report = client.generate_insights(
8936 chat_histories=chat_histories,
8937 name="Conversation Topics",
8938 instructions="What are the high-level topics of conversations users are having with the assistant?",
8939 openai_api_key=os.environ["OPENAI_API_KEY"],
8940 )
8942 # client.poll_insights(report=report)
8943 ```
8944 """
8945 model = self._ensure_insights_api_key(
8946 openai_api_key=openai_api_key,
8947 anthropic_api_key=anthropic_api_key,
8948 model=model,
8949 )
8950 project = self._ingest_insights_runs(chat_histories, name)
8951 config = {
8952 "name": name,
8953 "user_context": {
8954 "How are your agent traces structured?": "The run.outputs.messages field contains a chat history between the user and the agent. This is all the context you need.",
8955 "What would you like to learn about your agent?": instructions,
8956 },
8957 "last_n_hours": 1,
8958 "model": model,
8959 }
8960 response = self.request_with_retries(
8961 "POST", f"/sessions/{project.id}/insights", json=config
8962 )
8963 ls_utils.raise_for_status_with_text(response)
8964 res = response.json()
8965 report = ls_schemas.InsightsReport(
8966 **res,
8967 project_id=project.id,
8968 tenant_id=self._get_tenant_id(),
8969 host_url=self._host_url,
8970 )
8971 print( # noqa: T201
8972 "The Insights Agent is running! This can take up to 30 minutes to complete."
8973 " Once the report is completed, you'll be able to see results here: "
8974 f"{report.link}"
8975 )
8976 return report
8978 @warn_beta
8979 def poll_insights(
8980 self,
8981 *,
8982 report: ls_schemas.InsightsReport | None = None,
8983 id: str | uuid.UUID | None = None,
8984 project_id: str | uuid.UUID | None = None,
8985 rate: int = 30,
8986 timeout: int = 30 * 60,
8987 verbose: bool = False,
8988 ) -> ls_schemas.InsightsReport:
8989 """Poll the status of an Insights report.
8991 Args:
8992 report: THe InsightsReport.
8993 id: The Insights report ID. Should only specify if 'report' is not specified.
8994 project_id: The Tracing project ID. Should only specify if 'report' is not specified.
8995 """
8996 if not ((id and project_id) or report):
8997 raise ValueError("Must specify ('id' and 'project_id') or 'report'.")
8998 elif (id or project_id) and report:
8999 raise ValueError(
9000 "Must specify exactly one of ('id' and 'project_id') or 'report'."
9001 )
9002 elif report:
9003 id = report.id
9004 project_id = report.project_id
9006 max_tries = max(1, timeout // rate)
9007 for i in range(max_tries):
9008 response = self.request_with_retries(
9009 "GET", f"/sessions/{project_id}/insights/{id}"
9010 )
9011 ls_utils.raise_for_status_with_text(response)
9012 resp_json = response.json()
9013 if resp_json["status"] == "success":
9014 job = ls_schemas.InsightsReport(
9015 **resp_json,
9016 project_id=project_id, # type: ignore[arg-type]
9017 tenant_id=self._get_tenant_id(),
9018 host_url=self._host_url,
9019 )
9020 print( # noqa: T201
9021 "Insights report completed! View the results at %s",
9022 job.link,
9023 )
9024 return job
9025 elif resp_json["status"] == "error":
9026 raise ValueError(f"Failed to generate insights: {resp_json['error']}")
9027 elif verbose:
9028 print(f"Polling time: {i * rate}") # noqa: T201
9029 time.sleep(rate)
9030 raise TimeoutError("Insights still pending")
9032 def _ensure_insights_api_key(
9033 self,
9034 *,
9035 openai_api_key: str | None = None,
9036 anthropic_api_key: str | None = None,
9037 model: Literal["openai", "anthropic"] | None = None,
9038 ) -> Literal["openai", "anthropic"]:
9039 response = self.request_with_retries("GET", "/workspaces/current/secrets")
9040 ls_utils.raise_for_status_with_text(response)
9041 workspace_keys = {s.get("key") for s in response.json()}
9042 target_keys = set()
9043 if model in (None, "openai"):
9044 target_keys.add(_OPENAI_API_KEY)
9045 if model in (None, "anthropic"):
9046 target_keys.add(_ANTHROPIC_API_KEY)
9048 if existing_keys := workspace_keys.intersection(target_keys):
9049 return "openai" if _OPENAI_API_KEY in existing_keys else "anthropic"
9050 elif model == "openai":
9051 api_key = openai_api_key
9052 api_var = _OPENAI_API_KEY
9053 elif model == "anthropic":
9054 api_key = anthropic_api_key
9055 api_var = _ANTHROPIC_API_KEY
9056 elif openai_api_key or anthropic_api_key:
9057 api_key = openai_api_key or anthropic_api_key
9058 api_var = _OPENAI_API_KEY if openai_api_key else _ANTHROPIC_API_KEY
9059 else:
9060 raise ValueError("Must specify openai_api_key or anthropic_api_key.")
9061 response = self.request_with_retries(
9062 "POST",
9063 "/workspaces/current/secrets",
9064 json=[{"key": api_var, "value": api_key}],
9065 )
9066 ls_utils.raise_for_status_with_text(response)
9067 return "openai" if api_var == _OPENAI_API_KEY else "anthropic"
9069 def _ingest_insights_runs(self, data: list, name: str | None):
9070 if len(data) > 1000:
9071 warnings.warn(
9072 "Can only generate insights over 1000 data. Truncating to first 1000."
9073 )
9074 data = data[:1000]
9075 now = datetime.datetime.now(datetime.timezone.utc)
9076 project = self.create_project(
9077 name
9078 or ("insights " + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")),
9079 )
9080 run_ids = [str(uuid.uuid4()) for _ in range(len(data))]
9081 runs = [
9082 {
9083 "inputs": {"messages": x[:1]},
9084 "outputs": {"messages": x},
9085 "id": run_id,
9086 "trace_id": run_id,
9087 "dotted_order": f"{now.strftime('%Y%m%dT%H%M%S%fZ')}{str(run_id)}",
9088 "start_time": now - datetime.timedelta(seconds=1),
9089 "end_time": now,
9090 "run_type": "chain",
9091 "session_id": project.id,
9092 "name": "trace",
9093 }
9094 for run_id, x in zip(run_ids, data)
9095 ]
9096 self.batch_ingest_runs(create=runs)
9097 self.flush()
9098 return project
9101def convert_prompt_to_openai_format(
9102 messages: Any,
9103 model_kwargs: Optional[dict[str, Any]] = None,
9104) -> dict:
9105 """Convert a prompt to OpenAI format.
9107 Requires the `langchain_openai` package to be installed.
9109 Args:
9110 messages (Any): The messages to convert.
9111 model_kwargs (Optional[Dict[str, Any]]): Model configuration arguments including
9112 `stop` and any other required arguments.
9114 Returns:
9115 dict: The prompt in OpenAI format.
9117 Raises:
9118 ImportError: If the `langchain_openai` package is not installed.
9119 ls_utils.LangSmithError: If there is an error during the conversion process.
9120 """
9121 try:
9122 from langchain_openai import ChatOpenAI # type: ignore
9123 except ImportError:
9124 raise ImportError(
9125 "The convert_prompt_to_openai_format function requires the langchain_openai"
9126 "package to run.\nInstall with `pip install langchain_openai`"
9127 )
9129 openai = ChatOpenAI()
9131 model_kwargs = model_kwargs or {}
9132 stop = model_kwargs.pop("stop", None)
9134 try:
9135 return openai._get_request_payload(messages, stop=stop, **model_kwargs)
9136 except Exception as e:
9137 raise ls_utils.LangSmithError(f"Error converting to OpenAI format: {e}")
9140def convert_prompt_to_anthropic_format(
9141 messages: Any,
9142 model_kwargs: Optional[dict[str, Any]] = None,
9143) -> dict:
9144 """Convert a prompt to Anthropic format.
9146 Requires the `langchain_anthropic` package to be installed.
9148 Args:
9149 messages (Any): The messages to convert.
9150 model_kwargs (Optional[Dict[str, Any]]):
9151 Model configuration arguments including `model_name` and `stop`.
9153 Returns:
9154 dict: The prompt in Anthropic format.
9155 """
9156 try:
9157 from langchain_anthropic import ChatAnthropic # type: ignore
9158 except ImportError:
9159 raise ImportError(
9160 "The convert_prompt_to_anthropic_format function requires the "
9161 "langchain_anthropic package to run.\n"
9162 "Install with `pip install langchain_anthropic`"
9163 )
9165 model_kwargs = model_kwargs or {}
9166 model_name = model_kwargs.pop("model_name", "claude-3-haiku-20240307")
9167 stop = model_kwargs.pop("stop", None)
9168 timeout = model_kwargs.pop("timeout", None)
9170 anthropic = ChatAnthropic(
9171 model_name=model_name, timeout=timeout, stop=stop, **model_kwargs
9172 )
9174 try:
9175 return anthropic._get_request_payload(messages, stop=stop)
9176 except Exception as e:
9177 raise ls_utils.LangSmithError(f"Error converting to Anthropic format: {e}")
9180class _FailedAttachmentReader(io.BytesIO):
9181 """BytesIO that raises an error when read, for failed attachment downloads."""
9183 def __init__(self, error: Exception):
9184 super().__init__()
9185 self._error = error
9187 def read(self, size: Optional[int] = -1) -> bytes:
9188 raise ls_utils.LangSmithError(
9189 f"Failed to download attachment: {self._error}"
9190 ) from self._error
9193def _convert_stored_attachments_to_attachments_dict(
9194 data: dict, *, attachments_key: str, api_url: Optional[str] = None
9195) -> dict[str, AttachmentInfo]:
9196 """Convert attachments from the backend database format to the user facing format."""
9197 attachments_dict = {}
9198 if attachments_key in data and data[attachments_key]:
9199 for key, value in data[attachments_key].items():
9200 if not key.startswith("attachment."):
9201 continue
9202 if api_url is not None:
9203 full_url = _construct_url(api_url, value["presigned_url"])
9204 else:
9205 full_url = value["presigned_url"]
9206 try:
9207 response = requests.get(full_url, stream=True)
9208 response.raise_for_status()
9209 reader = io.BytesIO(response.content)
9210 except Exception as e:
9211 logger.warning(f"Error downloading attachment {key}: {e}")
9212 reader = _FailedAttachmentReader(e)
9213 attachments_dict[key.removeprefix("attachment.")] = AttachmentInfo(
9214 **{
9215 "presigned_url": value["presigned_url"],
9216 "reader": reader,
9217 "mime_type": value.get("mime_type"),
9218 }
9219 )
9220 return attachments_dict
9223def _close_files(files: list[io.BufferedReader]) -> None:
9224 """Close all opened files used in multipart requests."""
9225 for file in files:
9226 try:
9227 file.close()
9228 except Exception:
9229 logger.debug("Could not close file: %s", file.name)
9230 pass
9233def _dataset_examples_path(api_url: str, dataset_id: ID_TYPE) -> str:
9234 if api_url.rstrip("/").endswith("/v1"):
9235 return f"/platform/datasets/{dataset_id}/examples"
9236 else:
9237 return f"/v1/platform/datasets/{dataset_id}/examples"
9240def _platform_path(api_url: str, path: str) -> str:
9241 """Construct a platform API path based on the API URL structure."""
9242 if api_url.rstrip("/").endswith("/v1"):
9243 return f"/platform/{path}"
9244 else:
9245 return f"/v1/platform/{path}"
9248def _construct_url(api_url: str, pathname: str) -> str:
9249 if pathname.startswith("http"):
9250 return pathname
9251 if api_url.startswith("https://"):
9252 http = "https://"
9253 api_url = api_url[len("https://") :]
9254 elif api_url.startswith("http://"):
9255 http = "http://"
9256 api_url = api_url[len("http://") :]
9257 else:
9258 raise ValueError(
9259 f"api_url must start with 'http://' or 'https://'. Received {api_url=}"
9260 )
9262 api_parts = api_url.rstrip("/").split("/")
9263 path_parts = pathname.lstrip("/").split("/")
9265 if not api_parts:
9266 raise ValueError(
9267 "Must specify non-empty api_url or pathname must be a full url. "
9268 f"Received {api_url=}, {pathname=}"
9269 )
9270 if not path_parts:
9271 return api_url
9273 if path_parts[0] == "api":
9274 if api_parts[-1] == "api":
9275 api_parts = api_parts[:-1]
9276 elif api_parts[-2:] == ["api", "v1"]:
9277 api_parts = api_parts[:-2]
9278 parts = api_parts + path_parts
9279 return http + "/".join(p for p in parts if p)
9282def dump_model(model, *, exclude_none: bool = False) -> dict[str, Any]:
9283 """Dump model depending on pydantic version."""
9284 if hasattr(model, "model_dump"):
9285 return model.model_dump(exclude_none=exclude_none)
9286 elif hasattr(model, "dict"):
9287 return model.dict(exclude_none=exclude_none)
9288 else:
9289 raise TypeError("Unsupported model type")
9292def prep_obj_for_push(obj: Any) -> Any:
9293 """Format the object so its Prompt Hub compatible."""
9294 try:
9295 from langchain_core.prompts import ChatPromptTemplate
9296 from langchain_core.prompts.structured import StructuredPrompt
9297 from langchain_core.runnables import RunnableBinding, RunnableSequence
9298 except ImportError:
9299 raise ImportError(
9300 "The client.create_commit function requires the langchain-core"
9301 "package to run.\nInstall with `pip install langchain-core`"
9302 )
9304 # Transform 3-step RunnableSequence back to 2-step for structured prompts
9305 # See pull_prompt for the forward transformation
9306 chain_to_push = obj
9307 if (
9308 isinstance(obj, RunnableSequence)
9309 and isinstance(obj.first, ChatPromptTemplate)
9310 and isinstance(obj.steps[1], RunnableBinding)
9311 and 2 <= len(obj.steps) <= 3
9312 ):
9313 prompt = obj.first
9314 bound_model = obj.steps[1]
9315 model = bound_model.bound
9316 model_kwargs = bound_model.kwargs
9318 # have a sequence like:
9319 # ChatPromptTemplate | ChatModel.with_structured_output()
9320 if (
9321 not isinstance(prompt, StructuredPrompt)
9322 and "ls_structured_output_format" in bound_model.kwargs
9323 ):
9324 output_format = bound_model.kwargs["ls_structured_output_format"]
9325 prompt = StructuredPrompt(messages=prompt.messages, **output_format)
9327 # have a sequence like: StructuredPrompt | RunnableBinding(bound=ChatModel)
9328 if isinstance(prompt, StructuredPrompt):
9329 structured_kwargs = (prompt | model).steps[1].kwargs # type: ignore[attr-defined]
9330 # remove the kwargs that are bound by with_structured_output()
9331 bound_model.kwargs = {
9332 k: v for k, v in model_kwargs.items() if k not in structured_kwargs
9333 }
9334 # Can't pipe with | syntax bc StructuredPrompt defines special piping
9335 # behavior that'll cause bound_model.with_structured_output to be
9336 # called.
9337 chain_to_push = RunnableSequence(prompt, bound_model)
9338 return chain_to_push