{ "openapi": "3.1.0", "info": { "title": "FastAPI", "version": "0.1.0" }, "paths": { "/api/parea/v1/": { "get": { "summary": "Health Check", "operationId": "health_check_api_parea_v1__get", "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "additionalProperties": { "type": "string" }, "type": "object", "title": "Response Health Check Api Parea V1 Get" } } } } }, "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ] } }, "/api/parea/v1/completion": { "post": { "summary": "Completion", "description": "Get a completion response using either one of your organization\u00e2\u20ac\u2122s deployed prompts, or by providing completion details including prompt and inputs in the request.\nThis endpoint acts as a LLM gateway/proxy endpoint to generate completions from different LLMs.", "operationId": "completion_api_parea_v1_completion_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CompletionsSchema" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CompletionsResponseSchema" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ] } }, "/api/parea/v1/completion/stream": { "post": { "summary": "Stream Completion", "description": "Get a completion response using either one of your organization\u00e2\u20ac\u2122s deployed prompts, or by providing completion details including prompt and inputs in the request.\n\nThis endpoint acts as a LLM gateway/proxy endpoint to generate completions from different LLMs.", "operationId": "stream_completion_api_parea_v1_completion_stream_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CompletionsSchema" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": {} } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ] } }, "/api/parea/v1/deployed-prompt": { "post": { "summary": "Fetch Deployed Prompt", "description": "Given a deployment_id, fetches the deployed prompt and its details. Can be optionally used to fill-in the templated prompt with provided inputs.", "operationId": "fetch_deployed_prompt_api_parea_v1_deployed_prompt_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/DeploymentPromptIdSchema" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/GetDeployedPromptResponseSchema" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ] } }, "/api/parea/v1/feedback": { "post": { "summary": "Record Feedback", "description": "Record any (user) feedback & ground truth/correction of output for a log.", "operationId": "record_feedback_api_parea_v1_feedback_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/FeedbackRequestSchema" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": {} } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ] } }, "/api/parea/v1/trace_log": { "put": { "summary": "Update Trace Log", "description": "Update fields of a trace log.", "operationId": "update_trace_log_api_parea_v1_trace_log_put", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UpdateLogSchema" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": {} } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ] }, "post": { "summary": "Record Trace Log", "description": "Log a (LLM) span to visualize inference results, or chains.", "operationId": "record_trace_log_api_parea_v1_trace_log_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/TraceLogSchema" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": {} } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ] } }, "/api/parea/v1/trace_log/langchain": { "post": { "summary": "Record Langchain Trace Log", "operationId": "record_langchain_trace_log_api_parea_v1_trace_log_langchain_post", "requestBody": { "content": { "application/json": { "schema": { "type": "object", "title": "Run" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": {} } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ] } }, "/api/parea/v1/experiment": { "post": { "summary": "Create Experiment Api", "description": "Create an experiment and get the associated experiment uuid.", "operationId": "create_experiment_api_api_parea_v1_experiment_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CreateExperimentRequestSchema" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ExperimentSchema" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ] } }, "/api/parea/v1/experiment/{experiment_uuid}/stats": { "get": { "summary": "Get Experiment Stats Api", "description": "Fetches aggregated stats for every root-level trace log in an experiment.", "operationId": "get_experiment_stats_api_api_parea_v1_experiment__experiment_uuid__stats_get", "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ], "parameters": [ { "name": "experiment_uuid", "in": "path", "required": true, "schema": { "type": "string", "title": "Experiment Uuid" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ExperimentStatsSchema" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/api/parea/v1/experiment/{experiment_uuid}/finished": { "post": { "summary": "Finish Experiment Api", "description": "Finishes an experiment, calculates stats and returns all stats for root trace logs", "operationId": "finish_experiment_api_api_parea_v1_experiment__experiment_uuid__finished_post", "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ], "parameters": [ { "name": "experiment_uuid", "in": "path", "required": true, "schema": { "type": "string", "title": "Experiment Uuid" } } ], "requestBody": { "content": { "application/json": { "schema": { "anyOf": [ { "$ref": "#/components/schemas/FinishExperimentRequestSchema" }, { "type": "null" } ], "title": "Finish Experiment Request" } } } }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ExperimentStatsSchema" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/api/parea/v1/project": { "post": { "summary": "Create Or Get Project Api", "description": "Create a project or get the project if it already exists. Is used to find out UUID of project.", "operationId": "create_or_get_project_api_api_parea_v1_project_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CreateGetProjectRequestSchema" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CreateGetProjectResponseSchema" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ] } }, "/api/parea/v1/collection/{test_collection_identifier}": { "get": { "summary": "Get Test Case Collection By Identifier", "description": "Fetches dataset/test case collection by its ID or name.\n\nArgs:\n test_collection_identifier (Union[str, int]): ID or name of the test case collection/dataset.", "operationId": "get_test_case_collection_by_identifier_api_parea_v1_collection__test_collection_identifier__get", "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ], "parameters": [ { "name": "test_collection_identifier", "in": "path", "required": true, "schema": { "anyOf": [ { "type": "string" }, { "type": "integer" } ], "title": "Test Collection Identifier" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "anyOf": [ { "$ref": "#/components/schemas/GetTestCaseCollectionResponseSchema" }, { "type": "null" } ], "title": "Response Get Test Case Collection By Identifier Api Parea V1 Collection Test Collection Identifier Get" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/api/parea/v1/collection": { "post": { "summary": "Create Collection", "description": "Creates a dataset", "operationId": "create_collection_api_parea_v1_collection_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CreateTestCaseCollectionRequestSchema" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CreateTestCaseCollectionResponseSchema" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ] } }, "/api/parea/v1/testcases": { "post": { "summary": "Create Test Cases For Dataset", "description": "Adds items/test cases to a dataset. If the dataset does not exist, it will be created.\nReturns the IDs of the created items/test cases.\n\nReturns:\n List[int]: List of IDs of the created items/test cases.", "operationId": "create_test_cases_for_dataset_api_parea_v1_testcases_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/CreateTestCasesSchema" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "items": { "type": "integer" }, "type": "array", "title": "Response Create Test Cases For Dataset Api Parea V1 Testcases Post" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ] } }, "/api/parea/v1/trace_log/{trace_id}": { "get": { "summary": "Get Trace Log Api", "description": "Fetches a trace log by its UUID.", "operationId": "get_trace_log_api_api_parea_v1_trace_log__trace_id__get", "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ], "parameters": [ { "name": "trace_id", "in": "path", "required": true, "schema": { "type": "string", "title": "Trace Id" } } ], "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "anyOf": [ { "$ref": "#/components/schemas/TraceLogTreeSchema" }, { "type": "null" } ], "title": "Response Get Trace Log Api Api Parea V1 Trace Log Trace Id Get" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/api/parea/v1/experiments": { "post": { "summary": "List Experiments Api", "description": "Lists experiments given a set of filters incl. their high-level statistics.", "operationId": "list_experiments_api_api_parea_v1_experiments_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/ListExperimentsRequestSchema" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "items": { "$ref": "#/components/schemas/ExperimentWithStatsSchema" }, "type": "array", "title": "Response List Experiments Api Api Parea V1 Experiments Post" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ] } }, "/api/parea/v1/experiment/{experiment_uuid}/trace_logs": { "post": { "summary": "Get Experiment Trace Logs Api", "description": "Fetches all trace logs for an experiment.\n\nArgs:\n experiment_uuid (str): UUID of the experiment.\n\nReturns:\n list[TraceLogTreeSchema]: List of trace logs for the experiment.", "operationId": "get_experiment_trace_logs_api_api_parea_v1_experiment__experiment_uuid__trace_logs_post", "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ], "parameters": [ { "name": "experiment_uuid", "in": "path", "required": true, "schema": { "type": "string", "title": "Experiment Uuid" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/FilterBaseSchema" } } } }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/TraceLogTreeSchema" }, "title": "Response Get Experiment Trace Logs Api Api Parea V1 Experiment Experiment Uuid Trace Logs Post" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } }, "/api/parea/v1/get_trace_logs": { "post": { "summary": "Get Trace Logs Api", "description": "Fetches trace logs and returns them as a paginated response.", "operationId": "get_trace_logs_api_api_parea_v1_get_trace_logs_post", "requestBody": { "content": { "application/json": { "schema": { "$ref": "#/components/schemas/QueryParamsPaginationPublic" } } }, "required": true }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PaginatedResponse_TraceLogTreeSchema_" } } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } }, "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ] } }, "/api/parea/v1/update_test_case/{dataset_id}/{test_case_id}": { "post": { "summary": "Update Test Case", "description": "Update the item/test case of the dataset with any given fields.\n\nArgs:\n dataset_id (int): ID of the dataset.\n test_case_id (int): ID of the item/test case.\n update_test_case_request (UpdateTestCasePublicRequestSchema): Request body containing the fields to update.", "operationId": "update_test_case_api_parea_v1_update_test_case__dataset_id___test_case_id__post", "security": [ { "APIKeyHeader": [] }, { "APIKeyHeader": [] }, { "APIKeyHeader": [] } ], "parameters": [ { "name": "dataset_id", "in": "path", "required": true, "schema": { "type": "integer", "title": "Dataset Id" } }, { "name": "test_case_id", "in": "path", "required": true, "schema": { "type": "integer", "title": "Test Case Id" } } ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/UpdateTestCasePublicRequestSchema" } } } }, "responses": { "200": { "description": "Successful Response", "content": { "application/json": { "schema": {} } } }, "422": { "description": "Validation Error", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/HTTPValidationError" } } } } } } } }, "components": { "schemas": { "AddColumnsSchema": { "properties": { "columns": { "items": { "$ref": "#/components/schemas/NewColumn" }, "type": "array", "title": "Columns" }, "backfill": { "type": "boolean", "title": "Backfill", "default": true } }, "type": "object", "required": [ "columns" ], "title": "AddColumnsSchema" }, "AddExperimentTraceLogsToAnnotationQueuesSchema": { "properties": { "experiment_uuid": { "type": "string", "title": "Experiment Uuid" }, "annotation_queue_uuids": { "items": { "type": "string" }, "type": "array", "title": "Annotation Queue Uuids" } }, "type": "object", "required": [ "experiment_uuid", "annotation_queue_uuids" ], "title": "AddExperimentTraceLogsToAnnotationQueuesSchema" }, "AddTraceLogsToAnnotationQueuesSchema": { "properties": { "trace_ids": { "items": { "type": "string" }, "type": "array", "title": "Trace Ids" }, "annotation_queue_uuids": { "items": { "type": "string" }, "type": "array", "title": "Annotation Queue Uuids" } }, "type": "object", "required": [ "trace_ids", "annotation_queue_uuids" ], "title": "AddTraceLogsToAnnotationQueuesSchema" }, "AnalyticsRequestSchema": { "properties": { "name": { "type": "string", "title": "Name" }, "data": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Data" } }, "type": "object", "required": [ "name" ], "title": "AnalyticsRequestSchema" }, "AnnotationCriterionSchema": { "properties": { "name": { "type": "string", "title": "Name", "description": "Annotation name" }, "type": { "allOf": [ { "$ref": "#/components/schemas/DefinedAnnotationType" } ], "description": "Annotation type" }, "min_value": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Min Value", "description": "Minimum value" }, "max_value": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Max Value", "description": "Maximum value" }, "value_score_map": { "anyOf": [ { "additionalProperties": { "type": "number" }, "type": "object" }, { "type": "null" } ], "title": "Value Score Map", "description": "Value to score mapping" }, "id": { "anyOf": [ { "type": "string" }, { "type": "integer" } ], "title": "Id", "description": "Annotation ID" }, "organization_id": { "type": "string", "title": "Organization Id", "description": "Organization ID" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Description", "description": "Annotation description" }, "cur_bootstrapped_eval_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Cur Bootstrapped Eval Id", "description": "Current bootstrapped evaluation ID" }, "bootstrapped_evals": { "additionalProperties": { "$ref": "#/components/schemas/BootstrapEvalSchema" }, "type": "object", "title": "Bootstrapped Evals", "description": "Bootstrapped evaluations" } }, "type": "object", "required": [ "name", "type", "id", "organization_id" ], "title": "AnnotationCriterionSchema" }, "AnnotationQueueItemSchema": { "properties": { "annotation_queue_item_uuid": { "type": "string", "title": "Annotation Queue Item Uuid" }, "trace_id": { "type": "string", "title": "Trace Id" } }, "type": "object", "required": [ "annotation_queue_item_uuid", "trace_id" ], "title": "AnnotationQueueItemSchema" }, "AnnotationQueueSchema": { "properties": { "name": { "type": "string", "title": "Name", "description": "Annotation name" }, "uuid": { "type": "string", "title": "Uuid", "description": "Annotation ID" }, "created_at": { "type": "string", "title": "Created At", "description": "Annotation creation time" }, "updated_at": { "type": "string", "title": "Updated At", "description": "Annotation last update time" }, "organization_id": { "type": "string", "title": "Organization Id", "description": "Organization ID" } }, "type": "object", "required": [ "name", "uuid", "created_at", "updated_at", "organization_id" ], "title": "AnnotationQueueSchema" }, "Body_save_image_api_lab_save_image_post": { "properties": { "image": { "type": "string", "format": "binary", "title": "Image" } }, "type": "object", "required": [ "image" ], "title": "Body_save_image_api_lab_save_image_post" }, "BootstrapEvalFullResultSchema": { "properties": { "stats": { "additionalProperties": { "type": "number" }, "type": "object", "title": "Stats", "description": "Bootstrapped eval stats" }, "experiment_uuid": { "type": "string", "title": "Experiment Uuid", "description": "Experiment UUID" }, "source_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Source Code", "description": "Bootstrapped eval source code" }, "status": { "allOf": [ { "$ref": "#/components/schemas/BootstrappedEvalStatus" } ], "description": "Bootstrapped eval status" }, "output_target_pairs": { "items": { "$ref": "#/components/schemas/OutputTargetPairSchema" }, "type": "array", "title": "Output Target Pairs", "description": "Output target pairs", "default": [] } }, "type": "object", "title": "BootstrapEvalFullResultSchema" }, "BootstrapEvalRequestSchema": { "properties": { "criterion_id": { "type": "integer", "title": "Criterion Id" } }, "type": "object", "required": [ "criterion_id" ], "title": "BootstrapEvalRequestSchema" }, "BootstrapEvalSchema": { "properties": { "stats": { "additionalProperties": { "type": "number" }, "type": "object", "title": "Stats", "description": "Bootstrapped eval stats" }, "experiment_uuid": { "type": "string", "title": "Experiment Uuid", "description": "Experiment UUID" }, "source_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Source Code", "description": "Bootstrapped eval source code" }, "status": { "allOf": [ { "$ref": "#/components/schemas/BootstrappedEvalStatus" } ], "description": "Bootstrapped eval status" } }, "type": "object", "title": "BootstrapEvalSchema" }, "BootstrappedEvalStatus": { "type": "string", "enum": [ "pending", "running", "completed", "failed" ], "title": "BootstrappedEvalStatus" }, "BulkDeleteExperimentsSchema": { "properties": { "experiment_uuids": { "items": { "type": "string" }, "type": "array", "title": "Experiment Uuids" } }, "type": "object", "required": [ "experiment_uuids" ], "title": "BulkDeleteExperimentsSchema" }, "BuyCreditsRequestSchema": { "properties": { "plan": { "type": "string", "title": "Plan" }, "interval": { "type": "string", "title": "Interval", "default": "month" }, "organization_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Organization Id" } }, "type": "object", "required": [ "plan" ], "title": "BuyCreditsRequestSchema" }, "CUDAnnotationRequestSchema": { "properties": { "trace_id": { "type": "string", "title": "Trace Id", "description": "UUID of associated trace" }, "defined_annotation_id": { "type": "integer", "title": "Defined Annotation Id", "description": "Annotation criterion ID" }, "score": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Score", "description": "Annotation score" }, "value": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Value", "description": "Annotation value" } }, "type": "object", "required": [ "trace_id", "defined_annotation_id" ], "title": "CUDAnnotationRequestSchema" }, "CollectFeedbackRequestSchema": { "properties": { "result_id": { "type": "integer", "title": "Result Id" }, "feedback": { "$ref": "#/components/schemas/Rating" } }, "type": "object", "required": [ "result_id", "feedback" ], "title": "CollectFeedbackRequestSchema" }, "CompareExperimentsRequest": { "properties": { "project_name": { "type": "string", "title": "Project Name" }, "experiment_run_names": { "items": { "type": "string" }, "type": "array", "title": "Experiment Run Names" } }, "type": "object", "required": [ "project_name", "experiment_run_names" ], "title": "CompareExperimentsRequest" }, "ComparedExperimentAnnotationSchema": { "properties": { "trace_id": { "type": "string", "title": "Trace Id", "description": "UUID of associated trace" }, "defined_annotation_id": { "type": "integer", "title": "Defined Annotation Id", "description": "Annotation criterion ID" }, "score": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Score", "description": "Annotation score" }, "value": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Value", "description": "Annotation value" }, "name": { "type": "string", "title": "Name" } }, "type": "object", "required": [ "trace_id", "defined_annotation_id", "name" ], "title": "ComparedExperimentAnnotationSchema" }, "ComparedExperimentsOnSampleDetailsSchema": { "properties": { "trace_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Trace Id" }, "output": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Output" }, "scores": { "anyOf": [ { "items": { "$ref": "#/components/schemas/EvaluationScoreSchema" }, "type": "array" }, { "type": "null" } ], "title": "Scores" }, "annotations": { "anyOf": [ { "items": { "$ref": "#/components/schemas/ComparedExperimentAnnotationSchema" }, "type": "array" }, { "type": "null" } ], "title": "Annotations", "default": [] }, "latency": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Latency", "default": 0.0 }, "total_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Total Tokens", "default": 0 }, "cost": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Cost", "default": 0.0 }, "metadata": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Metadata" } }, "type": "object", "title": "ComparedExperimentsOnSampleDetailsSchema" }, "ComparedExperimentsRowDetailsResponse": { "properties": { "mean_or_single": { "$ref": "#/components/schemas/ComparedExperimentsOnSampleDetailsSchema" }, "all_results": { "anyOf": [ { "items": { "$ref": "#/components/schemas/ComparedExperimentsOnSampleDetailsSchema" }, "type": "array" }, { "type": "null" } ], "title": "All Results" } }, "type": "object", "required": [ "mean_or_single" ], "title": "ComparedExperimentsRowDetailsResponse" }, "ComparedExperimentsRowResponse": { "properties": { "inputs": { "type": "object", "title": "Inputs" }, "target": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Target" }, "eval_names": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Eval Names", "default": [] }, "annotation_names": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Annotation Names", "default": [] }, "experiments": { "additionalProperties": { "$ref": "#/components/schemas/ComparedExperimentsRowDetailsResponse" }, "type": "object", "title": "Experiments" } }, "type": "object", "required": [ "inputs", "experiments" ], "title": "ComparedExperimentsRowResponse" }, "ComparisonScoredResultSchema": { "properties": { "last_accessed": { "type": "string", "title": "Last Accessed" }, "url": { "type": "string", "title": "Url" }, "experiment_uuids": { "items": { "type": "string" }, "type": "array", "title": "Experiment Uuids", "default": [] } }, "type": "object", "required": [ "last_accessed", "url" ], "title": "ComparisonScoredResultSchema" }, "ComparisonScoredResultsSchema": { "properties": { "comparison_scores": { "anyOf": [ { "items": { "$ref": "#/components/schemas/ComparisonScoredResultSchema" }, "type": "array" }, { "type": "null" } ], "title": "Comparison Scores", "default": [] } }, "type": "object", "title": "ComparisonScoredResultsSchema" }, "ComparisonSetSchema": { "properties": { "url": { "type": "string", "title": "Url" }, "project_name": { "type": "string", "title": "Project Name", "default": "default" }, "experiment_uuids": { "items": { "type": "string" }, "type": "array", "title": "Experiment Uuids", "default": [] } }, "type": "object", "required": [ "url" ], "title": "ComparisonSetSchema" }, "CompletionsResponseSchema": { "properties": { "content": { "type": "string", "title": "Content", "description": "Generated completion content." }, "latency": { "type": "number", "title": "Latency", "description": "Latency of the completion in seconds." }, "input_tokens": { "type": "integer", "title": "Input Tokens", "description": "Number of tokens in the input." }, "output_tokens": { "type": "integer", "title": "Output Tokens", "description": "Number of tokens in the output." }, "total_tokens": { "type": "integer", "title": "Total Tokens", "description": "Total number of tokens in the input and output." }, "cost": { "type": "number", "title": "Cost", "description": "Cost of the completion in USD." }, "model": { "type": "string", "title": "Model", "description": "Model name." }, "provider": { "type": "string", "title": "Provider", "description": "Provider name." }, "cache_hit": { "type": "boolean", "title": "Cache Hit", "description": "If true, the completion was fetched from the cache." }, "status": { "type": "string", "title": "Status", "description": "Status of the completion. Either 'success' or 'error'." }, "error": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Error", "description": "Error message if the completion failed." }, "trace_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Trace Id", "description": "UUID of the log of the completion. Will be the same as the `trace_id` in the request if provided." }, "start_timestamp": { "type": "string", "title": "Start Timestamp", "description": "Start timestamp of the completion." }, "end_timestamp": { "type": "string", "title": "End Timestamp", "description": "End timestamp of the completion." }, "inference_id": { "type": "string", "title": "Inference Id", "description": "UUID of the log of the completion. The same as the `trace_id` in the request if provided." } }, "type": "object", "required": [ "content", "latency", "input_tokens", "output_tokens", "total_tokens", "cost", "model", "provider", "cache_hit", "status", "start_timestamp", "end_timestamp", "inference_id" ], "title": "CompletionsResponseSchema", "example": { "cache_hit": true, "content": "Sure! Here's a simple Hello World program...", "cost": 0.0008, "end_timestamp": "2024-07-24 18:05:17", "input_tokens": 22, "latency": 0.0, "model": "gpt-4o-mini-2024-07-18", "output_tokens": 189, "start_timestamp": "2024-07-24 18:05:17", "status": "success", "total_tokens": 211, "trace_id": "ush221-d23rfcasc-21cdsc" } }, "CompletionsSchema": { "properties": { "llm_inputs": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Llm Inputs", "description": "Key-value pairs as inputs to prompt template. Only needs to be provided if `deployment_id` is provided or `llm_configuration.messages` are templated." }, "llm_configuration": { "allOf": [ { "$ref": "#/components/schemas/LLMInputsSchema" } ], "description": "LLM configuration parameters such as messages, functions, etc.", "default": {} }, "project_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Project Name", "description": "Project name which is used to associate the log with a project.", "default": "default" }, "project_uuid": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Project Uuid", "description": "Project UUID which is used to associate the log with a project. Does not need to be provided if the `project_name` is provided." }, "experiment_uuid": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Experiment Uuid", "description": "Experiment UUID which is used to associate the log with an experiment." }, "parent_trace_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Parent Trace Id", "description": "UUID of the parent log. If given, will be used to associate the generation in a chain & create hierarchical nested logs." }, "root_trace_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Root Trace Id", "description": "UUID of the root log. If given, will be used to associate the generation in a chain & create hierarchical nested logs." }, "end_user_identifier": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "End User Identifier", "description": "Special field to track the end user which is interacting with your LLM app." }, "deployment_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Deployment Id", "description": "This is the ID for a specific deployed prompt. You can find your deployed prompts on the Deployments tab. If a deployment_id is provided, Parea will fetch all of the associated configuration including model name, model parameters, and any associated functions. Any information provided on the llm_configuration field will be used instead of the associated deployed prompts fields." }, "eval_metric_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "title": "Eval Metric Ids", "description": "List of evaluation metric IDs deployed on Parea which should be used to evaluate the completion output." }, "metadata": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Metadata", "description": "Key-value pairs to be associated with the log." }, "tags": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Tags", "description": "List of tags to be associated with the log." }, "target": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Target", "description": "Optional ground truth output for the inputs. Will be used for evaluation and can be used when creating a test case from the log." }, "trace_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Trace Id", "description": "UUID of the generation log. If not given, will be auto-generated." }, "trace_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Trace Name", "description": "Name of the generation log. If not given, will be auto-generated in the format `llm-{provider}`." }, "provider_api_key": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Provider Api Key", "description": "Provider API key to generate response. If not given, API keys saved on the platform will be used" }, "cache": { "type": "boolean", "title": "Cache", "description": "If true, the completion will be cached to avoid latency & cost for any subsequent completion using the same inputs.", "default": true }, "log_omit_inputs": { "type": "boolean", "title": "Log Omit Inputs", "description": "If true, the inputs, llm_configuration.messages, llm_configuration.functions, llm_configuration.model_params will not be logged.", "default": false }, "log_omit_outputs": { "type": "boolean", "title": "Log Omit Outputs", "description": "If true, the generated response will not be logged.", "default": false }, "log_omit": { "type": "boolean", "title": "Log Omit", "description": "Equivalent to setting both log_omit_inputs and log_omit_outputs to true.", "default": false }, "log_sample_rate": { "anyOf": [ { "type": "number", "maximum": 1.0, "minimum": 0.0 }, { "type": "null" } ], "title": "Log Sample Rate", "description": "If specified, this log and its entire associated trace will logged with this probability. Must be between 0 and 1 (incl.). Defaults to 1.0 (i.e., keeping all logs)", "default": 1.0 }, "inference_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Inference Id", "description": "Deprecated field which is the same `trace_id`" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name", "description": "Deprecated field" }, "retry": { "type": "boolean", "title": "Retry", "description": "Deprecated field", "default": false }, "fallback_strategy": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Fallback Strategy", "description": "Deprecated field" }, "stream": { "type": "boolean", "title": "Stream", "description": "Deprecated field. Use /completion/stream instead.", "default": false } }, "type": "object", "title": "CompletionsSchema", "examples": [ { "end_user_identifier": "2596@gmail.com", "inputs": { "city": "New York" }, "llm_configuration": { "function_call": "auto", "functions": [ { "description": "Get the current weather in a given location.", "name": "get_current_weather", "parameters": { "properties": { "location": { "description": "The city and state, e.g. San Francisco, CA", "type": "string" }, "unit": { "enum": [ "celsius", "fahrenheit" ], "type": "string" } }, "required": [ "location" ], "type": "object" } } ], "messages": [ { "content": "What is the weather in New York", "role": "user" } ], "model": "gpt-4o-mini", "model_params": { "max_length": 500, "model": "gpt-4o-mini", "temp": 0.0 }, "provider": "openai" }, "metadata": { "purpose": "testing", "user_id": "user_2Q9afRAKe6SZrc3U7" }, "tags": [ "weather", "forecast" ] }, { "deployment_id": "p-qZrnFesaeCpqcXJ_yL3wi", "llm_inputs": { "x": "Golang", "y": "Fiber" }, "metadata": { "purpose": "testing" } } ] }, "ConfigurationUnitSchema": { "properties": { "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model", "description": "Model name", "default": "" }, "temp": { "anyOf": [ { "type": "number", "maximum": 1.0, "minimum": 0.0 }, { "type": "null" } ], "title": "Temp", "description": "Temperature", "default": 1.0 }, "top_p": { "anyOf": [ { "type": "number", "maximum": 1.0, "minimum": 0.0 }, { "type": "null" } ], "title": "Top P", "description": "Top p", "default": 1.0 }, "frequency_penalty": { "anyOf": [ { "type": "number", "maximum": 2.0, "minimum": -2.0 }, { "type": "null" } ], "title": "Frequency Penalty", "description": "Frequency penalty", "default": 0.0 }, "presence_penalty": { "anyOf": [ { "type": "number", "maximum": 2.0, "minimum": -2.0 }, { "type": "null" } ], "title": "Presence Penalty", "description": "Presence penalty", "default": 0.0 }, "max_length": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Max Length", "description": "Max length of the output" }, "response_format": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Response Format", "description": "Response format" }, "safe_prompt": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Safe Prompt", "description": "Safe prompt. Only used for Mistral" }, "version_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Version Id", "description": "If not set, a new query & version will be created" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name", "description": "Name of the version" }, "content": { "type": "string", "title": "Content", "default": "" }, "example_inputs_completions": { "items": { "$ref": "#/components/schemas/LabSessionRow" }, "type": "array", "title": "Example Inputs Completions" }, "messages": { "items": { "$ref": "#/components/schemas/MessageSchema" }, "type": "array", "title": "Messages", "default": [] }, "function_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "title": "Function Ids" }, "function_call": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Function Call" } }, "type": "object", "required": [ "example_inputs_completions" ], "title": "ConfigurationUnitSchema" }, "CreateAnnotationCriterionSchema": { "properties": { "name": { "type": "string", "title": "Name", "description": "Annotation name" }, "type": { "allOf": [ { "$ref": "#/components/schemas/DefinedAnnotationType" } ], "description": "Annotation type" }, "min_value": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Min Value", "description": "Minimum value" }, "max_value": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Max Value", "description": "Maximum value" }, "value_score_map": { "anyOf": [ { "additionalProperties": { "type": "number" }, "type": "object" }, { "type": "null" } ], "title": "Value Score Map", "description": "Value to score mapping" } }, "type": "object", "required": [ "name", "type" ], "title": "CreateAnnotationCriterionSchema" }, "CreateAnnotationQueueSchema": { "properties": { "name": { "type": "string", "title": "Name", "description": "Annotation name" } }, "type": "object", "required": [ "name" ], "title": "CreateAnnotationQueueSchema" }, "CreateBulkTestCasesRequestSchema": { "properties": { "test_case_collection_id": { "type": "integer", "title": "Test Case Collection Id" }, "test_cases": { "items": { "$ref": "#/components/schemas/TestCaseSchema" }, "type": "array", "title": "Test Cases" } }, "type": "object", "required": [ "test_case_collection_id", "test_cases" ], "title": "CreateBulkTestCasesRequestSchema" }, "CreateDatasetReferenceSchema": { "properties": { "trace_id": { "type": "string", "title": "Trace Id" }, "collection_id": { "type": "integer", "title": "Collection Id" }, "collection_name": { "type": "string", "title": "Collection Name" }, "test_case_id": { "type": "integer", "title": "Test Case Id" }, "organization_id": { "type": "string", "title": "Organization Id" } }, "type": "object", "required": [ "trace_id", "collection_id", "collection_name", "test_case_id", "organization_id" ], "title": "CreateDatasetReferenceSchema" }, "CreateDeploymentModelSchema-Input": { "properties": { "deployment_id": { "type": "string", "title": "Deployment Id" }, "version_id": { "type": "integer", "title": "Version Id" }, "version_number": { "type": "number", "title": "Version Number" }, "model_type": { "type": "string", "title": "Model Type" }, "provider": { "type": "string", "title": "Provider" }, "model_parameters": { "$ref": "#/components/schemas/ModelParamsSchema" }, "active": { "type": "boolean", "title": "Active", "default": true }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "messages": { "items": { "$ref": "#/components/schemas/MessageSchema" }, "type": "array", "title": "Messages" }, "input_variables": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Input Variables" }, "function_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "title": "Function Ids" }, "evaluation_metric_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "title": "Evaluation Metric Ids" }, "function_call": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Function Call" }, "is_azure": { "type": "boolean", "title": "Is Azure", "default": false }, "deployer_name": { "type": "string", "title": "Deployer Name", "default": "Anonymous" }, "commit_message": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Commit Message" }, "event": { "anyOf": [ { "$ref": "#/components/schemas/HistoryEvent" }, { "type": "null" } ] }, "from_version": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "From Version" }, "organization_id": { "type": "string", "title": "Organization Id" }, "lab_sessions": { "anyOf": [ { "items": { "$ref": "#/components/schemas/LabSessionDetails" }, "type": "array" }, { "type": "null" } ], "title": "Lab Sessions" }, "history_items": { "anyOf": [ { "items": { "$ref": "#/components/schemas/HistoryItemSchema" }, "type": "array" }, { "type": "null" } ], "title": "History Items" }, "created_at": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Created At" }, "last_updated_at": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Last Updated At" } }, "type": "object", "required": [ "deployment_id", "version_id", "version_number", "model_type", "provider", "model_parameters", "messages", "organization_id" ], "title": "CreateDeploymentModelSchema" }, "CreateDeploymentModelSchema-Output": { "properties": { "deployment_id": { "type": "string", "title": "Deployment Id" }, "version_id": { "type": "integer", "title": "Version Id" }, "version_number": { "type": "number", "title": "Version Number" }, "model_type": { "type": "string", "title": "Model Type" }, "provider": { "type": "string", "title": "Provider" }, "model_parameters": { "$ref": "#/components/schemas/ModelParamsSchema" }, "active": { "type": "boolean", "title": "Active", "default": true }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "messages": { "items": { "$ref": "#/components/schemas/MessageSchema" }, "type": "array", "title": "Messages" }, "input_variables": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Input Variables" }, "function_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "title": "Function Ids" }, "evaluation_metric_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "title": "Evaluation Metric Ids" }, "function_call": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Function Call" }, "is_azure": { "type": "boolean", "title": "Is Azure", "default": false }, "deployer_name": { "type": "string", "title": "Deployer Name", "default": "Anonymous" }, "commit_message": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Commit Message" }, "event": { "anyOf": [ { "$ref": "#/components/schemas/HistoryEvent" }, { "type": "null" } ] }, "from_version": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "From Version" }, "organization_id": { "type": "string", "title": "Organization Id" }, "lab_sessions": { "anyOf": [ { "items": { "$ref": "#/components/schemas/LabSessionDetails" }, "type": "array" }, { "type": "null" } ], "title": "Lab Sessions" }, "history_items": { "anyOf": [ { "items": { "$ref": "#/components/schemas/HistoryItemSchema" }, "type": "array" }, { "type": "null" } ], "title": "History Items" }, "created_at": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Created At" }, "last_updated_at": { "anyOf": [ { "type": "string", "format": "date-time" }, { "type": "null" } ], "title": "Last Updated At" } }, "type": "object", "required": [ "deployment_id", "version_id", "version_number", "model_type", "provider", "model_parameters", "messages", "organization_id" ], "title": "CreateDeploymentModelSchema" }, "CreateEvalMetricAPIResponseSchema": { "properties": { "id": { "type": "integer", "title": "Id" }, "created_at": { "type": "string", "title": "Created At" }, "last_updated_at": { "type": "string", "title": "Last Updated At" }, "auto_eval_function_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Auto Eval Function Code" } }, "type": "object", "required": [ "id", "created_at", "last_updated_at" ], "title": "CreateEvalMetricAPIResponseSchema" }, "CreateEvalMetricRequestSchema": { "properties": { "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "webhook_url": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Webhook Url" }, "function_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Function Code" }, "type": { "$ref": "#/components/schemas/EvaluationMetricType" }, "is_auto_eval": { "type": "boolean", "title": "Is Auto Eval", "default": false }, "auto_eval_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Auto Eval Name" }, "auto_eval_inputs": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Auto Eval Inputs" } }, "type": "object", "required": [ "type" ], "title": "CreateEvalMetricRequestSchema" }, "CreateEvaluationJobRequestSchema": { "properties": { "organization_id": { "type": "string", "title": "Organization Id" }, "project_uuid": { "type": "string", "title": "Project Uuid" }, "playground_session_id": { "type": "integer", "title": "Playground Session Id" }, "test_case_collection_id": { "type": "integer", "title": "Test Case Collection Id" }, "test_case_ids": { "items": { "type": "integer" }, "type": "array", "title": "Test Case Ids" }, "evaluation_metric_ids": { "items": { "type": "integer" }, "type": "array", "title": "Evaluation Metric Ids" }, "version_ids": { "items": { "type": "integer" }, "type": "array", "title": "Version Ids" }, "n_trials": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "N Trials", "default": 1 } }, "type": "object", "required": [ "organization_id", "project_uuid", "playground_session_id", "test_case_collection_id", "test_case_ids", "evaluation_metric_ids", "version_ids" ], "title": "CreateEvaluationJobRequestSchema" }, "CreateExperimentRequestSchema": { "properties": { "name": { "type": "string", "title": "Name", "description": "Name of the experiment" }, "run_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Run Name", "description": "Name of the run. Must be unique for each experiment and can only contain alphanumeric characters, hyphens, and underscores." }, "project_uuid": { "type": "string", "title": "Project Uuid", "description": "UUID of the project associated with this experiment" }, "is_public": { "type": "boolean", "title": "Is Public", "description": "If experiment should be public", "default": false }, "metadata": { "anyOf": [ { "additionalProperties": { "type": "string" }, "type": "object" }, { "type": "null" } ], "title": "Metadata", "description": "Any additional key-value pairs which provide context or are useful for filtering." } }, "type": "object", "required": [ "name", "project_uuid" ], "title": "CreateExperimentRequestSchema", "examples": [ { "metadata": { "dataset": "dev 123" }, "name": "Test Experiment", "project_uuid": "...", "run_name": "test-experiment" } ] }, "CreateGetProjectRequestSchema": { "properties": { "name": { "type": "string", "title": "Name", "description": "Name of the project" } }, "type": "object", "required": [ "name" ], "title": "CreateGetProjectRequestSchema", "examples": [ { "name": "development" } ] }, "CreateGetProjectResponseSchema": { "properties": { "name": { "type": "string", "title": "Name", "description": "Name of the project" }, "uuid": { "type": "string", "title": "Uuid", "description": "UUID of the project" }, "created_at": { "type": "string", "title": "Created At", "description": "Timestamp the project was created" }, "was_created": { "type": "boolean", "title": "Was Created", "description": "If the project was created or not." } }, "type": "object", "required": [ "name", "uuid", "created_at", "was_created" ], "title": "CreateGetProjectResponseSchema", "examples": [ { "created_at": "2024-05-30 13:48:34", "name": "development", "uuid": "...", "was_created": false } ] }, "CreateLabSessionCommentSchema": { "properties": { "lab_session_id": { "type": "integer", "title": "Lab Session Id" }, "user_id": { "type": "string", "title": "User Id" }, "organization_id": { "type": "string", "title": "Organization Id" }, "author_name": { "type": "string", "title": "Author Name" }, "comment": { "type": "string", "title": "Comment" } }, "type": "object", "required": [ "lab_session_id", "user_id", "organization_id", "author_name", "comment" ], "title": "CreateLabSessionCommentSchema" }, "CreateTestCaseCollectionRequestSchema": { "properties": { "name": { "type": "string", "title": "Name", "description": "Name of the dataset" }, "column_names": { "items": { "type": "string" }, "type": "array", "title": "Column Names", "description": "Column names for the dataset" }, "test_cases": { "anyOf": [ { "items": { "$ref": "#/components/schemas/TestCaseSchema" }, "type": "array" }, { "type": "null" } ], "title": "Test Cases", "description": "Name of test cases/samples in the dataset" } }, "type": "object", "required": [ "name", "column_names" ], "title": "CreateTestCaseCollectionRequestSchema", "examples": [ { "column_names": [ "language", "framework" ], "name": "Hello World Programs", "test_cases": [ { "inputs": { "framework": "fastapi", "language": "python" }, "tags": [ "easy" ], "target": "a good program here" } ] } ] }, "CreateTestCaseCollectionResponseSchema": { "properties": { "id": { "type": "integer", "title": "Id", "description": "Unique identifier for the dataset" }, "last_updated_at": { "type": "string", "title": "Last Updated At", "description": "Timestamp when the dataset was last updated" }, "test_case_ids": { "items": { "type": "integer" }, "type": "array", "title": "Test Case Ids", "description": "List of created test case IDs in the dataset" } }, "type": "object", "required": [ "id", "last_updated_at", "test_case_ids" ], "title": "CreateTestCaseCollectionResponseSchema", "examples": [ { "id": 5000, "last_updated_at": "2021-06-01T00:00:00Z", "test_case_ids": [ 12345 ] } ] }, "CreateTestCasesSchema": { "properties": { "id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Id", "description": "ID for the dataset/test case collection. Either ID or name is required to identify the dataset." }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name", "description": "Name for the dataset/test case collection. Either ID or name is required to identify the dataset." }, "test_cases": { "items": { "$ref": "#/components/schemas/TestCaseSchema" }, "type": "array", "title": "Test Cases", "description": "List of items which should be added to dataset." } }, "type": "object", "required": [ "test_cases" ], "title": "CreateTestCasesSchema", "examples": [ { "name": "Hello World Programs", "test_cases": [ { "inputs": { "framework": "fastapi", "language": "python" }, "tags": [ "easy" ], "target": "a good program here" } ] }, { "id": 5000, "test_cases": [ { "inputs": { "framework": "fastapi", "language": "python" }, "tags": [ "easy" ], "target": "a good program here" } ] } ] }, "CreateUserIfMissing": { "properties": { "user_id": { "type": "string", "title": "User Id" }, "organization_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Organization Id" }, "org_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Org Name" } }, "type": "object", "required": [ "user_id" ], "title": "CreateUserIfMissing" }, "CreditResponseSchema": { "properties": { "credits_used": { "type": "number", "title": "Credits Used" }, "credits_added": { "type": "number", "title": "Credits Added" }, "has_stripe_customer": { "type": "boolean", "title": "Has Stripe Customer" }, "subscription_plan": { "anyOf": [ { "$ref": "#/components/schemas/SubscriptionEnum" }, { "type": "null" } ] } }, "type": "object", "required": [ "credits_used", "credits_added", "has_stripe_customer" ], "title": "CreditResponseSchema" }, "DashboardStatsResponse": { "properties": { "requests_stats": { "items": { "$ref": "#/components/schemas/MultiStatResponse" }, "type": "array", "title": "Requests Stats" }, "latency_stats": { "items": { "$ref": "#/components/schemas/MultiStatResponse" }, "type": "array", "title": "Latency Stats" }, "total_cost": { "items": { "$ref": "#/components/schemas/SingleStatResponse" }, "type": "array", "title": "Total Cost" }, "tokens_stats": { "items": { "$ref": "#/components/schemas/MultiStatResponse" }, "type": "array", "title": "Tokens Stats" }, "feedback_stats": { "items": { "$ref": "#/components/schemas/MultiStatResponse" }, "type": "array", "title": "Feedback Stats" }, "avg_scores": { "items": { "$ref": "#/components/schemas/MultiStatResponse" }, "type": "array", "title": "Avg Scores" } }, "type": "object", "required": [ "requests_stats", "latency_stats", "total_cost", "tokens_stats", "feedback_stats", "avg_scores" ], "title": "DashboardStatsResponse" }, "DatasetReferenceSchema": { "properties": { "trace_id": { "type": "string", "title": "Trace Id" }, "collection_id": { "type": "integer", "title": "Collection Id" }, "collection_name": { "type": "string", "title": "Collection Name" }, "test_case_id": { "type": "integer", "title": "Test Case Id" }, "organization_id": { "type": "string", "title": "Organization Id" }, "id": { "type": "integer", "title": "Id" }, "created_at": { "type": "string", "title": "Created At" } }, "type": "object", "required": [ "trace_id", "collection_id", "collection_name", "test_case_id", "organization_id", "id", "created_at" ], "title": "DatasetReferenceSchema" }, "DefinedAnnotationType": { "type": "string", "enum": [ "continuous", "categorical" ], "title": "DefinedAnnotationType" }, "DeleteExperimentStatRequest": { "properties": { "var1": { "type": "string", "title": "Var1", "description": "Name of the first variable" }, "var2": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Var2", "description": "Name of the second variable" }, "operation": { "allOf": [ { "$ref": "#/components/schemas/StatisticOperation" } ], "description": "What operation was executed on the variables to get value" } }, "type": "object", "required": [ "var1", "operation" ], "title": "DeleteExperimentStatRequest", "examples": [ { "operation": "mean", "var1": "Latency" } ] }, "DeleteRequestSchema": { "properties": { "version_id": { "type": "integer", "title": "Version Id" }, "cascade": { "type": "boolean", "title": "Cascade", "default": false } }, "type": "object", "required": [ "version_id" ], "title": "DeleteRequestSchema" }, "DeploymentPromptIdSchema": { "properties": { "deployment_id": { "type": "string", "title": "Deployment Id", "description": "ID of deployed prompt" }, "llm_inputs": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Llm Inputs", "description": "If provided, these key-value pairs will be used to replace the corresponding keys in the templated prompt" } }, "type": "object", "required": [ "deployment_id" ], "title": "DeploymentPromptIdSchema", "example": { "deployment_id": "p-qZrnFesaeCpqcXJ_yL3wi", "llm_inputs": { "x": "Python", "y": "Flask" } } }, "Detail": { "properties": { "result_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Result Id" }, "content": { "type": "string", "title": "Content" }, "tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Tokens" }, "cost": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Cost" }, "latency": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Latency" }, "scores": { "anyOf": [ { "additionalProperties": { "type": "number" }, "type": "object" }, { "type": "null" } ], "title": "Scores" }, "feedback": { "anyOf": [ { "$ref": "#/components/schemas/Rating" }, { "type": "null" } ] }, "logprobs": { "anyOf": [ { "items": { "$ref": "#/components/schemas/Logprob" }, "type": "array" }, { "type": "null" } ], "title": "Logprobs" }, "perplexity": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Perplexity" } }, "type": "object", "required": [ "content" ], "title": "Detail" }, "EvalMetricSchema": { "properties": { "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "webhook_url": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Webhook Url" }, "function_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Function Code" }, "type": { "$ref": "#/components/schemas/EvaluationMetricType" }, "is_auto_eval": { "type": "boolean", "title": "Is Auto Eval", "default": false }, "auto_eval_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Auto Eval Name" }, "auto_eval_inputs": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Auto Eval Inputs" }, "id": { "type": "integer", "title": "Id" }, "user_id": { "type": "string", "title": "User Id" }, "created_at": { "type": "string", "title": "Created At" }, "last_updated_at": { "type": "string", "title": "Last Updated At" } }, "type": "object", "required": [ "type", "id", "user_id", "created_at", "last_updated_at" ], "title": "EvalMetricSchema" }, "EvalMetricsSchema": { "properties": { "eval_metrics": { "additionalProperties": { "$ref": "#/components/schemas/EvalMetricSchema" }, "type": "object", "title": "Eval Metrics" } }, "type": "object", "required": [ "eval_metrics" ], "title": "EvalMetricsSchema" }, "EvaluateRequestSchema": { "properties": { "evaluation_metric_ids": { "items": { "type": "integer" }, "type": "array", "title": "Evaluation Metric Ids" }, "units": { "items": { "$ref": "#/components/schemas/LabEvaluateUnitSchema" }, "type": "array", "title": "Units" } }, "type": "object", "required": [ "evaluation_metric_ids", "units" ], "title": "EvaluateRequestSchema" }, "EvaluateSingleRequestSchema": { "properties": { "evaluation_metric_ids": { "items": { "type": "integer" }, "type": "array", "title": "Evaluation Metric Ids" }, "unit": { "$ref": "#/components/schemas/LabEvaluateUnitSchema" } }, "type": "object", "required": [ "evaluation_metric_ids", "unit" ], "title": "EvaluateSingleRequestSchema" }, "EvaluateUnitResponseSchema": { "properties": { "scores": { "additionalProperties": { "type": "number" }, "type": "object", "title": "Scores" }, "reasons": { "anyOf": [ { "additionalProperties": { "anyOf": [ { "type": "string" }, { "type": "null" } ] }, "type": "object" }, { "type": "null" } ], "title": "Reasons" }, "column": { "type": "integer", "title": "Column" }, "row": { "type": "integer", "title": "Row" } }, "type": "object", "required": [ "scores", "reasons", "column", "row" ], "title": "EvaluateUnitResponseSchema" }, "EvaluateUnitSchema": { "properties": { "inputs": { "anyOf": [ { "additionalProperties": { "type": "string" }, "type": "object" }, { "type": "null" } ], "title": "Inputs", "default": {} }, "target": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Target" }, "completion": { "type": "string", "title": "Completion" }, "result_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Result Id" } }, "type": "object", "required": [ "completion" ], "title": "EvaluateUnitSchema" }, "EvaluationMetricType": { "type": "string", "enum": [ "webhook", "function" ], "title": "EvaluationMetricType" }, "EvaluationScoreSchema": { "properties": { "name": { "type": "string", "title": "Name", "description": "Name of the score / evaluation" }, "score": { "type": "number", "title": "Score", "description": "Value of the score" }, "evaluation_metric_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Evaluation Metric Id", "description": "Will be automatically populated if this score was from a deployed evaluation metric." }, "reason": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Reason", "description": "Reason for this score" } }, "type": "object", "required": [ "name", "score" ], "title": "EvaluationScoreSchema", "examples": [ { "name": "Answer Matches Target", "reason": "The answer is mostly correct but lacks aspect A & B.", "score": 0.8 } ] }, "ExperimentPinnedStatistic": { "properties": { "var1": { "type": "string", "title": "Var1", "description": "Name of the first variable" }, "var2": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Var2", "description": "Name of the second variable" }, "operation": { "allOf": [ { "$ref": "#/components/schemas/StatisticOperation" } ], "description": "What operation was executed on the variables to get value" }, "value": { "type": "number", "title": "Value", "description": "Value of the statistic" } }, "type": "object", "required": [ "var1", "operation", "value" ], "title": "ExperimentPinnedStatistic", "examples": [ { "operation": "mean", "value": 0.23, "var1": "Latency" } ] }, "ExperimentSchema": { "properties": { "name": { "type": "string", "title": "Name", "description": "Name of the experiment" }, "run_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Run Name", "description": "Name of the run. Must be unique for each experiment and can only contain alphanumeric characters, hyphens, and underscores." }, "project_uuid": { "type": "string", "title": "Project Uuid", "description": "UUID of the project associated with this experiment" }, "is_public": { "type": "boolean", "title": "Is Public", "description": "If experiment should be public", "default": false }, "metadata": { "anyOf": [ { "additionalProperties": { "type": "string" }, "type": "object" }, { "type": "null" } ], "title": "Metadata", "description": "Any additional key-value pairs which provide context or are useful for filtering." }, "uuid": { "type": "string", "title": "Uuid", "description": "UUID of the created experiment" }, "created_at": { "type": "string", "title": "Created At", "description": "Timestamp the experiment was created" } }, "type": "object", "required": [ "name", "project_uuid", "uuid", "created_at" ], "title": "ExperimentSchema", "examples": [ { "created_at": "2024-05-30 13:48:34", "metadata": { "dataset": "dev 123" }, "name": "Test Experiment", "project_uuid": "...", "run_name": "test-experiment", "uuid": "..." } ] }, "ExperimentStats": { "properties": { "parent_trace_stats": { "items": { "$ref": "#/components/schemas/TraceStatsSchema" }, "type": "array", "title": "Parent Trace Stats", "description": "Stats of root-level trace logs of an experiment" }, "pinned_stats": { "items": { "$ref": "#/components/schemas/ExperimentPinnedStatistic" }, "type": "array", "title": "Pinned Stats", "description": "List of population/dataset-level statistics. Typically contains mean of any reported scores, latencies, etc." } }, "type": "object", "required": [ "parent_trace_stats", "pinned_stats" ], "title": "ExperimentStats", "examples": [ { "parent_trace_stats": [ { "cost": 0.5, "input_tokens": 100, "latency": 0.5, "output_tokens": 200, "scores": [ { "name": "accuracy", "score": 0.5 } ], "total_tokens": 300, "trace_id": "sasdc-csd-rnFesaeCpqcXJ_yL3wi" } ] } ] }, "ExperimentStatsSchema": { "properties": { "parent_trace_stats": { "items": { "$ref": "#/components/schemas/TraceStatsSchema" }, "type": "array", "title": "Parent Trace Stats", "description": "Stats of root-level trace logs of an experiment" } }, "type": "object", "required": [ "parent_trace_stats" ], "title": "ExperimentStatsSchema", "description": "Contains aggregated stats of all root trace logs of an experiment.", "examples": [ { "parent_trace_stats": [ { "cost": 0.5, "input_tokens": 100, "latency": 0.5, "output_tokens": 200, "scores": [ { "name": "accuracy", "score": 0.5 } ], "total_tokens": 300, "trace_id": "sasdc-csd-rnFesaeCpqcXJ_yL3wi" } ] } ] }, "ExperimentStatus": { "type": "string", "enum": [ "pending", "running", "completed", "failed" ], "title": "ExperimentStatus" }, "ExperimentWithStatsSchema": { "properties": { "name": { "type": "string", "title": "Name", "description": "Name of the experiment" }, "run_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Run Name", "description": "Name of the run. Must be unique for each experiment and can only contain alphanumeric characters, hyphens, and underscores." }, "project_uuid": { "type": "string", "title": "Project Uuid", "description": "UUID of the project associated with this experiment" }, "is_public": { "type": "boolean", "title": "Is Public", "description": "If experiment should be public", "default": false }, "metadata": { "anyOf": [ { "additionalProperties": { "type": "string" }, "type": "object" }, { "type": "null" } ], "title": "Metadata", "description": "Any additional key-value pairs which provide context or are useful for filtering." }, "uuid": { "type": "string", "title": "Uuid", "description": "UUID of the created experiment" }, "created_at": { "type": "string", "title": "Created At", "description": "Timestamp the experiment was created" }, "status": { "allOf": [ { "$ref": "#/components/schemas/ExperimentStatus" } ], "description": "Status of the experiment" }, "pinned_stats": { "items": { "$ref": "#/components/schemas/ExperimentPinnedStatistic" }, "type": "array", "title": "Pinned Stats", "description": "List of population/dataset-level statistics. Typically contains mean of any reported scores, latencies, etc." }, "num_samples": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Num Samples", "description": "Number of samples in the experiment" } }, "type": "object", "required": [ "name", "project_uuid", "uuid", "created_at", "status", "pinned_stats" ], "title": "ExperimentWithStatsSchema", "examples": [ { "created_at": "2024-04-15 22:19:13", "is_public": false, "metadata": { "dataset": "dev 123" }, "name": "greeting", "num_samples": 6, "pinned_stats": [ { "operation": "mean", "value": 0.176471, "var1": "levenshtein" }, { "operation": "mean", "value": 2e-05, "var1": "Cost" }, { "operation": "mean", "value": 11, "var1": "Total Tokens" }, { "operation": "mean", "value": 9, "var1": "Output Tokens" }, { "operation": "mean", "value": 2, "var1": "Input Tokens" }, { "operation": "mean", "value": 0.65273, "var1": "Latency" } ], "project_uuid": "...", "run_name": "bijou-kail", "status": "completed", "uuid": "..." } ] }, "FeedbackRequestSchema": { "properties": { "trace_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Trace Id", "description": "UUID of the trace log" }, "inference_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Inference Id", "description": "UUID of the trace log" }, "score": { "type": "number", "maximum": 1.0, "minimum": 0.0, "title": "Score", "description": "Feedback score" }, "comment": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Comment", "description": "Feedback comment" }, "target": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Target", "description": "Optional target or \u00e2\u20ac\u0153gold standard\u00e2\u20ac\u009d of log." }, "organization_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Organization Id", "description": "Field will be automatically filled-in." }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name", "description": "Deprecated field" } }, "type": "object", "required": [ "score" ], "title": "FeedbackRequestSchema", "examples": [ { "name": "test", "score": 0.5, "target": "perfect answer", "trace_id": "sasdc-csd-rnFesaeCpqcXJ_yL3wi" } ] }, "FilterBaseSchema": { "properties": { "filter_field": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Filter Field", "description": "Field to filter on.\nIf you want to filter by a score, you need to follow this format: 'score:{score_name}'.\nIf you want to filter by annotation, you need to follow this format: 'annotation:{annotation_type}:{annotation_id}'." }, "filter_key": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Filter Key", "description": "Filter by key when filtering on a map" }, "filter_operator": { "anyOf": [ { "$ref": "#/components/schemas/FilterOperator" }, { "type": "null" } ], "description": "Filter operator" }, "filter_value": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Filter Value", "description": "Filter value" } }, "type": "object", "title": "FilterBaseSchema", "examples": [ { "filter_field": "trace_name", "filter_operator": "like", "filter_value": "llm" } ] }, "FilterOperator": { "type": "string", "enum": [ "equals", "not_equals", "like", "greater_than_or_equal", "less_than_or_equal", "greater_than", "less_than", "is_null", "exists", "in", "between" ], "title": "FilterOperator" }, "FinishExperimentRequestSchema": { "properties": { "status": { "allOf": [ { "$ref": "#/components/schemas/ExperimentStatus" } ], "description": "Status of the experiment. Should be completed or failed", "default": "completed" }, "dataset_level_stats": { "anyOf": [ { "items": { "$ref": "#/components/schemas/EvaluationScoreSchema" }, "type": "array" }, { "type": "null" } ], "title": "Dataset Level Stats", "description": "List of dataset level evals/scores which should be added to the experiment. Examples are Pearson correlation between outputs and targets/ground truth values." } }, "type": "object", "title": "FinishExperimentRequestSchema", "description": "Any additional information to be added to the experiment", "examples": [ { "dataset_level_stats": [ { "name": "pearson_correlation", "score": 0.8 } ], "status": "completed" } ] }, "FromFieldEnum": { "type": "string", "enum": [ "trace_id", "collection_id", "test_case_id" ], "title": "FromFieldEnum" }, "FunctionSchema": { "properties": { "name": { "type": "string", "title": "Name" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Description" }, "signature": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Signature" }, "content": { "type": "string", "title": "Content" }, "tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Tokens" } }, "type": "object", "required": [ "name", "content" ], "title": "FunctionSchema" }, "FunctionVersionUpRequestSchema": { "properties": { "function_id": { "type": "integer", "title": "Function Id" }, "parent_id": { "type": "integer", "title": "Parent Id" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Description" }, "signature": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Signature" }, "content": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Content" }, "tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Tokens" } }, "type": "object", "required": [ "function_id", "parent_id" ], "title": "FunctionVersionUpRequestSchema" }, "FunctionsResponseSchema": { "properties": { "name": { "type": "string", "title": "Name" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Description" }, "signature": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Signature" }, "content": { "type": "string", "title": "Content" }, "tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Tokens" }, "id": { "type": "integer", "title": "Id" }, "parent_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Parent Id" }, "version_number": { "type": "number", "title": "Version Number" }, "created_at": { "type": "string", "format": "date-time", "title": "Created At" }, "versions_list": { "items": { "$ref": "#/components/schemas/VersionReference" }, "type": "array", "title": "Versions List" } }, "type": "object", "required": [ "name", "content", "id", "version_number", "created_at", "versions_list" ], "title": "FunctionsResponseSchema" }, "GetDatasetReferencesSchema": { "properties": { "organization_id": { "type": "string", "title": "Organization Id" }, "from_field": { "$ref": "#/components/schemas/FromFieldEnum" }, "trace_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Trace Id" }, "collection_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Collection Id" }, "test_case_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Test Case Id" } }, "type": "object", "required": [ "organization_id", "from_field" ], "title": "GetDatasetReferencesSchema" }, "GetDeployedPromptResponseSchema": { "properties": { "deployment_id": { "type": "string", "title": "Deployment Id", "description": "This is the ID for a deployed prompt. You can find your deployed prompts on the Deployments tab." }, "version_number": { "type": "number", "title": "Version Number", "description": "Version number of the deployed prompt" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name", "description": "Name of the deployed prompt" }, "functions": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Functions", "description": "If deployed prompt has function, these will appear as JSON strings." }, "function_call": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Function Call", "description": "If deployed prompt has a specified function call, it will appear." }, "prompt": { "anyOf": [ { "$ref": "#/components/schemas/PromptSchema" }, { "type": "null" } ], "description": "The messages of the deployed prompt" }, "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model", "description": "Model name of deployed prompt" }, "provider": { "anyOf": [ { "$ref": "#/components/schemas/ProviderName" }, { "type": "null" } ], "description": "Provider name of deployed prompt" }, "model_params": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Model Params", "description": "Model parameters of deployed prompt" } }, "type": "object", "required": [ "deployment_id", "version_number" ], "title": "GetDeployedPromptResponseSchema", "examples": [ { "deployment_id": "p-qZrnFesaeCpqcXJ_yL3wi", "functions": [], "model": "gpt-4o-mini", "model_params": { "frequency_penalty": 0.0, "presence_penalty": 0.0, "temp": 0.5, "top_p": 1.0 }, "name": "hello world", "prompt": { "inputs": { "x": "Python", "y": "Flask" }, "messages": [ { "content": "I want a Hello World program in Python. Using the Flask framework.", "role": "user" } ], "raw_messages": [ { "content": "I want a Hello World program in {{x}}. Using the {{y}} framework.", "role": "user" } ] }, "provider": "openai", "version_number": 1.0 } ] }, "GetLabSessionCommentsSchema": { "properties": { "lab_session_id": { "type": "integer", "title": "Lab Session Id" }, "organization_id": { "type": "string", "title": "Organization Id" } }, "type": "object", "required": [ "lab_session_id", "organization_id" ], "title": "GetLabSessionCommentsSchema" }, "GetLabSessionsResponseSchema": { "properties": { "id": { "type": "integer", "title": "Id" }, "user_id": { "type": "string", "title": "User Id" }, "name": { "type": "string", "title": "Name" }, "evaluation_metric_ids": { "items": { "type": "integer" }, "type": "array", "title": "Evaluation Metric Ids", "description": "Evaluation metric id" }, "created_at": { "type": "string", "title": "Created At" }, "last_updated_at": { "type": "string", "title": "Last Updated At" }, "version_ids": { "items": { "type": "integer" }, "type": "array", "title": "Version Ids" }, "result_ids": { "anyOf": [ { "items": { "items": { "type": "integer" }, "type": "array" }, "type": "array" }, { "type": "null" } ], "title": "Result Ids" } }, "type": "object", "required": [ "id", "user_id", "name", "created_at", "last_updated_at", "version_ids" ], "title": "GetLabSessionsResponseSchema" }, "GetLabVersionResponseSchema": { "properties": { "query_id": { "type": "integer", "title": "Query Id" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "version_id": { "type": "integer", "title": "Version Id" }, "version_number": { "type": "number", "title": "Version Number" }, "versions_list": { "items": { "$ref": "#/components/schemas/VersionReference" }, "type": "array", "title": "Versions List" }, "messages": { "items": { "$ref": "#/components/schemas/MessageSchema" }, "type": "array", "title": "Messages", "default": [] }, "result_params": { "$ref": "#/components/schemas/ModelParamsSchema" }, "function_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array", "uniqueItems": true }, { "type": "null" } ], "title": "Function Ids" }, "function_call": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Function Call" } }, "type": "object", "required": [ "query_id", "version_id", "version_number", "versions_list", "result_params" ], "title": "GetLabVersionResponseSchema" }, "GetLogsRequestSchema": { "properties": { "id": { "type": "string", "title": "Id" }, "is_chain": { "type": "boolean", "title": "Is Chain", "default": false } }, "type": "object", "required": [ "id" ], "title": "GetLogsRequestSchema" }, "GetPIDEModelResponseSchema": { "properties": { "id": { "type": "integer", "title": "Id" }, "user_id": { "type": "string", "title": "User Id" }, "name": { "type": "string", "title": "Name" }, "source_code": { "type": "string", "title": "Source Code" }, "test_case_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Test Case Id" }, "logs": { "items": { "items": { "$ref": "#/components/schemas/LogData" }, "type": "array" }, "type": "array", "title": "Logs" }, "created_at": { "type": "string", "title": "Created At" }, "last_updated_at": { "type": "string", "title": "Last Updated At" }, "inputs": { "anyOf": [ { "additionalProperties": { "type": "string" }, "type": "object" }, { "type": "null" } ], "title": "Inputs" } }, "type": "object", "required": [ "id", "user_id", "name", "source_code", "test_case_id", "logs", "created_at", "last_updated_at" ], "title": "GetPIDEModelResponseSchema" }, "GetResultsRequestSchema": { "properties": { "result_ids": { "items": { "items": { "type": "integer" }, "type": "array" }, "type": "array", "title": "Result Ids" } }, "type": "object", "required": [ "result_ids" ], "title": "GetResultsRequestSchema" }, "GetResultsResponseSchema": { "properties": { "input_units": { "items": { "$ref": "#/components/schemas/InputUnitSchema" }, "type": "array", "title": "Input Units" }, "results": { "items": { "items": { "$ref": "#/components/schemas/ResultSchema" }, "type": "array" }, "type": "array", "title": "Results" } }, "type": "object", "required": [ "input_units", "results" ], "title": "GetResultsResponseSchema" }, "GetTestCaseCollectionOverviewResponseSchema": { "properties": { "test_case_collections": { "additionalProperties": { "$ref": "#/components/schemas/TestCaseCollectionOverviewSchema" }, "type": "object", "title": "Test Case Collections" } }, "type": "object", "required": [ "test_case_collections" ], "title": "GetTestCaseCollectionOverviewResponseSchema" }, "GetTestCaseCollectionResponseSchema": { "properties": { "id": { "type": "integer", "title": "Id", "description": "Unique identifier for the test case collection" }, "name": { "type": "string", "title": "Name", "description": "Name of the test case collection" }, "column_names": { "items": { "type": "string" }, "type": "array", "title": "Column Names", "description": "Column names for the test case collection" }, "created_at": { "type": "string", "title": "Created At", "description": "Timestamp when the test case collection was created" }, "last_updated_at": { "type": "string", "title": "Last Updated At", "description": "Timestamp when the test case collection was last updated" }, "test_cases": { "additionalProperties": { "$ref": "#/components/schemas/GetTestCaseResponseSchema" }, "type": "object", "title": "Test Cases", "description": "Dictionary mapping test case ID to test case" } }, "type": "object", "required": [ "id", "name", "column_names", "created_at", "last_updated_at", "test_cases" ], "title": "GetTestCaseCollectionResponseSchema", "examples": [ { "column_names": [ "language", "framework" ], "created_at": "2021-06-01T00:00:00Z", "id": 5000, "last_updated_at": "2021-06-01T00:00:00Z", "name": "Hello World Programs", "test_cases": { "12345": { "id": 12345, "inputs": { "framework": "fastapi", "language": "python" }, "tags": [ "easy" ], "target": "a good program here", "test_case_collection_id": 5000 } } } ] }, "GetTestCaseResponseSchema": { "properties": { "inputs": { "additionalProperties": { "type": "string" }, "type": "object", "title": "Inputs", "description": "Dictionary mapping input names to values. Input names correspond to the column names of the test case collection." }, "target": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Target", "description": "Optional target/ground truth value for the test case/inputs" }, "tags": { "items": { "type": "string" }, "type": "array", "title": "Tags", "description": "List of tags associated with the test case", "default": [] }, "id": { "type": "integer", "title": "Id", "description": "Unique identifier for the test case" }, "test_case_collection_id": { "type": "integer", "title": "Test Case Collection Id", "description": "Unique identifier for the test case collection of this test case" } }, "type": "object", "required": [ "inputs", "id", "test_case_collection_id" ], "title": "GetTestCaseResponseSchema", "examples": [ { "id": 12345, "inputs": { "framework": "fastapi", "language": "python" }, "tags": [ "easy" ], "target": "a good program here", "test_case_collection_id": 5000 } ] }, "GetUniqueValuesRequestSchema": { "properties": { "organization_id": { "type": "string", "title": "Organization Id" }, "time_range": { "$ref": "#/components/schemas/TimeRange" }, "filter_field": { "$ref": "#/components/schemas/UniqueValueOptions" } }, "type": "object", "required": [ "organization_id", "time_range", "filter_field" ], "title": "GetUniqueValuesRequestSchema" }, "HTTPValidationError": { "properties": { "detail": { "items": { "$ref": "#/components/schemas/ValidationError" }, "type": "array", "title": "Detail" } }, "type": "object", "title": "HTTPValidationError" }, "HistoryEvent": { "type": "string", "enum": [ "create", "bump", "revert" ], "title": "HistoryEvent" }, "HistoryItemSchema": { "properties": { "deployment_id": { "type": "string", "title": "Deployment Id" }, "version_id": { "type": "integer", "title": "Version Id" }, "created_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Created At" }, "updated_at": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Updated At" }, "event": { "allOf": [ { "$ref": "#/components/schemas/HistoryEvent" } ], "default": "create" }, "to_version": { "type": "number", "title": "To Version", "default": 0.0 }, "from_version": { "type": "number", "title": "From Version", "default": 0.0 }, "deployer_name": { "type": "string", "title": "Deployer Name", "default": "Anonymous" }, "commit_message": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Commit Message" } }, "type": "object", "required": [ "deployment_id", "version_id" ], "title": "HistoryItemSchema" }, "InferenceRequestSchema": { "properties": { "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model", "description": "Model name", "default": "" }, "temp": { "anyOf": [ { "type": "number", "maximum": 1.0, "minimum": 0.0 }, { "type": "null" } ], "title": "Temp", "description": "Temperature", "default": 1.0 }, "top_p": { "anyOf": [ { "type": "number", "maximum": 1.0, "minimum": 0.0 }, { "type": "null" } ], "title": "Top P", "description": "Top p", "default": 1.0 }, "frequency_penalty": { "anyOf": [ { "type": "number", "maximum": 2.0, "minimum": -2.0 }, { "type": "null" } ], "title": "Frequency Penalty", "description": "Frequency penalty", "default": 0.0 }, "presence_penalty": { "anyOf": [ { "type": "number", "maximum": 2.0, "minimum": -2.0 }, { "type": "null" } ], "title": "Presence Penalty", "description": "Presence penalty", "default": 0.0 }, "max_length": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Max Length", "description": "Max length of the output" }, "response_format": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Response Format", "description": "Response format" }, "safe_prompt": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Safe Prompt", "description": "Safe prompt. Only used for Mistral" }, "column": { "type": "integer", "title": "Column" }, "row": { "type": "integer", "title": "Row" }, "unformatted_messages": { "items": { "$ref": "#/components/schemas/MessageSchema" }, "type": "array", "title": "Unformatted Messages", "default": [] }, "version_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Version Id" }, "example_inputs": { "anyOf": [ { "additionalProperties": { "type": "string" }, "type": "object" }, { "type": "null" } ], "title": "Example Inputs", "default": {} }, "test_case_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Test Case Id" }, "function_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "title": "Function Ids" }, "functions": { "anyOf": [ { "items": {}, "type": "array" }, { "type": "null" } ], "title": "Functions" }, "function_call": { "anyOf": [ { "type": "string" }, { "additionalProperties": { "type": "string" }, "type": "object" }, { "type": "null" } ], "title": "Function Call" }, "organization_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Organization Id" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" } }, "type": "object", "required": [ "column", "row" ], "title": "InferenceRequestSchema" }, "InputUnitSchema": { "properties": { "inputs": { "anyOf": [ { "additionalProperties": { "type": "string" }, "type": "object" }, { "type": "null" } ], "title": "Inputs", "default": {} }, "target": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Target" }, "test_case_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Test Case Id" }, "tags": { "items": { "type": "string" }, "type": "array", "title": "Tags", "default": [] }, "collection_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Collection Name" }, "collection_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Collection Id" } }, "type": "object", "title": "InputUnitSchema" }, "InputsSchema-Input": { "properties": { "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model", "description": "Model name" }, "provider": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Provider", "description": "Provider name" }, "model_params": { "anyOf": [ { "$ref": "#/components/schemas/ParamsSchema" }, { "type": "null" } ], "description": "Parameters such as temperature." }, "messages": { "anyOf": [ { "items": { "$ref": "#/components/schemas/MessageSchema" }, "type": "array" }, { "type": "null" } ], "title": "Messages", "description": "Messages to LLM" }, "functions": { "anyOf": [ { "items": {}, "type": "array" }, { "type": "null" } ], "title": "Functions", "description": "A list of functions the model may generate JSON inputs for. Assumes every item in list has key 'name', 'description' and 'parameters'" }, "function_call": { "anyOf": [ { "type": "string" }, { "type": "object" }, { "type": "null" } ], "title": "Function Call", "description": "Controls how the model responds to function calls. \u00e2\u20ac\u0153auto\u00e2\u20ac\u009d means the model can pick between an end-user or calling a function. To specifying a particular function name use a dictionary with key=\u00e2\u20ac\u0153name\u00e2\u20ac\u009d and value=\u00e2\u20ac\u0153my_function\u00e2\u20ac\u009d. This will force the model to call that function." } }, "type": "object", "title": "InputsSchema", "description": "Inputs for LLM inference such as messages, functions and additional parameters." }, "InputsSchema-Output": { "properties": { "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model", "description": "Model name" }, "provider": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Provider", "description": "Provider name" }, "model_params": { "anyOf": [ { "$ref": "#/components/schemas/ParamsSchema" }, { "type": "null" } ], "description": "Parameters such as temperature." }, "messages": { "anyOf": [ { "items": { "$ref": "#/components/schemas/MessageSchema" }, "type": "array" }, { "type": "null" } ], "title": "Messages", "description": "Messages to LLM" }, "functions": { "anyOf": [ { "items": {}, "type": "array" }, { "type": "null" } ], "title": "Functions", "description": "A list of functions the model may generate JSON inputs for. Assumes every item in list has key 'name', 'description' and 'parameters'" }, "function_call": { "anyOf": [ { "type": "string" }, { "type": "object" }, { "type": "null" } ], "title": "Function Call", "description": "Controls how the model responds to function calls. \u00e2\u20ac\u0153auto\u00e2\u20ac\u009d means the model can pick between an end-user or calling a function. To specifying a particular function name use a dictionary with key=\u00e2\u20ac\u0153name\u00e2\u20ac\u009d and value=\u00e2\u20ac\u0153my_function\u00e2\u20ac\u009d. This will force the model to call that function." } }, "type": "object", "title": "InputsSchema", "description": "Inputs for LLM inference such as messages, functions and additional parameters." }, "LLMInputsSchema": { "properties": { "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model", "description": "Model name" }, "provider": { "anyOf": [ { "$ref": "#/components/schemas/ProviderName" }, { "type": "null" } ], "description": "Provider name" }, "model_params": { "anyOf": [ { "$ref": "#/components/schemas/ModelParamsSchema" }, { "type": "null" } ], "description": "Model parameters" }, "messages": { "anyOf": [ { "items": { "$ref": "#/components/schemas/MessageSchema" }, "type": "array" }, { "type": "null" } ], "title": "Messages", "description": "List of messages" }, "history": { "anyOf": [ { "items": { "$ref": "#/components/schemas/MessageSchema" }, "type": "array" }, { "type": "null" } ], "title": "History", "description": "List of history messages which are prepended to messages. Useful for chatbot applications which rely on deployed prompts." }, "functions": { "anyOf": [ { "items": {}, "type": "array" }, { "type": "null" } ], "title": "Functions", "description": "List of function schemas, following the OpenAI API format" }, "function_call": { "anyOf": [ { "type": "string" }, { "additionalProperties": { "type": "string" }, "type": "object" }, { "type": "null" } ], "title": "Function Call", "description": "Optionally, specify the fnction call, following OpenAI API behavior." } }, "type": "object", "title": "LLMInputsSchema" }, "LabEvaluateUnitSchema": { "properties": { "inputs": { "anyOf": [ { "additionalProperties": { "type": "string" }, "type": "object" }, { "type": "null" } ], "title": "Inputs", "default": {} }, "target": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Target" }, "completion": { "type": "string", "title": "Completion" }, "result_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Result Id" }, "column": { "type": "integer", "title": "Column" }, "row": { "type": "integer", "title": "Row" }, "version_id": { "type": "integer", "title": "Version Id" } }, "type": "object", "required": [ "completion", "column", "row", "version_id" ], "title": "LabEvaluateUnitSchema" }, "LabFunctionsResponseSchema": { "properties": { "name": { "type": "string", "title": "Name" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Description" }, "signature": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Signature" }, "content": { "type": "string", "title": "Content" }, "tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Tokens" }, "id": { "type": "integer", "title": "Id" }, "parent_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Parent Id" }, "version_number": { "type": "number", "title": "Version Number" }, "created_at": { "type": "string", "format": "date-time", "title": "Created At" } }, "type": "object", "required": [ "name", "content", "id", "version_number", "created_at" ], "title": "LabFunctionsResponseSchema" }, "LabInferenceResponseSchema": { "properties": { "completion": { "type": "string", "title": "Completion" }, "column": { "type": "integer", "title": "Column" }, "row": { "type": "integer", "title": "Row" }, "latency": { "type": "number", "title": "Latency" }, "cost": { "type": "number", "title": "Cost" }, "completion_tokens": { "type": "integer", "title": "Completion Tokens" }, "result_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Result Id" }, "logprobs": { "anyOf": [ { "items": { "$ref": "#/components/schemas/Logprob" }, "type": "array" }, { "type": "null" } ], "title": "Logprobs" }, "perplexity": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Perplexity" } }, "type": "object", "required": [ "completion", "column", "row", "latency", "cost", "completion_tokens" ], "title": "LabInferenceResponseSchema" }, "LabSessionCommentSchema": { "properties": { "lab_session_id": { "type": "integer", "title": "Lab Session Id" }, "user_id": { "type": "string", "title": "User Id" }, "organization_id": { "type": "string", "title": "Organization Id" }, "author_name": { "type": "string", "title": "Author Name" }, "comment": { "type": "string", "title": "Comment" }, "id": { "type": "integer", "title": "Id" }, "created_at": { "type": "string", "title": "Created At" } }, "type": "object", "required": [ "lab_session_id", "user_id", "organization_id", "author_name", "comment", "id", "created_at" ], "title": "LabSessionCommentSchema" }, "LabSessionDetails": { "properties": { "session_id": { "type": "integer", "title": "Session Id" }, "name": { "type": "string", "title": "Name" } }, "type": "object", "required": [ "session_id", "name" ], "title": "LabSessionDetails" }, "LabSessionRow": { "properties": { "example_inputs": { "anyOf": [ { "additionalProperties": { "type": "string" }, "type": "object" }, { "type": "null" } ], "title": "Example Inputs", "default": {} }, "test_case_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Test Case Id" }, "completion": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Completion" }, "result_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Result Id" }, "scores": { "anyOf": [ { "additionalProperties": { "type": "number" }, "type": "object" }, { "type": "null" } ], "title": "Scores" }, "feedback": { "anyOf": [ { "$ref": "#/components/schemas/Rating" }, { "type": "null" } ] }, "logprobs": { "anyOf": [ { "items": { "$ref": "#/components/schemas/Logprob" }, "type": "array" }, { "type": "null" } ], "title": "Logprobs" }, "perplexity": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Perplexity" }, "latency": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Latency" }, "cost": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Cost" }, "completion_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Completion Tokens" } }, "type": "object", "title": "LabSessionRow" }, "ListExperimentsRequestSchema": { "properties": { "project_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Project Name", "description": "Name of the project to filter experiments by." }, "metadata_filter": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Metadata Filter", "description": "Filters experiments to contain the given metadata. The metadata_filter is a dictionary where the keys are the metadata keys and the values are the metadata values to filter by." }, "experiment_name_filter": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Experiment Name Filter", "description": "Filters experiments to contain the given string in their name." }, "run_name_filter": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Run Name Filter", "description": "Filters experiments to contain the given string in their run name." }, "experiment_uuids": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Experiment Uuids", "description": "Filters experiments to contain the given UUIDs." } }, "type": "object", "title": "ListExperimentsRequestSchema", "examples": [ { "experiment_name_filter": "Hello World", "metadata_filter": { "purpose": "testing" }, "project_name": "development", "run_name_filter": "v1" } ] }, "LogData": { "properties": { "inference_id": { "type": "string", "title": "Inference Id" }, "organization_id": { "type": "string", "title": "Organization Id" }, "trace_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Trace Id" }, "trace_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Trace Name" }, "session_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Session Id" }, "end_user_identifier": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "End User Identifier" }, "error": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Error" }, "status": { "type": "string", "title": "Status" }, "start_timestamp": { "type": "string", "title": "Start Timestamp" }, "end_timestamp": { "type": "string", "title": "End Timestamp" }, "duration": { "type": "number", "title": "Duration" }, "deployment_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Deployment Id" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "evaluation_metric_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "title": "Evaluation Metric Ids" }, "user_defined_metadata": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "User Defined Metadata" }, "cache_hit": { "type": "boolean", "title": "Cache Hit" }, "target": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Target" }, "tags": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Tags" }, "feedback_score": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Feedback Score" }, "llm_inputs": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Llm Inputs" }, "model_type": { "type": "string", "title": "Model Type" }, "provider": { "type": "string", "title": "Provider" }, "model_parameters": { "type": "object", "title": "Model Parameters" }, "messages": { "anyOf": [ { "items": { "type": "object" }, "type": "array" }, { "type": "null" } ], "title": "Messages" }, "functions": { "anyOf": [ { "items": {}, "type": "array" }, { "type": "null" } ], "title": "Functions" }, "function_call": { "anyOf": [ { "type": "string" }, { "additionalProperties": { "type": "string" }, "type": "object" }, { "type": "null" } ], "title": "Function Call" }, "output": { "type": "string", "title": "Output" }, "latency": { "type": "number", "title": "Latency" }, "input_tokens": { "type": "integer", "title": "Input Tokens" }, "output_tokens": { "type": "integer", "title": "Output Tokens" }, "total_tokens": { "type": "integer", "title": "Total Tokens" }, "cost": { "type": "number", "title": "Cost" } }, "type": "object", "required": [ "inference_id", "organization_id", "trace_id", "trace_name", "session_id", "end_user_identifier", "error", "status", "start_timestamp", "end_timestamp", "duration", "deployment_id", "name", "evaluation_metric_ids", "user_defined_metadata", "cache_hit", "target", "tags", "feedback_score", "llm_inputs", "model_type", "provider", "model_parameters", "messages", "functions", "function_call", "output", "latency", "input_tokens", "output_tokens", "total_tokens", "cost" ], "title": "LogData" }, "Logprob": { "properties": { "token": { "type": "string", "title": "Token" }, "logprob": { "type": "number", "title": "Logprob" } }, "type": "object", "required": [ "token", "logprob" ], "title": "Logprob" }, "MessageSchema": { "properties": { "content": { "anyOf": [ { "type": "string" }, { "items": {}, "type": "array" }, { "type": "null" } ], "title": "Content", "description": "Message content", "default": "" }, "role": { "allOf": [ { "$ref": "#/components/schemas/Role" } ], "description": "Message role", "default": "user" } }, "type": "object", "title": "MessageSchema" }, "ModelParamsSchema": { "properties": { "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model", "description": "Model name", "default": "" }, "temp": { "anyOf": [ { "type": "number", "maximum": 1.0, "minimum": 0.0 }, { "type": "null" } ], "title": "Temp", "description": "Temperature", "default": 1.0 }, "top_p": { "anyOf": [ { "type": "number", "maximum": 1.0, "minimum": 0.0 }, { "type": "null" } ], "title": "Top P", "description": "Top p", "default": 1.0 }, "frequency_penalty": { "anyOf": [ { "type": "number", "maximum": 2.0, "minimum": -2.0 }, { "type": "null" } ], "title": "Frequency Penalty", "description": "Frequency penalty", "default": 0.0 }, "presence_penalty": { "anyOf": [ { "type": "number", "maximum": 2.0, "minimum": -2.0 }, { "type": "null" } ], "title": "Presence Penalty", "description": "Presence penalty", "default": 0.0 }, "max_length": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Max Length", "description": "Max length of the output" }, "response_format": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Response Format", "description": "Response format" }, "safe_prompt": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Safe Prompt", "description": "Safe prompt. Only used for Mistral" } }, "type": "object", "title": "ModelParamsSchema" }, "MultiStatResponse": { "properties": { "time": { "type": "string", "title": "Time" }, "name_to_value": { "additionalProperties": { "anyOf": [ { "type": "number" }, { "type": "null" } ] }, "type": "object", "title": "Name To Value" } }, "type": "object", "required": [ "time", "name_to_value" ], "title": "MultiStatResponse" }, "NewColumn": { "properties": { "name": { "type": "string", "title": "Name" }, "value": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Value" }, "defaultValue": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Defaultvalue" } }, "type": "object", "required": [ "name" ], "title": "NewColumn" }, "NewVersionIdsResponseSchema": { "properties": { "query_id": { "type": "integer", "title": "Query Id" }, "version_id": { "type": "integer", "title": "Version Id" }, "version_number": { "type": "number", "title": "Version Number" }, "versions_list": { "items": { "$ref": "#/components/schemas/VersionReference" }, "type": "array", "title": "Versions List" } }, "type": "object", "required": [ "query_id", "version_id", "version_number", "versions_list" ], "title": "NewVersionIdsResponseSchema" }, "OrgAPIKeySchema": { "properties": { "organization_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Organization Id" } }, "type": "object", "title": "OrgAPIKeySchema" }, "OutputTargetPairSchema": { "properties": { "output": { "type": "number", "title": "Output" }, "target": { "type": "number", "title": "Target" } }, "type": "object", "required": [ "output", "target" ], "title": "OutputTargetPairSchema" }, "PIDECreateOrSaveRequestSchema": { "properties": { "id": { "type": "integer", "title": "Id" }, "name": { "type": "string", "title": "Name" }, "source_code": { "type": "string", "title": "Source Code" } }, "type": "object", "required": [ "id", "name", "source_code" ], "title": "PIDECreateOrSaveRequestSchema" }, "PIDECreateOrUpdateResponse": { "properties": { "id": { "type": "integer", "title": "Id" }, "created_at": { "type": "string", "title": "Created At" }, "last_updated_at": { "type": "string", "title": "Last Updated At" }, "source_code": { "type": "string", "title": "Source Code" } }, "type": "object", "required": [ "id", "created_at", "last_updated_at", "source_code" ], "title": "PIDECreateOrUpdateResponse" }, "PIDECreateRequestSchema": { "properties": { "name": { "type": "string", "title": "Name" }, "example_session": { "allOf": [ { "$ref": "#/components/schemas/PIDEExampleSession" } ], "default": "default" } }, "type": "object", "required": [ "name" ], "title": "PIDECreateRequestSchema" }, "PIDEExampleSession": { "type": "string", "enum": [ "default", "function_calling" ], "title": "PIDEExampleSession" }, "PIDERunRequestSchema": { "properties": { "id": { "type": "integer", "title": "Id" }, "name": { "type": "string", "title": "Name" }, "source_code": { "type": "string", "title": "Source Code" }, "inputs": { "additionalProperties": { "type": "string" }, "type": "object", "title": "Inputs" }, "test_case_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Test Case Id" }, "save_on_run": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Save On Run", "default": true } }, "type": "object", "required": [ "id", "name", "source_code", "inputs" ], "title": "PIDERunRequestSchema" }, "PIDERunResponseSchema": { "properties": { "trace_id": { "type": "string", "title": "Trace Id" } }, "type": "object", "required": [ "trace_id" ], "title": "PIDERunResponseSchema" }, "PaginatedResponse_TraceLogTreeSchema_": { "properties": { "total": { "type": "integer", "title": "Total" }, "page": { "type": "integer", "title": "Page" }, "total_pages": { "type": "integer", "title": "Total Pages" }, "page_size": { "type": "integer", "title": "Page Size" }, "results": { "items": { "$ref": "#/components/schemas/TraceLogTreeSchema" }, "type": "array", "title": "Results" } }, "type": "object", "required": [ "total", "page", "total_pages", "page_size", "results" ], "title": "PaginatedResponse[TraceLogTreeSchema]" }, "ParamsSchema": { "properties": { "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model", "description": "Model name" }, "temp": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Temp", "description": "Temperature" }, "top_p": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Top P", "description": "Top p" }, "frequency_penalty": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Frequency Penalty", "description": "Frequency penalty" }, "presence_penalty": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Presence Penalty", "description": "Presence penalty" }, "max_length": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Max Length", "description": "Max. number of completion tokens" }, "response_format": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Response Format", "description": "Response format. See OpenAI docs for definition" }, "safe_prompt": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Safe Prompt", "description": "Used for Mistral." } }, "type": "object", "title": "ParamsSchema", "description": "Additional parameters to run inference of LLMs, such as temperature, top_p, etc.\nFollowing the OpenAI API format." }, "PostCommentRequestSchema": { "properties": { "trace_id": { "type": "string", "title": "Trace Id", "description": "Trace ID" }, "comment": { "type": "string", "title": "Comment", "description": "Comment" } }, "type": "object", "required": [ "trace_id", "comment" ], "title": "PostCommentRequestSchema" }, "ProjectSchema": { "properties": { "name": { "type": "string", "title": "Name", "description": "Name of the project" }, "uuid": { "type": "string", "title": "Uuid", "description": "UUID of the project" }, "created_at": { "type": "string", "title": "Created At", "description": "Timestamp the project was created" } }, "type": "object", "required": [ "name", "uuid", "created_at" ], "title": "ProjectSchema", "examples": [ { "created_at": "2024-05-30 13:48:34", "name": "development", "uuid": "..." } ] }, "PromptSchema": { "properties": { "raw_messages": { "items": { "type": "object" }, "type": "array", "title": "Raw Messages", "description": "List of templated messages as they were created in the UI." }, "messages": { "items": { "type": "object" }, "type": "array", "title": "Messages", "description": "If `inputs` were given in request, these messages will be the filled-in version of `raw_messages`. Otherwise, they will be the same as `raw_messages`." }, "inputs": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Inputs", "description": "If you would like to fill in your prompt template with inputs you can provided then as a dictionary. The keys should match the names of the deployed prompt template\u00e2\u20ac\u2122s variables." } }, "type": "object", "required": [ "raw_messages", "messages" ], "title": "PromptSchema" }, "ProviderAPIKeySchema": { "properties": { "model_provider": { "type": "string", "title": "Model Provider" }, "api_key": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Api Key" }, "endpoint": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Endpoint" }, "organization_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Organization Id" }, "deployment_names": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Deployment Names" }, "aws_access_key_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Aws Access Key Id" }, "aws_region": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Aws Region" } }, "type": "object", "required": [ "model_provider" ], "title": "ProviderAPIKeySchema" }, "ProviderName": { "type": "string", "enum": [ "openai", "azure", "anthropic", "anyscale", "vertexai", "aws_bedrock", "openrouter", "mistral", "litellm", "groq", "fireworks", "cohere" ], "title": "ProviderName" }, "PublicKeyRequestSchema": { "properties": { "organization_id": { "type": "string", "title": "Organization Id" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "is_default": { "type": "boolean", "title": "Is Default", "default": false } }, "type": "object", "required": [ "organization_id" ], "title": "PublicKeyRequestSchema" }, "PublicKeyResponseSchema": { "properties": { "name": { "type": "string", "title": "Name" }, "key": { "type": "string", "title": "Key" }, "api_key": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Api Key" }, "created_at": { "type": "string", "format": "date-time", "title": "Created At" } }, "type": "object", "required": [ "name", "key", "created_at" ], "title": "PublicKeyResponseSchema" }, "QueriesResponseSchema": { "properties": { "queries": { "items": { "$ref": "#/components/schemas/QueryResponseSchema" }, "type": "array", "title": "Queries" } }, "type": "object", "required": [ "queries" ], "title": "QueriesResponseSchema" }, "QueryParamsPaginationPublic": { "properties": { "page": { "type": "integer", "exclusiveMinimum": 0.0, "title": "Page", "description": "Page number starting from 1", "default": 1 }, "page_size": { "type": "integer", "maximum": 100000.0, "exclusiveMinimum": 0.0, "title": "Page Size", "description": "Number of items per page", "default": 20 }, "filter_field": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Filter Field", "description": "Field to filter on.\nIf you want to filter by a score, you need to follow this format: 'score:{score_name}'.\nIf you want to filter by annotation, you need to follow this format: 'annotation:{annotation_type}:{annotation_id}'." }, "filter_key": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Filter Key", "description": "Filter by key when filtering on a map" }, "filter_operator": { "anyOf": [ { "$ref": "#/components/schemas/FilterOperator" }, { "type": "null" } ], "description": "Filter operator" }, "filter_value": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Filter Value", "description": "Filter value" }, "project_name": { "type": "string", "title": "Project Name", "description": "Filter by project name" }, "time_range": { "allOf": [ { "$ref": "#/components/schemas/TimeRange" } ], "description": "Time range for the logs: 1h,3h,6h,24h, 7d, 1m, 3m, 1y or na", "default": "na" }, "status": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Status", "description": "Filter by status: success, error" } }, "type": "object", "required": [ "project_name" ], "title": "QueryParamsPaginationPublic", "examples": [ { "filter_field": "trace_name", "filter_operator": "like", "filter_value": "llm" } ] }, "QueryResponseSchema": { "properties": { "query_id": { "type": "integer", "title": "Query Id" }, "version_id": { "type": "integer", "title": "Version Id" }, "version_number": { "type": "number", "title": "Version Number" }, "versions_list": { "items": { "$ref": "#/components/schemas/VersionReference" }, "type": "array", "title": "Versions List" }, "result_details": { "anyOf": [ { "$ref": "#/components/schemas/ResultDetailsSchema" }, { "type": "null" } ] }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "result_params": { "$ref": "#/components/schemas/ModelParamsSchema" }, "type": { "$ref": "#/components/schemas/QueryType" }, "rating": { "anyOf": [ { "$ref": "#/components/schemas/Rating" }, { "type": "null" } ] }, "tags": { "items": { "type": "string" }, "type": "array", "title": "Tags" }, "favorite": { "type": "boolean", "title": "Favorite" }, "score": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Score" }, "lab_session_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "title": "Lab Session Ids" }, "created_at": { "type": "string", "format": "date-time", "title": "Created At" }, "messages": { "items": { "$ref": "#/components/schemas/MessageSchema" }, "type": "array", "title": "Messages" }, "function_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array", "uniqueItems": true }, { "type": "null" } ], "title": "Function Ids" }, "function_call": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Function Call" } }, "type": "object", "required": [ "query_id", "version_id", "version_number", "versions_list", "result_params", "type", "tags", "favorite", "created_at", "messages" ], "title": "QueryResponseSchema" }, "QueryType": { "type": "string", "enum": [ "prompt", "template" ], "title": "QueryType" }, "Rating": { "type": "string", "enum": [ "good", "bad" ], "title": "Rating" }, "ResultDetailSchema": { "properties": { "example_inputs": { "anyOf": [ { "additionalProperties": { "type": "string" }, "type": "object" }, { "type": "null" } ], "title": "Example Inputs" }, "test_case_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Test Case Id" }, "original_details": { "anyOf": [ { "$ref": "#/components/schemas/Detail" }, { "type": "null" } ] }, "optimized_details": { "anyOf": [ { "$ref": "#/components/schemas/Detail" }, { "type": "null" } ] } }, "type": "object", "title": "ResultDetailSchema" }, "ResultDetailsSchema": { "properties": { "details": { "items": { "$ref": "#/components/schemas/ResultDetailSchema" }, "type": "array", "title": "Details" } }, "type": "object", "required": [ "details" ], "title": "ResultDetailsSchema" }, "ResultSchema": { "properties": { "id": { "type": "integer", "title": "Id" }, "content": { "type": "string", "title": "Content" }, "example_inputs": { "anyOf": [ { "additionalProperties": { "type": "string" }, "type": "object" }, { "type": "null" } ], "title": "Example Inputs" }, "test_case_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Test Case Id" }, "tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Tokens" }, "latency": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Latency" }, "cost": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Cost" }, "scores": { "anyOf": [ { "additionalProperties": { "type": "number" }, "type": "object" }, { "type": "null" } ], "title": "Scores" }, "feedback": { "anyOf": [ { "$ref": "#/components/schemas/Rating" }, { "type": "null" } ] }, "results_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Results Id" }, "logprobs": { "anyOf": [ { "items": { "$ref": "#/components/schemas/Logprob" }, "type": "array" }, { "type": "null" } ], "title": "Logprobs" } }, "type": "object", "required": [ "id", "content" ], "title": "ResultSchema" }, "Role": { "type": "string", "enum": [ "user", "assistant", "system", "user", "assistant", "function", "tool" ], "title": "Role" }, "RunEvalMetricRequestSchema": { "properties": { "function_code": { "type": "string", "title": "Function Code" }, "units": { "items": { "$ref": "#/components/schemas/EvaluateUnitSchema" }, "type": "array", "title": "Units" } }, "type": "object", "required": [ "function_code", "units" ], "title": "RunEvalMetricRequestSchema" }, "RunEvalMetricResponseSchema": { "properties": { "result_id_to_response": { "additionalProperties": { "$ref": "#/components/schemas/ScoreOrError" }, "type": "object", "title": "Result Id To Response" } }, "type": "object", "required": [ "result_id_to_response" ], "title": "RunEvalMetricResponseSchema" }, "SaveLabSessionRequestSchema": { "properties": { "id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Id", "description": "If not set, a new session will be created" }, "name": { "type": "string", "title": "Name", "description": "Name of the session" }, "evaluation_metric_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "title": "Evaluation Metric Ids", "description": "Evaluation metric ids" }, "configurations": { "items": { "$ref": "#/components/schemas/ConfigurationUnitSchema" }, "type": "array", "title": "Configurations", "description": "List of configurations" } }, "type": "object", "required": [ "name", "configurations" ], "title": "SaveLabSessionRequestSchema" }, "SaveLabSessionResponseSchema": { "properties": { "id": { "type": "integer", "title": "Id" }, "versions": { "items": { "$ref": "#/components/schemas/GetLabVersionResponseSchema" }, "type": "array", "title": "Versions" }, "result_ids": { "items": { "items": { "type": "integer" }, "type": "array" }, "type": "array", "title": "Result Ids" } }, "type": "object", "required": [ "id", "versions", "result_ids" ], "title": "SaveLabSessionResponseSchema" }, "ScoreOrError": { "properties": { "score": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Score" }, "error": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Error" } }, "type": "object", "title": "ScoreOrError" }, "ShareRequestSchema": { "properties": { "version_id": { "type": "integer", "title": "Version Id" }, "organization_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Organization Id" }, "shared_version_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Shared Version Id" }, "receiver_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Receiver Id" }, "is_public": { "type": "boolean", "title": "Is Public", "default": false } }, "type": "object", "required": [ "version_id" ], "title": "ShareRequestSchema" }, "SingleStatResponse": { "properties": { "time": { "type": "string", "title": "Time" }, "value": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Value" } }, "type": "object", "required": [ "time", "value" ], "title": "SingleStatResponse" }, "SortOrder": { "type": "string", "enum": [ "asc", "desc" ], "title": "SortOrder" }, "StatisticOperation": { "type": "string", "enum": [ "mean", "median", "variance", "standard_deviation", "min", "max", "mse", "mae", "correlation", "spearman_correlation", "accuracy", "custom" ], "title": "StatisticOperation" }, "StreamCompletionRequest": { "properties": { "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model", "description": "Model name", "default": "" }, "temp": { "anyOf": [ { "type": "number", "maximum": 1.0, "minimum": 0.0 }, { "type": "null" } ], "title": "Temp", "description": "Temperature", "default": 1.0 }, "top_p": { "anyOf": [ { "type": "number", "maximum": 1.0, "minimum": 0.0 }, { "type": "null" } ], "title": "Top P", "description": "Top p", "default": 1.0 }, "frequency_penalty": { "anyOf": [ { "type": "number", "maximum": 2.0, "minimum": -2.0 }, { "type": "null" } ], "title": "Frequency Penalty", "description": "Frequency penalty", "default": 0.0 }, "presence_penalty": { "anyOf": [ { "type": "number", "maximum": 2.0, "minimum": -2.0 }, { "type": "null" } ], "title": "Presence Penalty", "description": "Presence penalty", "default": 0.0 }, "max_length": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Max Length", "description": "Max length of the output" }, "response_format": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Response Format", "description": "Response format" }, "safe_prompt": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Safe Prompt", "description": "Safe prompt. Only used for Mistral" }, "messages": { "items": { "$ref": "#/components/schemas/MessageSchema" }, "type": "array", "title": "Messages", "default": [] }, "content": { "type": "string", "title": "Content", "default": "" }, "latency": { "type": "number", "title": "Latency", "default": 0.0 }, "unformatted_messages": { "items": { "$ref": "#/components/schemas/MessageSchema" }, "type": "array", "title": "Unformatted Messages", "default": [] }, "prev_messages_unformatted": { "items": { "$ref": "#/components/schemas/MessageSchema" }, "type": "array", "title": "Prev Messages Unformatted", "default": [] }, "version_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Version Id" }, "example_inputs": { "anyOf": [ { "additionalProperties": { "type": "string" }, "type": "object" }, { "type": "null" } ], "title": "Example Inputs", "default": {} }, "test_case_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Test Case Id" }, "function_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "title": "Function Ids" }, "logprobs": { "anyOf": [ { "items": { "$ref": "#/components/schemas/Logprob" }, "type": "array" }, { "type": "null" } ], "title": "Logprobs" }, "functions": { "anyOf": [ { "items": {}, "type": "array" }, { "type": "null" } ], "title": "Functions" }, "function_call": { "anyOf": [ { "type": "string" }, { "additionalProperties": { "type": "string" }, "type": "object" }, { "type": "null" } ], "title": "Function Call" }, "organization_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Organization Id" }, "inference_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Inference Id" }, "azure_endpoint": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Azure Endpoint" } }, "type": "object", "title": "StreamCompletionRequest" }, "StreamCompletionResponse": { "properties": { "tokens": { "type": "integer", "title": "Tokens" }, "tokens_prompt": { "type": "integer", "title": "Tokens Prompt" }, "tokens_completion": { "type": "integer", "title": "Tokens Completion" }, "cost": { "type": "number", "title": "Cost" }, "result_id": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Result Id" }, "perplexity": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Perplexity" } }, "type": "object", "required": [ "tokens", "tokens_prompt", "tokens_completion", "cost" ], "title": "StreamCompletionResponse" }, "StripeEndpointResponseSchema": { "properties": { "status": { "type": "integer", "title": "Status" }, "redirect_url": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Redirect Url" }, "error": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Error" } }, "type": "object", "required": [ "status" ], "title": "StripeEndpointResponseSchema" }, "SubscriptionEnum": { "type": "string", "enum": [ "team", "team6", "team12", "enterprise", "free" ], "title": "SubscriptionEnum" }, "SummaryResponseSchema": { "properties": { "prompts": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Prompts" }, "test_cases": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Test Cases" }, "functions": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Functions" }, "evaluations": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Evaluations" }, "trace_logs": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Trace Logs" }, "experiments": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Experiments" }, "deployments": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Deployments" } }, "type": "object", "title": "SummaryResponseSchema" }, "TestCaseCollectionOverviewSchema": { "properties": { "id": { "type": "integer", "title": "Id", "description": "Unique identifier for the test case collection" }, "name": { "type": "string", "title": "Name", "description": "Name of the test case collection" }, "column_names": { "items": { "type": "string" }, "type": "array", "title": "Column Names", "description": "Column names for the test case collection" }, "created_at": { "type": "string", "title": "Created At", "description": "Timestamp when the test case collection was created" }, "last_updated_at": { "type": "string", "title": "Last Updated At", "description": "Timestamp when the test case collection was last updated" } }, "type": "object", "required": [ "id", "name", "column_names", "created_at", "last_updated_at" ], "title": "TestCaseCollectionOverviewSchema", "examples": [ { "column_names": [ "input1", "input2" ], "created_at": "2021-06-01T00:00:00Z", "id": 5000, "last_updated_at": "2021-06-01T00:00:00Z", "name": "Hello World Programs" } ] }, "TestCaseSchema": { "properties": { "inputs": { "additionalProperties": { "type": "string" }, "type": "object", "title": "Inputs", "description": "Dictionary mapping input names to values. Input names correspond to the column names of the test case collection." }, "target": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Target", "description": "Optional target/ground truth value for the test case/inputs" }, "tags": { "items": { "type": "string" }, "type": "array", "title": "Tags", "description": "List of tags associated with the test case", "default": [] } }, "type": "object", "required": [ "inputs" ], "title": "TestCaseSchema", "examples": [ { "inputs": { "framework": "fastapi", "language": "python" }, "tags": [ "easy" ], "target": "a good program here" } ] }, "TimeRange": { "type": "string", "enum": [ "na", "1h", "3h", "6h", "24h", "7d", "1m", "3m", "6m", "1y" ], "title": "TimeRange" }, "TraceLogAnnotationSchema": { "properties": { "trace_id": { "type": "string", "title": "Trace Id", "description": "UUID of associated trace" }, "defined_annotation_id": { "type": "integer", "title": "Defined Annotation Id", "description": "Annotation criterion ID" }, "score": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Score", "description": "Annotation score" }, "value": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Value", "description": "Annotation value" }, "id": { "anyOf": [ { "type": "string" }, { "type": "integer" }, { "type": "null" } ], "title": "Id", "description": "Annotation ID" }, "created_at": { "type": "string", "title": "Created At", "description": "Annotation creation timestamp" }, "user_id": { "type": "string", "title": "User Id", "description": "Parea user ID" }, "user_email_address": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "User Email Address", "description": "User email address" }, "annotation_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Annotation Name", "description": "Annotation name" } }, "type": "object", "required": [ "trace_id", "defined_annotation_id", "created_at", "user_id" ], "title": "TraceLogAnnotationSchema" }, "TraceLogCommentSchema": { "properties": { "trace_id": { "type": "string", "title": "Trace Id", "description": "Trace ID" }, "comment": { "type": "string", "title": "Comment", "description": "Comment" }, "id": { "anyOf": [ { "type": "string" }, { "type": "integer" } ], "title": "Id", "description": "Comment ID" }, "user_id": { "type": "string", "title": "User Id", "description": "User ID" }, "user_email_address": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "User Email Address", "description": "User email address" }, "created_at": { "type": "string", "title": "Created At", "description": "Comment creation timestamp" } }, "type": "object", "required": [ "trace_id", "comment", "id", "user_id", "created_at" ], "title": "TraceLogCommentSchema" }, "TraceLogImage": { "properties": { "url": { "type": "string", "title": "Url", "description": "URL of image" }, "caption": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Caption", "description": "Caption of image" } }, "type": "object", "required": [ "url" ], "title": "TraceLogImage" }, "TraceLogSchema": { "properties": { "configuration": { "allOf": [ { "$ref": "#/components/schemas/InputsSchema-Input" } ], "description": "If this log was a LLM call, this will contain the configuration used for the call.", "default": {} }, "inputs": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Inputs", "description": "Key-value pair inputs of this trace. Note, there is a special field to capture messages in LLM calls. You can still use it in the case of LLM calls to track the key-value pairs for prompt templates." }, "output": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Output", "description": "Response of this step/log/function. If response isn\u00e2\u20ac\u2122t a string, it needs to be serialized to a string." }, "target": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Target", "description": "The target or \u00e2\u20ac\u0153gold standard\u00e2\u20ac\u009d response for the inputs of this log." }, "latency": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Latency", "description": "Latency of this log in seconds.", "default": 0.0 }, "time_to_first_token": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Time To First Token", "description": "If this was a LLM call, this will contain the time taken to generate the first token." }, "input_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Input Tokens", "description": "If this was a LLM call, this will contain the number of tokens in the input.", "default": 0 }, "output_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Output Tokens", "description": "If this was a LLM call, this will contain the number of tokens in the output.", "default": 0 }, "total_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Total Tokens", "description": "If this was a LLM call, this will contain the total number of tokens in the input and output.", "default": 0 }, "cost": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Cost", "description": "If this was a LLM call, this will contain the cost of the call.", "default": 0.0 }, "trace_id": { "type": "string", "title": "Trace Id", "description": "UUID of the trace log. Ex: e3267953-a16f-47f5-b37e-622dbb29d730" }, "start_timestamp": { "type": "string", "title": "Start Timestamp", "description": "Start timestamp" }, "parent_trace_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Parent Trace Id", "description": "If given, current trace will be a child of this trace. If current child is not a child, `parent_trace_id` should be equal to `trace_id`" }, "root_trace_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Root Trace Id", "description": "This is the UUID of the root trace/span of this trace. If current trace is the root trace, `root_trace_id` must be equal to `trace_id`" }, "project_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Project Name", "description": "Name of the project with which the trace/log should be associated with. Must be provided if `project_uuid` is not provided" }, "status": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Status", "description": "If the trace was a `success` or `error`" }, "error": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Error", "description": "If `status=error`, this should contain any additional information such as the stacktrace" }, "output_for_eval_metrics": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Output For Eval Metrics", "description": "If provided, will be used as output for any specified evaluation metric." }, "evaluation_metric_names": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Evaluation Metric Names", "description": "Names of evaluation metrics deployed on Parea which should be applied to this log." }, "scores": { "anyOf": [ { "items": { "$ref": "#/components/schemas/EvaluationScoreSchema" }, "type": "array" }, { "type": "null" } ], "title": "Scores", "description": "Any scores/eval results associated with this log." }, "feedback_score": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Feedback Score", "description": "Any captured (user) feedback on this log" }, "apply_eval_frac": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Apply Eval Frac", "description": "If specified, evals given with `evaluation_metric_names` will be applied to this log with this fraction.", "default": 1.0 }, "log_sample_rate": { "anyOf": [ { "type": "number", "maximum": 1.0, "minimum": 0.0 }, { "type": "null" } ], "title": "Log Sample Rate", "description": "If specified, this log and its entire associated trace will logged with this probability. Must be between 0 and 1 (incl.). Defaults to 1.0 (i.e., keeping all logs)", "default": 1.0 }, "deployment_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Deployment Id", "description": "Optionally, provide the ID of the used deployed prompt in this log." }, "cache_hit": { "type": "boolean", "title": "Cache Hit", "description": "If the cache was hit for this log.", "default": false }, "trace_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Trace Name", "description": "The name of this span." }, "children": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Children", "description": "UUIDs of any children." }, "children_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "title": "Children Ids", "description": "IDs of any children. Will be automatically populated." }, "end_timestamp": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "End Timestamp", "description": "End timestamp of span." }, "end_user_identifier": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "End User Identifier", "description": "Unique identifier for a end-user." }, "session_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Session Id", "description": "Unique identifier for a session. Can you be used to associated multiple logs in e.g. chat applications." }, "metadata": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Metadata", "description": "Any additional key-value pairs which provide context or are useful for filtering." }, "tags": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Tags", "description": "List of tags which provide additional context or are useful for filtering." }, "experiment_uuid": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Experiment Uuid", "description": "If given, will be used to associate this log with an experiment." }, "images": { "anyOf": [ { "items": { "$ref": "#/components/schemas/TraceLogImage" }, "type": "array" }, { "type": "null" } ], "title": "Images", "description": "Any images associated with trace." }, "comments": { "anyOf": [ { "items": { "$ref": "#/components/schemas/TraceLogCommentSchema" }, "type": "array" }, { "type": "null" } ], "title": "Comments", "description": "Any comments on log which were collected on Parea frontend." }, "annotations": { "anyOf": [ { "additionalProperties": { "additionalProperties": { "$ref": "#/components/schemas/TraceLogAnnotationSchema" }, "type": "object" }, "type": "object" }, { "type": "null" } ], "title": "Annotations", "description": "Any annotations on log which were collected on Parea frontend. It maps annoation criterion ID to a dictionary mapping user_id (Parea user ID) to annotation." }, "depth": { "type": "integer", "title": "Depth", "description": "Depth/level of nestedness of span in overall trace. Root-level trace is 0 and it always increments by 1.", "default": 0 }, "execution_order": { "type": "integer", "title": "Execution Order", "description": "The execution number of span in trace. It starts with 0 and increments by 1 with every span.", "default": 0 }, "evaluation_metric_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "title": "Evaluation Metric Ids", "description": "Deprecated" }, "fill_children": { "type": "boolean", "title": "Fill Children", "description": "Deprecated", "default": false }, "project_uuid": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Project Uuid", "description": "UUID of project with which this log is associated. Will be automatically filled-in by SDKs" }, "organization_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Organization Id", "description": "Organization ID associated with Parea API key. Will be automatically determined from API key" } }, "type": "object", "required": [ "trace_id", "start_timestamp" ], "title": "TraceLogSchema", "examples": [ { "depth": 0, "end_timestamp": "2024-05-30 13:48:35", "execution_order": 0, "inputs": { "x": "Golang", "y": "Fiber" }, "metadata": { "purpose": "testing" }, "output": "Some logged output", "parent_trace_id": "<>", "project_name": "default", "root_trace_id": "<>", "start_timestamp": "2024-05-30 13:48:34", "status": "success", "trace_id": "<>", "trace_name": "test" }, { "configuration": { "messages": [ { "content": "Some prompt", "role": "user" } ], "model": "gpt-4o", "provider": "openai" }, "depth": 0, "end_timestamp": "2024-05-30 13:48:35", "execution_order": 0, "inputs": { "promptTemplateVar1": "value", "promptTemplateVar2": "some other value" }, "metadata": { "purpose": "testing" }, "output": "LLM response", "parent_trace_id": "<>", "project_name": "default", "root_trace_id": "<>", "start_timestamp": "2024-05-30 13:48:34", "status": "success", "trace_id": "<>", "trace_name": "LLM" } ] }, "TraceLogTreeSchema": { "properties": { "configuration": { "allOf": [ { "$ref": "#/components/schemas/InputsSchema-Output" } ], "description": "If this log was a LLM call, this will contain the configuration used for the call.", "default": {} }, "inputs": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Inputs", "description": "Key-value pair inputs of this trace. Note, there is a special field to capture messages in LLM calls. You can still use it in the case of LLM calls to track the key-value pairs for prompt templates." }, "output": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Output", "description": "Response of this step/log/function. If response isn\u00e2\u20ac\u2122t a string, it needs to be serialized to a string." }, "target": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Target", "description": "The target or \u00e2\u20ac\u0153gold standard\u00e2\u20ac\u009d response for the inputs of this log." }, "latency": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Latency", "description": "Latency of this log in seconds.", "default": 0.0 }, "time_to_first_token": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Time To First Token", "description": "If this was a LLM call, this will contain the time taken to generate the first token." }, "input_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Input Tokens", "description": "If this was a LLM call, this will contain the number of tokens in the input.", "default": 0 }, "output_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Output Tokens", "description": "If this was a LLM call, this will contain the number of tokens in the output.", "default": 0 }, "total_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Total Tokens", "description": "If this was a LLM call, this will contain the total number of tokens in the input and output.", "default": 0 }, "cost": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Cost", "description": "If this was a LLM call, this will contain the cost of the call.", "default": 0.0 }, "trace_id": { "type": "string", "title": "Trace Id", "description": "UUID of the trace log. Ex: e3267953-a16f-47f5-b37e-622dbb29d730" }, "start_timestamp": { "type": "string", "title": "Start Timestamp", "description": "Start timestamp" }, "parent_trace_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Parent Trace Id", "description": "If given, current trace will be a child of this trace. If current child is not a child, `parent_trace_id` should be equal to `trace_id`" }, "root_trace_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Root Trace Id", "description": "This is the UUID of the root trace/span of this trace. If current trace is the root trace, `root_trace_id` must be equal to `trace_id`" }, "project_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Project Name", "description": "Name of the project with which the trace/log should be associated with. Must be provided if `project_uuid` is not provided" }, "status": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Status", "description": "If the trace was a `success` or `error`" }, "error": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Error", "description": "If `status=error`, this should contain any additional information such as the stacktrace" }, "output_for_eval_metrics": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Output For Eval Metrics", "description": "If provided, will be used as output for any specified evaluation metric." }, "evaluation_metric_names": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Evaluation Metric Names", "description": "Names of evaluation metrics deployed on Parea which should be applied to this log." }, "scores": { "anyOf": [ { "items": { "$ref": "#/components/schemas/EvaluationScoreSchema" }, "type": "array" }, { "type": "null" } ], "title": "Scores", "description": "Any scores/eval results associated with this log." }, "feedback_score": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Feedback Score", "description": "Any captured (user) feedback on this log" }, "apply_eval_frac": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Apply Eval Frac", "description": "If specified, evals given with `evaluation_metric_names` will be applied to this log with this fraction.", "default": 1.0 }, "log_sample_rate": { "anyOf": [ { "type": "number", "maximum": 1.0, "minimum": 0.0 }, { "type": "null" } ], "title": "Log Sample Rate", "description": "If specified, this log and its entire associated trace will logged with this probability. Must be between 0 and 1 (incl.). Defaults to 1.0 (i.e., keeping all logs)", "default": 1.0 }, "deployment_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Deployment Id", "description": "Optionally, provide the ID of the used deployed prompt in this log." }, "cache_hit": { "type": "boolean", "title": "Cache Hit", "description": "If the cache was hit for this log.", "default": false }, "trace_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Trace Name", "description": "The name of this span." }, "children": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Children", "description": "UUIDs of any children." }, "children_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "title": "Children Ids", "description": "IDs of any children. Will be automatically populated." }, "end_timestamp": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "End Timestamp", "description": "End timestamp of span." }, "end_user_identifier": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "End User Identifier", "description": "Unique identifier for a end-user." }, "session_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Session Id", "description": "Unique identifier for a session. Can you be used to associated multiple logs in e.g. chat applications." }, "metadata": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Metadata", "description": "Any additional key-value pairs which provide context or are useful for filtering." }, "tags": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Tags", "description": "List of tags which provide additional context or are useful for filtering." }, "experiment_uuid": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Experiment Uuid", "description": "If given, will be used to associate this log with an experiment." }, "images": { "anyOf": [ { "items": { "$ref": "#/components/schemas/TraceLogImage" }, "type": "array" }, { "type": "null" } ], "title": "Images", "description": "Any images associated with trace." }, "comments": { "anyOf": [ { "items": { "$ref": "#/components/schemas/TraceLogCommentSchema" }, "type": "array" }, { "type": "null" } ], "title": "Comments", "description": "Any comments on log which were collected on Parea frontend." }, "annotations": { "anyOf": [ { "additionalProperties": { "additionalProperties": { "$ref": "#/components/schemas/TraceLogAnnotationSchema" }, "type": "object" }, "type": "object" }, { "type": "null" } ], "title": "Annotations", "description": "Any annotations on log which were collected on Parea frontend. It maps annoation criterion ID to a dictionary mapping user_id (Parea user ID) to annotation." }, "depth": { "type": "integer", "title": "Depth", "description": "Depth/level of nestedness of span in overall trace. Root-level trace is 0 and it always increments by 1.", "default": 0 }, "execution_order": { "type": "integer", "title": "Execution Order", "description": "The execution number of span in trace. It starts with 0 and increments by 1 with every span.", "default": 0 }, "evaluation_metric_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "title": "Evaluation Metric Ids", "description": "Deprecated" }, "fill_children": { "type": "boolean", "title": "Fill Children", "description": "Deprecated", "default": false }, "project_uuid": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Project Uuid", "description": "UUID of project with which this log is associated. Will be automatically filled-in by SDKs" }, "organization_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Organization Id", "description": "Organization ID associated with Parea API key. Will be automatically determined from API key" }, "children_logs": { "items": { "$ref": "#/components/schemas/TraceLogTreeSchema" }, "type": "array", "title": "Children Logs", "description": "Children logs" } }, "type": "object", "required": [ "trace_id", "start_timestamp" ], "title": "TraceLogTreeSchema", "examples": [ { "depth": 0, "end_timestamp": "2024-05-30 13:48:35", "execution_order": 0, "inputs": { "x": "Golang", "y": "Fiber" }, "metadata": { "purpose": "testing" }, "output": "Some logged output", "parent_trace_id": "<>", "project_name": "default", "root_trace_id": "<>", "start_timestamp": "2024-05-30 13:48:34", "status": "success", "trace_id": "<>", "trace_name": "test" }, { "configuration": { "messages": [ { "content": "Some prompt", "role": "user" } ], "model": "gpt-4o", "provider": "openai" }, "depth": 0, "end_timestamp": "2024-05-30 13:48:35", "execution_order": 0, "inputs": { "promptTemplateVar1": "value", "promptTemplateVar2": "some other value" }, "metadata": { "purpose": "testing" }, "output": "LLM response", "parent_trace_id": "<>", "project_name": "default", "root_trace_id": "<>", "start_timestamp": "2024-05-30 13:48:34", "status": "success", "trace_id": "<>", "trace_name": "LLM" } ] }, "TraceLogTreeSchemaDEPRICATED": { "properties": { "configuration": { "allOf": [ { "$ref": "#/components/schemas/InputsSchema-Output" } ], "description": "If this log was a LLM call, this will contain the configuration used for the call.", "default": {} }, "inputs": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Inputs", "description": "Key-value pair inputs of this trace. Note, there is a special field to capture messages in LLM calls. You can still use it in the case of LLM calls to track the key-value pairs for prompt templates." }, "output": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Output", "description": "Response of this step/log/function. If response isn\u00e2\u20ac\u2122t a string, it needs to be serialized to a string." }, "target": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Target", "description": "The target or \u00e2\u20ac\u0153gold standard\u00e2\u20ac\u009d response for the inputs of this log." }, "latency": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Latency", "description": "Latency of this log in seconds.", "default": 0.0 }, "time_to_first_token": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Time To First Token", "description": "If this was a LLM call, this will contain the time taken to generate the first token." }, "input_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Input Tokens", "description": "If this was a LLM call, this will contain the number of tokens in the input.", "default": 0 }, "output_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Output Tokens", "description": "If this was a LLM call, this will contain the number of tokens in the output.", "default": 0 }, "total_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Total Tokens", "description": "If this was a LLM call, this will contain the total number of tokens in the input and output.", "default": 0 }, "cost": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Cost", "description": "If this was a LLM call, this will contain the cost of the call.", "default": 0.0 }, "trace_id": { "type": "string", "title": "Trace Id", "description": "UUID of the trace log. Ex: e3267953-a16f-47f5-b37e-622dbb29d730" }, "start_timestamp": { "type": "string", "title": "Start Timestamp", "description": "Start timestamp" }, "parent_trace_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Parent Trace Id", "description": "If given, current trace will be a child of this trace. If current child is not a child, `parent_trace_id` should be equal to `trace_id`" }, "root_trace_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Root Trace Id", "description": "This is the UUID of the root trace/span of this trace. If current trace is the root trace, `root_trace_id` must be equal to `trace_id`" }, "project_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Project Name", "description": "Name of the project with which the trace/log should be associated with. Must be provided if `project_uuid` is not provided" }, "status": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Status", "description": "If the trace was a `success` or `error`" }, "error": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Error", "description": "If `status=error`, this should contain any additional information such as the stacktrace" }, "output_for_eval_metrics": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Output For Eval Metrics", "description": "If provided, will be used as output for any specified evaluation metric." }, "evaluation_metric_names": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Evaluation Metric Names", "description": "Names of evaluation metrics deployed on Parea which should be applied to this log." }, "scores": { "anyOf": [ { "items": { "$ref": "#/components/schemas/EvaluationScoreSchema" }, "type": "array" }, { "type": "null" } ], "title": "Scores", "description": "Any scores/eval results associated with this log." }, "feedback_score": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Feedback Score", "description": "Any captured (user) feedback on this log" }, "apply_eval_frac": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Apply Eval Frac", "description": "If specified, evals given with `evaluation_metric_names` will be applied to this log with this fraction.", "default": 1.0 }, "log_sample_rate": { "anyOf": [ { "type": "number", "maximum": 1.0, "minimum": 0.0 }, { "type": "null" } ], "title": "Log Sample Rate", "description": "If specified, this log and its entire associated trace will logged with this probability. Must be between 0 and 1 (incl.). Defaults to 1.0 (i.e., keeping all logs)", "default": 1.0 }, "deployment_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Deployment Id", "description": "Optionally, provide the ID of the used deployed prompt in this log." }, "cache_hit": { "type": "boolean", "title": "Cache Hit", "description": "If the cache was hit for this log.", "default": false }, "trace_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Trace Name", "description": "The name of this span." }, "children": { "anyOf": [ { "items": { "$ref": "#/components/schemas/TraceLogTreeSchemaDEPRICATED" }, "type": "array" }, { "type": "null" } ], "title": "Children" }, "children_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "title": "Children Ids", "description": "IDs of any children. Will be automatically populated." }, "end_timestamp": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "End Timestamp", "description": "End timestamp of span." }, "end_user_identifier": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "End User Identifier", "description": "Unique identifier for a end-user." }, "session_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Session Id", "description": "Unique identifier for a session. Can you be used to associated multiple logs in e.g. chat applications." }, "metadata": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Metadata", "description": "Any additional key-value pairs which provide context or are useful for filtering." }, "tags": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Tags", "description": "List of tags which provide additional context or are useful for filtering." }, "experiment_uuid": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Experiment Uuid", "description": "If given, will be used to associate this log with an experiment." }, "images": { "anyOf": [ { "items": { "$ref": "#/components/schemas/TraceLogImage" }, "type": "array" }, { "type": "null" } ], "title": "Images", "description": "Any images associated with trace." }, "comments": { "anyOf": [ { "items": { "$ref": "#/components/schemas/TraceLogCommentSchema" }, "type": "array" }, { "type": "null" } ], "title": "Comments", "description": "Any comments on log which were collected on Parea frontend." }, "annotations": { "anyOf": [ { "additionalProperties": { "additionalProperties": { "$ref": "#/components/schemas/TraceLogAnnotationSchema" }, "type": "object" }, "type": "object" }, { "type": "null" } ], "title": "Annotations", "description": "Any annotations on log which were collected on Parea frontend. It maps annoation criterion ID to a dictionary mapping user_id (Parea user ID) to annotation." }, "depth": { "type": "integer", "title": "Depth", "description": "Depth/level of nestedness of span in overall trace. Root-level trace is 0 and it always increments by 1.", "default": 0 }, "execution_order": { "type": "integer", "title": "Execution Order", "description": "The execution number of span in trace. It starts with 0 and increments by 1 with every span.", "default": 0 }, "evaluation_metric_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "title": "Evaluation Metric Ids", "description": "Deprecated" }, "fill_children": { "type": "boolean", "title": "Fill Children", "description": "Deprecated", "default": false }, "project_uuid": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Project Uuid", "description": "UUID of project with which this log is associated. Will be automatically filled-in by SDKs" }, "organization_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Organization Id", "description": "Organization ID associated with Parea API key. Will be automatically determined from API key" }, "parent_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Parent Id" } }, "type": "object", "required": [ "trace_id", "start_timestamp" ], "title": "TraceLogTreeSchemaDEPRICATED", "examples": [ { "depth": 0, "end_timestamp": "2024-05-30 13:48:35", "execution_order": 0, "inputs": { "x": "Golang", "y": "Fiber" }, "metadata": { "purpose": "testing" }, "output": "Some logged output", "parent_trace_id": "<>", "project_name": "default", "root_trace_id": "<>", "start_timestamp": "2024-05-30 13:48:34", "status": "success", "trace_id": "<>", "trace_name": "test" }, { "configuration": { "messages": [ { "content": "Some prompt", "role": "user" } ], "model": "gpt-4o", "provider": "openai" }, "depth": 0, "end_timestamp": "2024-05-30 13:48:35", "execution_order": 0, "inputs": { "promptTemplateVar1": "value", "promptTemplateVar2": "some other value" }, "metadata": { "purpose": "testing" }, "output": "LLM response", "parent_trace_id": "<>", "project_name": "default", "root_trace_id": "<>", "start_timestamp": "2024-05-30 13:48:34", "status": "success", "trace_id": "<>", "trace_name": "LLM" } ] }, "TraceStatsSchema": { "properties": { "trace_id": { "type": "string", "title": "Trace Id", "description": "UUID of top-level trace log" }, "latency": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Latency", "description": "Latency of this log in seconds.", "default": 0.0 }, "input_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Input Tokens", "description": "Number of input tokens for all LLM calls in trace (incl. children).", "default": 0 }, "output_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Output Tokens", "description": "Number of output tokens for all LLM calls in trace (incl. children).", "default": 0 }, "total_tokens": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Total Tokens", "description": "Total number of tokens for all LLM calls in trace (incl. children).", "default": 0 }, "cost": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Cost", "description": "Total cost of all LLM calls in trace (incl. children)." }, "scores": { "anyOf": [ { "items": { "$ref": "#/components/schemas/EvaluationScoreSchema" }, "type": "array" }, { "type": "null" } ], "title": "Scores", "description": "Scores of all logs in trace (incl. children)." } }, "type": "object", "required": [ "trace_id" ], "title": "TraceStatsSchema", "description": "Aggregates stats for a trace and its children.", "examples": [ { "cost": 0.5, "input_tokens": 100, "latency": 0.5, "output_tokens": 200, "scores": [ { "name": "accuracy", "score": 0.5 } ], "total_tokens": 300, "trace_id": "sasdc-csd-rnFesaeCpqcXJ_yL3wi" } ] }, "UniqueValueOptions": { "type": "string", "enum": [ "deployment_id", "session_id", "end_user_identifier" ], "title": "UniqueValueOptions" }, "UpdateAnnotationCriterionPurpose": { "type": "string", "enum": [ "name", "description", "min_value", "max_value", "value_score_map" ], "title": "UpdateAnnotationCriterionPurpose" }, "UpdateAnnotationCriterionSchema": { "properties": { "id": { "anyOf": [ { "type": "string" }, { "type": "integer" } ], "title": "Id" }, "purpose": { "$ref": "#/components/schemas/UpdateAnnotationCriterionPurpose" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "description": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Description" }, "min_value": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Min Value" }, "max_value": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Max Value" }, "value_score_map": { "anyOf": [ { "additionalProperties": { "type": "number" }, "type": "object" }, { "type": "null" } ], "title": "Value Score Map" } }, "type": "object", "required": [ "id", "purpose" ], "title": "UpdateAnnotationCriterionSchema" }, "UpdateAnnotationQueuePurpose": { "type": "string", "const": "name", "title": "UpdateAnnotationQueuePurpose" }, "UpdateAnnotationQueueSchema": { "properties": { "uuid": { "type": "string", "title": "Uuid" }, "purpose": { "$ref": "#/components/schemas/UpdateAnnotationQueuePurpose" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" } }, "type": "object", "required": [ "uuid", "purpose" ], "title": "UpdateAnnotationQueueSchema" }, "UpdateColumnNamesSchema": { "properties": { "column_names": { "items": { "type": "string" }, "type": "array", "title": "Column Names" } }, "type": "object", "required": [ "column_names" ], "title": "UpdateColumnNamesSchema" }, "UpdateDeploymentModelSchema": { "properties": { "deployment_id": { "type": "string", "title": "Deployment Id" }, "version_id": { "type": "integer", "title": "Version Id" }, "version_number": { "type": "number", "title": "Version Number" }, "model_type": { "type": "string", "title": "Model Type" }, "provider": { "type": "string", "title": "Provider" }, "model_parameters": { "$ref": "#/components/schemas/ModelParamsSchema" }, "active": { "type": "boolean", "title": "Active", "default": true }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "messages": { "items": { "$ref": "#/components/schemas/MessageSchema" }, "type": "array", "title": "Messages" }, "input_variables": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Input Variables" }, "function_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "title": "Function Ids" }, "evaluation_metric_ids": { "anyOf": [ { "items": { "type": "integer" }, "type": "array" }, { "type": "null" } ], "title": "Evaluation Metric Ids" }, "function_call": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Function Call" }, "is_azure": { "type": "boolean", "title": "Is Azure", "default": false }, "deployer_name": { "type": "string", "title": "Deployer Name", "default": "Anonymous" }, "commit_message": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Commit Message" }, "event": { "anyOf": [ { "$ref": "#/components/schemas/HistoryEvent" }, { "type": "null" } ] }, "from_version": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "From Version" } }, "type": "object", "required": [ "deployment_id", "version_id", "version_number", "model_type", "provider", "model_parameters", "messages" ], "title": "UpdateDeploymentModelSchema" }, "UpdateEvalMetricRequestSchema": { "properties": { "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "webhook_url": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Webhook Url" }, "function_code": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Function Code" } }, "type": "object", "title": "UpdateEvalMetricRequestSchema" }, "UpdateEvaluationScoreSchema": { "properties": { "trace_id": { "type": "string", "title": "Trace Id" }, "name": { "type": "string", "title": "Name" }, "score": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Score" }, "reason": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Reason" }, "experiment_uuid": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Experiment Uuid" } }, "type": "object", "required": [ "trace_id", "name" ], "title": "UpdateEvaluationScoreSchema" }, "UpdateExperimentPurpose": { "type": "string", "enum": [ "name", "run_name" ], "title": "UpdateExperimentPurpose" }, "UpdateExperimentSchema": { "properties": { "uuid": { "type": "string", "title": "Uuid" }, "purpose": { "$ref": "#/components/schemas/UpdateExperimentPurpose" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "run_name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Run Name" } }, "type": "object", "required": [ "uuid", "purpose" ], "title": "UpdateExperimentSchema" }, "UpdateLogSchema": { "properties": { "trace_id": { "type": "string", "title": "Trace Id", "description": "UUID of the trace log which should be updated" }, "field_name_to_value_map": { "type": "object", "title": "Field Name To Value Map", "description": "Key-value pairs how each given field (`key`) of the trace log should be updated with `value`. For field names, see `TraceLogSchema`." }, "root_trace_id": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Root Trace Id", "description": "UUID of root trace" } }, "type": "object", "required": [ "trace_id", "field_name_to_value_map" ], "title": "UpdateLogSchema", "examples": [ { "field_name_to_value_map": { "error": "Some error message", "status": "error" }, "trace_id": "sasdc-csd-rnFesaeCpqcXJ_yL3wi" } ] }, "UpdatePideNameRequestSchema": { "properties": { "name": { "type": "string", "title": "Name" } }, "type": "object", "required": [ "name" ], "title": "UpdatePideNameRequestSchema" }, "UpdateSessionEvaluationMetricRequestSchema": { "properties": { "evaluation_metric_ids": { "items": { "type": "integer" }, "type": "array", "title": "Evaluation Metric Ids" } }, "type": "object", "required": [ "evaluation_metric_ids" ], "title": "UpdateSessionEvaluationMetricRequestSchema" }, "UpdateSessionNameRequestSchema": { "properties": { "name": { "type": "string", "title": "Name" } }, "type": "object", "required": [ "name" ], "title": "UpdateSessionNameRequestSchema" }, "UpdateTestCaseCollectionNameRequestSchema": { "properties": { "name": { "type": "string", "title": "Name" } }, "type": "object", "required": [ "name" ], "title": "UpdateTestCaseCollectionNameRequestSchema" }, "UpdateTestCasePublicRequestSchema": { "properties": { "inputs": { "anyOf": [ { "type": "object" }, { "type": "null" } ], "title": "Inputs", "description": "Key-value pairs for the inputs of that item" }, "target": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Target", "description": "Target/ground truth value for the item" }, "tags": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Tags", "description": "Tags for the item" } }, "type": "object", "title": "UpdateTestCasePublicRequestSchema", "description": "Any set field will be used to update the item in the dataset.", "examples": [ { "inputs": { "framework": "gin", "language": "golang" } } ] }, "UpdateTestCaseRequestSchema": { "properties": { "test_case_id": { "type": "integer", "title": "Test Case Id" }, "collection_id": { "type": "integer", "title": "Collection Id" }, "update_request": { "type": "object", "title": "Update Request" } }, "type": "object", "required": [ "test_case_id", "collection_id", "update_request" ], "title": "UpdateTestCaseRequestSchema" }, "UpdateTestCaseTagsRequestSchema": { "properties": { "test_case_id": { "type": "integer", "title": "Test Case Id" }, "collection_id": { "type": "integer", "title": "Collection Id" }, "tags": { "items": { "type": "string" }, "type": "array", "title": "Tags", "default": [] } }, "type": "object", "required": [ "test_case_id", "collection_id" ], "title": "UpdateTestCaseTagsRequestSchema" }, "UpdateVersionPurpose": { "type": "string", "enum": [ "tag", "name", "favorite", "rating", "messages" ], "title": "UpdateVersionPurpose" }, "UpdateVersionSchema": { "properties": { "version_id": { "type": "integer", "title": "Version Id" }, "purpose": { "$ref": "#/components/schemas/UpdateVersionPurpose" }, "name": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Name" }, "tags": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "title": "Tags" }, "favorite": { "anyOf": [ { "type": "boolean" }, { "type": "null" } ], "title": "Favorite" }, "rating": { "anyOf": [ { "$ref": "#/components/schemas/Rating" }, { "type": "null" } ] }, "messages": { "anyOf": [ { "items": { "additionalProperties": { "type": "string" }, "type": "object" }, "type": "array" }, { "type": "null" } ], "title": "Messages" } }, "type": "object", "required": [ "version_id", "purpose" ], "title": "UpdateVersionSchema" }, "UploadAnnotationsSchema": { "properties": { "annotation_criterion_id": { "type": "integer", "title": "Annotation Criterion Id" }, "annotations": { "items": { "$ref": "#/components/schemas/UploadedAnnotationRowSchema" }, "type": "array", "title": "Annotations" } }, "type": "object", "required": [ "annotation_criterion_id", "annotations" ], "title": "UploadAnnotationsSchema" }, "UploadedAnnotationRowSchema": { "properties": { "inputs": { "type": "object", "title": "Inputs" }, "output": { "type": "string", "title": "Output" }, "score": { "type": "number", "title": "Score" }, "value": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Value" } }, "type": "object", "required": [ "inputs", "output", "score" ], "title": "UploadedAnnotationRowSchema" }, "UserInfoResponseSchema": { "properties": { "id": { "type": "string", "title": "Id" }, "name": { "type": "string", "title": "Name" }, "profileUrl": { "type": "string", "title": "Profileurl" }, "email": { "type": "string", "title": "Email" } }, "type": "object", "required": [ "id", "name", "profileUrl", "email" ], "title": "UserInfoResponseSchema" }, "ValidationError": { "properties": { "loc": { "items": { "anyOf": [ { "type": "string" }, { "type": "integer" } ] }, "type": "array", "title": "Location" }, "msg": { "type": "string", "title": "Message" }, "type": { "type": "string", "title": "Error Type" } }, "type": "object", "required": [ "loc", "msg", "type" ], "title": "ValidationError" }, "VersionReference": { "properties": { "version_id": { "type": "integer", "title": "Version Id" }, "version": { "type": "number", "title": "Version" } }, "type": "object", "required": [ "version_id", "version" ], "title": "VersionReference" }, "VersionUpRequestSchema": { "properties": { "version_id": { "type": "integer", "title": "Version Id" }, "new_messages": { "items": { "$ref": "#/components/schemas/MessageSchema" }, "type": "array", "title": "New Messages" }, "model": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "title": "Model" }, "temp": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Temp" }, "top_p": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Top P" }, "frequency_penalty": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Frequency Penalty" }, "presence_penalty": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "title": "Presence Penalty" }, "max_length": { "anyOf": [ { "type": "integer" }, { "type": "null" } ], "title": "Max Length" }, "kwargs_samples": { "anyOf": [ { "items": { "additionalProperties": { "type": "string" }, "type": "object" }, "type": "array" }, { "type": "null" } ], "title": "Kwargs Samples" } }, "type": "object", "required": [ "version_id", "new_messages" ], "title": "VersionUpRequestSchema" } }, "securitySchemes": { "OAuth2PasswordBearer": { "type": "oauth2", "flows": { "password": { "scopes": {}, "tokenUrl": "token" } } }, "APIKeyHeader": { "type": "apiKey", "in": "header", "name": "x-user-id" } } } }