{"openapi":"3.1.0","info":{"title":"Fireworks AI evaluators, evaluation jobs and evaluations (your key) API","version":"0.1.0","description":"Fireworks AI evaluators, evaluation jobs and evaluations (your key) lets an AI assistant read your own Fireworks Evaluators data through the Fireworks Evaluators API with a key you supply (Fireworks API key). Read-only: 7 tools (list_evaluators, get_evaluator, list_evaluator_versions, list_evaluation_jobs, get_evaluation_job, list_evaluations, get_evaluation). The BYOK key travels with each...\n\nEvery operation is read-only or compute-only. The same tools are available over MCP (streamable HTTP) at https://byok-fireworks-evaluators.connectorbay.com/mcp.","contact":{"name":"Unify Dynamics LLC","email":"agent@connectorbay.com","url":"https://connectorbay.com/byok-fireworks-evaluators/docs"},"termsOfService":"https://connectorbay.com/byok-fireworks-evaluators/terms"},"servers":[{"url":"https://byok-fireworks-evaluators.connectorbay.com"}],"externalDocs":{"url":"https://connectorbay.com/byok-fireworks-evaluators/docs"},"security":[{"upstreamKey":[]}],"components":{"securitySchemes":{"bearer":{"type":"http","scheme":"bearer","description":"Optional. A Pro API key raises rate limits; no credentials are required."},"upstreamKey":{"type":"apiKey","in":"header","name":"X-Upstream-Key","description":"Your own key for the service this connector wraps (bring your own key). Used for that one call only; never stored, logged or cached."}},"schemas":{"Error":{"type":"object","properties":{"ok":{"const":false},"error":{"type":"object"}}}}},"paths":{"/v1/tools/list_evaluators":{"post":{"operationId":"list_evaluators","summary":"List evaluators with state, entry point, commit hash, default dataset and times; source code is not returned. Reads eval","description":"List evaluators with state, entry point, commit hash, default dataset and times; source code is not returned. Reads evaluation metadata only: it runs no evaluation and no model. Use for \"What evaluators do I have on Fireworks\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"account_id":{"description":"Your Fireworks account id (the id in your account's resource names, for example 'my-account').","maxLength":100,"title":"Account Id","type":"string"},"page_size":{"anyOf":[{"maximum":200,"minimum":1,"type":"integer"},{"type":"null"}],"default":50,"description":"Items per page (default 50, values above 200 are treated as 200).","title":"Page Size"},"page_token":{"anyOf":[{"maxLength":200,"type":"string"},{"type":"null"}],"default":null,"description":"Token from the previous answer's nextPageToken to get the following page.","title":"Page Token"},"filter":{"anyOf":[{"maxLength":300,"type":"string"},{"type":"null"}],"default":null,"description":"Optional filter expression in Google AIP-160 syntax, for example 'state=ACTIVE'.","title":"Filter"},"order_by":{"anyOf":[{"maxLength":100,"type":"string"},{"type":"null"}],"default":null,"description":"Comma-separated fields to order by; add ' desc' after a field for descending.","title":"Order By"}},"required":["account_id"],"title":"list_evaluatorsArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/get_evaluator":{"post":{"operationId":"get_evaluator","summary":"Get one evaluator by id with state, entry point, commit hash and times. Reads evaluation metadata only: it runs no evalu","description":"Get one evaluator by id with state, entry point, commit hash and times. Reads evaluation metadata only: it runs no evaluation and no model. Use for \"Show Fireworks evaluator math-check\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"account_id":{"description":"Your Fireworks account id (the id in your account's resource names, for example 'my-account').","maxLength":100,"title":"Account Id","type":"string"},"evaluator_id":{"description":"Evaluator id, the last part of its name.","maxLength":100,"title":"Evaluator Id","type":"string"}},"required":["account_id","evaluator_id"],"title":"get_evaluatorArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_evaluator_versions":{"post":{"operationId":"list_evaluator_versions","summary":"List the stored versions of an evaluator with creation time and alternate ids. Reads evaluation metadata only: it runs n","description":"List the stored versions of an evaluator with creation time and alternate ids. Reads evaluation metadata only: it runs no evaluation and no model. Use for \"How many versions does Fireworks evaluator math-check have\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"account_id":{"description":"Your Fireworks account id (the id in your account's resource names, for example 'my-account').","maxLength":100,"title":"Account Id","type":"string"},"evaluator_id":{"description":"Evaluator id, the last part of its name.","maxLength":100,"title":"Evaluator Id","type":"string"},"page_size":{"anyOf":[{"maximum":200,"minimum":1,"type":"integer"},{"type":"null"}],"default":50,"description":"Items per page (default 50, values above 200 are treated as 200).","title":"Page Size"},"page_token":{"anyOf":[{"maxLength":200,"type":"string"},{"type":"null"}],"default":null,"description":"Token from the previous answer's nextPageToken to get the following page.","title":"Page Token"}},"required":["account_id","evaluator_id"],"title":"list_evaluator_versionsArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_evaluation_jobs":{"post":{"operationId":"list_evaluation_jobs","summary":"List evaluation jobs with state, evaluator, input and output datasets, metrics and times. Reads evaluation metadata only","description":"List evaluation jobs with state, evaluator, input and output datasets, metrics and times. Reads evaluation metadata only: it runs no evaluation and no model. Use for \"Which Fireworks evaluation jobs finished and what were their metrics\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"account_id":{"description":"Your Fireworks account id (the id in your account's resource names, for example 'my-account').","maxLength":100,"title":"Account Id","type":"string"},"page_size":{"anyOf":[{"maximum":200,"minimum":1,"type":"integer"},{"type":"null"}],"default":50,"description":"Items per page (default 50, values above 200 are treated as 200).","title":"Page Size"},"page_token":{"anyOf":[{"maxLength":200,"type":"string"},{"type":"null"}],"default":null,"description":"Token from the previous answer's nextPageToken to get the following page.","title":"Page Token"},"filter":{"anyOf":[{"maxLength":300,"type":"string"},{"type":"null"}],"default":null,"description":"Optional filter expression in Google AIP-160 syntax, for example 'state=ACTIVE'.","title":"Filter"},"order_by":{"anyOf":[{"maxLength":100,"type":"string"},{"type":"null"}],"default":null,"description":"Comma-separated fields to order by; add ' desc' after a field for descending.","title":"Order By"}},"required":["account_id"],"title":"list_evaluation_jobsArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/get_evaluation_job":{"post":{"operationId":"get_evaluation_job","summary":"Get one evaluation job by id with state, datasets, metrics and times. Reads evaluation metadata only: it runs no evaluat","description":"Get one evaluation job by id with state, datasets, metrics and times. Reads evaluation metadata only: it runs no evaluation and no model. Use for \"Show Fireworks evaluation job ej1\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"account_id":{"description":"Your Fireworks account id (the id in your account's resource names, for example 'my-account').","maxLength":100,"title":"Account Id","type":"string"},"evaluation_job_id":{"description":"Evaluation job id, the last part of its name.","maxLength":100,"title":"Evaluation Job Id","type":"string"}},"required":["account_id","evaluation_job_id"],"title":"get_evaluation_jobArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/list_evaluations":{"post":{"operationId":"list_evaluations","summary":"List saved evaluations (provider and assertion setups) with type, description and times. Reads evaluation metadata only:","description":"List saved evaluations (provider and assertion setups) with type, description and times. Reads evaluation metadata only: it runs no evaluation and no model. Use for \"What saved evaluations do I have on Fireworks\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"account_id":{"description":"Your Fireworks account id (the id in your account's resource names, for example 'my-account').","maxLength":100,"title":"Account Id","type":"string"},"page_size":{"anyOf":[{"maximum":200,"minimum":1,"type":"integer"},{"type":"null"}],"default":50,"description":"Items per page (default 50, values above 200 are treated as 200).","title":"Page Size"},"page_token":{"anyOf":[{"maxLength":200,"type":"string"},{"type":"null"}],"default":null,"description":"Token from the previous answer's nextPageToken to get the following page.","title":"Page Token"},"filter":{"anyOf":[{"maxLength":300,"type":"string"},{"type":"null"}],"default":null,"description":"Optional filter expression in Google AIP-160 syntax, for example 'state=ACTIVE'.","title":"Filter"},"order_by":{"anyOf":[{"maxLength":100,"type":"string"},{"type":"null"}],"default":null,"description":"Comma-separated fields to order by; add ' desc' after a field for descending.","title":"Order By"}},"required":["account_id"],"title":"list_evaluationsArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}},"/v1/tools/get_evaluation":{"post":{"operationId":"get_evaluation","summary":"Get one saved evaluation by id with type, description and times. Reads evaluation metadata only: it runs no evaluation a","description":"Get one saved evaluation by id with type, description and times. Reads evaluation metadata only: it runs no evaluation and no model. Use for \"Show Fireworks evaluation e1\".","tags":["tools"],"requestBody":{"required":true,"content":{"application/json":{"schema":{"properties":{"account_id":{"description":"Your Fireworks account id (the id in your account's resource names, for example 'my-account').","maxLength":100,"title":"Account Id","type":"string"},"evaluation_id":{"description":"Evaluation id, the last part of its name.","maxLength":100,"title":"Evaluation Id","type":"string"}},"required":["account_id","evaluation_id"],"title":"get_evaluationArguments","type":"object"}}}},"responses":{"200":{"description":"Tool result. `ok: true` on success; `ok: false` with a structured `error` (provider_unavailable, upstream_error, bad_input) when the tool could not answer.","content":{"application/json":{"schema":{"type":"object","additionalProperties":true}}}},"422":{"description":"Arguments failed validation","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}},"429":{"description":"Rate limit exceeded (see Retry-After)","content":{"application/json":{"schema":{"$ref":"#/components/schemas/Error"}}}}},"x-read-only":true}}}}