diff --git a/.github/workflows/config-types.yml b/.github/workflows/config-types.yml new file mode 100644 index 0000000..eccc02f --- /dev/null +++ b/.github/workflows/config-types.yml @@ -0,0 +1,34 @@ +name: Config types + +on: + pull_request: + paths: + - "config/metadata/**" + - "config-types/**" + - "package.json" + - "package-lock.json" + - ".github/workflows/config-types.yml" + push: + branches: [notion-main] + paths: + - "config/metadata/**" + - "config-types/**" + - "package.json" + - "package-lock.json" + - ".github/workflows/config-types.yml" + +permissions: + contents: read + +jobs: + config-types: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "22" + cache: npm + - run: npm ci + - run: npm test + - run: npm pack --dry-run diff --git a/.gitignore b/.gitignore index 61a7038..711f3eb 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,7 @@ ko ko*.tar.gz crane crane*.tar.gz + +# TypeScript configuration package tooling +node_modules/ +*.tgz diff --git a/NOTION_FORK.md b/NOTION_FORK.md index 8aff68f..74e2a3a 100644 --- a/NOTION_FORK.md +++ b/NOTION_FORK.md @@ -59,3 +59,12 @@ repo, then consumed by `docker/refinery/Dockerfile` in notion-next: The pubsub package's Redis tests require a local Redis on `:6379`; all other tests run standalone. + +## TypeScript configuration types + +This fork also owns the `@notionhq/refinery-config-types` package, generated +from the canonical configuration metadata. When upgrading metadata, run +`npm ci && npm run generate && npm test` and include the updated declarations. +See [config-types/README.md](config-types/README.md) for consumption, provenance, +and validation limitations. Carry the package and its CI check forward when +rebasing the Notion patch set onto a new upstream release. diff --git a/config-types/README.md b/config-types/README.md new file mode 100644 index 0000000..26651d0 --- /dev/null +++ b/config-types/README.md @@ -0,0 +1,70 @@ +# Refinery configuration types + +`@notionhq/refinery-config-types` is a types-only package generated from this +fork's canonical `config/metadata/configMeta.yaml` and `rulesMeta.yaml` files. +It exports `RefineryConfig`, `RefineryRules`, and individual section/sampler types. +There are no runtime dependencies or install/build hooks. + +```typescript +import type { RefineryConfig, RefineryRules } from "@notionhq/refinery-config-types" + +const config = { + General: { ConfigurationVersion: 2 }, +} satisfies RefineryConfig + +const rules = { + RulesVersion: 2, + Samplers: { __default__: { DeterministicSampler: { SampleRate: 1 } } }, +} satisfies RefineryRules +``` + +## Regeneration and verification + +From the repository root, with Node 22.13 or newer: + +```sh +npm ci +npm run generate +npm test +npm pack --dry-run +``` + +Commit regenerated `config-types/index.d.ts` whenever the metadata or generator +changes. CI checks that generation is reproducible and compiles both valid and +intentionally invalid consumer examples. Generator fixture tests also compile +small schemas to check required fields, conflicting fields, conditional +requirements (including defaults), and failure on unknown field types or invalid +constraint references. Run them alone with: + +```sh +node --experimental-strip-types --test config-types/generate.test.mjs +``` + +The declaration header records a hash +of both metadata files so packaging-only commits do not imply a schema change. + +The small structural mapping in `generate.ts` connects rules, conditions, and +samplers because the metadata describes those object fields without their full +TypeScript structure. Unknown field types fail generation and must be mapped +when upgrading Refinery. + +## Consumption and upgrades + +The root package manifest supports an authenticated Git dependency pinned to a +full immutable commit SHA in `makenotion/refinery`. Generated declarations are +checked in, so consumers do not need Go, the generator, or its development tools. +The package is private to prevent accidental registry publication. Registry +publishing and notion-next dependency integration are separate changes. + +Choose a package revision whose metadata matches the deployed Refinery version; +update the pin and generated consumer config when upgrading that version. A +packaging-only commit can differ from the image commit while having identical +metadata. Do not depend on a moving branch. Private Git authentication must be +verified in the consuming repository's install and CI environment. + +These types check structure, choices, required fields, conflicting fields and +exactly one sampler per target. They do not replace `refinery --validate`: +numeric ranges, duration syntax, secret expansion and other runtime constraints +remain the binary's responsibility. Metadata may also include deprecated options +or differ from runtime implementation; the package follows that metadata rather +than silently introducing its own configuration schema. diff --git a/config-types/generate.test.mjs b/config-types/generate.test.mjs new file mode 100644 index 0000000..df88425 --- /dev/null +++ b/config-types/generate.test.mjs @@ -0,0 +1,110 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { spawnSync } from "node:child_process"; +import { test } from "node:test"; +import { generateConfigTypes } from "./generate.ts"; + +// Compile the generated fixture as a consumer would. @ts-expect-error also +// fails compilation if a constraint is accidentally weakened. +function checkConsumer(fields, examples) { + const directory = mkdtempSync(join(tmpdir(), "refinery-types-fixture-")); + try { + const declarations = generateConfigTypes({ + groups: [{ name: "Example", fields }], + }); + writeFileSync(join(directory, "index.d.ts"), declarations); + writeFileSync( + join(directory, "consumer.ts"), + 'import type { RefineryConfig } from "./index";\n' + examples, + ); + const result = spawnSync(process.execPath, [ + fileURLToPath(new URL("../node_modules/typescript/bin/tsc", import.meta.url)), + "--noEmit", "--strict", "--exactOptionalPropertyTypes", + "--skipLibCheck", "--types", "node", + "--typeRoots", fileURLToPath(new URL("../node_modules/@types", import.meta.url)), + join(directory, "consumer.ts"), + ], { encoding: "utf8" }); + assert.equal(result.status, 0, result.error?.message ?? result.stdout + result.stderr); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +} + +test("required fields require the containing group", () => { + checkConsumer([ + { name: "Value", type: "string", validations: [{ type: "required" }] }, + ], ` + const valid: RefineryConfig = { Example: { Value: "ok" } }; + // @ts-expect-error globally required group is missing + const missingGroup: RefineryConfig = {}; + // @ts-expect-error required field is missing + const missingField: RefineryConfig = { Example: {} }; + `); +}); + +test("requiredInGroup permits omission of the group, but not its field", () => { + checkConsumer([ + { name: "Value", type: "string", validations: [{ type: "requiredInGroup" }] }, + ], ` + const omitted: RefineryConfig = {}; + const valid: RefineryConfig = { Example: { Value: "ok" } }; + // @ts-expect-error group is present without its required field + const invalid: RefineryConfig = { Example: {} }; + `); +}); + +test("conflictsWith rejects both fields, including through a variable", () => { + checkConsumer([ + { name: "Left", type: "string", validations: [{ type: "conflictsWith", arg: "Right" }] }, + { name: "Right", type: "string" }, + ], ` + const neither: RefineryConfig = { Example: {} }; + const left: RefineryConfig = { Example: { Left: "a" } }; + const right: RefineryConfig = { Example: { Right: "b" } }; + const both = { Left: "a", Right: "b" }; + // @ts-expect-error mutually exclusive fields + const invalid: RefineryConfig = { Example: both }; + `); +}); + +test("requiredWith requires its dependent field only when the trigger is set", () => { + checkConsumer([ + { name: "Trigger", type: "bool" }, + { name: "Dependent", type: "string", validations: [{ type: "requiredWith", arg: "Trigger" }] }, + ], ` + const neither: RefineryConfig = { Example: {} }; + const dependent: RefineryConfig = { Example: { Dependent: "a" } }; + const both: RefineryConfig = { Example: { Trigger: true, Dependent: "a" } }; + // @ts-expect-error missing dependent field + const invalid: RefineryConfig = { Example: { Trigger: true } }; + `); +}); + +test("a default satisfies requiredWith without an explicit dependent field", () => { + checkConsumer([ + { name: "Trigger", type: "bool" }, + { name: "Dependent", type: "string", default: "default", + validations: [{ type: "requiredWith", arg: "Trigger" }] }, + ], ` + const valid: RefineryConfig = { Example: { Trigger: true } }; + `); +}); + +test("unknown field types fail generation instead of silently weakening types", () => { + assert.throws(() => generateConfigTypes({ + groups: [{ name: "Example", fields: [{ name: "Value", type: "futuretype" }] }], + }), /Unhandled metadata field type "futuretype" for Example.Value/); +}); + +test("invalid constraint references fail generation", () => { + for (const type of ["requiredWith", "conflictsWith"]) { + assert.throws(() => generateConfigTypes({ + groups: [{ name: "Example", fields: [ + { name: "Value", type: "string", validations: [{ type, arg: "Missing" }] }, + ] }], + }), new RegExp("Invalid " + type + " reference for Example.Value")); + } +}); diff --git a/config-types/generate.ts b/config-types/generate.ts new file mode 100644 index 0000000..a097b2b --- /dev/null +++ b/config-types/generate.ts @@ -0,0 +1,457 @@ +import { readFileSync, writeFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { fileURLToPath } from "node:url"; +import { resolve } from "node:path"; +import YAML from "yaml"; + +type MetadataValidation = { + type: string; + arg?: unknown; +}; + +type MetadataField = { + name: string; + type: string; + valuetype?: string; + choices?: string[]; + default?: unknown; + summary?: string | undefined; + description?: string | undefined; + // A bare `validations:` key parses as null rather than absent. + validations?: MetadataValidation[] | null; +}; + +type MetadataGroup = { + name: string; + title?: string; + type?: string; + summary?: string | undefined; + description?: string | undefined; + fields?: MetadataField[]; +}; + +export type MetadataFile = { + groups: MetadataGroup[]; +}; + +/** + * The samplers that may appear nested under a rule via `Rules.Sampler` are + * listed in the metadata itself (the `validChildren` validation); the full + * set allowed at the top level of the `Samplers` map is every sampler group. + */ +const RULES_GROUP_TYPE_NAMES: Record = { + Rules: "RulesBasedSamplerRule", + Conditions: "RulesBasedSamplerCondition", +}; + +/** + * Structural field types the flat metadata can't express: which object type an + * `object`/`objectarray`/`sliceorscalar` field actually holds. Keyed by + * `.`. `Rules.Sampler` is handled separately via `validChildren`. + */ +const STRUCTURAL_FIELD_TYPES: Record = { + "RulesBasedSampler.Rules": "RulesBasedSamplerRule[]", + "Rules.Conditions": "RulesBasedSamplerCondition[]", + "Conditions.Value": + "string | number | boolean | Array", +}; + +const metadataDirectory = new URL("../config/metadata/", import.meta.url); +const output = new URL("./index.d.ts", import.meta.url); + +export function generateRefineryConfigTypes(): string { + const configMeta = readMetadata("configMeta.yaml"); + const rulesMeta = readMetadata("rulesMeta.yaml"); + const hash = createHash("sha256"); + for (const name of ["configMeta.yaml", "rulesMeta.yaml"]) { + hash + .update(name) + .update("\0") + .update(readFileSync(new URL(name, metadataDirectory))); + } + return [ + `// Generated by npm run generate. Do not edit.\n// Source: config/metadata/{configMeta,rulesMeta}.yaml\n// Metadata SHA-256: ${hash.digest("hex")}`, + generateConfigTypes(configMeta), + generateRulesTypes(rulesMeta), + "", + ].join("\n\n"); +} + +function readMetadata(filename: string): MetadataFile { + const raw: unknown = YAML.parse( + readFileSync(new URL(filename, metadataDirectory), "utf8"), + ); + // Validate the portion of the metadata consumed here; additional documentation + // and runtime-validation keys remain owned by Refinery. + if ( + !raw || + typeof raw !== "object" || + !("groups" in raw) || + !Array.isArray(raw.groups) + ) { + throw new Error(`Invalid metadata groups in ${filename}`); + } + if (!raw.groups.every(isMetadataGroup)) { + throw new Error(`Invalid metadata group or field in ${filename}`); + } + return { groups: raw.groups }; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function optionalStrings( + value: Record, + names: string[], +): boolean { + return names.every( + (name) => value[name] === undefined || typeof value[name] === "string", + ); +} + +function isMetadataField(value: unknown): value is MetadataField { + return ( + isRecord(value) && + typeof value.name === "string" && + typeof value.type === "string" && + optionalStrings(value, ["valuetype", "summary", "description"]) && + (value.choices === undefined || + (Array.isArray(value.choices) && + value.choices.every((choice) => typeof choice === "string"))) && + (value.validations == null || + (Array.isArray(value.validations) && + value.validations.every( + (validation) => + isRecord(validation) && typeof validation.type === "string", + ))) + ); +} + +function isMetadataGroup(value: unknown): value is MetadataGroup { + return ( + isRecord(value) && + typeof value.name === "string" && + optionalStrings(value, ["title", "type", "summary", "description"]) && + (value.fields === undefined || + (Array.isArray(value.fields) && value.fields.every(isMetadataField))) + ); +} + +export function generateConfigTypes(meta: MetadataFile): string { + const sections: string[] = []; + const rootFields: string[] = []; + + for (const group of meta.groups) { + const typeName = `${group.name}Config`; + sections.push(generateGroupType({ group, typeName })); + rootFields.push( + [ + jsdoc({ summary: group.summary, description: group.description }), + `\t${group.name}${group.fields?.some((field) => field.validations?.some((validation) => validation.type === "required")) ? "" : "?"}: ${typeName}`, + ] + .filter(Boolean) + .join("\n"), + ); + } + + sections.push( + [ + jsdoc({ + summary: "is the root of a Refinery config.yaml file.", + description: + "Groups containing globally required fields must be present; other groups use Refinery defaults.", + }), + `export type RefineryConfig = {`, + rootFields.join("\n"), + `}`, + ].join("\n"), + ); + + return sections.join("\n\n"); +} + +function generateRulesTypes(meta: MetadataFile): string { + const sections: string[] = []; + const samplerGroupNames: string[] = []; + let samplersGroup: MetadataGroup | undefined; + let downstreamSamplerNames: string[] = []; + + for (const group of meta.groups) { + if (group.type === "samplerList") { + samplersGroup = group; + continue; + } + + if (group.name.endsWith("Sampler")) { + samplerGroupNames.push(group.name); + } + + const samplerField = group.fields?.find( + (field) => field.name === "Sampler", + ); + const validChildren = samplerField?.validations?.find( + (validation) => validation.type === "validChildren", + )?.arg; + if (Array.isArray(validChildren)) { + downstreamSamplerNames = validChildren.map(String); + } + + const typeName = RULES_GROUP_TYPE_NAMES[group.name] ?? group.name; + sections.push(generateGroupType({ group, typeName })); + } + + if (samplerGroupNames.length === 0 || downstreamSamplerNames.length === 0) { + throw new Error( + "rulesMeta.yaml is missing sampler groups or the Rules.Sampler validChildren list", + ); + } + + const singleKey = (name: string, names: string[]) => + `{ ${names.map((key) => (key === name ? `${key}: ${key}` : `${key}?: never`)).join("; ")} }`; + + sections.push( + [ + jsdoc({ + summary: + "is a sampler allowed as the downstream `Sampler` of a rules-based sampler rule.", + description: + "Derived from the `validChildren` validation on `Rules.Sampler` in rulesMeta.yaml.", + }), + `export type RulesBasedDownstreamSampler =`, + downstreamSamplerNames + .map((name) => `\t| ${singleKey(name, samplerGroupNames)}`) + .join("\n"), + ].join("\n"), + ); + + sections.push( + [ + jsdoc({ + summary: + "is the sampler configuration for a single target (Honeycomb environment).", + description: + "Exactly one sampler type must be set per target; other sampler keys are explicitly forbidden in each union member.", + }), + `export type RefinerySamplerChoice =`, + samplerGroupNames + .map((name) => `\t| ${singleKey(name, samplerGroupNames)}`) + .join("\n"), + ].join("\n"), + ); + + sections.push( + [ + jsdoc({ + summary: "is the root of a Refinery rules.yaml file.", + description: samplersGroup?.description, + }), + `export type RefineryRules = {`, + `\tRulesVersion: 2`, + `\tSamplers: { __default__: RefinerySamplerChoice } & Record<`, + `\t\tstring,`, + `\t\tRefinerySamplerChoice`, + `\t>`, + `}`, + ].join("\n"), + ); + + return sections.join("\n\n"); +} + +function generateGroupType(args: { + group: MetadataGroup; + typeName: string; +}): string { + const { group, typeName } = args; + const lines: string[] = []; + + const groupDoc = jsdoc({ + summary: group.summary ?? group.title, + description: group.description, + }); + if (groupDoc) { + lines.push(groupDoc); + } + + lines.push(`export type ${typeName} = {`); + for (const field of group.fields ?? []) { + const fieldDoc = jsdoc({ + summary: field.summary, + description: field.description, + default: field.default, + }); + if (fieldDoc) { + lines.push(indent(fieldDoc)); + } + const optional = isRequired(field) ? "" : "?"; + lines.push(`\t${field.name}${optional}: ${fieldType({ group, field })}`); + } + lines.push(`}`); + const constraints = new Set(); + for (const field of group.fields ?? []) { + for (const validation of field.validations ?? []) { + if ( + validation.type !== "conflictsWith" && + validation.type !== "requiredWith" + ) + continue; + const other = validation.arg; + if ( + typeof other !== "string" || + !group.fields?.some((candidate) => candidate.name === other) + ) { + throw new Error( + `Invalid ${validation.type} reference for ${group.name}.${field.name}`, + ); + } + if (validation.type === "conflictsWith") { + const [left, right] = [field.name, other].sort(); + constraints.add(` & ({ ${left}?: never } | { ${right}?: never })`); + } else if (field.default === undefined) { + // Runtime validation accepts a missing dependent field when it has a default. + constraints.add( + ` & ({ ${other}?: never } | { ${field.name}: ${fieldType({ group, field })} })`, + ); + } + } + } + lines.push(...constraints); + + return lines.join("\n"); +} + +function isRequired(field: MetadataField): boolean { + return ( + field.validations?.some( + (validation) => + validation.type === "required" || validation.type === "requiredInGroup", + ) ?? false + ); +} + +function fieldType(args: { + group: MetadataGroup; + field: MetadataField; +}): string { + const { group, field } = args; + + const structural = STRUCTURAL_FIELD_TYPES[`${group.name}.${field.name}`]; + if (structural) { + return structural; + } + + if (group.name === "Rules" && field.name === "Sampler") { + return "RulesBasedDownstreamSampler"; + } + + if (field.choices?.length) { + return field.choices.map((choice) => JSON.stringify(choice)).join(" | "); + } + + switch (field.type) { + case "int": + case "float": + case "percentage": + return "number"; + case "bool": + case "defaulttrue": + return "boolean"; + case "string": + case "duration": + case "hostport": + case "url": + return "string"; + case "memorysize": + return "string | number"; + case "stringarray": + return "string[]"; + case "map": + return "Record"; + default: + throw new Error( + `Unhandled metadata field type "${field.type}" for ${group.name}.${field.name}. ` + + `Add a mapping in config-types/generate.ts.`, + ); + } +} + +function jsdoc(args: { + summary?: string | undefined; + description?: string | undefined; + default?: unknown; +}): string { + const bodyLines: string[] = []; + if (args.summary) { + bodyLines.push(...wrapText(args.summary.trim())); + } + if (args.description) { + if (bodyLines.length) { + bodyLines.push(""); + } + for (const paragraph of args.description.trim().split(/\n{2,}/)) { + if (bodyLines.length && bodyLines[bodyLines.length - 1] !== "") { + bodyLines.push(""); + } + bodyLines.push(...wrapText(paragraph.replaceAll(/\s+/g, " ").trim())); + } + } + if (args.default !== undefined && args.default !== "") { + if (bodyLines.length) { + bodyLines.push(""); + } + bodyLines.push(`@default ${JSON.stringify(args.default)}`); + } + + if (bodyLines.length === 0) { + return ""; + } + + return [ + "/**", + ...bodyLines.map((line) => ` * ${line}`.trimEnd()), + " */", + ].join("\n"); +} + +function wrapText(text: string, width = 76): string[] { + // Comment terminators inside metadata descriptions would break the JSDoc. + const safe = text.replaceAll("*/", "*\\/"); + const words = safe.split(" "); + const lines: string[] = []; + let current = ""; + for (const word of words) { + if (current && current.length + 1 + word.length > width) { + lines.push(current); + current = word; + } else { + current = current ? `${current} ${word}` : word; + } + } + if (current) { + lines.push(current); + } + return lines; +} + +function indent(block: string): string { + return block + .split("\n") + .map((line) => `\t${line}`) + .join("\n"); +} + +// Importing the generator for fixture tests must not rewrite the declarations. +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const generated = generateRefineryConfigTypes(); + if (process.argv.includes("--check")) { + if (readFileSync(output, "utf8") !== generated) { + throw new Error( + "Generated Refinery types are stale. Run npm run generate.", + ); + } + } else { + writeFileSync(output, generated); + console.log(`Generated ${fileURLToPath(output)}`); + } +} diff --git a/config-types/index.d.ts b/config-types/index.d.ts new file mode 100644 index 0000000..38b8fd1 --- /dev/null +++ b/config-types/index.d.ts @@ -0,0 +1,2459 @@ +// Generated by npm run generate. Do not edit. +// Source: config/metadata/{configMeta,rulesMeta}.yaml +// Metadata SHA-256: 72e1f0411df7643ce7783736ff77e482ba0e782854309753e6adecfbad9e10ce + +/** + * General Configuration + * + * contains general configuration options that apply to the entire Refinery + * process. + */ +export type GeneralConfig = { + /** + * is the file format of this particular configuration file. + * + * This file is version 2. This field is required. It exists to allow the + * configuration system to adapt to future changes in the configuration file + * format. + * + * @default 2 + */ + ConfigurationVersion: number + /** + * is the minimum version of Refinery that can load this configuration file. + * + * This setting specifies the lowest Refinery version capable of loading all of + * the features used in this file. If this value is present, then Refinery will + * refuse to start if its version is less than this setting. + * + * @default "v2.0" + */ + MinRefineryVersion?: string + /** + * is a prefix that can be used to distinguish a dataset from an environment in + * the rules. + * + * If telemetry is being sent to both a classic dataset and a new environment + * called the same thing, such as `production`, then this parameter can be used + * to distinguish these cases. When Refinery receives telemetry using an API + * key associated with a Honeycomb Classic dataset, it will then use the prefix + * in the form `{prefix}. {dataset}` when trying to resolve the rules + * definition. + */ + DatasetPrefix?: string + /** + * is the average interval between attempts at reloading the configuration + * file. + * + * Refinery will attempt to read its configuration and check for changes at + * approximately this interval. This time is varied by a random amount up to + * 10% to avoid all instances refreshing together. In installations where + * configuration changes are handled by restarting Refinery, which is often the + * case when using Kubernetes, disable this feature with a value of `0s`. As of + * Refinery v2.7, news of a configuration change is immediately propagated to + * all peers, and they will attempt to reload their configurations. Note that + * external factors (for example, Kubernetes ConfigMaps) may cause delays in + * propagating configuration changes. + * + * @default "15s" + */ + ConfigReloadInterval?: string +} + +/** + * Network Configuration + * + * contains network configuration options. + */ +export type NetworkConfig = { + /** + * is the address where Refinery listens for incoming requests. + * + * This setting is the IP and port on which Refinery listens for incoming HTTP + * requests. These requests include traffic formatted as Honeycomb events, + * proxied requests to the Honeycomb API, and OpenTelemetry data using the + * `http` protocol. Incoming traffic is expected to be HTTP, so if SSL is a + * requirement, put something like `nginx` in front to do the decryption. + * + * @default "0.0.0.0:8080" + */ + ListenAddr?: string + /** + * is the IP and port on which to listen for traffic being rerouted from a + * peer. + * + * Incoming traffic is expected to be HTTP, so if using SSL use something like + * nginx or a load balancer to do the decryption. + * + * @default "0.0.0.0:8081" + */ + PeerListenAddr?: string + /** + * is the duration the http server waits for activity on the connection. + * + * This is the amount of time after which if the http server does not see any + * activity, then it pings the client to see if the transport is still alive. + * "0s" means no timeout. + * + * @default "0s" + */ + HTTPIdleTimeout?: string + /** + * is the URL of the upstream Honeycomb API where the data will be sent. + * + * This setting is the destination to which Refinery sends all events that it + * decides to keep. + * + * @default "https://api.honeycomb.io" + */ + HoneycombAPI?: string + /** + * is a map of additional HTTP headers to add to all upstream Honeycomb API + * requests. + * + * These headers will be added to all HTTP requests made to the upstream + * Honeycomb API endpoint, including trace data, OTel metrics, OTel traces, and + * logs. This is useful for scenarios where requests need to pass through an + * mTLS proxy that requires additional headers like FORWARD_TO_URL. Both keys + * and values must be strings. Reserved Honeycomb header prefixes + * ("x-honeycomb-" and "x-hny-") cannot be set. + */ + AdditionalHeaders?: Record +} + +/** + * OpAMP Configuration + * + * contains OpAMP configuration options. OpAMP support is an experimental + * feature, subject to breaking changes at any time. Not intended or supported + * for customer use. **FOR INTERNAL USE ONLY** + */ +export type OpAMPConfig = { + /** + * controls whether to enable OpAMP support. + * + * OpAMP support is an experimental feature, subject to breaking changes at any + * time. Not intended or supported for customer use. **FOR INTERNAL USE ONLY** + * + * @default false + */ + Enabled?: boolean + /** + * controls whether to record usage metrics. + * + * OpAMP support is an experimental feature, subject to breaking changes at any + * time. Not intended or supported for customer use. **FOR INTERNAL USE ONLY** + * + * @default true + */ + RecordUsage?: boolean + /** + * is the URL of the OpAMP server for this client. + * + * OpAMP support is an experimental feature, subject to breaking changes at any + * time. Not intended or supported for customer use. **FOR INTERNAL USE ONLY** + * + * @default "wss://127.0.0.1:4320/v1/opamp" + */ + Endpoint?: string +} + +/** + * Access Key Configuration + * + * contains access keys -- API keys that the proxy will treat specially, and + * other flags that control how the proxy handles API keys. + */ +export type AccessKeysConfig = { + /** + * is a set of Honeycomb API keys that the proxy will treat specially. + * + * This list only applies to span traffic - other Honeycomb API actions will be + * proxied through to the upstream API directly without modifying keys. + */ + ReceiveKeys?: string[] + /** + * is a set of Honeycomb Ingest Key IDs that the proxy will treat specially. + * + * When `AcceptOnlyListedKeys` is `true`, traffic using an API key whose + * Honeycomb ingest key ID matches an entry in this list will be accepted. The + * key ID is the `id` field returned by the Honeycomb `/1/auth` endpoint; it is + * distinct from the full API key value. This allows authorization based on key + * IDs rather than full key values, which avoids storing secret key material in + * the configuration file. Both `ReceiveKeys` and `ReceiveKeyIDs` may be used + * simultaneously. Note: This feature does not support legacy API keys. Only + * Honeycomb Ingest Keys (which have a key ID) are compatible with this + * setting. + */ + ReceiveKeyIDs?: string[] + /** + * is a boolean flag that causes events arriving with API keys not in the + * `ReceiveKeys` list to be rejected. + * + * If `true`, then only traffic using the keys listed in `ReceiveKeys` or whose + * key ID is listed in `ReceiveKeyIDs` is accepted. Events arriving with API + * keys not in either list will be rejected with an HTTP `401` error. If + * `false`, then all traffic is accepted and `ReceiveKeys` is ignored. This + * setting is applied **before** the `SendKey` and `SendKeyMode` settings. + * + * @default false + */ + AcceptOnlyListedKeys?: boolean + /** + * is an optional Honeycomb API key that Refinery can use to send data to + * Honeycomb, depending on configuration. Setting this value via a command line + * flag may expose credentials - it is recommended to use the environment + * variable or a configuration file. + * + * If `SendKey` is set to a valid Honeycomb key, then Refinery can use the + * listed key to send data. The exact behavior depends on the value of + * `SendKeyMode`. + */ + SendKey?: string + /** + * controls how SendKey is used to replace or augment API keys used in incoming + * telemetry. + * + * controls how SendKey is used to replace or supply API keys used in incoming + * telemetry. If `AcceptOnlyListedKeys` is `true`, then `SendKeys` will only be + * used for events with keys listed in `ReceiveKeys`. `none` uses the incoming + * key for all telemetry (default). `all` overwrites all keys, even missing + * ones, with `SendKey`. `nonblank` overwrites all supplied keys but will not + * inject `SendKey` if the incoming key is blank. `listedonly` overwrites only + * the keys listed in `ReceiveKeys`. `unlisted` uses the `SendKey` for all + * events *except* those with keys listed in `ReceiveKeys`, which use their + * original keys. `missingonly` uses the SendKey only to inject keys into + * events with blank keys. All other events use their original keys. + * + * @default "none" + */ + SendKeyMode?: "none" | "all" | "nonblank" | "listedonly" | "unlisted" | "missingonly" +} + +/** + * Refinery Telemetry + * + * contains configuration information for the telemetry that Refinery uses to + * record its own operation. + */ +export type RefineryTelemetryConfig = { + /** + * controls whether to decorate traces with Refinery rule evaluation results. + * + * When enabled, this setting causes traces that are sent to Honeycomb to + * include the field `meta.refinery.reason`. This field contains text + * indicating which rule was evaluated that caused the trace to be included. + * This setting also includes the field `meta.refinery.send_reason`, which + * contains the reason that the trace was sent. Possible values of this field + * are `trace_send_got_root`, which means that the root span arrived; + * `trace_send_expired`, which means that `TraceTimeout` was reached; + * `trace_send_ejected_full`, which means that the trace cache was full; and + * `trace_send_ejected_memsize`, which means that Refinery was out of memory. + * These names are also the names of metrics that Refinery tracks. We recommend + * enabling this setting whenever a rules-based sampler is in use, as it is + * useful for debugging and understanding the behavior of your Refinery + * installation. + * + * @default false + */ + AddRuleReasonToTrace?: boolean + /** + * controls whether to add a metadata field to root spans that indicates the + * number of child elements in a trace. + * + * The added metadata field, `meta.span_count`, indicates the number of child + * elements on the trace at the time the sampling decision was made. If `true` + * and `AddCountsToRoot` is set to false, then Refinery will add + * `meta.span_count` to the root span. + * + * @default true + */ + AddSpanCountToRoot?: boolean + /** + * controls whether to add metadata fields to root spans that indicates the + * number of child spans, span events, span links, and honeycomb events. + * + * If `true`, then Refinery will ignore the `AddSpanCountToRoot` setting and + * add the following fields to the root span based on the values at the time + * the sampling decision was made: - `meta.span_count`: the number of child + * spans on the trace - `meta.span_event_count`: the number of span events on + * the trace - `meta.span_link_count`: the number of span links on the trace - + * `meta.event_count`: the number of honeycomb events on the trace + * + * @default false + */ + AddCountsToRoot?: boolean + /** + * specifies whether to add host metadata to traces. + * + * If `true`, then Refinery will add the following tag to all traces: - + * `meta.refinery.local_hostname`: the hostname of the Refinery node + * + * @default true + */ + AddHostMetadataToTrace?: boolean +} + +/** + * Traces + * + * contains configuration for how traces are managed. + */ +export type TracesConfig = { + /** + * is the duration to wait after the root span arrives before sending a trace. + * + * This setting is a short timer that is triggered when a trace is marked + * complete by the arrival of the root span. Refinery waits for this duration + * before sending the trace. This setting exists to allow for asynchronous + * spans and small network delays to elapse before sending the trace. + * `SendDelay` is not applied if the `TraceTimeout` expires or the `SpanLimit` + * is reached. + * + * @default "2s" + */ + SendDelay?: string + /** + * is how frequently Refinery sends unfulfilled batches. + * + * By default, this setting uses the `DefaultBatchTimeout` in `libhoney` as its + * value, which is `100ms`. + */ + BatchTimeout?: string + /** + * is the duration to wait before making the trace decision on an incomplete + * trace. + * + * A long timer; it represents the outside boundary of how long to wait before + * making the trace decision about an incomplete trace. Normally trace + * decisions (send or drop) are made when the root span arrives. Sometimes the + * root span never arrives (for example, due to crashes). Once this timer + * fires, Refinery will make a trace decision based on the spans that have + * arrived so far. This ensures sending a trace even when the root span never + * arrives. After the trace decision has been made, Refinery retains a record + * of that decision for a period of time. When additional spans (including the + * root span) arrive, they will be kept or dropped based on the original + * decision. If particularly long-lived traces are present in your data, then + * you should increase this timer. Note that this increase will also increase + * the memory requirements for Refinery. + * + * @default "60s" + */ + TraceTimeout?: string + /** + * is the number of spans after which a trace becomes eligible for a trace + * decision. + * + * This setting helps to keep memory usage under control. If a trace has more + * than this set number of spans, then it becomes eligible for a trace + * decision. It's most helpful in a situation where a sudden burst of many + * spans in a large trace hits Refinery all at once, causing memory usage to + * spike and possibly crashing Refinery. + * + * @default 32000 + */ + SpanLimit?: number + /** + * is the maximum number of events to be included in each batch for sending. + * + * This value is used to set the `BatchSize` field in the `libhoney` library + * used to send data to Honeycomb. If you have particularly large traces, then + * you should increase this value. Note that this will also increase the memory + * requirements for Refinery. + * + * @default 500 + */ + MaxBatchSize?: number + /** + * is the interval between checks for traces to send. + * + * A short timer that determines the duration between trace cache review runs + * to send. Increasing this will spend more time processing incoming events to + * reduce `incoming_` or `peer_router_dropped` spikes. Decreasing this will + * check the trace cache for timeouts more frequently. + * + * @default "100ms" + */ + SendTicker?: string + /** + * is the maximum number of expired traces to process. + * + * This setting controls how many traces are processed when it is time to make + * a sampling decision. Up to this many traces will be processed every + * `SendTicker` duration. If this number is too small it will mean Refinery is + * spending less time calculating sampling decisions, resulting in data + * arriving at Honeycomb slower. Additionally, MaxExpiredTraces indirectly + * affects the system’s health check behavior. The HealthCheckTimeout will be + * automatically adjusted based on this value to ensure health checks remain + * accurate relative to the configured processing load. If your + * `collector_collect_loop_duration_ms` is above 3 seconds it is recommended to + * reduce this value and the `SendTicker` duration. This will mean Refinery + * makes fewer sampling decision calculations each `SendTicker` tick, but gets + * the chance to make decisions more often. + * + * @default 3000 + */ + MaxExpiredTraces?: number +} + +/** + * Debugging + * + * contains configuration values used when setting up and debugging Refinery. + */ +export type DebuggingConfig = { + /** + * is the IP and port where the debug service runs. + * + * The debug service is generally only used when debugging Refinery itself, and + * will only run if the command line option `-d` is specified. If this value is + * not specified, then the debug service runs on the first open port between + * `localhost:6060` and `localhost:6069`. + */ + DebugServiceAddr?: string + /** + * is the token that must be specified to access the `/query` endpoint. Setting + * this value via a command line flag may expose credentials - it is + * recommended to use the environment variable or a configuration file. + * + * This token must be specified with the header "X-Honeycomb-Refinery-Query" in + * order for a `/query` request to succeed. These `/query` requests are + * intended for debugging Refinery during setup and are not typically needed in + * normal operation. If not specified, then the `/query` endpoints are + * inaccessible. + */ + QueryAuthToken?: string + /** + * is a list of span fields to include when logging errors happen during the + * ingestion of events. + * + * For example, the span too large error. This is primarily useful in trying to + * track down misbehaving senders in a large installation. The fields + * `dataset`, `apihost`, and `environment` are always included. If a field is + * not present in the span, then it will not be present in the error log. + */ + AdditionalErrorFields?: string[] + /** + * controls whether sampling is applied to incoming traces. + * + * If enabled, then Refinery marks the traces that would be dropped given the + * current sampling rules, and sends all traces regardless of the sampling + * decision. This is useful for evaluating sampling rules. When DryRun is + * enabled, traces is decorated with `meta.refinery. dryrun.kept` that is set + * to `true` or `false`, based on whether the trace would be kept or dropped. + * In addition, `SampleRate` will be set to the incoming rate for all traces, + * and the field `meta.refinery.dryrun.sample_rate` will be set to the sample + * rate that would have been used. NOTE: This setting is not compatible with + * `TraceCache=distributed`, because drop trace decisions shared among peers do + * not contain all the relevant information needed to send traces to Honeycomb. + * + * @default false + */ + DryRun?: boolean +} + +/** + * Refinery Logger + * + * contains configuration for logging. + */ +export type LoggerConfig = { + /** + * is the type of logger to use. + * + * The setting specifies where (and if) Refinery sends logs. `none` means that + * logs are discarded. `honeycomb` means that logs will be forwarded to + * Honeycomb as events according to the set Logging settings. `stdout` means + * that logs will be written to `stdout`. + * + * @default "stdout" + */ + Type?: "stdout" | "honeycomb" | "none" + /** + * is the logging level above which Refinery should send a log to the logger. + * + * `warn` is the recommended level for production. `debug` is very verbose, and + * should not be used in production environments. + * + * @default "warn" + */ + Level?: "debug" | "info" | "warn" | "error" | "panic" +} + +/** + * Honeycomb Logger + * + * contains configuration for logging to Honeycomb. Only used if `Logger.Type` + * is "honeycomb". + */ +export type HoneycombLoggerConfig = { + /** + * is the URL of the Honeycomb API where Refinery sends its logs. + * + * Refinery's internal logs will be sent to this host using the standard + * Honeycomb Events API. + * + * @default "https://api.honeycomb.io" + */ + APIHost?: string + /** + * is the API key used to send Refinery's logs to Honeycomb. Setting this value + * via a command line flag may expose credentials - it is recommended to use + * the environment variable or a configuration file. + * + * It is recommended that you create a separate team and key for Refinery logs. + */ + APIKey?: string + /** + * is the dataset to which logs will be sent. + * + * Only used if `APIKey` is specified. + * + * @default "Refinery Logs" + */ + Dataset?: string + /** + * controls whether logs are sampled before sending to Honeycomb. + * + * The sample rate is controlled by the `SamplerThroughput` setting. The + * sampler used throttles the rate of logs sent to Honeycomb from any given + * source within Refinery -- it should effectively limit the rate of redundant + * messages. + * + * @default true + */ + SamplerEnabled?: boolean + /** + * is the sampling throughput for logs in events per second. + * + * The sampling algorithm attempts to make sure that the average throughput + * approximates this value, while also ensuring that all unique logs arrive at + * Honeycomb at least once per sampling period. + * + * @default 10 + */ + SamplerThroughput?: number + /** + * adds the provided attributes to all logs written by the Honeycomb logger. + * + * When supplying via a environment variable, the value should be a string of + * comma-separated key-value pairs. When supplying via the command line, the + * value should be a key value pair. If multiple key-value pairs are needed, + * each should be supplied via its own command line flag. The key-value pairs + * must use ':' as the separator. + */ + AdditionalAttributes?: Record +} + +/** + * Stdout Logger + * + * contains configuration for logging to `stdout`. Only used if `Logger.Type` + * is "stdout". + */ +export type StdoutLoggerConfig = { + /** + * controls whether to use structured logging. + * + * `true` generates structured logs (JSON). `false` generates plain text logs. + * + * @default false + */ + Structured?: boolean + /** + * controls whether logs are sampled before sending to `stdout`. + * + * The sample rate is controlled by the `SamplerThroughput` setting. + * + * @default false + */ + SamplerEnabled?: boolean + /** + * is the sampling throughput for logs in events per second. + * + * The sampling algorithm attempts to make sure that the average throughput + * approximates this value, while also ensuring that all unique logs arrive at + * `stdout` at least once per sampling period. + * + * @default 10 + */ + SamplerThroughput?: number +} + +/** + * Prometheus Metrics + * + * contains configuration for Refinery's internally-generated metrics as made + * available through Prometheus. + */ +export type PrometheusMetricsConfig = { + /** + * controls whether to expose Refinery metrics over the `PrometheusListenAddr` + * port. + * + * Each of the metrics providers can be enabled or disabled independently. + * Metrics can be sent to multiple destinations. + * + * @default false + */ + Enabled?: boolean + /** + * is the IP and port the Prometheus Metrics server will run on. + * + * Determines the interface and port on which Prometheus will listen for + * requests for `/metrics`. Must be different from the main Refinery listener. + * Only used if `Enabled` is `true` in `PrometheusMetrics`. + * + * @default "localhost:2112" + */ + ListenAddr?: string +} + +/** + * Legacy Metrics + * + * contains configuration for Refinery's legacy metrics. Version 1.x of + * Refinery used this format for sending Metrics to Honeycomb. The metrics + * generated that way are nonstandard and will be deprecated in a future + * release. New installations should prefer `OTelMetrics`. + */ +export type LegacyMetricsConfig = { + /** + * controls whether to send legacy-formatted metrics to Honeycomb. + * + * Each of the metrics providers can be enabled or disabled independently. + * Metrics can be sent to multiple destinations. + * + * @default false + */ + Enabled?: boolean + /** + * is the URL of the Honeycomb API where legacy metrics are sent. + * + * Refinery's internal metrics will be sent to this host using the standard + * Honeycomb Events API. + * + * @default "https://api.honeycomb.io" + */ + APIHost?: string + /** + * is the API key used by Refinery to send its metrics to Honeycomb. Setting + * this value via a command line flag may expose credentials - it is + * recommended to use the environment variable or a configuration file. + * + * It is recommended that you create a separate team and key for Refinery + * metrics. + */ + APIKey?: string + /** + * is the Honeycomb dataset where Refinery sends its metrics. + * + * Only used if `APIKey` is specified. + * + * @default "Refinery Metrics" + */ + Dataset?: string + /** + * is the interval between sending legacy metrics to Honeycomb. + * + * Between 1 and 60 seconds is typical. + * + * @default "30s" + */ + ReportingInterval?: string +} + +/** + * OpenTelemetry Metrics + * + * contains configuration for Refinery's OpenTelemetry (OTel) metrics. This is + * the preferred way to send metrics to Honeycomb. New installations should + * prefer `OTelMetrics`. + */ +export type OTelMetricsConfig = { + /** + * controls whether to send metrics via OpenTelemetry. + * + * Each of the metrics providers can be enabled or disabled independently. + * Metrics can be sent to multiple destinations. + * + * @default false + */ + Enabled?: boolean + /** + * is the URL of the OpenTelemetry API to which metrics will be sent. + * + * Refinery's internal metrics will be sent to the `/v1/metrics` endpoint on + * this host. + * + * @default "https://api.honeycomb.io" + */ + APIHost?: string + /** + * is the API key used to send Honeycomb metrics via OpenTelemetry. Setting + * this value via a command line flag may expose credentials - it is + * recommended to use the environment variable or a configuration file. + * + * It is recommended that you create a separate team and key for Refinery + * metrics. If this is blank, then Refinery will not set the Honeycomb-specific + * headers for OpenTelemetry, and your `APIHost` must be set to a valid + * OpenTelemetry endpoint. + */ + APIKey?: string + /** + * is the Honeycomb dataset that Refinery sends its OpenTelemetry metrics. + * + * Only used if `APIKey` is specified. + * + * @default "Refinery Metrics" + */ + Dataset?: string + /** + * is the interval between sending OpenTelemetry metrics to Honeycomb. + * + * Between `1` and `60` seconds is typical. + * + * @default "30s" + */ + ReportingInterval?: string + /** + * is the compression algorithm to use when sending OpenTelemetry metrics to + * Honeycomb. + * + * `gzip` is the default and recommended value. In rare circumstances, + * compression costs may outweigh the benefits, in which case `none` may be + * used. + * + * @default "gzip" + */ + Compression?: "none" | "gzip" + /** + * adds the provided attributes as resource attributes on all OpenTelemetry + * metrics emitted by Refinery. + * + * This is useful for injecting deployment-specific metadata (such as a cluster + * ID or environment name) into metrics so they can be filtered or grouped in + * the metrics backend. Both keys and values must be strings. When supplying + * via a environment variable, the value should be a string of comma-separated + * key-value pairs. When supplying via the command line, the value should be a + * key value pair. If multiple key-value pairs are needed, each should be + * supplied via its own command line flag. The key-value pairs must use ':' as + * the separator. + */ + AdditionalAttributes?: Record +} + +/** + * OpenTelemetry Tracing + * + * contains configuration for Refinery's own tracing. + */ +export type OTelTracingConfig = { + /** + * controls whether to send Refinery's own OpenTelemetry traces. + * + * The setting specifies if Refinery sends traces. + * + * @default false + */ + Enabled?: boolean + /** + * is the URL of the OpenTelemetry API to which traces will be sent. + * + * Refinery's internal traces will be sent to the `/v1/traces` endpoint on this + * host. + * + * @default "https://api.honeycomb.io" + */ + APIHost?: string + /** + * is the API key used to send Refinery's traces to Honeycomb. Setting this + * value via a command line flag may expose credentials - it is recommended to + * use the environment variable or a configuration file. + * + * It is recommended that you create a separate team and key for Refinery + * telemetry. If this value is blank, then Refinery will not set the + * Honeycomb-specific headers for OpenTelemetry, and your `APIHost` must be set + * to a valid OpenTelemetry endpoint. + */ + APIKey?: string + /** + * is the Honeycomb dataset to which Refinery sends its OpenTelemetry metrics. + * + * Only used if `APIKey` is specified. + * + * @default "Refinery Traces" + */ + Dataset?: string + /** + * is the rate at which Refinery samples its own traces. + * + * This is the Honeycomb sample rate used to sample traces sent by Refinery. + * Since each incoming span generates multiple outgoing spans, a minimum sample + * rate of `100` is strongly advised. + * + * @default 100 + */ + SampleRate?: number + /** + * controls whether to send Refinery's own OpenTelemetry traces via http + * instead of https. + * + * When true Refinery will export its internal traces over http instead of + * https. Useful if you plan on sending your traces to a different refinery + * instance for tail sampling. + * + * @default false + */ + Insecure?: boolean +} + +/** + * Peer Management + * + * controls how the Refinery cluster communicates between peers. + */ +export type PeerManagementConfig = { + /** + * is the type of peer management to use. + * + * Peer management is the mechanism by which Refinery locates its peers. `file` + * means that Refinery gets its peer list from the Peers list in this config + * file. It also prevents Refinery from using a publish/subscribe mechanism to + * propagate peer lists, stress levels, and configuration changes. `redis` + * means that Refinery uses a Publish/Subscribe mechanism, implemented on + * Redis, to propagate peer lists, stress levels, and notification of + * configuration changes much more quickly than the legacy mechanism. The + * recommended setting is `redis`, especially for new installations. If `redis` + * is specified, fields in `RedisPeerManagement` must also be set. + * + * @default "file" + */ + Type?: "redis" | "file" + /** + * specifies the identifier to use when registering itself with peers. + * + * By default, when using a peer registry, Refinery will use the local hostname + * to identify itself to other peers. If your environment requires something + * else, (for example, if peers cannot resolve each other by name), then you + * can specify the exact identifier, such as an IP address, to use here. + * Overrides `IdentifierInterfaceName`, if both are set. + */ + Identifier?: string + /** + * specifies a network interface to use when finding a local hostname. + * + * By default, when using a peer registry, Refinery will use the local hostname + * to identify itself to other peers. If your environment requires that you use + * IPs as identifiers (for example, if peers cannot resolve each other by + * name), then you can specify the network interface that Refinery is listening + * on here. Refinery will use the first unicast address that it finds on the + * specified network interface as its identifier. + */ + IdentifierInterfaceName?: string + /** + * specifies that Refinery should use an IPV6 address as its identifier. + * + * If using `IdentifierInterfaceName`, Refinery will default to the first IPv4 + * unicast address it finds for the specified interface. If this value is + * specified, then Refinery will use the first IPV6 unicast address found. + * + * @default false + */ + UseIPV6Identifier?: boolean + /** + * is the list of peers to use when Type is "file", excluding self. + * + * This list is ignored when Type is "redis". The format is a list of strings + * of the form "scheme://host:port". + */ + Peers?: string[] +} + +/** + * Redis Peer Management + * + * controls how the Refinery cluster communicates between peers when using + * Redis. Does not apply when `PeerManagement.Type` is "file". + */ +export type RedisPeerManagementConfig = { + /** + * is the host and port of the Redis instance to use for peer cluster + * membership management. + * + * Must be in the form `host:port`. + */ + Host?: string + /** + * is a list of host and port pairs for the instances in a Redis Cluster, and + * used for managing peer cluster membership. + * + * This configuration enables Refinery to connect to a Redis deployment setup + * in Cluster Mode. Each entry in the list should follow the format + * `host:port`. If `ClusterHosts` is specified, the `Host` setting will be + * ignored. + */ + ClusterHosts?: string[] + /** + * is the username used to connect to Redis for peer cluster membership + * management. Setting this value via a command line flag may expose + * credentials - it is recommended to use the environment variable or a + * configuration file. + * + * Many Redis installations do not use this field. + */ + Username?: string + /** + * is the password used to connect to Redis for peer cluster membership + * management. Setting this value via a command line flag may expose + * credentials - it is recommended to use the environment variable or a + * configuration file. + * + * Many Redis installations do not use this field. + */ + Password?: string + /** + * is the string used to connect to Redis for peer cluster membership + * management using an explicit AUTH command. Setting this value via a command + * line flag may expose credentials - it is recommended to use the environment + * variable or a configuration file. + * + * Many Redis installations do not use this field. + */ + AuthCode?: string + /** + * is a cluster identifier used as a prefix for Redis pubsub channel names. + * + * ClusterName is used to namespace Redis pubsub channels when multiple + * Refinery clusters share the same Redis instance. If set, all pubsub channel + * names will be prefixed with this value (e.g., "production:peers"). This + * allows multiple clusters to coexist without interfering with each other. + * Must be alphanumeric if specified. + */ + ClusterName?: string + /** + * is a string used as a prefix for the keys in Redis while storing the peer + * membership. + * + * DEPRECATED: Use ClusterName instead. It might be useful to override this in + * any situation where multiple Refinery clusters or multiple applications want + * to share a single Redis instance. It may not be blank. + * + * @default "refinery" + */ + Prefix?: string + /** + * is the database number to use for the Redis instance storing the peer + * membership. + * + * An integer from 0-15 indicating the database number to use for the Redis + * instance storing the peer membership. It might be useful to set this in any + * situation where multiple Refinery clusters or multiple applications want to + * share a single Redis instance. + * + * @default 0 + */ + Database?: number + /** + * enables TLS when connecting to Redis for peer cluster membership management. + * + * When enabled, this setting sets the `MinVersion` in the TLS configuration to + * `1.2`. + * + * @default false + */ + UseTLS?: boolean + /** + * disables certificate checks when connecting to Redis for peer cluster + * membership management. + * + * This setting is intended for use with self-signed certificates and sets the + * `InsecureSkipVerify` flag within Redis. + * + * @default false + */ + UseTLSInsecure?: boolean + /** + * is the timeout to use when communicating with Redis. + * + * It is rarely necessary to adjust this value. + * + * @default "5s" + */ + Timeout?: string + /** + * controls the way that traces are assigned to Refinery nodes. + * + * The `hash` strategy is strongly recommended, as only 1/N traces (where N is + * the number of nodes) are disrupted when the node count changes. The `legacy` + * strategy uses a simple algorithm that unfortunately causes 1/2 of the + * in-flight traces to be assigned to a different node whenever the number of + * nodes changes. The legacy strategy is **deprecated** and is intended to be + * removed in a future release. In v2.0, the `legacy` strategy will be removed + * along with this setting. + * + * @default "hash" + */ + Strategy?: "legacy" | "hash" +} + +/** + * Collection Settings + * + * contains the settings that are relevant to collecting spans together to make + * traces. If none of the memory settings are used, then Refinery will not + * attempt to limit its memory usage. This is not recommended for production + * use since a burst of traffic could cause Refinery to run out of memory and + * crash. + */ +export type CollectionConfig = { + /** + * Deprecated. Set PeerQueueSize and IncomingQueueSize instead. + * + * NOTE: This setting is now deprecated and no longer controls the cache size. + * Instead the maximum memory usage is controlled by `MaxMemoryPercentage` and + * `MaxAlloc`. Setting this value does still set the default PeerQueueSize to + * (3 x CacheCapacity) and IncomingQueueSize to (3 x CacheCapacity) if they are + * not set. This behavior will be removed in the future. Instead, set + * PeerQueueSize and IncomingQueueSize to the exact values you want. + * + * @default "10_000" + */ + CacheCapacity?: number + /** + * is the maximum number of in-flight spans redirected from other peers stored + * in the peer span queue. + * + * The peer span queue serves as a buffer for spans redirected from other peers + * before they are processed. In the event that this queue reaches its + * capacity, any subsequent spans will be discarded. The size of this queue is + * contingent upon the number of peers within the cluster. Specifically, with N + * peers, the queue's span capacity is determined by (N-1)/N of the total + * number of spans. Its minimum value should be at least three times the + * `CacheCapacity`. + * + * @default "30_000" + */ + PeerQueueSize?: number + /** + * is the number of in-flight spans to keep in the incoming span queue. + * + * The incoming span queue is used to buffer spans before they are processed. + * If this queue fills up, then subsequent spans will be dropped. Its minimum + * value should be at least three times the `CacheCapacity`. + * + * @default "30_000" + */ + IncomingQueueSize?: number + /** + * is the amount of system memory available to the Refinery process. + * + * This value will typically be set through an environment variable controlled + * by the container or deploy script. If this value is zero or not set, then + * `MaxMemoryPercentage` cannot be used to calculate the maximum allocation and + * `MaxAlloc` will be used instead. If set, then this must be a memory size. + * Sizes with standard unit suffixes (such as `MB` and `GiB`) and Kubernetes + * units (such as `M` and `Gi`) are supported. Fractional values with a suffix + * are supported. If `AvailableMemory` is set, `Collections.MaxAlloc` must not + * be defined. A useful value for this setting will leave 1-2GB of pod memory + * for overages. For typical configurations this may be about 85%-90% of the + * pod's total memory. + */ + AvailableMemory?: string | number + /** + * is the maximum percentage of memory that should be allocated by the span + * collector. + * + * If nonzero, then it must be an integer value between 1 and 100, representing + * the target maximum percentage of memory that should be allocated by the span + * collector. If set to a non-zero value, then once per tick (see `SendTicker`) + * the collector will compare total allocated bytes to this calculated value. + * If allocation is too high, then traces will be ejected from the cache early + * to reduce memory. Useful values for this setting are generally in the range + * of 70-90. + * + * @default 75 + */ + MaxMemoryPercentage?: number + /** + * is the maximum number of bytes that should be allocated by the Collector. + * + * If set, then this must be a memory size. Sizes with standard unit suffixes + * (such as `MB` and `GiB`) and Kubernetes units (such as `M` and `Gi`) are + * supported. Fractional values with a suffix are supported. See + * `MaxMemoryPercentage` for more details. If set, + * `Collections.AvailableMemory` must not be defined. + * + * @default 0 + */ + MaxAlloc?: string | number + /** + * controls whether to transmit traces in cache to remaining peers during + * cluster scaling event. + * + * Redistribution is intended to help prevent data loss by reconsolidating + * traces onto the appropriate node after a scaling event. During scale down + * events, all stored spans are forwarded to the new owning Refinery peer + * instance, so it can shut down without dropping spans. During scale up + * events, only stored spans that belong to another Refinery peer instance are + * forwarded, so that instance can make whole trace decisions. Refinery uses + * additional system resources during scale up/down events. If the cluster does + * not have enough resource capacity headroom, a redistribution event can cause + * the cluster to go into stress relief (if enabled). If `true`, Refinery will + * NOT forward live traces in its cache to the rest of the peers when peers + * join or leave the cluster. Disabling redistribution can help to prevent + * disruptive bursts of network traffic when large traces with long + * `TraceTimeout` are redistributed. However, disabling redistribution may also + * cause partial data loss of traces that were in flight when scaling occurred. + * + * @default true + */ + DisableRedistribution?: boolean + /** + * controls the amount of time Refinery waits after each cluster scaling event + * before redistributing in-memory traces. + * + * This value should be longer than the amount of time between individual pod + * changes in a bulk scaling operation (changing the cluster size by more than + * one pod). Each redistribution generates additional traffic between peers. If + * this value is too short, multiple consecutive redistributions will occur and + * the resulting traffic may overwhelm the cluster. + * + * @default "30s" + */ + RedistributionDelay?: string + /** + * controls the maximum time Refinery can use while draining traces at + * shutdown. + * + * This setting controls the duration that Refinery expects to have to drain + * in-process traces before shutting down an instance. When asked to shut down + * gracefully, Refinery stops accepting new spans immediately and drains the + * remaining traces by sending them to remaining peers. This value should be + * set to a bit less than the normal timeout period for shutting down without + * forcibly terminating the process. + * + * @default "15s" + */ + ShutdownDelay?: string + /** + * controls how Refinery handles spans that belong to the same trace in a + * clustered environment. + * + * This feature is experimental, in active development, and not intended for + * broad customer use at this time. **Distributed mode is UNSUPPORTED without + * prior coordination with the Honeycomb product engineering team.** When + * `concentrated`, Refinery will route all spans that belong to the same trace + * to a single peer. This is the default behavior ("Trace Locality") and the + * way Refinery has worked in the past. When `distributed`, Refinery will + * instead keep spans on the node where they were received, and forward proxy + * spans that contain only the key information needed to make a trace decision. + * This can reduce the amount of traffic between peers, and can help avoid a + * situation where a single large trace can cause a memory overrun on a single + * node. If `distributed`, the amount of traffic between peers will be reduced, + * but the amount of traffic between Refinery and Redis will significantly + * increase, because Refinery uses Redis to distribute the trace decisions to + * all nodes in the cluster. It is important to adjust the size of the Redis + * cluster in this case. The total volume of network traffic in `distributed` + * mode should be expected to decrease unless the cluster size is very large + * (hundreds of nodes). NOTE: This setting is not compatible with `DryRun` when + * set to `distributed`. See `DryRun` for more information. + * + * @default "concentrated" + */ + TraceLocalityMode?: string + /** + * controls the maximum duration allowed for collection health checks to + * complete. + * + * The `HealthCheckTimeout` setting specifies the maximum duration allowed for + * the health checks of the collection subsystems to complete. If a subsystem + * does not respond within this timeout period, it will be marked as unhealthy. + * This timeout value should be set carefully to ensure that transient delays + * do not lead to unnecessary failure detection while still allowing for timely + * identification of actual health issues. This timeout should be configured to + * balance responsiveness and stability — allowing for timely detection of real + * health issues without being overly sensitive to brief or harmless delays. + * Refinery will adjust the timeout based on the configured `MaxExpiredTraces`, + * so that health checks remain effective under varying system conditions. + * + * @default "15s" + */ + HealthCheckTimeout?: string + /** + * Maximum size for batching drop decisions. + * + * Specifies the maximum number of drop decisions that can be batched before + * triggering processing. This ensures that drop decisions do not accumulate + * indefinitely, allowing timely and efficient management of spans by + * processing them in bulk. If the number of drop decisions reaches this limit, + * the batch will be processed immediately, even if the timeout has not been + * reached. + * + * @default 1000 + */ + MaxDropDecisionBatchSize?: number + /** + * Interval for sending drop decisions in batches. + * + * Sets the time interval for sending accumulated drop decisions in batches. + * This ensures that drop decisions are processed at regular intervals, + * improving efficiency by reducing the frequency of network transmissions. If + * the maximum batch size (MaxDropDecisionBatchSize) is reached before this + * interval elapses, the batch will be sent immediately. + * + * @default "1s" + */ + DropDecisionSendInterval?: string + /** + * Maximum size for batching kept decisions. + * + * Specifies the maximum number of kept decisions that can be batched before + * triggering processing. This ensures that kept decisions do not accumulate + * indefinitely, allowing timely and efficient management of spans by + * processing them in bulk. If the number of kept decisions reaches this limit, + * the batch will be processed immediately, even if the timeout has not been + * reached. + * + * @default 1000 + */ + MaxKeptDecisionBatchSize?: number + /** + * Interval for sending kept decisions in batches. + * + * Sets the time interval for sending accumulated kept decisions in batches. + * This ensures that kept decisions are processed at regular intervals, + * improving efficiency by reducing the frequency of network transmissions. If + * the maximum batch size (MaxKeptDecisionBatchSize) is reached before this + * interval elapses, the batch will be sent immediately. + * + * @default "1s" + */ + KeptDecisionSendInterval?: string + /** + * is the number of parallel collection workers to run for trace processing. + * + * Controls the number of parallel collection workers used for processing + * traces. Each worker processes a subset of traces independently using + * consistent hashing. Values greater than 1 enable parallel processing which + * can improve throughput on multi-core systems. The default of 0 means + * Refinery will automatically set this value to the number of logical CPUs + * available at startup. + * + * @default 0 + */ + WorkerCount?: number +} + +/** + * Buffer Sizes + * + * contains the settings that are relevant to the sizes of communications + * buffers. + */ +export type BufferSizesConfig = { + /** + * is the size of the queue used to buffer spans to send to the upstream + * Collector. + * + * The size of the buffer is measured in spans. If the buffer fills up, then + * performance will degrade because Refinery will block while waiting for space + * to become available. If this happens, then you should increase the buffer + * size. + * + * @default "10_000" + */ + UpstreamBufferSize?: number + /** + * is the size of the queue used to buffer spans to send to peer nodes. + * + * The size of the buffer is measured in spans. If the buffer fills up, then + * performance will degrade because Refinery will block while waiting for space + * to become available. If this happens, then you should increase this buffer + * size. + * + * @default "100_000" + */ + PeerBufferSize?: number +} + +/** + * Specialized Configuration + * + * contains special-purpose configuration options that are not typically + * needed. + */ +export type SpecializedConfig = { + /** + * is the duration for which environment information is cached. + * + * This is the amount of time for which Refinery caches environment + * information, which it looks up from Honeycomb for each different `APIKey`. + * This information is used when making sampling decisions. If you have a very + * large number of environments, then you may want to increase this value. + * + * @default "1h" + */ + EnvironmentCacheTTL?: string + /** + * determines whether Refinery will compress span data it forwards to peers. + * + * If it costs money to transmit data between Refinery instances (for example, + * when spread across AWS availability zones), then you almost certainly want + * compression enabled to reduce your bill. The option to disable it is + * provided as an escape hatch for deployments that value lower CPU utilization + * over data transfer costs. + * + * @default true + */ + CompressPeerCommunication?: boolean + /** + * is the collector to use for collecting traces. + * + * Currently, the only supported value is `InMemCollector`. + * + * @default "InMemCollector" + */ + Collector?: "InMemCollector" + /** + * controls how the cache is managed when memory pressure is high. + * + * `resize` means that when a cache overrun occurs, the cache is shrunk and + * never grows again, which is generally not helpful unless it occurs because + * of a permanent change in traffic patterns. In the `impact` strategy, the + * items having the most impact on the cache size are ejected from the cache + * earlier than normal but the cache is not resized. In all cases, it only + * applies if `MaxAlloc` is nonzero. Default is `resize` for compatibility but + * `impact` is recommended for most installations. In v2.0, the only available + * strategy is `impact`, and this setting will be removed. + * + * @default "resize" + */ + CacheOverrunStrategy?: "resize" | "impact" + /** + * is a map that can be used for injecting user-defined attributes into every + * span. + * + * For example, it could be used for naming a Refinery cluster. Both keys and + * values must be strings. + */ + AdditionalAttributes?: Record +} + +/** + * ID Fields + * + * controls the field names to use for the event ID fields. These fields are + * used to identify events that are part of the same trace. + */ +export type IDFieldsConfig = { + /** + * is the list of field names to use for the trace ID. + * + * The first field in the list that is present in an incoming span will be used + * as the trace ID. If none of the fields are present, then Refinery treats the + * span as not being part of a trace and forwards it immediately to Honeycomb. + */ + TraceNames?: string[] + /** + * is the list of field names to use for the parent ID. + * + * The first field in the list that is present in an event will be used as the + * parent ID. A trace without a `parent_id` is assumed to be a root span. + */ + ParentNames?: string[] +} + +/** + * gRPC Server Parameters + * + * controls the parameters of the gRPC server used to receive OpenTelemetry + * data in gRPC format. + */ +export type GRPCServerParametersConfig = { + /** + * specifies whether the gRPC server is enabled. + * + * If `false`, then the gRPC server is not started and no gRPC traffic is + * accepted. + * + * @default true + */ + Enabled?: boolean + /** + * is the address Refinery listens to for incoming GRPC OpenTelemetry events. + * + * Incoming traffic is expected to be unencrypted, so if using SSL, then put + * something like `nginx` in front to do the decryption. + */ + ListenAddr?: string + /** + * is the amount of time to permit an idle connection. + * + * A duration for the amount of time after which an idle connection will be + * closed by sending a GoAway. "Idle" means that there are no active RPCs. "0s" + * sets duration to infinity, but this is not recommended for Refinery + * deployments behind a load balancer, because it will prevent the load + * balancer from distributing load evenly among peers. + * + * @default "1m" + */ + MaxConnectionIdle?: string + /** + * is the maximum amount of time a gRPC connection may exist. + * + * After this duration, the gRPC connection is closed by sending a `GoAway`. A + * random jitter of +/-10% will be added to `MaxConnectionAge` to spread out + * connection storms. `0s` sets duration to infinity; a value measured in low + * minutes will help load balancers to distribute load among peers more evenly. + * + * @default "3m" + */ + MaxConnectionAge?: string + /** + * is the duration beyond `MaxConnectionAge` after which the connection will be + * forcibly closed. + * + * This setting is in case the upstream node ignores the `GoAway` request. "0s" + * sets duration to infinity. + * + * @default "1m" + */ + MaxConnectionAgeGrace?: string + /** + * is the duration between keep-alive pings. + * + * After this amount of time, if the client does not see any activity, then it + * pings the server to see if the transport is still alive. "0s" sets duration + * to 2 hours. + * + * @default "1m" + */ + KeepAlive?: string + /** + * is the duration the server waits for activity on the connection. + * + * This is the amount of time after which if the server does not see any + * activity, then it pings the client to see if the transport is still alive. + * "0s" sets duration to 20 seconds. + * + * @default "20s" + */ + KeepAliveTimeout?: string + /** + * is the maximum message size the server can send. + * + * The server enforces a maximum message size to avoid exhausting the memory + * available to the process by a single request. The size is expressed in + * bytes. + * + * @default "15MB" + */ + MaxSendMsgSize?: string | number + /** + * is the maximum message size the server can receive. + * + * The server enforces a maximum message size to avoid exhausting the memory + * available to the process by a single request. The size is expressed in + * bytes. + * + * @default "15MB" + */ + MaxRecvMsgSize?: string | number +} + +/** + * Sample Cache + * + * controls the sample cache used to retain information about trace status + * after the sampling decision has been made. + */ +export type SampleCacheConfig = { + /** + * is the type of sample cache. + * + * "cuckoo" is a strategy where dropped traces are preserved in a "Cuckoo + * Filter", which can remember a much larger number of dropped traces, leaving + * capacity to retain a much larger number of kept traces. It is also more + * configurable. The "cuckoo" filter is recommended for most installations and + * is the only option available in Refinery 2.0 and later. "legacy" is a + * strategy where both keep and drop decisions are stored in a circular buffer + * that is 5x the size of the trace cache. This is Refinery's original sample + * cache strategy. + * + * @default "cuckoo" + */ + Type?: string + /** + * is the number of traces preserved in the cuckoo kept traces cache. + * + * Refinery keeps a record of each trace that was kept and sent to Honeycomb, + * along with some statistical information. This is most useful in cases where + * the trace was sent before sending the root span, so that the root span can + * be decorated with accurate metadata. Default is `10_000` traces. Each trace + * in this cache consumes roughly 200 bytes. + * + * @default "10_000" + */ + KeptSize?: number + /** + * is the size of the cuckoo dropped traces cache. + * + * This cache consumes 4-6 bytes per trace at a scale of millions of traces. + * Changing its size with live reload sets a future limit, but does not have an + * immediate effect. + * + * @default "1_000_000" + */ + DroppedSize?: number + /** + * controls how often the cuckoo cache re-evaluates its remaining capacity. + * + * This cache is quite resilient so it does not need to happen very often, but + * the operation is also inexpensive. Default is 10 seconds. + * + * @default "10s" + */ + SizeCheckInterval?: string +} + +/** + * Stress Relief + * + * controls the Stress Relief mechanism, which is used to prevent Refinery from + * being overwhelmed by a large number of traces. There is a metric called + * `stress_level` that is emitted as part of Refinery metrics. It is a measure + * of Refinery's throughput rate relative to its processing rate, combined with + * the amount of room in its internal queues, and ranges from `0` to `100`. + * `stress_level` is generally expected to be `0` except under heavy load. When + * stress levels reach `100`, there is an increased chance that Refinery will + * become unstable. To avoid this problem, the Stress Relief system can do + * deterministic sampling on new trace traffic based solely on `TraceID`, + * without having to store traces in the cache or take the time processing + * sampling rules. Existing traces in flight will be processed normally, but + * when Stress Relief is active, trace decisions are made deterministically on + * a per-span basis; all spans will be sampled according to the `SamplingRate` + * specified here. Once Stress Relief activates (by exceeding the + * `ActivationLevel`), it will not deactivate until `stress_level` falls below + * the `DeactivationLevel`. When it deactivates, normal trace decisions are + * made -- and any additional spans that arrive for traces that were active + * during Stress Relief will respect the decisions made during that time. The + * measurement of stress is a lagging indicator and is highly dependent on + * Refinery configuration and scaling. Other configuration values should be + * well tuned first, before adjusting the Stress Relief Activation parameters. + * Stress Relief is not a substitute for proper configuration and scaling, but + * it can be used as a safety valve to prevent Refinery from becoming unstable + * under heavy load. + */ +export type StressReliefConfig = { + /** + * is a string indicating how to use Stress Relief. + * + * This setting sets the Stress Relief mode. "never" means that Stress Relief + * will never activate. "monitor" is the recommended setting, and means that + * Stress Relief will monitor the status of Refinery and activate according to + * the levels set by fields such as `ActivationLevel`. "always" means that + * Stress Relief is always on, which may be useful in an emergency situation. + * + * @default "never" + */ + Mode?: string + /** + * is the `stress_level` (from 0-100) at which Stress Relief is triggered. + * + * This value must be greater than `DeactivationLevel` and should be high + * enough that it is not reached in normal operation. + * + * @default 90 + */ + ActivationLevel?: number + /** + * is the `stress_level` (from 0-100) at which Stress Relief is turned off. + * + * This setting is subject to `MinimumActivationDuration`. The value must be + * less than `ActivationLevel`. + * + * @default 75 + */ + DeactivationLevel?: number + /** + * is the sampling rate to use when Stress Relief is activated. + * + * All new traces will be deterministically sampled at this rate based only on + * the `traceID`. It should be chosen to be a rate that sends fewer samples + * than the average sampling rate Refinery is expected to generate. For + * example, if Refinery is configured to normally sample at a rate of 1 in 10, + * then Stress Relief should be configured to sample at a rate of at least 1 in + * 30. If this value is configured to send more data to Honeycomb during stress + * relief than the normal average sampling strategy, it may overwhelm the + * upstream send queue and have the opposite of the desired effect. + * + * @default 100 + */ + SamplingRate?: number + /** + * is the minimum time that Stress Relief will stay enabled once activated. + * + * This setting helps to prevent oscillations. + * + * @default "10s" + */ + MinimumActivationDuration?: string + /** + * is the minimum time that Stress Relief will stay enabled. + * + * This setting is used when switching into Monitor mode. When Stress Relief is + * enabled, it will start up in stressed mode for at least this set duration of + * time to try to make sure that Refinery can handle the load before it begins + * processing it in earnest. This is to help address the problem of trying to + * bring a new node into an already-overloaded cluster. If this duration is + * `0`, then Refinery will not start in stressed mode, which will provide + * faster startup at the possible cost of startup instability. + * + * @default "3s" + */ + MinimumStartupDuration?: string +} + +/** + * is the root of a Refinery config.yaml file. + * + * Groups containing globally required fields must be present; other groups use + * Refinery defaults. + */ +export type RefineryConfig = { +/** + * contains general configuration options that apply to the entire Refinery + * process. + */ + General: GeneralConfig +/** + * contains network configuration options. + */ + Network?: NetworkConfig +/** + * contains OpAMP configuration options. OpAMP support is an experimental + * feature, subject to breaking changes at any time. Not intended or supported + * for customer use. **FOR INTERNAL USE ONLY** + */ + OpAMP?: OpAMPConfig +/** + * contains access keys -- API keys that the proxy will treat specially, and + * other flags that control how the proxy handles API keys. + */ + AccessKeys?: AccessKeysConfig +/** + * contains configuration information for the telemetry that Refinery uses to + * record its own operation. + */ + RefineryTelemetry?: RefineryTelemetryConfig +/** + * contains configuration for how traces are managed. + */ + Traces?: TracesConfig +/** + * contains configuration values used when setting up and debugging Refinery. + */ + Debugging?: DebuggingConfig +/** + * contains configuration for logging. + */ + Logger?: LoggerConfig +/** + * contains configuration for logging to Honeycomb. Only used if `Logger.Type` + * is "honeycomb". + */ + HoneycombLogger?: HoneycombLoggerConfig +/** + * contains configuration for logging to `stdout`. Only used if `Logger.Type` + * is "stdout". + */ + StdoutLogger?: StdoutLoggerConfig +/** + * contains configuration for Refinery's internally-generated metrics as made + * available through Prometheus. + */ + PrometheusMetrics?: PrometheusMetricsConfig +/** + * contains configuration for Refinery's legacy metrics. Version 1.x of + * Refinery used this format for sending Metrics to Honeycomb. The metrics + * generated that way are nonstandard and will be deprecated in a future + * release. New installations should prefer `OTelMetrics`. + */ + LegacyMetrics?: LegacyMetricsConfig +/** + * contains configuration for Refinery's OpenTelemetry (OTel) metrics. This is + * the preferred way to send metrics to Honeycomb. New installations should + * prefer `OTelMetrics`. + */ + OTelMetrics?: OTelMetricsConfig +/** + * contains configuration for Refinery's own tracing. + */ + OTelTracing?: OTelTracingConfig +/** + * controls how the Refinery cluster communicates between peers. + */ + PeerManagement?: PeerManagementConfig +/** + * controls how the Refinery cluster communicates between peers when using + * Redis. Does not apply when `PeerManagement.Type` is "file". + */ + RedisPeerManagement?: RedisPeerManagementConfig +/** + * contains the settings that are relevant to collecting spans together to make + * traces. If none of the memory settings are used, then Refinery will not + * attempt to limit its memory usage. This is not recommended for production + * use since a burst of traffic could cause Refinery to run out of memory and + * crash. + */ + Collection?: CollectionConfig +/** + * contains the settings that are relevant to the sizes of communications + * buffers. + */ + BufferSizes?: BufferSizesConfig +/** + * contains special-purpose configuration options that are not typically + * needed. + */ + Specialized?: SpecializedConfig +/** + * controls the field names to use for the event ID fields. These fields are + * used to identify events that are part of the same trace. + */ + IDFields?: IDFieldsConfig +/** + * controls the parameters of the gRPC server used to receive OpenTelemetry + * data in gRPC format. + */ + GRPCServerParameters?: GRPCServerParametersConfig +/** + * controls the sample cache used to retain information about trace status + * after the sampling decision has been made. + */ + SampleCache?: SampleCacheConfig +/** + * controls the Stress Relief mechanism, which is used to prevent Refinery from + * being overwhelmed by a large number of traces. There is a metric called + * `stress_level` that is emitted as part of Refinery metrics. It is a measure + * of Refinery's throughput rate relative to its processing rate, combined with + * the amount of room in its internal queues, and ranges from `0` to `100`. + * `stress_level` is generally expected to be `0` except under heavy load. When + * stress levels reach `100`, there is an increased chance that Refinery will + * become unstable. To avoid this problem, the Stress Relief system can do + * deterministic sampling on new trace traffic based solely on `TraceID`, + * without having to store traces in the cache or take the time processing + * sampling rules. Existing traces in flight will be processed normally, but + * when Stress Relief is active, trace decisions are made deterministically on + * a per-span basis; all spans will be sampled according to the `SamplingRate` + * specified here. Once Stress Relief activates (by exceeding the + * `ActivationLevel`), it will not deactivate until `stress_level` falls below + * the `DeactivationLevel`. When it deactivates, normal trace decisions are + * made -- and any additional spans that arrive for traces that were active + * during Stress Relief will respect the decisions made during that time. The + * measurement of stress is a lagging indicator and is highly dependent on + * Refinery configuration and scaling. Other configuration values should be + * well tuned first, before adjusting the Stress Relief Activation parameters. + * Stress Relief is not a substitute for proper configuration and scaling, but + * it can be used as a safety valve to prevent Refinery from becoming unstable + * under heavy load. + */ + StressRelief?: StressReliefConfig +} + +/** + * Deterministic Sampler + * + * The Deterministic Sampler (`DeterministicSampler`) uses a fixed sample rate + * to sample traces based on their trace ID. This is the simplest sampling + * algorithm - it is a static sample rate, choosing traces randomly to either + * keep or send (at the appropriate rate). It is not influenced by the contents + * of the trace other than the trace ID. + */ +export type DeterministicSampler = { + /** + * is the sample rate to use. + * + * The sample rate to use. It indicates a ratio, where one sample trace is kept + * for every N traces seen. For example, a `SampleRate` of `30` will keep 1 out + * of every 30 traces. The choice on whether to keep any specific trace is + * random, so the rate is approximate. The sample rate is calculated from the + * trace ID, so all spans with the same trace ID will be sampled or not sampled + * together. A `SampleRate` of `1` or less will keep all traces. Specifying + * this value is required. + */ + SampleRate: number +} + +/** + * Dynamic Sampler + * + * The Dynamic Sampler (`DynamicSampler`) is the basic Dynamic Sampler + * implementation. Most installations will find the EMA Dynamic Sampler to be a + * better choice. This sampler collects the values of a number of fields from a + * trace and uses them to form a key. This key is handed to the standard + * dynamic sampler algorithm, which generates a sample rate based on the + * frequency with which that key has appeared during the previous + * `ClearFrequency`. See https://github.com/honeycombio/dynsampler-go for more + * detail on the mechanics of the Dynamic Sampler. This sampler uses the + * `AvgSampleRate` algorithm from that package. + */ +export type DynamicSampler = { + /** + * is the sample rate to use. + * + * $DeterministicSampler.SampleRate + */ + SampleRate: number + /** + * is the duration over which the sampler will calculate the sample rate. + * + * The duration after which the Dynamic Sampler should reset its internal + * counters. It should be specified as a duration string. For example, "30s" or + * "1m". Defaults to "30s". + */ + ClearFrequency?: string + /** + * is the list of fields to use to create the key for the Dynamic Sampler. + * + * A list of all the field names to use to form the key that will be handed to + * the Dynamic Sampler. The combination of values from all of these fields + * should reflect how interesting the trace is compared to another. When + * choosing field names for `FieldList`, a good field selection has consistent + * values for high-frequency, boring traffic, and unique values for outliers + * and interesting traffic. Including an error field, or something like `HTTP + * status code`, is an excellent choice. Using fields with very high + * cardinality, like `k8s.pod.id`, is a bad choice. If the combination of + * fields essentially makes each trace unique, then the Dynamic Sampler will + * sample everything. If the combination of fields is not unique enough, then + * you will not be guaranteed samples of the most interesting traces. If a + * trace does not contain any of the fields specified in the FieldList, it will + * still be evaluated by the sampler. However, since it lacks all key fields, + * it will be grouped under a single empty (blank) key. This means that all + * such traces will share the same sample rate, determined by that one shared + * key. As an example, consider as a good set of fields: the combination of + * `HTTP endpoint` (high-frequency and boring), `HTTP method`, and `status + * code` (normally boring but can become interesting when indicating an error) + * since it will allowing proper sampling of all endpoints under normal traffic + * and call out when there is failing traffic to any endpoint. As of Refinery + * 2.8.0, the `root.` prefix can be used to limit the field value to that of + * the root span. For example, `root.http.response.status_code` will only + * consider the `http.response.status_code` field from the root span rather + * than a combination of all the spans in the trace. This is useful when you + * want to sample based on the root span's properties rather than the entire + * trace, and helps to reduce the cardinality of the sampler key. In contrast, + * for example, consider as a bad set of fields: a combination of `HTTP + * endpoint`, `status code`, and `pod id`, since it would result in keys that + * are all unique, and therefore result in sampling 100% of traces. For + * example, rather than a set of fields, using only the `HTTP endpoint` field + * is a **bad** choice, as it is not unique enough, and therefore interesting + * traces, like traces that experienced a `500`, might not be sampled. Field + * names may come from any span in the trace; if they occur on multiple spans, + * then all unique values will be included in the key. + */ + FieldList: string[] + /** + * is the maximum number of keys to track. + * + * Limits the number of distinct keys tracked by the sampler. Once `MaxKeys` is + * reached, new keys will not be included in the sample rate map, but existing + * keys will continue to be be counted. Use this field to keep the sample rate + * map size under control. Defaults to `500`; Dynamic Samplers will rarely + * achieve their sampling goals with more keys than this. + */ + MaxKeys?: number + /** + * indicates whether to include the trace length as part of the key. + * + * Indicates whether to include the trace length (number of spans in the trace) + * as part of the key. The number of spans is exact, so if there are normally + * small variations in trace length, we recommend setting this field to `false` + * (the default). If your traces are consistent lengths and changes in trace + * length is a useful indicator to view in Honeycomb, then set this field to + * `true`. + */ + UseTraceLength?: boolean +} + +/** + * EMA Dynamic Sampler + * + * The Exponential Moving Average (EMA) Dynamic Sampler (`EMADynamicSampler`) + * attempts to average a given sample rate, weighting rare traffic and frequent + * traffic differently so as to end up with the correct average. + * `EMADynamicSampler` is an improvement upon the simple `DynamicSampler` and + * is recommended for many use cases. Based on the `DynamicSampler`, + * `EMADynamicSampler` differs in that rather than compute rate based on a + * periodic sample of traffic, it maintains an Exponential Moving Average of + * counts seen per key, and adjusts this average at regular intervals. The + * weight applied to more recent intervals is defined by `weight`, a number + * between (0, 1). Larger values weight the average more toward recent + * observations. In other words, a larger weight will cause sample rates more + * quickly adapt to traffic patterns, while a smaller weight will result in + * sample rates that are less sensitive to bursts or drops in traffic and thus + * more consistent over time. Keys that are not already present in the EMA will + * always have a sample rate of `1`. Keys that occur more frequently will be + * sampled on a logarithmic curve. Every key will be represented at least once + * in any given window and more frequent keys will have their sample rate + * increased proportionally to trend towards the goal sample rate. + */ +export type EMADynamicSampler = { + /** + * is the sample rate to use. + * + * $DeterministicSampler.SampleRate + */ + GoalSampleRate: number + /** + * is how often the sampler will recalculate the sample rate. + * + * The duration after which the EMA Dynamic Sampler should recalculate its + * internal counters. It should be specified as a duration string. For example, + * `30s` or `1m`. Defaults to `15s`. + */ + AdjustmentInterval?: string + /** + * is the weight to use when calculating the EMA. + * + * The weight to use when calculating the EMA. It should be a number between + * `0` and `1`. Larger values weight the average more toward recent + * observations. In other words, a larger weight will cause sample rates more + * quickly adapt to traffic patterns, while a smaller weight will result in + * sample rates that are less sensitive to bursts or drops in traffic and thus + * more consistent over time. The default value is `0.5`. + */ + Weight?: number + /** + * is the value to use when aging out keys. + * + * Indicates the threshold for removing keys from the EMA. The EMA of any key + * will approach `0` if it is not repeatedly observed, but will never truly + * reach it, so this field determines what constitutes "zero". Keys with + * averages below this threshold will be removed from the EMA. Default is the + * value of `Weight`, as this prevents a key with the smallest integer value + * (1) from being aged out immediately. This value should generally be less + * than (<=) `Weight`, unless you have very specific reasons to set it higher. + */ + AgeOutValue?: number + /** + * is the multiple of the sample rate to use when detecting bursts. + * + * If set, then this value is multiplied by the sum of the running average of + * counts to dynamically define the burst detection threshold. If total counts + * observed for a given interval exceed this threshold, then EMA is updated + * immediately, rather than waiting on the `AdjustmentInterval`. Defaults to + * `2`; a negative value disables. With the default of `2`, if your traffic + * suddenly doubles, then burst detection will kick in. + */ + BurstMultiple?: number + /** + * the number of `AdjustmentIntervals` to wait before detecting bursts. + * + * Indicates the number of intervals to run before burst detection kicks in. + * Defaults to `3`. + */ + BurstDetectionDelay?: number + /** + * is the list of fields to use to create the key for the Dynamic Sampler. + * + * $DynamicSampler.FieldList + */ + FieldList: string[] + /** + * is the maximum number of keys to track. + * + * $DynamicSampler.MaxKeys + */ + MaxKeys?: number + /** + * indicates whether to include the trace length as part of the key. + * + * $DynamicSampler.UseTraceLength + */ + UseTraceLength?: boolean +} + +/** + * EMA Throughput Sampler + * + * The Exponential Moving Average (EMA) Throughput Sampler + * (`EMAThroughputSampler`) attempts to achieve a given throughput -- number of + * spans per second -- weighting rare traffic and frequent traffic differently + * so as to end up with the correct rate. The `EMAThroughputSampler` is an + * improvement upon the Total Throughput Sampler and is recommended for most + * throughput-based use cases. Because it like the `EMADynamicSampler`, + * `EMAThroughputSampler` maintains an Exponential Moving Average of counts + * seen per key, and adjusts this average at regular intervals. The weight + * applied to more recent intervals is defined by `weight`, a number between + * (0, 1) - larger values weight the average more toward recent observations. + * In other words, a larger weight will cause sample rates more quickly adapt + * to traffic patterns, while a smaller weight will result in sample rates that + * are less sensitive to bursts or drops in traffic and thus more consistent + * over time. New keys that are not already present in the EMA will always have + * a sample rate of `1`. Keys that occur more frequently will be sampled on a + * logarithmic curve. Every key will be represented at least once in any given + * window and more frequent keys will have their sample rate increased + * proportionally to trend towards the goal throughput. + */ +export type EMAThroughputSampler = { + /** + * is the desired throughput per second of events sent to Honeycomb. + * + * The desired throughput **per second**. This is the number of events per + * second you want to send to Honeycomb. The sampler will adjust sample rates + * to try to achieve this desired throughput. This value is calculated for the + * individual instance, not for the cluster; if your cluster has multiple + * instances, then you will need to divide your total desired sample rate by + * the number of instances to get this value. + */ + GoalThroughputPerSec: number + /** + * indicates whether to use the cluster size to calculate the goal throughput. + * + * Indicates whether to use the cluster size to calculate the goal throughput. + * If `true`, then the goal throughput will be divided by the number of + * instances in the cluster. If `false` (the default), then the goal throughput + * will be the value specified in `GoalThroughputPerSec`. + */ + UseClusterSize?: boolean + /** + * is the sample rate to use. + * + * `InitialSampleRate` is the sample rate to use during startup, before the + * sampler has accumulated enough data to calculate a reasonable throughput. + * This is mainly useful in situations where unsampled throughput is high + * enough to cause problems. Default value is `10`. + */ + InitialSampleRate?: number + /** + * is how often the sampler will recalculate the sample rate. + * + * $EMADynamicSampler.AdjustmentInterval + */ + AdjustmentInterval?: string + /** + * is the weight to use when calculating the EMA. + * + * $EMADynamicSampler.Weight + */ + Weight?: number + /** + * is the value to use when aging out keys. + * + * $EMADynamicSampler.AgeOutValue + */ + AgeOutValue?: number + /** + * is the multiple of the sample rate to use when detecting bursts. + * + * $EMADynamicSampler.BurstMultiple + */ + BurstMultiple?: number + /** + * the number of `AdjustmentIntervals` to wait before detecting bursts. + * + * $EMADynamicSampler.BurstDetectionDelay + */ + BurstDetectionDelay?: number + /** + * is the list of fields to use to create the key for the Dynamic Sampler. + * + * $DynamicSampler.FieldList + */ + FieldList: string[] + /** + * is the maximum number of keys to track. + * + * $DynamicSampler.MaxKeys + */ + MaxKeys?: number + /** + * indicates whether to include the trace length as part of the key. + * + * $DynamicSampler.UseTraceLength + */ + UseTraceLength?: boolean +} + +/** + * Windowed Throughput Sampler + * + * Windowed Throughput Sampler (`WindowedThroughputSampler`) is an enhanced + * version of total throughput sampling. Just like the `TotalThroughput` + * Sampler, `WindowedThroughputSampler` attempts to meet the goal of fixed + * number of events per second sent to Honeycomb. The original throughput + * sampler updates the sampling rate every "ClearFrequency" seconds. While this + * parameter is configurable, it suffers from the following tradeoff: - + * Decreasing it is more responsive to load spikes, but with the cost of making + * the sampling decision on less data. - Increasing it is less responsive to + * load spikes, but sample rates will be more stable because they are made with + * more data. + * + * The Windowed Throughput Sampler resolves this by introducing two different, + * tunable parameters: - `UpdateFrequency`: how often the sampling rate is + * recomputed - `LookbackFrequency`: how much total time is considered when + * recomputing sampling rate. + * + * A standard configuration would be to set `UpdateFrequency` to `1s` and + * `LookbackFrequency` to `30s`. In this configuration, for every second, we + * lookback at the last 30 seconds of data in order to compute the new sampling + * rate. The actual sampling rate computation is nearly identical to the + * original Throughput Sampler, but this variant has better support for + * floating point numbers and does a better job with less-common keys. + */ +export type WindowedThroughputSampler = { + /** + * is the desired throughput per second of events sent to Honeycomb. + * + * The desired throughput **per second**. This is the number of events per + * second you want to send to Honeycomb. The sampler will adjust sample rates + * to try to achieve this desired throughput. This value is calculated for the + * individual instance, not for the cluster; if your cluster has multiple + * instances, then you will need to divide your total desired sample rate by + * the number of instances to get this value. + */ + GoalThroughputPerSec: number + /** + * indicates whether to use the cluster size to calculate the goal throughput. + * + * Indicates whether to use the cluster size to calculate the goal throughput. + * If `true`, then the goal throughput will be divided by the number of + * instances in the cluster. If `false` (the default), then the goal throughput + * will be the value specified in `GoalThroughputPerSec`. + */ + UseClusterSize?: boolean + /** + * is how often the sampling rate is recomputed. + * + * The duration between sampling rate computations. It should be specified as a + * duration string. For example, `30s` or `1m`. Defaults to `1s`. + */ + UpdateFrequency?: string + /** + * how far back in time to look when computing the sampling rate. + * + * This controls how far back in time to lookback to dynamically adjust the + * sampling rate. Default is `30 * UpdateFrequencyDuration`. This field is + * forced to be an **integer multiple** of `UpdateFrequencyDuration`. + */ + LookbackFrequency?: string + /** + * is the list of fields to use to create the key for the Dynamic Sampler. + * + * $DynamicSampler.FieldList + */ + FieldList: string[] + /** + * is the maximum number of keys to track. + * + * $DynamicSampler.MaxKeys + */ + MaxKeys?: number + /** + * indicates whether to include the trace length as part of the key. + * + * $DynamicSampler.UseTraceLength + */ + UseTraceLength?: boolean +} + +/** + * Total Throughput Sampler + * + * Total Throughput Sampler (`TotalThroughputSampler`) attempts to meet a goal + * of a fixed number of events per second sent to Honeycomb. This sampler is + * **deprecated** and present mainly for compatibility. Consider using either + * `EMAThroughputSampler` or `WindowedThroughputSampler` instead. If your key + * space is sharded across different servers, then this is a good method for + * making sure each server sends roughly the same volume of content to + * Honeycomb. It performs poorly when the active keyspace is very large. + * `GoalThroughputPerSec` * `ClearFrequency` defines the upper limit of the + * number of keys that can be reported and stay under the goal, but with that + * many keys, you'll only get one event per key per `ClearFrequencySec`, which + * is very coarse. Aim for at least 1 event per key per sec to 1 event per key + * per 10sec to get reasonable data. In other words, the number of active keys + * should be less than 10 * `GoalThroughputPerSec`. + */ +export type TotalThroughputSampler = { + /** + * is the desired throughput per second of events sent to Honeycomb. + * + * The desired throughput per second of events sent to Honeycomb. This is the + * number of events per second you want to send. This is not the same as the + * Sample Rate. + */ + GoalThroughputPerSec: number + /** + * indicates whether to use the cluster size to calculate the goal throughput. + * + * Indicates whether to use the cluster size to calculate the goal throughput. + * If `true`, then the goal throughput will be divided by the number of + * instances in the cluster. If `false` (the default), then the goal throughput + * will be the value specified in `GoalThroughputPerSec`. + */ + UseClusterSize?: boolean + /** + * is the duration over which the sampler will calculate the throughput. It + * should be specified as a duration string. For example, `30s` or `1m`. + * Defaults to `30s`. + * + * $DynamicSampler.ClearFrequency + */ + ClearFrequency?: string + /** + * is the list of fields to use to create the key for the Dynamic Sampler. + * + * $DynamicSampler.FieldList + */ + FieldList: string[] + /** + * is the maximum number of keys to track. + * + * $DynamicSampler.MaxKeys + */ + MaxKeys?: number + /** + * indicates whether to include the trace length as part of the key. + * + * $DynamicSampler.UseTraceLength + */ + UseTraceLength?: boolean +} + +/** + * Rules-based Sampler + * + * The Rules-based sampler allows you to specify a set of rules that will + * determine whether a trace should be sampled or not. Rules are evaluated in + * order, and the first rule that matches will be used to determine the sample + * rate. If no rules match, then the `SampleRate` defaults to `1` and all + * traces will be kept. Rules-based samplers will usually be configured to have + * the last rule be a default rule with no conditions that uses a downstream + * Dynamic Sampler to keep overall sample rate under control. + */ +export type RulesBasedSampler = { + /** + * is the list of rules to use. + * + * `Rules` is a list of rules to use to determine the sample rate. + */ + Rules?: RulesBasedSamplerRule[] + /** + * indicates whether to expand nested JSON when evaluating rules. + * + * Indicates whether to expand nested JSON when evaluating rules. If false (the + * default), nested JSON will be treated as a string. If `true`, nested JSON + * will be expanded into a `map[string]interface{}` and the value of the field + * will be the value of the nested field. For example, if you have a field + * called `http.request.headers` and you want to check the value of the + * `User-Agent` header, then you would set this to `true` and use + * `http.request.headers.User-Agent` as the field name in your rule. This is a + * computationally expensive option and may cause performance problems if you + * have a large number of spans with nested JSON. + */ + CheckNestedFields?: boolean +} + +/** + * Rules for Rules-based Samplers + * + * Rules are evaluated in order, and the first rule that matches will be used + * to determine the sample rate. If no rules match, then the `SampleRate` will + * be `1` and all traces will be kept. If a rule matches, one of three things + * happens, and they are evaluated in this order: a) if the rule specifies a + * downstream Sampler, that sampler is used to determine the sample rate; b) if + * the rule has the `Drop` flag set to `true`, the trace is dropped; c) the + * rule's sample rate is used. + */ +export type RulesBasedSamplerRule = { + /** + * is the name of the rule. + * + * The name of the rule. This field is used for debugging and will appear in + * the trace metadata if `AddRuleReasonToTrace` is set to `true`. + */ + Name?: string + /** + * is the Dynamic Sampler to use if the rule matches. + * + * The sampler to use if the rule matches. If this is set, the sample rate will + * be determined by this downstream sampler. If this is not set, the sample + * rate will be determined by the `Drop` flag or the `SampleRate` field. + */ + Sampler?: RulesBasedDownstreamSampler + /** + * indicates whether to drop the trace. + * + * Indicates whether to drop the trace if it matches this rule. If `true`, then + * the trace will be dropped. If `false`, then the trace will be kept. + */ + Drop?: boolean + /** + * is the sample rate to use. + * + * If the rule is matched, there is no Sampler specified, and the `Drop` flag + * is `false`, then this is the sample rate to use. + */ + SampleRate?: number + /** + * is the list of conditions to use to determine whether the rule matches. + * + * Conditions is a list of conditions to use to determine whether the rule + * matches. All conditions must be met for the rule to match. If there are no + * conditions, then the rule will always match. A no-condition rule is + * typically used for the last rule to provide a default behavior. + */ + Conditions?: RulesBasedSamplerCondition[] + /** + * controls the scope of the rule. + * + * Controls the scope of the rule evaluation. If set to `trace` (the default), + * then each condition can apply to any span in the trace independently. If set + * to `span`, then all of the conditions in the rule will be evaluated against + * each span in the trace and the rule only succeeds if all of the conditions + * match on a single span together. + * + * WARNING: The `has-root-span` operator cannot be used with `Scope: span`. The + * `has-root-span` operator is a trace-level condition that checks whether the + * trace has a root span. When using `Scope: span`, all conditions must match + * on a single span, which is incompatible with trace-level operators like + * `has-root-span`. Combining them will cause the rule to fail evaluation and + * be skipped. Traces that should match the rule will not be sampled as + * expected. + */ + Scope?: "trace" | "span" +} + +/** + * Conditions for the Rules in Rules-based Samplers + * + * Conditions are evaluated in order, and the first condition that does not + * match will cause the rule to not match. If all conditions match, then the + * rule will match. If there are no conditions, then the rule will always + * match. + */ +export type RulesBasedSamplerCondition = { + /** + * is the field to check. + * + * The field to check. This can name any field in the trace. If the field is + * not present, then the condition will not match. The comparison is + * case-sensitive. The field can also include a prefix that changes the span + * used for evaluation of the field. The only prefix currently supported is + * `root`, as in `root.http.status`. Specifying `root.` causes the condition to + * be evaluated against the root span. For example, if the `Field` is + * `root.url`, then the condition will be processed using the url field from + * the root span. The setting `Scope: span` for a rule does not change the + * meaning of this prefix -- the condition is still evaluated on the root span + * and is treated as if it were part of the span being processed. When using + * the `root.` prefix on a field with a `not-exists` operator, include the + * `has-root-span: true` condition in the rule. The `not-exists` condition on a + * `root.`-prefixed field will evaluate to false if the existence of the root + * span is not checked and the root span does not exist. The primary reason a + * root span is not present on a trace when a sampling decision is being made + * is when the root span takes longer to complete than the configured + * TraceTimeout. + */ + Field?: string + /** + * An array of field names to check. These can name any field in the trace. The + * fields are checked in the order defined here, and the first named field that + * contains a value will be used for the condition. Only the first populated + * field will be used, even if the condition fails. If a `root.` prefix is + * present on a field, but the root span is not on the trace, that field will + * be skipped. If none of the fields are present, then the condition will not + * match. The comparison is case-sensitive. All fields are checked as + * individual fields before any of them are checked as nested fields (see + * `CheckNestedFields`). + */ + Fields?: string[] + /** + * is the comparison operator to use. + * + * The comparison operator to use. String comparisons are case-sensitive. For + * most cases, only use negative operators (`!=`, `does-not-contain`, + * `not-exists`, and `not-in`) in a rule with a scope of "span". WARNING: Rules + * can have `Scope: trace` or `Scope: span`. Using a negative operator with + * `Scope: trace` will cause the condition be true if **any** single span in + * the entire trace matches. Use `Scope: span` with negative operators. As a + * general rule, negative operators are more difficult to use correctly and + * should be used sparingly. It is usually more effective to use positive + * operators and be more explicit about rules applied. + */ + Operator?: "=" | "!=" | ">" | "<" | ">=" | "<=" | "starts-with" | "contains" | "does-not-contain" | "exists" | "not-exists" | "has-root-span" | "matches" | "in" | "not-in" + /** + * is the value to compare against. + * + * The value to compare against. If `Datatype` is not specified, then the value + * and the field will be compared based on the type of the field. The `in` and + * `not-in` operators can accept a list of values, which should all be of the + * same datatype. + */ + Value?: string | number | boolean | Array + /** + * is the datatype to use when comparing the value and the field. + * + * The datatype to use when comparing the value and the field. If `Datatype` is + * specified, then both values will be converted (best-effort) to that type and + * then compared. Errors in conversion will result in the comparison evaluating + * to `false`. This is especially useful when a field like `http status code` + * may be rendered as strings by some environments and as numbers or booleans + * by others. The best practice is to always specify `Datatype`; this avoids + * ambiguity, allows for more accurate comparisons, and offers a minor + * performance improvement. + */ + Datatype?: "string" | "int" | "float" | "bool" +} + & ({ Field?: never } | { Fields?: never }) + +/** + * is a sampler allowed as the downstream `Sampler` of a rules-based sampler + * rule. + * + * Derived from the `validChildren` validation on `Rules.Sampler` in + * rulesMeta.yaml. + */ +export type RulesBasedDownstreamSampler = + | { DeterministicSampler?: never; DynamicSampler: DynamicSampler; EMADynamicSampler?: never; EMAThroughputSampler?: never; WindowedThroughputSampler?: never; TotalThroughputSampler?: never; RulesBasedSampler?: never } + | { DeterministicSampler?: never; DynamicSampler?: never; EMADynamicSampler: EMADynamicSampler; EMAThroughputSampler?: never; WindowedThroughputSampler?: never; TotalThroughputSampler?: never; RulesBasedSampler?: never } + | { DeterministicSampler?: never; DynamicSampler?: never; EMADynamicSampler?: never; EMAThroughputSampler: EMAThroughputSampler; WindowedThroughputSampler?: never; TotalThroughputSampler?: never; RulesBasedSampler?: never } + | { DeterministicSampler?: never; DynamicSampler?: never; EMADynamicSampler?: never; EMAThroughputSampler?: never; WindowedThroughputSampler: WindowedThroughputSampler; TotalThroughputSampler?: never; RulesBasedSampler?: never } + | { DeterministicSampler?: never; DynamicSampler?: never; EMADynamicSampler?: never; EMAThroughputSampler?: never; WindowedThroughputSampler?: never; TotalThroughputSampler: TotalThroughputSampler; RulesBasedSampler?: never } + +/** + * is the sampler configuration for a single target (Honeycomb environment). + * + * Exactly one sampler type must be set per target; other sampler keys are + * explicitly forbidden in each union member. + */ +export type RefinerySamplerChoice = + | { DeterministicSampler: DeterministicSampler; DynamicSampler?: never; EMADynamicSampler?: never; EMAThroughputSampler?: never; WindowedThroughputSampler?: never; TotalThroughputSampler?: never; RulesBasedSampler?: never } + | { DeterministicSampler?: never; DynamicSampler: DynamicSampler; EMADynamicSampler?: never; EMAThroughputSampler?: never; WindowedThroughputSampler?: never; TotalThroughputSampler?: never; RulesBasedSampler?: never } + | { DeterministicSampler?: never; DynamicSampler?: never; EMADynamicSampler: EMADynamicSampler; EMAThroughputSampler?: never; WindowedThroughputSampler?: never; TotalThroughputSampler?: never; RulesBasedSampler?: never } + | { DeterministicSampler?: never; DynamicSampler?: never; EMADynamicSampler?: never; EMAThroughputSampler: EMAThroughputSampler; WindowedThroughputSampler?: never; TotalThroughputSampler?: never; RulesBasedSampler?: never } + | { DeterministicSampler?: never; DynamicSampler?: never; EMADynamicSampler?: never; EMAThroughputSampler?: never; WindowedThroughputSampler: WindowedThroughputSampler; TotalThroughputSampler?: never; RulesBasedSampler?: never } + | { DeterministicSampler?: never; DynamicSampler?: never; EMADynamicSampler?: never; EMAThroughputSampler?: never; WindowedThroughputSampler?: never; TotalThroughputSampler: TotalThroughputSampler; RulesBasedSampler?: never } + | { DeterministicSampler?: never; DynamicSampler?: never; EMADynamicSampler?: never; EMAThroughputSampler?: never; WindowedThroughputSampler?: never; TotalThroughputSampler?: never; RulesBasedSampler: RulesBasedSampler } + +/** + * is the root of a Refinery rules.yaml file. + * + * `Samplers` maps targets to sampler configurations. Each target is a + * Honeycomb environment (or a dataset for Honeycomb Classic keys). The value + * is the sampler to use for that target. A `__default__` target is required. + * The target called `__default__` will be used for any target that is not + * explicitly listed. The targets are determined by examining the API key used + * to send the trace. If the API key is a Honeycomb Classic key with a + * 32-character hexadecimal value, then the specified dataset name is used as + * the target. If the API key is a key with 20-23 alphanumeric characters, then + * the key's environment name is used as the target. + */ +export type RefineryRules = { + RulesVersion: 2 + Samplers: { __default__: RefinerySamplerChoice } & Record< + string, + RefinerySamplerChoice + > +} + diff --git a/config-types/tsconfig.json b/config-types/tsconfig.json new file mode 100644 index 0000000..53903ea --- /dev/null +++ b/config-types/tsconfig.json @@ -0,0 +1,11 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "strict": true, + "noEmit": true, + "exactOptionalPropertyTypes": true, + "skipLibCheck": false + }, + "include": ["./*.ts"] +} diff --git a/config-types/types.test.ts b/config-types/types.test.ts new file mode 100644 index 0000000..7e74552 --- /dev/null +++ b/config-types/types.test.ts @@ -0,0 +1,110 @@ +import type { + RefineryConfig, + RefineryRules, + RefinerySamplerChoice, + RulesBasedDownstreamSampler, + RulesBasedSamplerCondition, +} from "@notionhq/refinery-config-types"; + +const config: RefineryConfig = { + General: { ConfigurationVersion: 2 }, + AccessKeys: { ReceiveKeys: ["${HONEYCOMB_API_KEY}"] }, + Collection: { AvailableMemory: "8Gi", MaxMemoryPercentage: 85 }, +}; +const rules: RefineryRules = { + RulesVersion: 2, + Samplers: { + __default__: { + RulesBasedSampler: { + Rules: [ + { + Name: "Full Traces", + Conditions: [ + { Field: "sampling.full_trace", Operator: "=", Value: true }, + ], + Sampler: { + EMADynamicSampler: { + GoalSampleRate: 10, + FieldList: ["sampling.reason"], + MaxKeys: 1000, + }, + }, + }, + ], + }, + }, + development: { DeterministicSampler: { SampleRate: 1 } }, + }, +}; + +// @ts-expect-error A globally required configuration group cannot be omitted. +const missingGeneral: RefineryConfig = {}; +// @ts-expect-error A sampler is required. +const noSampler: RefinerySamplerChoice = {}; +// @ts-expect-error Required sampler fields cannot be omitted. +const missingRate: RefinerySamplerChoice = { DeterministicSampler: {} }; +const twoSamplers = { + DeterministicSampler: { SampleRate: 1 }, + EMADynamicSampler: { GoalSampleRate: 10, FieldList: ["name"] }, +}; +// @ts-expect-error Multiple sampler types are invalid even through an intermediate variable. +const multipleSamplers: RefinerySamplerChoice = twoSamplers; +// @ts-expect-error The downstream choice also requires exactly one sampler. +const multipleDownstreamSamplers: RulesBasedDownstreamSampler = twoSamplers; +const nestedRules: RulesBasedDownstreamSampler = { + // @ts-expect-error Rules-based samplers cannot be nested downstream. + RulesBasedSampler: { Rules: [] }, +}; +// @ts-expect-error Field and Fields are mutually exclusive. +const conflictingFields: RulesBasedSamplerCondition = { + Field: "name", + Fields: ["name"], + Operator: "=", +}; +const singleField: RulesBasedSamplerCondition = { + Field: "name", + Operator: "=", +}; +const multipleFields: RulesBasedSamplerCondition = { + Fields: ["name", "other"], + Operator: "=", +}; +const invalidOperator: RulesBasedSamplerCondition = { + Field: "name", + // @ts-expect-error Unknown operators are rejected. + Operator: "typo", +}; +const noDefault: RefineryRules = { + RulesVersion: 2, + // @ts-expect-error The default target is required. + Samplers: { development: { DeterministicSampler: { SampleRate: 1 } } }, +}; +const invalidVersion: RefineryRules = { + // @ts-expect-error RulesVersion must be 2. + RulesVersion: 3, + Samplers: rules.Samplers, +}; + +void [ + config, + rules, + missingGeneral, + noSampler, + missingRate, + multipleSamplers, + multipleDownstreamSamplers, + nestedRules, + conflictingFields, + singleField, + multipleFields, + invalidOperator, + noDefault, + invalidVersion, +]; + +// Refinery accepts memory sizes as either unit strings or numeric byte counts. +const numericMemory: RefineryConfig = { + General: { ConfigurationVersion: 2 }, + Collection: { AvailableMemory: 8589934592 }, +}; +void numericMemory; diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..95d4cf3 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,68 @@ +{ + "name": "@notionhq/refinery-config-types", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@notionhq/refinery-config-types", + "version": "0.0.0", + "license": "Apache-2.0", + "devDependencies": { + "@types/node": "22.20.4", + "typescript": "5.9.3", + "yaml": "2.9.1" + }, + "engines": { + "node": ">=22.13.0" + } + }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/yaml": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.1.tgz", + "integrity": "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==", + "dev": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..9da1d76 --- /dev/null +++ b/package.json @@ -0,0 +1,32 @@ +{ + "name": "@notionhq/refinery-config-types", + "version": "0.0.0", + "private": true, + "description": "TypeScript configuration types generated from Notion Refinery metadata", + "license": "Apache-2.0", + "type": "module", + "types": "./config-types/index.d.ts", + "exports": { + ".": { + "types": "./config-types/index.d.ts" + } + }, + "files": [ + "config-types/index.d.ts", + "config-types/README.md", + "LICENSE" + ], + "scripts": { + "generate": "node --experimental-strip-types config-types/generate.ts", + "check:generated": "node --experimental-strip-types config-types/generate.ts --check", + "test": "npm run check:generated && node --experimental-strip-types --test config-types/generate.test.mjs && tsc --project config-types/tsconfig.json" + }, + "engines": { + "node": ">=22.13.0" + }, + "devDependencies": { + "@types/node": "22.20.4", + "typescript": "5.9.3", + "yaml": "2.9.1" + } +}