fix(explore): hold back what is still owed below a clustered render (CG-31)
Carry-forward slack let a file spend what the files ABOVE it left on the
table. Nothing held back what was promised BELOW it. The whole-file BUY arm
has always refused that trade (`owedBelow`); the cluster path read `headroom`
— what is left before the hard ceiling — instead of what is still owed, so
`fileBudget` and `SPINE_CEILING` could pay a 1.5x overshoot out of another
file's reservation.
`fundedHeadroom` is the same inequality in the units the cluster path spends
in: source PLUS the per-section overhead each unreached file will charge.
Floored at the file's own reservation — a kept promise is not a displacement —
and it is <= `headroom` by construction, so it is the only bound the three
render sites need. The skeleton path's `bodyCap` takes it too.
Measured on `__tests__/fixtures/displacement-ts` (a 4-stage pipeline padded
past 500 files, where the 24K envelope genuinely saturates the 24.4K render
ceiling):
before ingest.ts emitted 9,301 on a 6,289 spendable, then lost the whole
section to the final ceiling — 0 delivered. types.ts and sink.ts
skipped `budget-whole-file`. 3 of 6 admitted files delivered.
after ingest.ts bounded to the 4,913 actually free. 6 of 6 delivered,
envelope 14,908 -> 22,066.
The self-query allocation fixture flips back to PASS with it, on a clean full
rebuild of this repo's index (CG-33). Its `afterCG30` verdict blamed an
over-RESERVED incidental file; the reservation was identical in both arms —
the file was over-SPENDING. Recorded honestly in `afterCG31`.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Colby McHenry
co-authored by
Claude Opus 5
Claude Opus 5
parent
0d014a6582
commit
089dcc276f
@@ -0,0 +1,234 @@
|
||||
/**
|
||||
* Regression fixture for CG-31 — a clustered render may not spend a reservation
|
||||
* still owed to a file the loop has not reached.
|
||||
*
|
||||
* The allocator hands every admitted file a reservation (CG-12), and the render
|
||||
* loop then walks the files in rank order. Carry-forward slack lets a file spend
|
||||
* what the files ABOVE it left on the table, which is right; what was missing is
|
||||
* the other half — nothing was held back for the files BELOW it. The whole-file
|
||||
* BUY arm has always refused that trade (`owedBelow`, `tools.ts`); the cluster
|
||||
* path had no equivalent, so `fileBudget`/`SPINE_CEILING` read what was left
|
||||
* before the hard ceiling rather than what was still promised, and the first
|
||||
* oversize file could take the response.
|
||||
*
|
||||
* `__tests__/fixtures/displacement-ts/` reproduces it. Four pipeline stages
|
||||
* compete for one envelope; the first, `ingest.ts`, is a single ~20K function —
|
||||
* one cluster member far bigger than any reservation it can earn — so it takes
|
||||
* the bounded overshoot CG-30 left it. The fixture is padded to >500 indexed
|
||||
* files on purpose: the displacement only exists on the 24K tier, where the
|
||||
* reservations plus the response preamble genuinely saturate the hard ceiling.
|
||||
*
|
||||
* Measured against the pre-fix build (CG-30 landed, CG-31 not):
|
||||
*
|
||||
* ingest.ts 9,301 chars emitted on a 6,289 spendable — then dropped whole
|
||||
* by the final ceiling, so it cost the response and delivered 0
|
||||
* types.ts skipped `budget-whole-file`
|
||||
* sink.ts skipped `budget-whole-file`
|
||||
* delivered 3 of 6 admitted files, 14,908-char envelope
|
||||
*
|
||||
* With the guard: 6 of 6, 22,066-char envelope, and `ingest.ts` bounded to the
|
||||
* 4,913 that were actually still free.
|
||||
*/
|
||||
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import CodeGraph from '../src/index';
|
||||
import { ToolHandler } from '../src/mcp/tools';
|
||||
import { attributeSourceBytes } from '../src/mcp/explore-diagnostics';
|
||||
import type { ExploreDiagnosticReport, ExploreDiagnosticFile } from '../src/mcp/explore-diagnostics';
|
||||
|
||||
const FIXTURE_SRC = path.join(__dirname, 'fixtures', 'displacement-ts');
|
||||
|
||||
/**
|
||||
* Padding modules, written into the temp copy rather than checked in. The
|
||||
* output tier is chosen by INDEXED FILE COUNT, and the displacement this test
|
||||
* pins only exists at >=500 files (24K envelope against a 24.4K render ceiling
|
||||
* that also has to hold the response preamble). Below that the ceiling has
|
||||
* enough slack to absorb an overshoot and the bug is invisible.
|
||||
*/
|
||||
const FILLER_FILES = 520;
|
||||
|
||||
/** A symbol bag spanning all four stages — they compete for one envelope. */
|
||||
const QUERY = 'ingestRecords normalizeRecords enrichRecords publishRecords';
|
||||
/** One symbol, one file — the concentration case the guard must not flatten. */
|
||||
const PRECISE_QUERY = 'ingestRecords';
|
||||
|
||||
/** The giant: one ~20K function, the file that used to take the response. */
|
||||
const GIANT = 'src/pipeline/ingest.ts';
|
||||
/** Ranked below the giant and dropped by it pre-fix. */
|
||||
const STARVED = ['src/pipeline/types.ts', 'src/pipeline/sink.ts'];
|
||||
|
||||
interface Probe {
|
||||
response: string;
|
||||
report: ExploreDiagnosticReport;
|
||||
bytes: Map<string, number>;
|
||||
}
|
||||
|
||||
describe('CG-31 — the cluster path holds back what is still owed below it', () => {
|
||||
let testDir: string;
|
||||
let cg: CodeGraph;
|
||||
let spread: Probe;
|
||||
let precise: Probe;
|
||||
|
||||
const fileOf = (probe: Probe, p: string): ExploreDiagnosticFile => {
|
||||
const rec = probe.report.files.find((f) => f.path === p);
|
||||
if (!rec) throw new Error(`${p} absent from the diagnostic report`);
|
||||
return rec;
|
||||
};
|
||||
/** Admitted = the allocator reserved bytes for it. */
|
||||
const admitted = (probe: Probe): ExploreDiagnosticFile[] =>
|
||||
probe.report.files.filter((f) => (f.allowance ?? 0) > 0);
|
||||
|
||||
beforeAll(async () => {
|
||||
testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-cg31-'));
|
||||
fs.cpSync(FIXTURE_SRC, testDir, { recursive: true });
|
||||
fs.rmSync(path.join(testDir, '.codegraph'), { recursive: true, force: true });
|
||||
|
||||
const filler = path.join(testDir, 'src', 'generated');
|
||||
fs.mkdirSync(filler, { recursive: true });
|
||||
for (let i = 0; i < FILLER_FILES; i++) {
|
||||
// Deterministic, unrelated to the query — these pad the file count, they
|
||||
// must never rank.
|
||||
fs.writeFileSync(
|
||||
path.join(filler, `unit${i}.ts`),
|
||||
`export const seed${i} = ${i};\n`
|
||||
+ `export function widget${i}(n: number): number {\n return n * ${i + 1} + seed${i};\n}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
cg = CodeGraph.initSync(testDir);
|
||||
await cg.indexAll();
|
||||
|
||||
// The per-file bounds are only observable through the diagnostic sidecar.
|
||||
const sidecar = path.join(testDir, 'explore-diag.jsonl');
|
||||
const previous = process.env.CODEGRAPH_EXPLORE_DEBUG;
|
||||
process.env.CODEGRAPH_EXPLORE_DEBUG = sidecar;
|
||||
const run = async (handler: ToolHandler, query: string): Promise<Probe> => {
|
||||
const result = await handler.execute('codegraph_explore', { query });
|
||||
const response = result.content?.[0]?.text ?? '';
|
||||
const written = fs.readFileSync(sidecar, 'utf-8').trim().split('\n').filter(Boolean);
|
||||
return {
|
||||
response,
|
||||
report: JSON.parse(written[written.length - 1]!) as ExploreDiagnosticReport,
|
||||
bytes: attributeSourceBytes(response),
|
||||
};
|
||||
};
|
||||
try {
|
||||
const handler = new ToolHandler(cg);
|
||||
spread = await run(handler, QUERY);
|
||||
precise = await run(handler, PRECISE_QUERY);
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.CODEGRAPH_EXPLORE_DEBUG;
|
||||
else process.env.CODEGRAPH_EXPLORE_DEBUG = previous;
|
||||
}
|
||||
}, 180_000);
|
||||
|
||||
afterAll(() => {
|
||||
if (cg) cg.destroy();
|
||||
if (testDir && fs.existsSync(testDir)) fs.rmSync(testDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
// ── Fixture shape — if these rot, the gate below means nothing ─────────────
|
||||
|
||||
describe('fixture shape', () => {
|
||||
it('sits on the 24K tier, where the reservations saturate the ceiling', () => {
|
||||
expect(cg.getStats().fileCount).toBeGreaterThanOrEqual(500);
|
||||
expect(spread.report.budget.maxOutputChars).toBe(24000);
|
||||
});
|
||||
|
||||
it('admits every stage file, so there is something to displace', () => {
|
||||
const paths = admitted(spread).map((f) => f.path);
|
||||
expect(paths).toContain(GIANT);
|
||||
for (const p of STARVED) expect(paths).toContain(p);
|
||||
expect(paths.length).toBeGreaterThanOrEqual(5);
|
||||
});
|
||||
|
||||
it('renders the giant through the CLUSTER path, over its reservation', () => {
|
||||
const rec = fileOf(spread, GIANT);
|
||||
expect(rec.render).toBe('clusters');
|
||||
// One member bigger than anything it can earn beside its siblings — the
|
||||
// shape that makes the bounded overshoot fire at all.
|
||||
const source = fs.readFileSync(path.join(testDir, GIANT), 'utf-8');
|
||||
expect(source.length).toBeGreaterThan((rec.spendable ?? 0) * 2);
|
||||
// And the guard actually bit — a vacuous pass here would hide a regression.
|
||||
expect(rec.funded).not.toBeNull();
|
||||
expect(rec.funded!).toBeLessThan(rec.spendable!);
|
||||
});
|
||||
});
|
||||
|
||||
// ── The gate ──────────────────────────────────────────────────────────────
|
||||
|
||||
describe('displacement refusal', () => {
|
||||
it('CG-31 GATE: no clustered file emits past what was still free to spend', () => {
|
||||
for (const probe of [spread, precise]) {
|
||||
const over = probe.report.files
|
||||
.filter((f) => f.render === 'clusters' && f.funded !== null)
|
||||
// +1 for the render loop's own rounding on the windowed cut.
|
||||
.filter((f) => f.emittedChars > f.funded! + 1)
|
||||
.map((f) => `${f.path}: ${f.emittedChars} of ${f.funded}`);
|
||||
expect(over).toEqual([]);
|
||||
}
|
||||
});
|
||||
|
||||
it('CG-31 GATE: every admitted file below the top one is delivered', () => {
|
||||
// Pre-fix: 3 of 6 — `ingest.ts` overshot, was itself cut by the final
|
||||
// ceiling, and took `types.ts` + `sink.ts` down with it.
|
||||
for (const rec of admitted(spread)) {
|
||||
expect(rec.skipped, `${rec.path} skipped`).toBeNull();
|
||||
expect(spread.bytes.get(rec.path) ?? 0, `${rec.path} bytes`).toBeGreaterThan(0);
|
||||
}
|
||||
for (const p of STARVED) expect(spread.bytes.get(p) ?? 0).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it('the guard is symmetric — it is about ORDER, not rank', () => {
|
||||
// Nothing here protects rank #1 specifically: the LAST admitted file, the
|
||||
// only one with no reservation owed below it, is delivered too.
|
||||
const files = admitted(spread);
|
||||
const last = files[files.length - 1]!;
|
||||
expect(last.skipped).toBeNull();
|
||||
expect(spread.bytes.get(last.path) ?? 0).toBeGreaterThan(0);
|
||||
// And the last file is never itself cut by the guard — nothing is owed
|
||||
// below it, so `funded` may not sit under its own reservation.
|
||||
expect(last.funded!).toBeGreaterThanOrEqual(Math.min(last.allowance!, last.emittedChars));
|
||||
});
|
||||
|
||||
it('a kept promise is not a displacement — no file is cut below its reservation', () => {
|
||||
for (const probe of [spread, precise]) {
|
||||
for (const rec of admitted(probe)) {
|
||||
if (rec.funded === null) continue;
|
||||
expect(rec.funded, rec.path).toBeGreaterThanOrEqual(
|
||||
Math.min(rec.allowance!, rec.emittedChars));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps the response inside the hard ceiling', () => {
|
||||
for (const probe of [spread, precise]) {
|
||||
expect(probe.report.envelope.chars).toBeLessThanOrEqual(probe.report.budget.hardCeiling);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── The thing the guard must NOT become ───────────────────────────────────
|
||||
|
||||
describe('concentration survives', () => {
|
||||
it('a precise symbol query still puts the most source in the named file', () => {
|
||||
const mine = precise.bytes.get(GIANT) ?? 0;
|
||||
const others = [...precise.bytes.entries()].filter(([p]) => p !== GIANT);
|
||||
expect(mine).toBeGreaterThan(0);
|
||||
for (const [p, n] of others) {
|
||||
expect(mine, `${GIANT} vs ${p}`).toBeGreaterThan(n);
|
||||
}
|
||||
// Not a forced even split: the named file takes a clear plurality.
|
||||
const total = [...precise.bytes.values()].reduce((s, n) => s + n, 0);
|
||||
expect(mine / total).toBeGreaterThan(1 / precise.bytes.size);
|
||||
});
|
||||
|
||||
it('the named file still outspends what it would get from an even split', () => {
|
||||
const rec = fileOf(precise, GIANT);
|
||||
const even = precise.report.budget.maxOutputChars / admitted(precise).length;
|
||||
expect(rec.emittedChars).toBeGreaterThan(even);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"name": "displacement-fixture",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"type": "module"
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
import { ingestRecords } from './pipeline/ingest';
|
||||
import { normalizeRecords } from './pipeline/normalize';
|
||||
import { enrichRecords } from './pipeline/enrich';
|
||||
import { publishRecords } from './pipeline/publish';
|
||||
import type { PipelineOptions, PipelineRecord, RawRecord } from './pipeline/types';
|
||||
|
||||
/** Run one batch through every pipeline stage, in order. */
|
||||
export function runPipeline(batch: RawRecord[], options: PipelineOptions): PipelineRecord[] {
|
||||
return publishRecords(enrichRecords(normalizeRecords(ingestRecords(batch, options), options), options), options);
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
import { writeBatch } from './sink';
|
||||
import type { PipelineOptions, PipelineRecord } from './types';
|
||||
|
||||
/** Enrich every record in a batch. */
|
||||
export function enrichRecords(records: PipelineRecord[], options: PipelineOptions): PipelineRecord[] {
|
||||
const out: PipelineRecord[] = [];
|
||||
for (const record of records) {
|
||||
const tags = [...record.tags];
|
||||
const warnings = [...record.warnings];
|
||||
let value = record.value;
|
||||
|
||||
// 1. segment
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('segment:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('segment: missing after enrich');
|
||||
} else {
|
||||
value = weightFacet(value, hit.length);
|
||||
tags.push('segment.enri');
|
||||
}
|
||||
}
|
||||
|
||||
// 2. referrer
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('referrer:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('referrer: missing after enrich');
|
||||
} else {
|
||||
value = blendFacet(value, hit.length);
|
||||
tags.push('referrer.enri');
|
||||
}
|
||||
}
|
||||
|
||||
// 3. experiment
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('experiment:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('experiment: missing after enrich');
|
||||
} else {
|
||||
value = weightFacet(value, hit.length);
|
||||
tags.push('experiment.enri');
|
||||
}
|
||||
}
|
||||
|
||||
// 4. subscription
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('subscription:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('subscription: missing after enrich');
|
||||
} else {
|
||||
value = blendFacet(value, hit.length);
|
||||
tags.push('subscription.enri');
|
||||
}
|
||||
}
|
||||
|
||||
// 5. entitlement
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('entitlement:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('entitlement: missing after enrich');
|
||||
} else {
|
||||
value = weightFacet(value, hit.length);
|
||||
tags.push('entitlement.enri');
|
||||
}
|
||||
}
|
||||
|
||||
// 6. invoice
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('invoice:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('invoice: missing after enrich');
|
||||
} else {
|
||||
value = blendFacet(value, hit.length);
|
||||
tags.push('invoice.enri');
|
||||
}
|
||||
}
|
||||
|
||||
// 7. refund
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('refund:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('refund: missing after enrich');
|
||||
} else {
|
||||
value = weightFacet(value, hit.length);
|
||||
tags.push('refund.enri');
|
||||
}
|
||||
}
|
||||
|
||||
// 8. dispute
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('dispute:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('dispute: missing after enrich');
|
||||
} else {
|
||||
value = blendFacet(value, hit.length);
|
||||
tags.push('dispute.enri');
|
||||
}
|
||||
}
|
||||
|
||||
// 9. payout
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('payout:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('payout: missing after enrich');
|
||||
} else {
|
||||
value = weightFacet(value, hit.length);
|
||||
tags.push('payout.enri');
|
||||
}
|
||||
}
|
||||
|
||||
out.push({ ...record, value, tags: tags.slice(0, options.maxTags), warnings });
|
||||
}
|
||||
writeBatch('enrichRecords', out);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** weightFacet — a small deterministic helper. */
|
||||
export function weightFacet(base: number, width: number): number {
|
||||
const scaled = base + width * 3 - (width % 7);
|
||||
return scaled < 0 ? 0 : scaled;
|
||||
}
|
||||
|
||||
/** blendFacet — a small deterministic helper. */
|
||||
export function blendFacet(base: number, width: number): number {
|
||||
const scaled = base + width * 3 - (width % 7);
|
||||
return scaled < 0 ? 0 : scaled;
|
||||
}
|
||||
@@ -0,0 +1,541 @@
|
||||
import { scaleFacet, clampFacet } from './normalize';
|
||||
import { writeBatch } from './sink';
|
||||
import type { PipelineOptions, PipelineRecord, RawRecord } from './types';
|
||||
|
||||
/**
|
||||
* Ingest one batch of raw records.
|
||||
*
|
||||
* Every facet is unpacked in its own block so an on-call engineer can read the
|
||||
* ingest end-to-end in one place. The shape is deliberately flat: this single
|
||||
* function is the whole stage, which is exactly the shape that makes it the
|
||||
* biggest cluster member in the file.
|
||||
*/
|
||||
export function ingestRecords(batch: RawRecord[], options: PipelineOptions): PipelineRecord[] {
|
||||
const out: PipelineRecord[] = [];
|
||||
for (const record of batch) {
|
||||
const tags: string[] = [];
|
||||
const warnings: string[] = [];
|
||||
let value = 0;
|
||||
|
||||
// 1. identity — normalise the identity facet of the record.
|
||||
{
|
||||
const raw = record.payload['identity'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('identity: empty, dropped');
|
||||
} else {
|
||||
const scaled = scaleFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('identity:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('identity: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 2. geography — normalise the geography facet of the record.
|
||||
{
|
||||
const raw = record.payload['geography'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('geography: empty, dropped');
|
||||
} else {
|
||||
const scaled = clampFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('geography:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('geography: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. currency — normalise the currency facet of the record.
|
||||
{
|
||||
const raw = record.payload['currency'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('currency: empty, dropped');
|
||||
} else {
|
||||
const scaled = scaleFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('currency:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('currency: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. timestamp — normalise the timestamp facet of the record.
|
||||
{
|
||||
const raw = record.payload['timestamp'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('timestamp: empty, dropped');
|
||||
} else {
|
||||
const scaled = clampFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('timestamp:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('timestamp: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 5. channel — normalise the channel facet of the record.
|
||||
{
|
||||
const raw = record.payload['channel'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('channel: empty, dropped');
|
||||
} else {
|
||||
const scaled = scaleFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('channel:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('channel: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 6. campaign — normalise the campaign facet of the record.
|
||||
{
|
||||
const raw = record.payload['campaign'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('campaign: empty, dropped');
|
||||
} else {
|
||||
const scaled = clampFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('campaign:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('campaign: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 7. device — normalise the device facet of the record.
|
||||
{
|
||||
const raw = record.payload['device'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('device: empty, dropped');
|
||||
} else {
|
||||
const scaled = scaleFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('device:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('device: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 8. locale — normalise the locale facet of the record.
|
||||
{
|
||||
const raw = record.payload['locale'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('locale: empty, dropped');
|
||||
} else {
|
||||
const scaled = clampFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('locale:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('locale: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 9. consent — normalise the consent facet of the record.
|
||||
{
|
||||
const raw = record.payload['consent'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('consent: empty, dropped');
|
||||
} else {
|
||||
const scaled = scaleFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('consent:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('consent: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 10. segment — normalise the segment facet of the record.
|
||||
{
|
||||
const raw = record.payload['segment'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('segment: empty, dropped');
|
||||
} else {
|
||||
const scaled = clampFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('segment:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('segment: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 11. referrer — normalise the referrer facet of the record.
|
||||
{
|
||||
const raw = record.payload['referrer'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('referrer: empty, dropped');
|
||||
} else {
|
||||
const scaled = scaleFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('referrer:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('referrer: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 12. experiment — normalise the experiment facet of the record.
|
||||
{
|
||||
const raw = record.payload['experiment'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('experiment: empty, dropped');
|
||||
} else {
|
||||
const scaled = clampFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('experiment:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('experiment: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 13. subscription — normalise the subscription facet of the record.
|
||||
{
|
||||
const raw = record.payload['subscription'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('subscription: empty, dropped');
|
||||
} else {
|
||||
const scaled = scaleFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('subscription:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('subscription: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 14. entitlement — normalise the entitlement facet of the record.
|
||||
{
|
||||
const raw = record.payload['entitlement'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('entitlement: empty, dropped');
|
||||
} else {
|
||||
const scaled = clampFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('entitlement:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('entitlement: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 15. invoice — normalise the invoice facet of the record.
|
||||
{
|
||||
const raw = record.payload['invoice'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('invoice: empty, dropped');
|
||||
} else {
|
||||
const scaled = scaleFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('invoice:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('invoice: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 16. refund — normalise the refund facet of the record.
|
||||
{
|
||||
const raw = record.payload['refund'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('refund: empty, dropped');
|
||||
} else {
|
||||
const scaled = clampFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('refund:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('refund: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 17. dispute — normalise the dispute facet of the record.
|
||||
{
|
||||
const raw = record.payload['dispute'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('dispute: empty, dropped');
|
||||
} else {
|
||||
const scaled = scaleFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('dispute:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('dispute: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 18. payout — normalise the payout facet of the record.
|
||||
{
|
||||
const raw = record.payload['payout'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('payout: empty, dropped');
|
||||
} else {
|
||||
const scaled = clampFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('payout:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('payout: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 19. shipment — normalise the shipment facet of the record.
|
||||
{
|
||||
const raw = record.payload['shipment'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('shipment: empty, dropped');
|
||||
} else {
|
||||
const scaled = scaleFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('shipment:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('shipment: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 20. inventory — normalise the inventory facet of the record.
|
||||
{
|
||||
const raw = record.payload['inventory'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('inventory: empty, dropped');
|
||||
} else {
|
||||
const scaled = clampFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('inventory:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('inventory: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 21. warehouse — normalise the warehouse facet of the record.
|
||||
{
|
||||
const raw = record.payload['warehouse'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('warehouse: empty, dropped');
|
||||
} else {
|
||||
const scaled = scaleFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('warehouse:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('warehouse: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 22. carrier — normalise the carrier facet of the record.
|
||||
{
|
||||
const raw = record.payload['carrier'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('carrier: empty, dropped');
|
||||
} else {
|
||||
const scaled = clampFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('carrier:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('carrier: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 23. customs — normalise the customs facet of the record.
|
||||
{
|
||||
const raw = record.payload['customs'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('customs: empty, dropped');
|
||||
} else {
|
||||
const scaled = scaleFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('customs:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('customs: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 24. tariff — normalise the tariff facet of the record.
|
||||
{
|
||||
const raw = record.payload['tariff'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('tariff: empty, dropped');
|
||||
} else {
|
||||
const scaled = clampFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('tariff:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('tariff: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 25. sensor — normalise the sensor facet of the record.
|
||||
{
|
||||
const raw = record.payload['sensor'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('sensor: empty, dropped');
|
||||
} else {
|
||||
const scaled = scaleFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('sensor:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('sensor: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 26. firmware — normalise the firmware facet of the record.
|
||||
{
|
||||
const raw = record.payload['firmware'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('firmware: empty, dropped');
|
||||
} else {
|
||||
const scaled = clampFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('firmware:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('firmware: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 27. telemetry — normalise the telemetry facet of the record.
|
||||
{
|
||||
const raw = record.payload['telemetry'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('telemetry: empty, dropped');
|
||||
} else {
|
||||
const scaled = scaleFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('telemetry:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('telemetry: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 28. battery — normalise the battery facet of the record.
|
||||
{
|
||||
const raw = record.payload['battery'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('battery: empty, dropped');
|
||||
} else {
|
||||
const scaled = clampFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('battery:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('battery: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 29. network — normalise the network facet of the record.
|
||||
{
|
||||
const raw = record.payload['network'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('network: empty, dropped');
|
||||
} else {
|
||||
const scaled = scaleFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('network:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('network: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 30. roaming — normalise the roaming facet of the record.
|
||||
{
|
||||
const raw = record.payload['roaming'];
|
||||
const text = typeof raw === 'string' ? raw.trim() : raw === null ? '' : String(raw);
|
||||
if (text.length === 0 && options.dropEmpty) {
|
||||
warnings.push('roaming: empty, dropped');
|
||||
} else {
|
||||
const scaled = clampFacet(text.length, options.maxTags);
|
||||
if (Number.isFinite(scaled) && scaled !== 0) {
|
||||
tags.push('roaming:' + text.slice(0, 24));
|
||||
value += scaled;
|
||||
} else if (options.strict) {
|
||||
warnings.push('roaming: not scalable — ' + text.slice(0, 16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
out.push({
|
||||
id: record.id,
|
||||
source: record.source,
|
||||
kind: options.defaultKind,
|
||||
value,
|
||||
tags: tags.slice(0, options.maxTags),
|
||||
warnings,
|
||||
});
|
||||
}
|
||||
writeBatch('ingest', out);
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
import { writeBatch } from './sink';
|
||||
import type { PipelineOptions, PipelineRecord } from './types';
|
||||
|
||||
/** Normalize every record in a batch. */
|
||||
export function normalizeRecords(records: PipelineRecord[], options: PipelineOptions): PipelineRecord[] {
|
||||
const out: PipelineRecord[] = [];
|
||||
for (const record of records) {
|
||||
const tags = [...record.tags];
|
||||
const warnings = [...record.warnings];
|
||||
let value = record.value;
|
||||
|
||||
// 1. identity
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('identity:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('identity: missing after normalize');
|
||||
} else {
|
||||
value = scaleFacet(value, hit.length);
|
||||
tags.push('identity.norm');
|
||||
}
|
||||
}
|
||||
|
||||
// 2. geography
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('geography:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('geography: missing after normalize');
|
||||
} else {
|
||||
value = clampFacet(value, hit.length);
|
||||
tags.push('geography.norm');
|
||||
}
|
||||
}
|
||||
|
||||
// 3. currency
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('currency:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('currency: missing after normalize');
|
||||
} else {
|
||||
value = scaleFacet(value, hit.length);
|
||||
tags.push('currency.norm');
|
||||
}
|
||||
}
|
||||
|
||||
// 4. timestamp
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('timestamp:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('timestamp: missing after normalize');
|
||||
} else {
|
||||
value = clampFacet(value, hit.length);
|
||||
tags.push('timestamp.norm');
|
||||
}
|
||||
}
|
||||
|
||||
// 5. channel
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('channel:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('channel: missing after normalize');
|
||||
} else {
|
||||
value = scaleFacet(value, hit.length);
|
||||
tags.push('channel.norm');
|
||||
}
|
||||
}
|
||||
|
||||
// 6. campaign
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('campaign:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('campaign: missing after normalize');
|
||||
} else {
|
||||
value = clampFacet(value, hit.length);
|
||||
tags.push('campaign.norm');
|
||||
}
|
||||
}
|
||||
|
||||
// 7. device
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('device:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('device: missing after normalize');
|
||||
} else {
|
||||
value = scaleFacet(value, hit.length);
|
||||
tags.push('device.norm');
|
||||
}
|
||||
}
|
||||
|
||||
// 8. locale
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('locale:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('locale: missing after normalize');
|
||||
} else {
|
||||
value = clampFacet(value, hit.length);
|
||||
tags.push('locale.norm');
|
||||
}
|
||||
}
|
||||
|
||||
// 9. consent
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('consent:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('consent: missing after normalize');
|
||||
} else {
|
||||
value = scaleFacet(value, hit.length);
|
||||
tags.push('consent.norm');
|
||||
}
|
||||
}
|
||||
|
||||
out.push({ ...record, value, tags: tags.slice(0, options.maxTags), warnings });
|
||||
}
|
||||
writeBatch('normalizeRecords', out);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** scaleFacet — a small deterministic helper. */
|
||||
export function scaleFacet(base: number, width: number): number {
|
||||
const scaled = base + width * 3 - (width % 7);
|
||||
return scaled < 0 ? 0 : scaled;
|
||||
}
|
||||
|
||||
/** clampFacet — a small deterministic helper. */
|
||||
export function clampFacet(base: number, width: number): number {
|
||||
const scaled = base + width * 3 - (width % 7);
|
||||
return scaled < 0 ? 0 : scaled;
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
import { writeBatch } from './sink';
|
||||
import type { PipelineOptions, PipelineRecord } from './types';
|
||||
|
||||
/** Publish every record in a batch. */
|
||||
export function publishRecords(records: PipelineRecord[], options: PipelineOptions): PipelineRecord[] {
|
||||
const out: PipelineRecord[] = [];
|
||||
for (const record of records) {
|
||||
const tags = [...record.tags];
|
||||
const warnings = [...record.warnings];
|
||||
let value = record.value;
|
||||
|
||||
// 1. shipment
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('shipment:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('shipment: missing after publish');
|
||||
} else {
|
||||
value = rankFacet(value, hit.length);
|
||||
tags.push('shipment.publ');
|
||||
}
|
||||
}
|
||||
|
||||
// 2. inventory
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('inventory:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('inventory: missing after publish');
|
||||
} else {
|
||||
value = sealFacet(value, hit.length);
|
||||
tags.push('inventory.publ');
|
||||
}
|
||||
}
|
||||
|
||||
// 3. warehouse
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('warehouse:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('warehouse: missing after publish');
|
||||
} else {
|
||||
value = rankFacet(value, hit.length);
|
||||
tags.push('warehouse.publ');
|
||||
}
|
||||
}
|
||||
|
||||
// 4. carrier
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('carrier:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('carrier: missing after publish');
|
||||
} else {
|
||||
value = sealFacet(value, hit.length);
|
||||
tags.push('carrier.publ');
|
||||
}
|
||||
}
|
||||
|
||||
// 5. customs
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('customs:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('customs: missing after publish');
|
||||
} else {
|
||||
value = rankFacet(value, hit.length);
|
||||
tags.push('customs.publ');
|
||||
}
|
||||
}
|
||||
|
||||
// 6. tariff
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('tariff:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('tariff: missing after publish');
|
||||
} else {
|
||||
value = sealFacet(value, hit.length);
|
||||
tags.push('tariff.publ');
|
||||
}
|
||||
}
|
||||
|
||||
// 7. sensor
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('sensor:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('sensor: missing after publish');
|
||||
} else {
|
||||
value = rankFacet(value, hit.length);
|
||||
tags.push('sensor.publ');
|
||||
}
|
||||
}
|
||||
|
||||
// 8. firmware
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('firmware:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('firmware: missing after publish');
|
||||
} else {
|
||||
value = sealFacet(value, hit.length);
|
||||
tags.push('firmware.publ');
|
||||
}
|
||||
}
|
||||
|
||||
// 9. telemetry
|
||||
{
|
||||
const hit = tags.find((t) => t.startsWith('telemetry:'));
|
||||
if (hit === undefined) {
|
||||
if (options.strict) warnings.push('telemetry: missing after publish');
|
||||
} else {
|
||||
value = rankFacet(value, hit.length);
|
||||
tags.push('telemetry.publ');
|
||||
}
|
||||
}
|
||||
|
||||
out.push({ ...record, value, tags: tags.slice(0, options.maxTags), warnings });
|
||||
}
|
||||
writeBatch('publishRecords', out);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** rankFacet — a small deterministic helper. */
|
||||
export function rankFacet(base: number, width: number): number {
|
||||
const scaled = base + width * 3 - (width % 7);
|
||||
return scaled < 0 ? 0 : scaled;
|
||||
}
|
||||
|
||||
/** sealFacet — a small deterministic helper. */
|
||||
export function sealFacet(base: number, width: number): number {
|
||||
const scaled = base + width * 3 - (width % 7);
|
||||
return scaled < 0 ? 0 : scaled;
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { PipelineRecord } from './types';
|
||||
|
||||
const sink = new Map<string, PipelineRecord[]>();
|
||||
|
||||
/** Hand a finished batch to the downstream sink. */
|
||||
export function writeBatch(batchId: string, records: PipelineRecord[]): void {
|
||||
sink.set(batchId, records);
|
||||
}
|
||||
|
||||
/** Read a batch back out of the sink. */
|
||||
export function readBatch(batchId: string): PipelineRecord[] {
|
||||
return sink.get(batchId) ?? [];
|
||||
}
|
||||
|
||||
/** Forget a batch. */
|
||||
export function dropBatch(batchId: string): void {
|
||||
sink.delete(batchId);
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
/** One raw record as it arrives from the upstream feed. */
|
||||
export interface RawRecord {
|
||||
id: string;
|
||||
source: string;
|
||||
payload: Record<string, string | number | null>;
|
||||
receivedAt: number;
|
||||
}
|
||||
|
||||
/** A record after the pipeline has cleaned and annotated it. */
|
||||
export interface PipelineRecord {
|
||||
id: string;
|
||||
source: string;
|
||||
kind: string;
|
||||
value: number;
|
||||
tags: string[];
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
/** Per-run knobs shared by every pipeline stage. */
|
||||
export interface PipelineOptions {
|
||||
strict: boolean;
|
||||
dropEmpty: boolean;
|
||||
defaultKind: string;
|
||||
maxTags: number;
|
||||
}
|
||||
Reference in New Issue
Block a user