{"ok":true,"service":"callcurb-lookup-api","environment":"production","buildVersion":"0.1.0","timestamp":"2026-10-07T15:52:14.650Z","uptimeSeconds":82412,"observability":{"window_minutes":15,"request_count":4,"request_rate_per_second":0,"last_request_at":"2026-10-07T15:51:38.984Z","latency_ms":{"average":3.25,"p50":5,"p95":5,"p99":5,"max":5,"target_p95":30,"target_p99":80},"response_size_bytes":{"average":null,"p95":null,"max":null},"cache":{"cacheable_request_count":0,"hit_count":0,"mixed_count":0,"hit_rate":null},"status_codes":{"success_2xx":0,"client_error_4xx":4,"server_error_5xx":0},"top_errors":[{"code":"NUMBER_NOT_FOUND","statusCode":404,"count":4,"lastOccurredAt":"2026-10-07T15:51:38.984Z"}],"key_validation_failures":{"count":0,"last_occurred_at":null},"buffered_rollups":{"started_at":"2026-10-06T16:58:42.645Z","totals":{"request_count":136,"average_latency_ms":4.35,"max_latency_ms":15,"last_request_at":"2026-10-07T15:51:38.984Z","methods":{"GET":136},"endpoints":{"/v1/lookup/{e164}":136},"status_codes":{"200":1,"400":1,"404":134},"status_classes":{"success2xx":1,"clientError4xx":135,"serverError5xx":0},"cache_status":{"hit":0,"miss":1,"mixed":0,"bypass":135},"latency_histogram":[{"upperBoundMs":1,"label":"<=1","count":1},{"upperBoundMs":2,"label":"<=2","count":1},{"upperBoundMs":5,"label":"<=5","count":120},{"upperBoundMs":10,"label":"<=10","count":9},{"upperBoundMs":20,"label":"<=20","count":5},{"upperBoundMs":30,"label":"<=30","count":0},{"upperBoundMs":50,"label":"<=50","count":0},{"upperBoundMs":75,"label":"<=75","count":0},{"upperBoundMs":100,"label":"<=100","count":0},{"upperBoundMs":250,"label":"<=250","count":0},{"upperBoundMs":500,"label":"<=500","count":0},{"upperBoundMs":1000,"label":"<=1000","count":0},{"upperBoundMs":2000,"label":"<=2000","count":0},{"upperBoundMs":null,"label":"+Inf","count":0}]},"key_validation_failures":{"count":0,"last_occurred_at":null},"scylla_queries":{"total_count":2,"operations":[{"operation_id":"serving_row_single","count":1,"average_duration_ms":2,"max_duration_ms":2,"last_observed_at":"2026-10-07T13:29:20.887Z","total_rows_observed":1,"latency_histogram":[{"upperBoundMs":1,"label":"<=1","count":0},{"upperBoundMs":2,"label":"<=2","count":1},{"upperBoundMs":5,"label":"<=5","count":0},{"upperBoundMs":10,"label":"<=10","count":0},{"upperBoundMs":20,"label":"<=20","count":0},{"upperBoundMs":30,"label":"<=30","count":0},{"upperBoundMs":50,"label":"<=50","count":0},{"upperBoundMs":75,"label":"<=75","count":0},{"upperBoundMs":100,"label":"<=100","count":0},{"upperBoundMs":250,"label":"<=250","count":0},{"upperBoundMs":500,"label":"<=500","count":0},{"upperBoundMs":1000,"label":"<=1000","count":0},{"upperBoundMs":2000,"label":"<=2000","count":0},{"upperBoundMs":null,"label":"+Inf","count":0}]},{"operation_id":"canonical_row_single_location_backfill","count":1,"average_duration_ms":1,"max_duration_ms":1,"last_observed_at":"2026-10-07T13:29:20.887Z","total_rows_observed":0,"latency_histogram":[{"upperBoundMs":1,"label":"<=1","count":1},{"upperBoundMs":2,"label":"<=2","count":0},{"upperBoundMs":5,"label":"<=5","count":0},{"upperBoundMs":10,"label":"<=10","count":0},{"upperBoundMs":20,"label":"<=20","count":0},{"upperBoundMs":30,"label":"<=30","count":0},{"upperBoundMs":50,"label":"<=50","count":0},{"upperBoundMs":75,"label":"<=75","count":0},{"upperBoundMs":100,"label":"<=100","count":0},{"upperBoundMs":250,"label":"<=250","count":0},{"upperBoundMs":500,"label":"<=500","count":0},{"upperBoundMs":1000,"label":"<=1000","count":0},{"upperBoundMs":2000,"label":"<=2000","count":0},{"upperBoundMs":null,"label":"+Inf","count":0}]}]},"resources":{"latest":{"capturedAt":"2026-10-07T15:51:43.250Z","processUptimeSeconds":82382,"rssMb":784.3,"heapUsedMb":351.7,"heapTotalMb":369,"externalMb":81,"hostMemoryUsedMb":22338.9,"hostMemoryTotalMb":31335.3,"loadAverage":[0,0.03,0.05],"cpuCount":8,"nodeVersion":"v22.22.2","platform":"linux/x64","pid":746166,"activeWorkerCount":4},"history":[]},"flush":{"export_path":"/var/lib/callcurb/lookup-api/lookup-api-observability-rollups.json","flush_interval_ms":5000,"resource_snapshot_interval_ms":30000,"pending_changes":0,"last_attempt_at":"2026-10-07T15:52:13.110Z","last_success_at":"2026-10-07T15:52:13.110Z","last_error":null}}},"worker_runtime":{"authority_snapshot":{"refreshIntervalMs":5000,"freshnessWindowMs":5000,"ageMs":82407489,"minAgeMs":82193420,"maxAgeMs":82407489,"staleWorkerCount":4,"workerCount":4,"lastLoadedAt":"2026-10-06T17:02:17.797Z","lastLoadedAtNewest":"2026-10-06T17:02:17.797Z","lastLoadedAtOldest":"2026-10-06T16:58:42.644Z","lastRefreshError":null,"lastRefreshErrorCount":0},"usage_queue":{"persistenceMode":"primary_ipc","queuedCount":0,"maxDepth":10000,"workersAtMaxDepth":0,"totalPersisted":1,"totalDropped":0,"totalRecovered":0,"totalShutdownFlushed":0,"lastPersistedAt":"2026-10-06T17:02:16.812Z","lastFlushStartedAt":"2026-10-07T15:52:10.130Z","lastFlushCompletedAt":"2026-10-07T15:52:10.129Z","lastFlushDurationMs":5000,"lastFlushBatchSize":37,"lastFlushError":"Timed out waiting for primary usage persistence ack for batch 746179-1791388323106-15947."},"cache":{"posture":"per_worker_lru","maxEntriesPerWorker":1000,"configuredWorkerCount":4,"estimatedClusterEntryCapacity":4000,"activeEntryCount":1,"approxValueBytes":1419,"utilizationPercent":0.03,"totalEvictions":0,"totalExpiredOnRead":0,"lastSetAt":"2026-10-07T13:29:20.885Z"},"workers":[{"workerId":1,"pid":746179,"capturedAt":"2026-10-07T15:52:11.217Z","requestCount":1,"requestRatePerSecond":0,"p95LatencyMs":4,"cacheEntryCount":0,"cacheApproxValueBytes":0,"usageQueuedCount":0,"usageDroppedTotal":0,"authorityAgeMs":82193420,"rssMb":197.1,"topErrors":[{"code":"NUMBER_NOT_FOUND","count":1}]},{"workerId":2,"pid":746180,"capturedAt":"2026-10-07T15:52:10.134Z","requestCount":1,"requestRatePerSecond":0,"p95LatencyMs":3,"cacheEntryCount":0,"cacheApproxValueBytes":0,"usageQueuedCount":0,"usageDroppedTotal":0,"authorityAgeMs":82407483,"rssMb":196.1,"topErrors":[{"code":"NUMBER_NOT_FOUND","count":1}]},{"workerId":3,"pid":746186,"capturedAt":"2026-10-07T15:52:10.133Z","requestCount":1,"requestRatePerSecond":0,"p95LatencyMs":3,"cacheEntryCount":1,"cacheApproxValueBytes":1419,"usageQueuedCount":0,"usageDroppedTotal":0,"authorityAgeMs":82407489,"rssMb":194.2,"topErrors":[{"code":"NUMBER_NOT_FOUND","count":1}]},{"workerId":4,"pid":746192,"capturedAt":"2026-10-07T15:52:10.132Z","requestCount":1,"requestRatePerSecond":0,"p95LatencyMs":3,"cacheEntryCount":0,"cacheApproxValueBytes":0,"usageQueuedCount":0,"usageDroppedTotal":0,"authorityAgeMs":82407479,"rssMb":196.9,"topErrors":[{"code":"NUMBER_NOT_FOUND","count":1}]}]},"runtime_telemetry_source":"primary_coordinated","cache_posture":{"model":"per_worker_lru","workerDuplicationAccepted":true,"maxEntriesPerWorker":1000,"configuredWorkerCount":4,"estimatedClusterEntryCapacity":4000,"activeEntryCount":1,"approxValueBytes":1419,"summary":"The runtime keeps one per-worker LRU cache and sizes memory explicitly as max entries per worker multiplied by configured workers. This keeps cache hits local and avoids adding a shared cache dependency to the hot path."},"coordinator_status":{"state":"active","role":"primary","configuredWorkerCount":4,"activeWorkerCount":4,"workerRestartCount":0,"backgroundSyncOwner":"primary","primaryPid":746166,"workerPids":[746179,746180,746186,746192],"lastUpdatedAt":"2026-10-07T15:52:13.109Z","lastWorkerForkedAt":null,"lastWorkerOnlineAt":null,"lastWorkerExitAt":null,"lastWorkerExitReason":null,"usageWriter":{"mode":"primary_serialized_sqlite","queuedBatchCount":64994,"queuedEventCount":623845,"inFlightBatchCount":0,"lastBatchReceivedAt":"2026-10-07T15:52:11.334Z","lastPersistedAt":"2026-10-06T17:02:17.796Z","lastBatchSize":1,"lastDurationMs":1,"lastError":null,"totalReceivedBatches":64995,"totalReceivedEvents":623846,"totalPersistedBatches":1,"totalPersistedEvents":1,"totalFailedBatches":0}},"consistency_contract":{"authoritativeSqliteCounters":["api-key lifecycle status and revocation state","rotation lineage and allowlists","persisted daily quota-window counters","persisted long-window usage history and exports"],"workerLocalProtections":["bounded authority snapshots for hot-path auth reads","worker-local minute rate-limit buckets","worker-local queued usage shadow counters","worker-local queue backpressure and drop protection"],"boundedSkew":{"authorityFreshnessWindowMs":5000,"maxExtraQuotaRequestsPerWindow":3,"maxExtraRateLimitRequestsPerMinute":3},"quotaGuard":{"configuredWorkerCount":4,"strategy":"Each worker may admit only ceil((remaining_authoritative_quota) / configured_workers) additional requests before the next SQLite-backed snapshot refresh."},"singleWriterDurability":{"workerPersistenceMode":"primary_ipc","coordinatorWriterMode":"primary_serialized_sqlite","strategy":"Workers batch usage locally, forward flush batches to the primary process over IPC, and the primary coalesces those worker batches into one serialized SQLite persistence lane."}},"scylla_connection":{"kind":"loopback","host":"127.0.0.1","port":9042,"preferredForHetzner":true,"requiresManualVerification":false,"detail":"Scylla is targeted through loopback at 127.0.0.1:9042, which is the preferred same-host Hetzner path."},"dependency_summary":{"total":2,"degraded_count":0},"degraded_dependencies":[],"readiness":{"state":"degraded","summary":"The local key authority has a non-blocking warning.","blockers":[],"warnings":["The local key authority has a non-blocking warning."]}}