Runlayer Operator (Kubernetes)

spec:
  gcp:
    projectId: acme-prod   # sets GCP_PROJECT_ID once on platform workloads

spec:
  serviceAccount:
    backend:
      annotations:
        eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/runlayer-acme-backend
    worker:
      annotations:
        eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/runlayer-acme-worker
    auditConsumer:
      annotations:
        eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/runlayer-acme-audit-consumer
    siemExport:
      annotations:
        eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/runlayer-acme-siem-export
    sessionMaterializer:
      annotations:
        eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/runlayer-acme-session-materializer

spec:
  serviceAccount:
    backend:
      annotations:
        iam.gke.io/gcp-service-account: runlayer-acme-backend@PROJECT.iam.gserviceaccount.com
    worker:
      annotations:
        iam.gke.io/gcp-service-account: runlayer-acme-worker@PROJECT.iam.gserviceaccount.com
    auditConsumer:
      annotations:
        iam.gke.io/gcp-service-account: runlayer-acme-audit@PROJECT.iam.gserviceaccount.com
    siemExport:
      annotations:
        iam.gke.io/gcp-service-account: runlayer-acme-siem@PROJECT.iam.gserviceaccount.com
    sessionMaterializer:
      annotations:
        iam.gke.io/gcp-service-account: runlayer-acme-session-materializer@PROJECT.iam.gserviceaccount.com

# After `helm pull` / chart unpack, copy values.example.yaml from the chart
cp values.example.yaml values-runlayer-operator.yaml

helm upgrade --install runlayer-operator \
  oci://088332244652.dkr.ecr.us-east-1.amazonaws.com/runlayer-helm-charts/runlayer-operator \
  --version <chart-version> \
  --namespace runlayer-system --create-namespace \
  -f values-runlayer-operator.yaml

helm upgrade --install runlayer-operator \
  oci://412677576004.dkr.ecr.us-east-1.amazonaws.com/runlayer-helm-charts/runlayer-operator \
  --version <chart-version> \
  --namespace runlayer-system --create-namespace \
  -f values-runlayer-operator.yaml \
  --set image.repository=412677576004.dkr.ecr.us-east-1.amazonaws.com/runlayer-operator

# Extract CRDs from the same chart version you are upgrading to, then:
kubectl apply --server-side --force-conflicts \
  -f crds/platform.runlayer.com_runlayerinstances.yaml
kubectl apply --server-side --force-conflicts \
  -f crds/deploy.runlayer.com_mcpservers.yaml

helm upgrade runlayer-operator \
  oci://088332244652.dkr.ecr.us-east-1.amazonaws.com/runlayer-helm-charts/runlayer-operator \
  --version <new-chart-version> \
  --namespace runlayer-system \
  -f values-runlayer-operator.yaml

kubectl create namespace runlayer-acme

kubectl create secret generic runlayer-db -n runlayer-acme \
  --from-literal=username='runlayer' \
  --from-literal=password='…'

kubectl create secret generic runlayer-redis -n runlayer-acme \
  --from-literal=password='…'

kubectl create secret generic runlayer-auth -n runlayer-acme \
  --from-literal=AUTH_CLIENT_ID='…' \
  --from-literal=AUTH_API_KEY='…'

kubectl create secret generic runlayer-app -n runlayer-acme \
  --from-literal=SECRET_KEY='…' \
  --from-literal=MASTER_SALT='…' \
  --from-literal=MCP_CATALOG_API_KEY='…'   # required — from Runlayer

apiVersion: platform.runlayer.com/v1alpha1
kind: RunlayerInstance
metadata:
  name: acme-prod
  namespace: runlayer-system
  labels:
    runlayer.com/customer-id: acme
spec:
  customerId: acme
  domain: acme.example.com
  environment: production
  version: "1.0.0"
  images:
    backend: 123456789012.dkr.ecr.us-east-1.amazonaws.com/runlayer/backend:1.0.0
    frontend: 123456789012.dkr.ecr.us-east-1.amazonaws.com/runlayer/frontend:1.0.0
    worker: 123456789012.dkr.ecr.us-east-1.amazonaws.com/runlayer/worker:1.0.0
  database:
    host: acme-db.cluster-xxx.us-east-1.rds.amazonaws.com
    port: 5432
    name: runlayer
    credentials:
      usernameFrom: { name: runlayer-db, key: username }
      passwordFrom: { name: runlayer-db, key: password }
  redis:
    host: acme-redis.xxx.cache.amazonaws.com
    port: 6379
    tls: true
    credentials:
      passwordFrom: { name: runlayer-redis, key: password }
  auth:
    clientIdFrom: { name: runlayer-auth, key: AUTH_CLIENT_ID }
    apiKeyFrom: { name: runlayer-auth, key: AUTH_API_KEY }
  appSecretRef:
    name: runlayer-app
  components:
    backend: { enabled: true, replicas: 2 }
    frontend: { enabled: true, replicas: 2 }
    worker: { enabled: true, replicas: 1 }
  deploy:
    enabled: true
  serviceAccount:
    backend:
      annotations:
        eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/runlayer-acme-backend
    worker:
      annotations:
        eks.amazonaws.com/role-arn: arn:aws:iam::123456789012:role/runlayer-acme-worker
  ingress:
    enabled: true
    ingressClassName: alb
    controllerNamespace: aws-load-balancer-controller
    annotations:
      alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-1:123456789012:certificate/…
      alb.ingress.kubernetes.io/scheme: internet-facing
      alb.ingress.kubernetes.io/target-type: ip

apiVersion: platform.runlayer.com/v1alpha1
kind: RunlayerInstance
metadata:
  name: acme-prod
  namespace: runlayer-system
spec:
  customerId: acme
  domain: acme.example.com
  environment: production
  version: "1.0.0"
  images:
    backend: us-east1-docker.pkg.dev/PROJECT/runlayer/backend:1.0.0
    frontend: us-east1-docker.pkg.dev/PROJECT/runlayer/frontend:1.0.0
    worker: us-east1-docker.pkg.dev/PROJECT/runlayer/worker:1.0.0
  database:
    host: 10.x.x.x   # Cloud SQL private IP or PSC endpoint
    credentials:
      usernameFrom: { name: runlayer-db, key: username }
      passwordFrom: { name: runlayer-db, key: password }
  redis:
    host: 10.y.y.y   # Memorystore
    port: 6379
    tls: true
  auth:
    clientIdFrom: { name: runlayer-auth, key: AUTH_CLIENT_ID }
    apiKeyFrom: { name: runlayer-auth, key: AUTH_API_KEY }
  appSecretRef:
    name: runlayer-app
  components:
    backend: { enabled: true, replicas: 2 }
    frontend: { enabled: true, replicas: 2 }
    worker: { enabled: true, replicas: 1 }
  deploy:
    enabled: true
  serviceAccount:
    backend:
      annotations:
        iam.gke.io/gcp-service-account: runlayer-acme-backend@PROJECT.iam.gserviceaccount.com
    worker:
      annotations:
        iam.gke.io/gcp-service-account: runlayer-acme-worker@PROJECT.iam.gserviceaccount.com
  ingress:
    enabled: true
    ingressClassName: gce
    gcp:
      managedCertificate:
        enabled: true
      staticIPName: acme-prod-ip

spec:
  images:
    mcpCatalogApi: REGISTRY/mcp-catalog-api:TAG
    sentryRelay: REGISTRY/sentry-relay:TAG
    toolguard: REGISTRY/toolguard:TAG   # or 088332244652.dkr.ecr.us-east-1.amazonaws.com/runlayer/toolguard:…
    topicCpu: REGISTRY/toolguard:runlayer-multimodel-guard-TAG   # or 088332244652.dkr.ecr.us-east-1.amazonaws.com/runlayer/toolguard:…
    topicCpuRev: REGISTRY/toolguard:runlayer-rev-guard-TAG       # Rev-only image; required when topicCpu.rev.enabled
    llmGateway: REGISTRY/runlayer/llm-gateway:TAG
  components:
    mcpCatalogApi:
      enabled: true
      apiKeyFrom: { name: runlayer-catalog, key: API_KEY }
    toolguard:
      enabled: true
      gpu:
        enabled: false   # CPU-only / Autopilot without GPU capacity
    topicCpu:
      enabled: true
      # gunicornWorkers: 1          # slow pool (memory >= workers * 2048 MiB)
      # fast:
      #   gunicornWorkers: 3        # memory >= max(workers * 1024, 2048) MiB
      # rev:                        # Rev self-hosted decision model (opt-in; POST /predict/runlayer-rev)
      #   enabled: true             # requires images.topicCpuRev; backend gets RUNLAYER_REV_ENDPOINT_URL + RUNLAYER_REV_TIMEOUT
      #   gunicornWorkers: 1        # memory >= (workers + 1) * 2560 MiB; recycler briefly runs workers+1
    llmGateway:
      enabled: true
      apiKeyFrom: { name: runlayer-llm-gateway, key: apiKey }
    auditConsumer:
      enabled: true
      streamBackend: kinesis   # or pubsub
      kinesis:
        streamArn: arn:aws:kinesis:…
        consumerArn: arn:aws:kinesis:…/consumer/…
        checkpointTableName: runlayer-audit-checkpoints
        deadLetterBucket: runlayer-audit-dlq
      # pubsub:
      #   subscriptionId: audit-log-sub
      #   deadLetterBucket: acme-audit-dlq
    siemExport:
      enabled: true
      streamBackend: kinesis
      kinesis:
        streamArn: arn:aws:kinesis:…
        consumerArn: arn:aws:kinesis:…/consumer/…
        checkpointTableName: runlayer-siem-checkpoints
        deadLetterBucket: runlayer-siem-dlq
      export:
        s3Bucket: customer-siem-bucket
        # gcsBucket: customer-siem-gcs   # when streamBackend: pubsub
    sessionMaterializer:
      enabled: true
      streamBackend: kinesis   # or pubsub
      sessionsReadFromPostgres: true
      kinesis:
        streamArn: arn:aws:kinesis:…:stream/hook-events
        checkpointTableName: runlayer-session-materializer-checkpoints
        deadLetterBucket: runlayer-session-materializer-dlq
      payload:
        s3Bucket: runlayer-session-payloads
      hookEvents:
        streamName: hook-events
        region: us-east-1
        s3Bucket: runlayer-hook-events-parquet
      # pubsub:
      #   subscriptionId: hook-events-sub
      #   deadLetterBucket: acme-session-materializer-dlq
      # payload:
      #   gcsBucket: acme-session-payloads
      # hookEvents:
      #   topicId: hook-events
      #   publishTimeoutSeconds: 5  # optional; backend default is 5s
  agents:
    sandboxMode: agentcore   # or k8s
    filesBucket: acme-agent-sandbox-files
    agentcore:
      runtimeArn: arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/acme_prod_agent_sandbox
    # k8s:
    #   warmPool: runlayer-agent-pool
    #   namespace: runlayer-acme-agents   # optional; defaults to platform NS
  serviceAccount:
    sessionMaterializer:
      annotations:
        eks.amazonaws.com/role-arn: arn:aws:iam::…:role/runlayer-session-materializer
        # GKE: iam.gke.io/gcp-service-account: …@PROJECT.iam.gserviceaccount.com

kubectl wait runlayerinstance/acme-prod -n runlayer-system \
  --for=jsonpath='{.status.phase}'=Ready --timeout=20m

kubectl get runlayerinstance acme-prod -n runlayer-system -o yaml

helm upgrade --install acme-prod \
  oci://088332244652.dkr.ecr.us-east-1.amazonaws.com/runlayer-helm-charts/runlayer-instance \
  --version <chart-version> \
  --namespace runlayer-system \
  -f values-acme.yaml

spec:
  security:
    networkPolicy:
      enabled: true
      additionalEgressRules:
        - to:
            - ipBlock: { cidr: 10.0.0.0/8 }
          ports:
            - protocol: TCP
              port: 5432
            - protocol: TCP
              port: 6379

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: runlayer-audit-spool
  labels:
    release: prometheus # match your Prometheus rule selector
spec:
  groups:
    - name: runlayer-audit-spool
      # Severity split: fallback ratio = hot-path latency regression (page); dead-letters pruned = quantified audit-record loss (page); oldest-age stays warning as the early-drift signal.
      rules:
        # More than 1% of spool-eligible audit emits are falling back to
        # inline delivery, as a % against delivered + fallen-back records —
        # the denominator is confirmed deliveries, so the ratio deflates
        # while a post-outage backlog drains. spool_closed is excluded: it
        # only occurs when a request races a graceful shutdown, i.e.
        # ordinary deploy traffic.
        # The >= 5 fallbacks / 10m floor keeps one-off blips in quiet hours
        # from paging; on a deployment emitting fewer than 5 spool-eligible
        # audit events per 10 minutes it suppresses the alert entirely, so
        # lower or drop that clause on low-traffic deployments.
        # No counter-birth guard on the fallback counter: current backend
        # builds pre-seed every reason series to 0 at process start, so
        # increase() already sees the 0 -> N step on a fresh process — a
        # birth term here would double-count a young process's fallbacks.
        # (On older builds without the zero baselines, a fallback series
        # born by its first loss event hides from increase() until its
        # second export; the dead-letters rule below shows the birth-branch
        # pattern if you must cover such builds.)
        # The delivery histogram is NOT pre-seeded: until the first
        # successful delivery its _count has no series at all, and an empty
        # denominator would silence the alert in exactly the worst case
        # (everything falling back). The "or <fallbacks> * 0" branch keeps
        # the denominator non-empty then, so the ratio pins to 100 instead
        # of going missing.
        - alert: RunlayerAuditSpoolFallbackRatioHigh
          expr: |
            (
              100 * sum(increase(runlayer_audit_spool_fallback_count_total{reason!="spool_closed"}[10m]))
              /
              (
                (
                  sum(increase(runlayer_audit_spool_delivery_duration_milliseconds_count[10m]))
                  or
                  sum(increase(runlayer_audit_spool_fallback_count_total{reason!="spool_closed"}[10m])) * 0
                )
                + sum(increase(runlayer_audit_spool_fallback_count_total{reason!="spool_closed"}[10m]))
              )
              > 1
            )
            and
            (
              sum(increase(runlayer_audit_spool_fallback_count_total{reason!="spool_closed"}[10m])) >= 5
            )
          for: 10m
          labels:
            severity: critical
          annotations:
            summary: "Runlayer audit spool is rejecting appends; delivery is back on the request hot path"
            description: "Split runlayer_audit_spool_fallback_count_total by the reason label — each reason has its own first move (see the Runlayer Operator docs, Monitoring the audit log spool)."

        # The oldest undelivered spool record is more than 5 minutes old:
        # the tailer is not draining (wedged leader, or delivery target
        # slow/unreachable). Expected to spike briefly after a database or
        # delivery-target outage while backlog drains.
        - alert: RunlayerAuditSpoolBacklogStale
          expr: max(runlayer_audit_spool_oldest_age_seconds) > 300
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Runlayer audit spool backlog is not draining"
            description: "Oldest undelivered audit record is over 5 minutes old. Check backend logs for tailer/delivery errors and runlayer_audit_spool_depth_bytes for backlog growth."

        # The age gauge is emitted on every tailer pass while a leader is
        # measuring (an explicit 0 when the spool is empty), so a gap in the
        # series is itself a signal: no tailer leader is measuring, or the
        # spool head is unreadable with backlog present. The rule above
        # evaluates to an empty vector in exactly those states and stays
        # green. Each backend pod's tailer leader emits its own series, so
        # count the reporting pods against your replica count rather than
        # testing global absence — with absent(), one healthy pod hides
        # every broken one. Set the threshold to your backend replica count
        # (spec.components.backend.replicas; join against
        # kube-state-metrics' kube_deployment_spec_replicas if you want it
        # to track scaling automatically). Only deploy this companion rule
        # where the spool and OTEL export are enabled — otherwise the gauge
        # is legitimately absent and it will fire constantly.
        - alert: RunlayerAuditSpoolAgeGaugeMissing
          expr: (count(runlayer_audit_spool_oldest_age_seconds) or vector(0)) < 2 # match spec.components.backend.replicas
          for: 15m
          labels:
            severity: warning
          annotations:
            summary: "Runlayer audit spool age gauge reporting from fewer pods than expected"
            description: "At least one backend pod has no tailer leader measuring its spool (or an unreadable spool head). That pod's backlog delivery may be stopped entirely. Check backend logs for tailer leadership and audit_spool_gauge_emit_failed errors."

        # Records were deleted from the dead-letter directory to keep it
        # bounded. Unlike fallbacks, this is real audit-log loss.
        # dead_prune_uncounted covers files the pruner deleted without being
        # able to read a record count first (counts files, not records) —
        # the same irreversible loss, kept on a separate series so
        # dead_pruned stays record-denominated.
        # Birth branches: current backend builds pre-seed both counters to
        # 0 at process start, so increase() catches a first prune on its
        # own. The "unless last_over_time(...)" branches cover builds
        # without the zero baselines, where a prune counter is born already
        # positive and increase() reports nothing until a second prune in
        # the same process — which may never come, since prunes are rare.
        # The per-series unless runs BEFORE the sum: a recycled worker's
        # pre-seeded zero baseline (new series, first sample 0) sums to 0
        # and is dropped by the > 0 filter, and the windowed presence check
        # (last_over_time over (t-20m, t-5m], not an instant offset) means
        # a false birth needs a 15m+ metrics-ingestion gap, not one missed
        # scrape, before old series could masquerade as newborn.
        - alert: RunlayerAuditSpoolDeadLettersPruned
          expr: |
            sum(increase(runlayer_audit_spool_dead_pruned_count_total[1h])) > 0
            or sum(runlayer_audit_spool_dead_pruned_count_total unless last_over_time(runlayer_audit_spool_dead_pruned_count_total[15m] offset 5m)) > 0
            or sum(increase(runlayer_audit_spool_dead_prune_uncounted_count_total[1h])) > 0
            or sum(runlayer_audit_spool_dead_prune_uncounted_count_total unless last_over_time(runlayer_audit_spool_dead_prune_uncounted_count_total[15m] offset 5m)) > 0
          labels:
            severity: critical
          annotations:
            summary: "Runlayer audit records pruned from the dead-letter directory (audit-log loss)"
            description: "Poison records were deleted to keep dead/ bounded. Investigate what is producing poison records (audit_spool_record_dead_lettered backend logs) before retention erases the evidence."