Troubleshooting & Recovery

bashnpm run veilnetx:doctor
# or directly:
node aevion-globus-backend/scripts/veilnetx-chain-doctor.js

# Output:
#   chain length: 6
#   walking entries…
#   ✗ break at index 1: stored hash a5d6571627f4… expected ce8f01abff32…
#     entry id: f3e2-…  kind: deposit  ts: 2026-05-12T09:41:00Z
#     suspect: canonical-JSON key order (meta has {b, a} vs {a, b})

bashnpm run veilnetx:rebuild-dry
# or:
node aevion-globus-backend/scripts/rebuild-veilnetx-chain.js --dry-run

# Inspect the diff. If only canonicalization is changing, proceed.
# If amounts or payloads changed, STOP — that indicates tampering, not drift.

bashpg_dump $DATABASE_URL --table='VeilNetXEntry' --data-only -f veilnetx-backup-$(date +%Y%m%d-%H%M%S).sql

bashnpm run veilnetx:rebuild

# Then verify:
curl -s $BASE/api/veilnetx-ledger/chain/verify | jq .verified
# expected: true

sqlSELECT id, "maskId", "paymentRef", "amountCents", status, "createdAt"
  FROM "QMaskCardCharge"
 WHERE "paymentRef" = $1
 ORDER BY "createdAt";

json// expected response shape from POST /api/qmaskcard/charges replay:
{ id: "<original-uuid>", status: "authorized", idempotent: true, amountCents: 500 }

sql-- Verify the index exists
SELECT indexname, indexdef
  FROM pg_indexes
 WHERE tablename = 'QMaskCardCharge'
   AND indexname LIKE '%paymentRef%';

-- Expected:
--   qmaskcard_charge_idempotency_idx
--   CREATE UNIQUE INDEX … ON "QMaskCardCharge" ("maskId", "paymentRef") WHERE status = 'authorized'

sqlCREATE UNIQUE INDEX CONCURRENTLY qmaskcard_charge_idempotency_idx
    ON "QMaskCardCharge" ("maskId", "paymentRef")
 WHERE status = 'authorized';

-- Then find historical doubles:
SELECT "maskId", "paymentRef", COUNT(*) AS n, SUM("amountCents") AS total
  FROM "QMaskCardCharge"
 WHERE status = 'authorized'
 GROUP BY 1, 2
HAVING COUNT(*) > 1;

sql-- Example: QGood donations vs VeilNetX-anchored donations
SELECT (SELECT COUNT(*) FROM "QGoodDonation") AS qgood_count,
       (SELECT COUNT(*) FROM "VeilNetXEntry" WHERE kind = 'qgood:donation') AS anchored_count;

sqlSELECT d.id, d."campaignId", d."amountCents", d."createdAt"
  FROM "QGoodDonation" d
  LEFT JOIN "VeilNetXEntry" v
    ON v."sourceId" = d.id::text AND v.kind = 'qgood:donation'
 WHERE v.id IS NULL
 ORDER BY d."createdAt" DESC
 LIMIT 100;

ts// scripts/reanchor-qgood.ts
import { emitVeilNetX } from "../src/lib/ecosystemEvents";
// fetch un-anchored donations, then:
for (const d of missing) {
  await emitVeilNetX({
    kind: "qgood:donation",
    sourceId: d.id,
    actorEmail: d.donor_email_hash ?? null,
    amountCents: d.amountCents,
    currency: d.currency,
    meta: { campaignId: d.campaignId },
  });
}

bashcurl -s $BASE/api/veilnetx-ledger/chain/verify | jq .verified  # expect true
npm run veilnetx:stats   # confirm count matches qgood total

httpPATCH /api/qpaynet/me/webhook
Authorization: Bearer <jwt>

{ "secret": "<new-32-byte-hex>" }

# AEVION starts signing with NEW immediately; OLD is no longer used outbound.

http# Revert outbound secret to the prior value:
PATCH /api/qpaynet/me/webhook

{ "secret": "<the-old-value-still-in-AEVION_WEBHOOK_SECRET_OLD>" }

# Receiver still verifies because both are in previousSecrets.

sqlSELECT (SELECT COALESCE(SUM(weight),0) FROM "ZTideEvent") AS event_sum,
       (SELECT COALESCE(SUM(score),0)  FROM "ZTideScore") AS score_sum;

sqlSELECT s."userId", s.score AS score_now,
       COALESCE(SUM(e.weight), 0) AS expected
  FROM "ZTideScore" s
  LEFT JOIN "ZTideEvent" e ON e."userId" = s."userId"
 GROUP BY s."userId", s.score
HAVING s.score != COALESCE(SUM(e.weight), 0)
 ORDER BY ABS(s.score - COALESCE(SUM(e.weight), 0)) DESC;

sqlBEGIN;
UPDATE "ZTideScore" s
   SET score = COALESCE(sub.s, 0),
       "updatedAt" = NOW()
  FROM (
    SELECT "userId", SUM(weight) AS s
      FROM "ZTideEvent"
     GROUP BY "userId"
  ) AS sub
 WHERE sub."userId" = s."userId";
-- Verify before commit:
SELECT SUM(score) FROM "ZTideScore";
-- Should now equal SUM(weight) from ZTideEvent.
COMMIT;

bashBASE=$YOUR_BASE node aevion-globus-backend/scripts/fintech-prod-smoke.js | grep "Z-Tide:"