{
    "ok": true,
    "level": "deg",
    "status": "degraded",
    "label": "Degraded performance: Singapore",
    "checked_at": "2026-11-04T13:59:17+00:00",
    "checked_ts": 1793800757,
    "checked_at_hms": "13:59:17",
    "probe_interval_seconds": 60,
    "sla": "99.9%",
    "regions": [
        {
            "code": "REY",
            "city": "Reykjav\u00edk",
            "status": "operational",
            "since": "2024-03-04",
            "uptime_90d_pct": 99.997,
            "downtime_90d_minutes": 4,
            "probe": {
                "from": "AMS",
                "rtt_ms": 27.9,
                "rtt_24h_min_ms": 27.4,
                "rtt_24h_max_ms": 35.8
            },
            "last_incident": {
                "id": "i-2026-09-05-rey",
                "title": "Core switch supervisor failover in Reykjav\u00edk",
                "started_at": "2026-09-05 20:39:00Z"
            }
        },
        {
            "code": "HEL",
            "city": "Helsinki",
            "status": "operational",
            "since": "2025-02-10",
            "uptime_90d_pct": 100,
            "downtime_90d_minutes": 0,
            "probe": {
                "from": "AMS",
                "rtt_ms": 26,
                "rtt_24h_min_ms": 26,
                "rtt_24h_max_ms": 31.8
            },
            "last_incident": {
                "id": "i-2026-08-20-hel",
                "title": "Hardware fault on one GPU node in Helsinki: card replaced",
                "started_at": "2026-08-20 07:11:00Z"
            }
        },
        {
            "code": "AMS",
            "city": "Amsterdam",
            "status": "operational",
            "since": "2025-06-16",
            "uptime_90d_pct": 99.992,
            "downtime_90d_minutes": 11,
            "probe": {
                "from": "HEL",
                "rtt_ms": 26.7,
                "rtt_24h_min_ms": 26,
                "rtt_24h_max_ms": 33.5
            },
            "last_incident": {
                "id": "i-2026-09-05-ams",
                "title": "NVLink fabric manager restart on 4 nodes in Amsterdam",
                "started_at": "2026-09-05 10:17:00Z"
            }
        },
        {
            "code": "SIN",
            "city": "Singapore",
            "status": "degraded",
            "since": "2026-02-02",
            "uptime_90d_pct": 100,
            "downtime_90d_minutes": 0,
            "probe": {
                "from": "AMS",
                "rtt_ms": 163.3,
                "rtt_24h_min_ms": 155.6,
                "rtt_24h_max_ms": 200.7
            },
            "last_incident": {
                "id": "i-2026-11-04-sin",
                "title": "Utility feed transfer to generator in Singapore",
                "started_at": "2026-11-04 13:27:00Z"
            }
        }
    ],
    "shared": [
        {
            "id": "console",
            "name": "Console & API",
            "status": "operational",
            "uptime_90d_pct": 100
        },
        {
            "id": "billing",
            "name": "Crypto top-ups",
            "status": "operational",
            "uptime_90d_pct": 100
        }
    ],
    "ongoing": [
        {
            "id": "i-2026-11-04-sin",
            "kind": "incident",
            "region": "SIN",
            "component": "power",
            "title": "Utility feed transfer to generator in Singapore",
            "impact": "None. Servers ran on generator power throughout.",
            "started_at": "2026-11-04 13:27:00Z",
            "duration_minutes": 78,
            "counts_as_downtime": false,
            "resolved": false,
            "updates": [
                {
                    "at": "2026-11-04 13:27:00Z",
                    "status": "investigating",
                    "text": "The data centre in Singapore transferred to generator after a disturbance on the utility feed. No server was affected; UPS bridged the transfer."
                },
                {
                    "at": "2026-11-04 13:54:00Z",
                    "status": "identified",
                    "text": "Generators carrying the full load, fuel for 48 hours on site."
                }
            ],
            "post_incident_review": null
        }
    ],
    "scheduled": [
        {
            "id": "m-2026-11-24-hel",
            "kind": "maintenance",
            "region": "HEL",
            "component": "compute",
            "title": "BMC firmware update on PCIe chassis in Helsinki",
            "impact": "IPMI console unavailable for about two minutes per server; servers keep running.",
            "started_at": "2026-11-24 01:45:00Z",
            "duration_minutes": 25,
            "counts_as_downtime": false,
            "resolved": false,
            "updates": [],
            "post_incident_review": null
        },
        {
            "id": "m-2026-11-24-rey",
            "kind": "maintenance",
            "region": "REY",
            "component": "network",
            "title": "Switch software upgrade in Reykjav\u00edk",
            "impact": "Up to 60 seconds of packet loss per server during the failover, twice.",
            "started_at": "2026-11-24 02:15:00Z",
            "duration_minutes": 32,
            "counts_as_downtime": false,
            "resolved": false,
            "updates": [],
            "post_incident_review": null
        },
        {
            "id": "m-2026-11-25-console",
            "kind": "maintenance",
            "region": "ALL",
            "component": "console",
            "title": "Console deployment",
            "impact": "Console read-only for under a minute; servers and API unaffected.",
            "started_at": "2026-11-25 01:30:00Z",
            "duration_minutes": 8,
            "counts_as_downtime": false,
            "resolved": false,
            "updates": [],
            "post_incident_review": null
        },
        {
            "id": "m-2026-12-03-rey",
            "kind": "maintenance",
            "region": "REY",
            "component": "compute",
            "title": "NVLink firmware update on H200 nodes in Reykjav\u00edk",
            "impact": "Each node reboots once; no re-image, data untouched.",
            "started_at": "2026-12-03 01:30:00Z",
            "duration_minutes": 53,
            "counts_as_downtime": false,
            "resolved": false,
            "updates": [],
            "post_incident_review": null
        }
    ],
    "incidents": [
        {
            "id": "i-2026-11-04-sin",
            "kind": "incident",
            "region": "SIN",
            "component": "power",
            "title": "Utility feed transfer to generator in Singapore",
            "impact": "None. Servers ran on generator power throughout.",
            "started_at": "2026-11-04 13:27:00Z",
            "duration_minutes": 78,
            "counts_as_downtime": false,
            "resolved": false,
            "updates": [
                {
                    "at": "2026-11-04 13:27:00Z",
                    "status": "investigating",
                    "text": "The data centre in Singapore transferred to generator after a disturbance on the utility feed. No server was affected; UPS bridged the transfer."
                },
                {
                    "at": "2026-11-04 13:54:00Z",
                    "status": "identified",
                    "text": "Generators carrying the full load, fuel for 48 hours on site."
                }
            ],
            "post_incident_review": null
        },
        {
            "id": "m-2026-11-03-rey",
            "kind": "maintenance",
            "region": "REY",
            "component": "network",
            "title": "Switch software upgrade in Reykjav\u00edk",
            "impact": "Up to 60 seconds of packet loss per server during the failover, twice.",
            "started_at": "2026-11-03 01:45:00Z",
            "duration_minutes": 40,
            "counts_as_downtime": false,
            "resolved": true,
            "updates": [
                {
                    "at": "2026-10-27 08:45:00Z",
                    "status": "scheduled",
                    "text": "Announced for 3 November, 01:45\u201302:25 UTC. Affected tenants notified in the console. Up to 60 seconds of packet loss per server during the failover, twice."
                },
                {
                    "at": "2026-11-03 01:45:00Z",
                    "status": "in progress",
                    "text": "Window open. Work proceeds one device at a time."
                },
                {
                    "at": "2026-11-03 02:25:00Z",
                    "status": "completed",
                    "text": "Completed in 40 minutes, inside the announced window. Announced maintenance under the monthly allowance: no SLA impact."
                }
            ],
            "post_incident_review": null
        },
        {
            "id": "m-2026-10-22-rey",
            "kind": "maintenance",
            "region": "REY",
            "component": "compute",
            "title": "NVLink firmware update on B200 nodes in Reykjav\u00edk",
            "impact": "Each node reboots once; no re-image, data untouched.",
            "started_at": "2026-10-22 01:15:00Z",
            "duration_minutes": 30,
            "counts_as_downtime": false,
            "resolved": true,
            "updates": [
                {
                    "at": "2026-10-14 14:15:00Z",
                    "status": "scheduled",
                    "text": "Announced for 22 October, 01:15\u201301:45 UTC. Affected tenants notified in the console. Each node reboots once; no re-image, data untouched."
                },
                {
                    "at": "2026-10-22 01:15:00Z",
                    "status": "in progress",
                    "text": "Window open. Work proceeds one device at a time."
                },
                {
                    "at": "2026-10-22 01:45:00Z",
                    "status": "completed",
                    "text": "Completed in 30 minutes, inside the announced window. Announced maintenance under the monthly allowance: no SLA impact."
                }
            ],
            "post_incident_review": null
        },
        {
            "id": "m-2026-10-22-hel",
            "kind": "maintenance",
            "region": "HEL",
            "component": "compute",
            "title": "NVLink firmware update on H200 nodes in Helsinki",
            "impact": "Each node reboots once; no re-image, data untouched.",
            "started_at": "2026-10-22 01:00:00Z",
            "duration_minutes": 57,
            "counts_as_downtime": false,
            "resolved": true,
            "updates": [
                {
                    "at": "2026-10-14 14:00:00Z",
                    "status": "scheduled",
                    "text": "Announced for 22 October, 01:00\u201301:57 UTC. Affected tenants notified in the console. Each node reboots once; no re-image, data untouched."
                },
                {
                    "at": "2026-10-22 01:00:00Z",
                    "status": "in progress",
                    "text": "Window open. Work proceeds one device at a time."
                },
                {
                    "at": "2026-10-22 01:57:00Z",
                    "status": "completed",
                    "text": "Completed in 57 minutes, inside the announced window. Announced maintenance under the monthly allowance: no SLA impact."
                }
            ],
            "post_incident_review": null
        },
        {
            "id": "i-2026-10-13-billing",
            "kind": "incident",
            "region": "ALL",
            "component": "billing",
            "title": "Payment processor maintenance: new invoices paused",
            "impact": "New top-up invoices could not be created for a short time; open invoices kept confirming.",
            "started_at": "2026-10-13 21:51:00Z",
            "duration_minutes": 37,
            "counts_as_downtime": false,
            "resolved": true,
            "updates": [
                {
                    "at": "2026-10-13 21:51:00Z",
                    "status": "investigating",
                    "text": "Our payment processor is in maintenance; creating a new invoice fails with a clear message. Open invoices and confirmations continue."
                },
                {
                    "at": "2026-10-13 22:04:00Z",
                    "status": "identified",
                    "text": "Processor back; invoice creation working from every probe."
                },
                {
                    "at": "2026-10-13 22:18:00Z",
                    "status": "monitoring",
                    "text": "Watching for an hour."
                },
                {
                    "at": "2026-10-13 22:28:00Z",
                    "status": "resolved",
                    "text": "Resolved."
                }
            ],
            "post_incident_review": null
        },
        {
            "id": "i-2026-10-02-console",
            "kind": "incident",
            "region": "ALL",
            "component": "console",
            "title": "Elevated latency on the console and API",
            "impact": "Console pages and API calls slower than usual; servers unaffected.",
            "started_at": "2026-10-02 21:13:00Z",
            "duration_minutes": 23,
            "counts_as_downtime": false,
            "resolved": true,
            "updates": [
                {
                    "at": "2026-10-02 21:13:00Z",
                    "status": "investigating",
                    "text": "Console and API responses are taking one to three seconds instead of under 200 ms."
                },
                {
                    "at": "2026-10-02 21:21:00Z",
                    "status": "identified",
                    "text": "A database index rebuild is contending with live traffic. The rebuild is being throttled."
                },
                {
                    "at": "2026-10-02 21:30:00Z",
                    "status": "monitoring",
                    "text": "Latency back to normal; the rebuild finishes in the background."
                },
                {
                    "at": "2026-10-02 21:36:00Z",
                    "status": "resolved",
                    "text": "Resolved. Servers and billing were not affected."
                }
            ],
            "post_incident_review": null
        },
        {
            "id": "m-2026-09-30-rey",
            "kind": "maintenance",
            "region": "REY",
            "component": "provision",
            "title": "Provisioning host OS patching in Reykjav\u00edk",
            "impact": "New orders in Reykjav\u00edk queue for the window; running servers unaffected.",
            "started_at": "2026-09-30 01:30:00Z",
            "duration_minutes": 32,
            "counts_as_downtime": false,
            "resolved": true,
            "updates": [
                {
                    "at": "2026-09-26 13:30:00Z",
                    "status": "scheduled",
                    "text": "Announced for 30 September, 01:30\u201302:02 UTC. Affected tenants notified in the console. New orders in Reykjav\u00edk queue for the window; running servers unaffected."
                },
                {
                    "at": "2026-09-30 01:30:00Z",
                    "status": "in progress",
                    "text": "Window open. Work proceeds one device at a time."
                },
                {
                    "at": "2026-09-30 02:02:00Z",
                    "status": "completed",
                    "text": "Completed in 32 minutes, inside the announced window. Announced maintenance under the monthly allowance: no SLA impact."
                }
            ],
            "post_incident_review": null
        },
        {
            "id": "m-2026-09-29-billing",
            "kind": "maintenance",
            "region": "ALL",
            "component": "billing",
            "title": "Payment processor upgrade",
            "impact": "New invoices paused for the window; open invoices keep confirming.",
            "started_at": "2026-09-29 02:15:00Z",
            "duration_minutes": 25,
            "counts_as_downtime": false,
            "resolved": true,
            "updates": [
                {
                    "at": "2026-09-21 18:15:00Z",
                    "status": "scheduled",
                    "text": "Announced for 29 September, 02:15\u201302:40 UTC. Affected tenants notified in the console. New invoices paused for the window; open invoices keep confirming."
                },
                {
                    "at": "2026-09-29 02:15:00Z",
                    "status": "in progress",
                    "text": "Window open. Work proceeds one device at a time."
                },
                {
                    "at": "2026-09-29 02:40:00Z",
                    "status": "completed",
                    "text": "Completed in 25 minutes, inside the announced window. Announced maintenance under the monthly allowance: no SLA impact."
                }
            ],
            "post_incident_review": null
        },
        {
            "id": "m-2026-09-15-sin",
            "kind": "maintenance",
            "region": "SIN",
            "component": "network",
            "title": "Transit port migration in Singapore",
            "impact": "No downtime expected: traffic drains to the second provider first.",
            "started_at": "2026-09-15 18:45:00Z",
            "duration_minutes": 23,
            "counts_as_downtime": false,
            "resolved": true,
            "updates": [
                {
                    "at": "2026-09-12 05:45:00Z",
                    "status": "scheduled",
                    "text": "Announced for 15 September, 18:45\u201319:08 UTC. Affected tenants notified in the console. No downtime expected: traffic drains to the second provider first."
                },
                {
                    "at": "2026-09-15 18:45:00Z",
                    "status": "in progress",
                    "text": "Window open. Work proceeds one device at a time."
                },
                {
                    "at": "2026-09-15 19:08:00Z",
                    "status": "completed",
                    "text": "Completed in 23 minutes, inside the announced window. Announced maintenance under the monthly allowance: no SLA impact."
                }
            ],
            "post_incident_review": null
        },
        {
            "id": "i-2026-09-08-console",
            "kind": "incident",
            "region": "ALL",
            "component": "console",
            "title": "Intermittent errors on the console",
            "impact": "Some console requests failed for a few minutes; the API and servers were unaffected.",
            "started_at": "2026-09-08 13:37:00Z",
            "duration_minutes": 12,
            "counts_as_downtime": false,
            "resolved": true,
            "updates": [
                {
                    "at": "2026-09-08 13:37:00Z",
                    "status": "investigating",
                    "text": "A share of console requests return an error page after a deployment."
                },
                {
                    "at": "2026-09-08 13:41:00Z",
                    "status": "identified",
                    "text": "Deployment rolled back."
                },
                {
                    "at": "2026-09-08 13:49:00Z",
                    "status": "resolved",
                    "text": "Resolved. The change ships again after review."
                }
            ],
            "post_incident_review": null
        },
        {
            "id": "i-2026-09-05-rey",
            "kind": "incident",
            "region": "REY",
            "component": "network",
            "title": "Core switch supervisor failover in Reykjav\u00edk",
            "impact": "Brief loss of connectivity for every server in Reykjav\u00edk during the failover.",
            "started_at": "2026-09-05 20:39:00Z",
            "duration_minutes": 4,
            "counts_as_downtime": true,
            "resolved": true,
            "updates": [
                {
                    "at": "2026-09-05 20:39:00Z",
                    "status": "investigating",
                    "text": "The active supervisor of a core switch in Reykjav\u00edk rebooted unexpectedly; the standby took over."
                },
                {
                    "at": "2026-09-05 20:43:00Z",
                    "status": "resolved",
                    "text": "Resolved. 4 minutes of downtime credited automatically to every server in Reykjav\u00edk. The faulty supervisor module was replaced the same day."
                }
            ],
            "post_incident_review": null
        },
        {
            "id": "i-2026-09-05-ams",
            "kind": "incident",
            "region": "AMS",
            "component": "compute",
            "title": "NVLink fabric manager restart on 4 nodes in Amsterdam",
            "impact": "Tenants on 4 SXM nodes in Amsterdam lost NVLink for the restart; single-card servers unaffected.",
            "started_at": "2026-09-05 10:17:00Z",
            "duration_minutes": 11,
            "counts_as_downtime": true,
            "resolved": true,
            "updates": [
                {
                    "at": "2026-09-05 10:17:00Z",
                    "status": "investigating",
                    "text": "The fabric manager on 4 HGX nodes in Amsterdam stopped responding; NVLink links reported down."
                },
                {
                    "at": "2026-09-05 10:21:00Z",
                    "status": "identified",
                    "text": "Service restarted on the affected nodes; links renegotiating."
                },
                {
                    "at": "2026-09-05 10:28:00Z",
                    "status": "resolved",
                    "text": "Resolved. The affected servers were credited automatically."
                }
            ],
            "post_incident_review": "The fabric manager service on 4 nodes hit a known deadlock after a driver update and stopped answering; NVLink went down for 11 minutes on those nodes. The affected tenants were credited. The fix was to pin the fabric manager to the driver version it was validated with, and to add a watchdog that restarts it within 60 seconds."
        },
        {
            "id": "m-2026-09-02-rey",
            "kind": "maintenance",
            "region": "REY",
            "component": "power",
            "title": "Annual UPS battery test in Reykjav\u00edk",
            "impact": "None expected. Servers stay on utility power throughout.",
            "started_at": "2026-09-02 02:00:00Z",
            "duration_minutes": 61,
            "counts_as_downtime": false,
            "resolved": true,
            "updates": [
                {
                    "at": "2026-08-29 09:00:00Z",
                    "status": "scheduled",
                    "text": "Announced for 2 September, 02:00\u201303:01 UTC. Affected tenants notified in the console. None expected. Servers stay on utility power throughout."
                },
                {
                    "at": "2026-09-02 02:00:00Z",
                    "status": "in progress",
                    "text": "Window open. Work proceeds one device at a time."
                },
                {
                    "at": "2026-09-02 03:01:00Z",
                    "status": "completed",
                    "text": "Completed in 61 minutes, inside the announced window. Announced maintenance under the monthly allowance: no SLA impact."
                }
            ],
            "post_incident_review": null
        },
        {
            "id": "i-2026-08-29-console",
            "kind": "incident",
            "region": "ALL",
            "component": "console",
            "title": "Elevated latency on the console and API",
            "impact": "Console pages and API calls slower than usual; servers unaffected.",
            "started_at": "2026-08-29 00:07:00Z",
            "duration_minutes": 18,
            "counts_as_downtime": false,
            "resolved": true,
            "updates": [
                {
                    "at": "2026-08-29 00:07:00Z",
                    "status": "investigating",
                    "text": "Console and API responses are taking one to three seconds instead of under 200 ms."
                },
                {
                    "at": "2026-08-29 00:13:00Z",
                    "status": "identified",
                    "text": "A database index rebuild is contending with live traffic. The rebuild is being throttled."
                },
                {
                    "at": "2026-08-29 00:25:00Z",
                    "status": "resolved",
                    "text": "Resolved. Servers and billing were not affected."
                }
            ],
            "post_incident_review": null
        },
        {
            "id": "m-2026-08-25-console",
            "kind": "maintenance",
            "region": "ALL",
            "component": "console",
            "title": "Console deployment",
            "impact": "Console read-only for under a minute; servers and API unaffected.",
            "started_at": "2026-08-25 02:00:00Z",
            "duration_minutes": 10,
            "counts_as_downtime": false,
            "resolved": true,
            "updates": [
                {
                    "at": "2026-08-21 13:00:00Z",
                    "status": "scheduled",
                    "text": "Announced for 25 August, 02:00\u201302:10 UTC. Affected tenants notified in the console. Console read-only for under a minute; servers and API unaffected."
                },
                {
                    "at": "2026-08-25 02:00:00Z",
                    "status": "in progress",
                    "text": "Window open. Work proceeds one device at a time."
                },
                {
                    "at": "2026-08-25 02:10:00Z",
                    "status": "completed",
                    "text": "Completed in 10 minutes, inside the announced window. Announced maintenance under the monthly allowance: no SLA impact."
                }
            ],
            "post_incident_review": null
        },
        {
            "id": "m-2026-08-25-ams",
            "kind": "maintenance",
            "region": "AMS",
            "component": "network",
            "title": "Transit port migration in Amsterdam",
            "impact": "No downtime expected: traffic drains to the second provider first.",
            "started_at": "2026-08-25 01:30:00Z",
            "duration_minutes": 26,
            "counts_as_downtime": false,
            "resolved": true,
            "updates": [
                {
                    "at": "2026-08-19 15:30:00Z",
                    "status": "scheduled",
                    "text": "Announced for 25 August, 01:30\u201301:56 UTC. Affected tenants notified in the console. No downtime expected: traffic drains to the second provider first."
                },
                {
                    "at": "2026-08-25 01:30:00Z",
                    "status": "in progress",
                    "text": "Window open. Work proceeds one device at a time."
                },
                {
                    "at": "2026-08-25 01:56:00Z",
                    "status": "completed",
                    "text": "Completed in 26 minutes, inside the announced window. Announced maintenance under the monthly allowance: no SLA impact."
                }
            ],
            "post_incident_review": null
        },
        {
            "id": "i-2026-08-20-hel",
            "kind": "incident",
            "region": "HEL",
            "component": "compute",
            "title": "Hardware fault on one GPU node in Helsinki: card replaced",
            "impact": "One server offline while a failed card was swapped from local spares; the tenant was credited.",
            "started_at": "2026-08-20 07:11:00Z",
            "duration_minutes": 181,
            "counts_as_downtime": false,
            "resolved": true,
            "updates": [
                {
                    "at": "2026-08-20 07:11:00Z",
                    "status": "investigating",
                    "text": "Monitoring flagged an uncorrectable memory error on one card of a node in Helsinki; the server was taken offline with the tenant informed."
                },
                {
                    "at": "2026-08-20 08:14:00Z",
                    "status": "identified",
                    "text": "The card is being swapped from the spares shelf in Helsinki."
                },
                {
                    "at": "2026-08-20 09:21:00Z",
                    "status": "monitoring",
                    "text": "Card replaced, node back online and burn-in passed."
                },
                {
                    "at": "2026-08-20 10:12:00Z",
                    "status": "resolved",
                    "text": "Resolved. Downtime credited ten times over to the affected server. No other tenant was involved."
                }
            ],
            "post_incident_review": null
        },
        {
            "id": "i-2026-08-10-billing",
            "kind": "incident",
            "region": "ALL",
            "component": "billing",
            "title": "Confirmation delays on Litecoin top-ups",
            "impact": "Top-ups in Litecoin took longer than usual to confirm on chain; other coins unaffected.",
            "started_at": "2026-08-10 09:58:00Z",
            "duration_minutes": 42,
            "counts_as_downtime": false,
            "resolved": true,
            "updates": [
                {
                    "at": "2026-08-10 09:58:00Z",
                    "status": "investigating",
                    "text": "Litecoin transactions are confirming slowly because of congestion on the network itself. Invoices stay valid and will credit once confirmed."
                },
                {
                    "at": "2026-08-10 10:13:00Z",
                    "status": "identified",
                    "text": "Congestion easing; confirmations arriving with a delay of about an hour."
                },
                {
                    "at": "2026-08-10 10:28:00Z",
                    "status": "monitoring",
                    "text": "Backlog cleared; new payments confirm at the usual pace."
                },
                {
                    "at": "2026-08-10 10:40:00Z",
                    "status": "resolved",
                    "text": "Resolved. Every delayed payment was credited at the invoice rate."
                }
            ],
            "post_incident_review": null
        }
    ]
}