{"page":{"id":"01HHF88M5X743YJG251VX5099P","name":"Tinybird","url":"https://status.tinybird.co/","updated_at":"2026-10-01T12:04:54Z"},"incidents":[{"id":"01M3VK1WYB2776SWM83CNNKEP7","name":"Ingestion issue in public-a","status":"resolved","created_at":"2026-10-01T11:19:35Z","updated_at":"2026-10-01T12:04:54Z","resolved_at":"2026-10-01T12:04:54Z","impact":"minor","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01M3VNMW5665MSYK6KHZXHS48P","body":"Ingestion has been recovered. We're ingesting the remaining data from the ingestion queue.","created_at":"2026-10-01T12:04:54Z","display_at":"2026-10-01T12:04:54Z","incident_id":"01M3VK1WYB2776SWM83CNNKEP7","status":"resolved","updated_at":"2026-10-01T12:04:54Z"},{"id":"01M3VK1WYB13XS92BF79P1JZJ9","body":"We're investigating an ingestion issue while rotating some ClickHouse replicas","created_at":"2026-10-01T11:19:35Z","display_at":"2026-10-01T11:19:35Z","incident_id":"01M3VK1WYB2776SWM83CNNKEP7","status":"investigating","updated_at":"2026-10-01T11:19:35Z"}]},{"id":"01M39H4C38P9PWXJHXPYW70GQ8","name":"eu-west-1 degraded","status":"resolved","created_at":"2026-09-24T10:59:39Z","updated_at":"2026-09-25T08:44:30Z","resolved_at":"2026-09-24T14:00:00Z","impact":"minor","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01M3BVSM4WCS494QNV1DM6YCG2","body":"Since we deployed the fix yesterday everything has been working correctly.","created_at":"2026-09-25T08:44:30Z","display_at":"2026-09-24T14:00:00Z","incident_id":"01M39H4C38P9PWXJHXPYW70GQ8","status":"resolved","updated_at":"2026-09-25T08:44:45Z"},{"id":"01M39KWD8Q4211DB4H1WVAJT8K","body":"The fix is rolled out, and everything is back to normal. We keep monitoring to ensure everything works as expected.","created_at":"2026-09-24T11:47:44Z","display_at":"2026-09-24T11:47:44Z","incident_id":"01M39H4C38P9PWXJHXPYW70GQ8","status":"monitoring","updated_at":"2026-09-24T11:47:44Z"},{"id":"01M39JV1CS0J2RH6QVMRQN3QV8","body":"The root cause was identified. A major traffic spike hit an endpoint with synchronous metadata database queries, affecting some applications' performance and causing a cascading database failure, which has already recovered. A fix is being deployed and should restore the region's performance.","created_at":"2026-09-24T11:29:31Z","display_at":"2026-09-24T11:29:31Z","incident_id":"01M39H4C38P9PWXJHXPYW70GQ8","status":"identified","updated_at":"2026-09-24T11:29:31Z"},{"id":"01M39H4C384CBQ8VQQDFTG67MC","body":"We detected some degradation in the `eu-west-1` region due to a metadata database failure. The team is looking into it, and we'll update you with more information soon.","created_at":"2026-09-24T10:59:39Z","display_at":"2026-09-24T10:59:39Z","incident_id":"01M39H4C38P9PWXJHXPYW70GQ8","status":"investigating","updated_at":"2026-09-24T10:59:39Z"}]},{"id":"01M2SS0MKTSDKAGDY63YC1A8X3","name":"Network issues in gcp-europe-west3","status":"resolved","created_at":"2026-09-18T08:09:35Z","updated_at":"2026-09-18T15:41:04Z","resolved_at":"2026-09-18T15:41:04Z","impact":"minor","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01M2TJVBE2TVM9KNM2ZHF275TF","body":"GCP marked as resolved the incident impacting europe-west3: _\"The issue with Virtual Private Cloud (VPC) has been resolved for all affected users as of Friday, 2026-09-18 05:35 UTC+2.\"_ ","created_at":"2026-09-18T15:41:04Z","display_at":"2026-09-18T15:41:04Z","incident_id":"01M2SS0MKTSDKAGDY63YC1A8X3","status":"resolved","updated_at":"2026-09-18T15:41:04Z"},{"id":"01M2T7MQZGVBES3YA7RX5PQY8Q","body":"Since 11:10utc we are not detecting more errors. However gcp incident still open so we keep monitoring","created_at":"2026-09-18T12:25:13Z","display_at":"2026-09-18T12:25:13Z","incident_id":"01M2SS0MKTSDKAGDY63YC1A8X3","status":"monitoring","updated_at":"2026-09-18T12:25:13Z"},{"id":"01M2STGY9BJVEN0SY0KTQY7YZP","body":"We've detected degradation across all components in the region due to network issues related to an ongoing incident on GCP: \"Virtual Private Cloud (VPC) experiencing intermittent packet loss and elevated latency in europe-west3-c zone. Event ID: LJCMWTL \"","created_at":"2026-09-18T08:35:57Z","display_at":"2026-09-18T08:35:57Z","incident_id":"01M2SS0MKTSDKAGDY63YC1A8X3","status":"monitoring","updated_at":"2026-09-18T08:35:57Z"},{"id":"01M2SSGVKK6PN91DPXC61X75C7","body":"We've detected degradation across all components in the region due to network issues related to an ongoing incident on GCP: \"Virtual Private Cloud (VPC) experiencing intermittent packet loss and elevated latency in europe-west3-c zone\"","created_at":"2026-09-18T08:18:26Z","display_at":"2026-09-18T08:18:26Z","incident_id":"01M2SS0MKTSDKAGDY63YC1A8X3","status":"monitoring","updated_at":"2026-09-18T08:18:26Z"},{"id":"01M2SS0MKTWKP1EK36FY8NAHMG","body":"We've detected degradation across all components in the region due to network issues.","created_at":"2026-09-18T08:09:35Z","display_at":"2026-09-18T08:09:35Z","incident_id":"01M2SS0MKTSDKAGDY63YC1A8X3","status":"investigating","updated_at":"2026-09-18T08:09:35Z"}]},{"id":"01M24W8BNNZ0A8B9Y67WZ41XW8","name":"Service Disruption – AWS eu-west-1","status":"resolved","created_at":"2026-09-10T05:22:10Z","updated_at":"2026-09-10T06:32:26Z","resolved_at":"2026-09-10T06:32:26Z","impact":"minor","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01M25090ZARQ6D2S63MVKJ52CZ","body":"We have identified the root cause of this morning's disruption: a fault in an internal monitoring component caused resource exhaustion on some infrastructure hosts, impacting service availability in the region. The issue has been resolved and the region is operating as normal.","created_at":"2026-09-10T06:32:26Z","display_at":"2026-09-10T06:32:26Z","incident_id":"01M24W8BNNZ0A8B9Y67WZ41XW8","status":"resolved","updated_at":"2026-09-10T06:32:26Z"},{"id":"01M24XMZ8DCRZKF20HFPJSQ2DJ","body":"The service disruption affecting our AWS eu-west-1 region has been resolved and the region is operating as normal. We are continuing to investigate the root cause and will share any relevant updates. All other regions were unaffected.","created_at":"2026-09-10T05:46:32Z","display_at":"2026-09-10T05:46:32Z","incident_id":"01M24W8BNNZ0A8B9Y67WZ41XW8","status":"investigating","updated_at":"2026-09-10T05:46:32Z"},{"id":"01M24W8BNNRQC6J86SCA173XN7","body":"Our team is actively investigating the cause and working to restore normal service as quickly as possible.\nAll other regions are operating normally.","created_at":"2026-09-10T05:22:10Z","display_at":"2026-09-10T05:22:10Z","incident_id":"01M24W8BNNZ0A8B9Y67WZ41XW8","status":"investigating","updated_at":"2026-09-10T05:22:10Z"}]},{"id":"01M085P9A3YEMFXN1T8XN1NVYG","name":"On-demand copies are failing","status":"resolved","created_at":"2026-08-17T15:33:23Z","updated_at":"2026-08-17T15:33:23Z","resolved_at":"2026-08-15T12:20:00Z","impact":"minor","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01M085P9A3MGABMA5H5KJGHNP4","body":"A fix was rolled out to the replica provisioning path. Subsequent on-demand replica creations completed successfully. We are conducting a follow-up review to prevent recurrence.\n\nWe apologize for the disruption and any inconvenience caused.","created_at":"2026-08-17T15:33:23Z","display_at":"2026-08-15T12:20:00Z","incident_id":"01M085P9A3YEMFXN1T8XN1NVYG","status":"resolved","updated_at":"2026-08-17T15:33:23Z"},{"id":"01M085P9A30833QYYZ4AX2D30E","body":"Some customers were unable to provision new on-demand replicas. Existing replicas and normal (non on-demand) replica provisioning were not affected. Queries and ingestion on already-running replicas continued to operate normally throughout the window.","created_at":"2026-08-17T15:33:23Z","display_at":"2026-08-15T09:15:00Z","incident_id":"01M085P9A3YEMFXN1T8XN1NVYG","status":"identified","updated_at":"2026-08-17T15:33:23Z"}]},{"id":"01KZTRTJTQE56PEDZMYV80QNSC","name":"Reads and writes degraded on US East (AWS) public cluster","status":"resolved","created_at":"2026-08-12T10:38:25Z","updated_at":"2026-08-12T11:34:20Z","resolved_at":"2026-08-12T11:34:20Z","impact":"minor","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KZTW0YJ2AJJBA4Y7HHA73W20","body":"We have successfully identified the root cause of the US East (AWS) public cluster disruption and fully restored both read and write operations; our team is currently processing the backlog of queued inserts accumulated during the incident.","created_at":"2026-08-12T11:34:20Z","display_at":"2026-08-12T11:34:20Z","incident_id":"01KZTRTJTQE56PEDZMYV80QNSC","status":"resolved","updated_at":"2026-08-12T11:34:20Z"},{"id":"01KZTSVB9HSF4PEB58DZ76VKX5","body":"We're investigating a service disruption in our US East (AWS) public cluster that began at 10:25 UTC and is affecting both reads and writes: API queries may fail or return errors, and incoming data is being queued. Our team is actively investigating and working to restore stability.","created_at":"2026-08-12T10:56:19Z","display_at":"2026-08-12T10:56:19Z","incident_id":"01KZTRTJTQE56PEDZMYV80QNSC","status":"investigating","updated_at":"2026-08-12T10:56:19Z"},{"id":"01KZTRTJTQMQVS55X7WP2TB2HF","body":"We're investigating an issue causing delays in data ingestion for some workspaces in the the **US East (AWS)** region","created_at":"2026-08-12T10:38:25Z","display_at":"2026-08-12T10:38:25Z","incident_id":"01KZTRTJTQE56PEDZMYV80QNSC","status":"investigating","updated_at":"2026-08-12T10:38:25Z"}]},{"id":"01KZB1BNN51K2GZFCTYQ8Z77FX","name":"Degradation on GCP europe-west2 public cluster","status":"resolved","created_at":"2026-08-06T07:59:43Z","updated_at":"2026-08-06T08:20:57Z","resolved_at":"2026-08-06T08:20:57Z","impact":"minor","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KZB2JJ6KD6ZK8NKJFHDNRF9H","body":"The incident is now resolved.","created_at":"2026-08-06T08:20:57Z","display_at":"2026-08-06T08:20:57Z","incident_id":"01KZB1BNN51K2GZFCTYQ8Z77FX","status":"resolved","updated_at":"2026-08-06T08:20:57Z"},{"id":"01KZB20CFBHR0K3CAZG5XBVWYG","body":"The issue was caused due to a peak of heavy queries causing OOM in one of the replicas. This is now fixed and everything should be working properly now. We'll keep monitoring.","created_at":"2026-08-06T08:11:02Z","display_at":"2026-08-06T08:11:02Z","incident_id":"01KZB1BNN51K2GZFCTYQ8Z77FX","status":"monitoring","updated_at":"2026-08-06T08:11:02Z"},{"id":"01KZB1BNN5WM3ZKXX0QVNJ8BRK","body":"We're investigating an issue causing degradation on GCP europe-west2 public-a cluster. We'll keep you updated as soon as we find the issue.","created_at":"2026-08-06T07:59:43Z","display_at":"2026-08-06T07:59:43Z","incident_id":"01KZB1BNN51K2GZFCTYQ8Z77FX","status":"investigating","updated_at":"2026-08-06T07:59:43Z"}]},{"id":"01KYS9RSTQ872TATVP7M9FYGJC","name":"Errors when creating and deleting replicas","status":"resolved","created_at":"2026-07-30T10:40:22Z","updated_at":"2026-07-30T11:13:52Z","resolved_at":"2026-07-30T11:13:52Z","impact":"major","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KYSBP4RD5QSYBJRC2CN6HBTW","body":"The incident is now resolved.","created_at":"2026-07-30T11:13:52Z","display_at":"2026-07-30T11:13:52Z","incident_id":"01KYS9RSTQ872TATVP7M9FYGJC","status":"resolved","updated_at":"2026-07-30T11:13:52Z"},{"id":"01KYSAP0B5WAREN2GEFYCHVS04","body":"We have identified the issue and it should be fixed now. We're already seeing some new replicas being provisioned. We'll keep monitoring.","created_at":"2026-07-30T10:56:19Z","display_at":"2026-07-30T10:56:19Z","incident_id":"01KYS9RSTQ872TATVP7M9FYGJC","status":"monitoring","updated_at":"2026-07-30T10:56:19Z"},{"id":"01KYS9RSTQN1NVS038Q8MBGDMT","body":"We're investigating an issue when creating or deleting new replicas. This affects cluster replicas and jobs on-demand (populates and copies).","created_at":"2026-07-30T10:40:22Z","display_at":"2026-07-30T10:40:22Z","incident_id":"01KYS9RSTQ872TATVP7M9FYGJC","status":"investigating","updated_at":"2026-07-30T10:40:22Z"}]},{"id":"01KYKQFHR96KCJTZ8G3FN0K8W1","name":"A significant fraction of Clickhouse replicas marked as unhealthy","status":"resolved","created_at":"2026-07-28T06:44:32Z","updated_at":"2026-07-28T08:04:43Z","resolved_at":"2026-07-28T08:04:43Z","impact":"minor","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KYKW2BXB2Y07MS1TF5J4D2RC","body":"We've confirmed that service has been restored fully.","created_at":"2026-07-28T08:04:43Z","display_at":"2026-07-28T08:04:43Z","incident_id":"01KYKQFHR96KCJTZ8G3FN0K8W1","status":"resolved","updated_at":"2026-07-28T08:04:43Z"},{"id":"01KYKTJWWGS4G3AQMA6H31A6VG","body":"We have narrowed the root cause to an automatic Ubuntu package upgrade triggering the restart of our database services on many servers, and those being marked unhealthy as a result.\n\nService should be generally restored already.","created_at":"2026-07-28T07:38:48Z","display_at":"2026-07-28T07:38:48Z","incident_id":"01KYKQFHR96KCJTZ8G3FN0K8W1","status":"monitoring","updated_at":"2026-07-28T07:38:48Z"},{"id":"01KYKSHJFC89B3Q21F6P8Q0NVQ","body":"We've marked as healthy the database replicas that were incorrectly marked as unhealthy. We're confirming the cause and have a few suspects.\n\n \n\nWe keep monitoring the situation.","created_at":"2026-07-28T07:20:36Z","display_at":"2026-07-28T07:20:36Z","incident_id":"01KYKQFHR96KCJTZ8G3FN0K8W1","status":"monitoring","updated_at":"2026-07-28T07:23:04Z"},{"id":"01KYKQFHR9DVQ77J0MMCB3ETJ7","body":"Some database replicas have been marked as unhealthy incorrectly. This is affecting how traffic is distributed among healthy replicas, and in some clusters it's resulting in degraded performance and some errors.\n\n \n\n ","created_at":"2026-07-28T06:44:32Z","display_at":"2026-07-28T06:44:32Z","incident_id":"01KYKQFHR96KCJTZ8G3FN0K8W1","status":"investigating","updated_at":"2026-07-28T07:21:47Z"}]},{"id":"01KXQJYDZ3WJZ6WE18A2GRKW77","name":"Kafka connector is working well, ingestion is catching up","status":"resolved","created_at":"2026-07-17T08:26:33Z","updated_at":"2026-07-17T09:03:12Z","resolved_at":"2026-07-17T09:03:12Z","impact":"minor","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KXQN1HKEH72DRJ1YHM024YPW","body":"Ingestion is up to date now","created_at":"2026-07-17T09:03:12Z","display_at":"2026-07-17T09:03:12Z","incident_id":"01KXQJYDZ3WJZ6WE18A2GRKW77","status":"resolved","updated_at":"2026-07-17T09:03:12Z"},{"id":"01KXQM9Y8XN7GM491E51FNAGA2","body":"GCP europe-west3 still having problems, we're redeploying the region","created_at":"2026-07-17T08:50:19Z","display_at":"2026-07-17T08:50:19Z","incident_id":"01KXQJYDZ3WJZ6WE18A2GRKW77","status":"monitoring","updated_at":"2026-07-17T08:50:19Z"},{"id":"01KXQKRS54VSDS3DGKYPTD8FV8","body":"The problem is fixed now ingestion is recovered","created_at":"2026-07-17T08:40:56Z","display_at":"2026-07-17T08:40:56Z","incident_id":"01KXQJYDZ3WJZ6WE18A2GRKW77","status":"monitoring","updated_at":"2026-07-17T08:40:56Z"},{"id":"01KXQK6ZMK6J9TZ379QH5YTCNE","body":"We've identified the issue and are deploying a fix","created_at":"2026-07-17T08:31:13Z","display_at":"2026-07-17T08:31:13Z","incident_id":"01KXQJYDZ3WJZ6WE18A2GRKW77","status":"identified","updated_at":"2026-07-17T08:31:13Z"},{"id":"01KXQK4FP5CAVRQBQNRMR7AJ4D","body":"A Kafka connector update is affecting message consumption","created_at":"2026-07-17T08:29:51Z","display_at":"2026-07-17T08:29:51Z","incident_id":"01KXQJYDZ3WJZ6WE18A2GRKW77","status":"investigating","updated_at":"2026-07-17T08:29:51Z"},{"id":"01KXQJYDZ34XBH3SJE7WD1QZ5X","body":"Kafka connector is not consuming messages","created_at":"2026-07-17T08:26:33Z","display_at":"2026-07-17T08:26:33Z","incident_id":"01KXQJYDZ3WJZ6WE18A2GRKW77","status":"investigating","updated_at":"2026-07-17T08:26:33Z"}]},{"id":"01KWDYHKN65K8CZ1WFFVF51CWJ","name":"High error rate on GCP europe-west-3","status":"resolved","created_at":"2026-07-01T04:21:09Z","updated_at":"2026-07-01T05:20:35Z","resolved_at":"2026-07-01T05:20:35Z","impact":"major","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KWE1YDJA616WTRVE88PCHJ7M","body":"The situation is now resolved. There shouldn't be more errors.","created_at":"2026-07-01T05:20:35Z","display_at":"2026-07-01T05:20:35Z","incident_id":"01KWDYHKN65K8CZ1WFFVF51CWJ","status":"resolved","updated_at":"2026-07-01T05:20:35Z"},{"id":"01KWE17NDCAHER2E6FWS2TAX04","body":"We've identified the issue and applied a fix. We're monitoring the situation.","created_at":"2026-07-01T05:08:09Z","display_at":"2026-07-01T05:08:09Z","incident_id":"01KWDYHKN65K8CZ1WFFVF51CWJ","status":"monitoring","updated_at":"2026-07-01T05:08:09Z"},{"id":"01KWDYHKN60H395NAXCN3HYZ10","body":"We're investigating a sudden increase in error rate in the region.","created_at":"2026-07-01T04:21:09Z","display_at":"2026-07-01T04:21:09Z","incident_id":"01KWDYHKN65K8CZ1WFFVF51CWJ","status":"investigating","updated_at":"2026-07-01T04:21:09Z"}]},{"id":"01KVR3WK2GK8ZXE6V17WPFHMR3","name":"Degraded performance in the AWS us-east public cluster.","status":"resolved","created_at":"2026-06-22T16:51:15Z","updated_at":"2026-06-24T15:43:30Z","resolved_at":"2026-06-24T15:43:30Z","impact":"major","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KVX4SZTHR632MRZRE7W5Y8X5","body":"All data has been recovered and the system health is back to normal","created_at":"2026-06-24T15:43:30Z","display_at":"2026-06-24T15:43:30Z","incident_id":"01KVR3WK2GK8ZXE6V17WPFHMR3","status":"resolved","updated_at":"2026-06-24T15:43:30Z"},{"id":"01KVW9ZTTVKMKVZ03V29RHKE1S","body":"We are still getting rate-limited by our upstream storage provider but we are draining retries faster than before because of some optimizations we have made on the system. We are working on accelerating them as much as possible but we still have some retries in the queue.\n\nNo data has been lost, all affected inserts are being safely retried.","created_at":"2026-06-24T07:54:50Z","display_at":"2026-06-24T07:54:50Z","incident_id":"01KVR3WK2GK8ZXE6V17WPFHMR3","status":"monitoring","updated_at":"2026-06-24T07:54:50Z"},{"id":"01KVTCNBXJN314D4C5NGDFEACJ","body":"The bulk of the earlier retries has now been recovered, and no new ingestion errors are being introduced on the customer side. Recovery is progressing more slowly than we'd like because we are currently being rate-limited by our upstream storage provider, which is limiting how quickly we can drain the remaining retries.\n\nNo data has been lost, all affected inserts are being safely retried. We'll provide the next update as soon as we see meaningful progress on either front.","created_at":"2026-06-23T14:03:04Z","display_at":"2026-06-23T14:03:04Z","incident_id":"01KVR3WK2GK8ZXE6V17WPFHMR3","status":"monitoring","updated_at":"2026-06-23T14:03:04Z"},{"id":"01KVT07X5F727GA8SVB9T1FQB4","body":"We're continuing to work through a backlog of delayed ingestion on our AWS US East 1 shared cluster following yesterday's degradation. Our coordination layer was resized last night and the bulk of the impact has cleared. Recovery is taking longer than expected, so affected workspaces may continue to see ingestion lag while we work through the remaining queue.","created_at":"2026-06-23T10:26:00Z","display_at":"2026-06-23T10:26:00Z","incident_id":"01KVR3WK2GK8ZXE6V17WPFHMR3","status":"monitoring","updated_at":"2026-06-23T10:26:00Z"},{"id":"01KVRNWWJ1RRZW8A9S6DFC6XRT","body":"We keep ingesting the delayed real time data","created_at":"2026-06-22T22:05:59Z","display_at":"2026-06-22T22:05:59Z","incident_id":"01KVR3WK2GK8ZXE6V17WPFHMR3","status":"monitoring","updated_at":"2026-06-22T22:05:59Z"},{"id":"01KVRNBF6HAZFMQN73WE26QMW8","body":"We've identify the root cause and we are recovering the retries from the HFI ingestion at the moment. The platform should be stable now","created_at":"2026-06-22T21:56:28Z","display_at":"2026-06-22T21:56:28Z","incident_id":"01KVR3WK2GK8ZXE6V17WPFHMR3","status":"monitoring","updated_at":"2026-06-22T21:56:28Z"},{"id":"01KVRKVGSHFGPSPVV0MPX94F10","body":"We've already identified the root cause and we're working through the fix. Real-time ingestion is now fully available, we're working on ingesting past data that hasn't been processed yet.","created_at":"2026-06-22T21:30:17Z","display_at":"2026-06-22T21:30:17Z","incident_id":"01KVR3WK2GK8ZXE6V17WPFHMR3","status":"identified","updated_at":"2026-06-22T21:30:17Z"},{"id":"01KVRAV5W9AXTEPQAADMZ4SVJ0","body":"We're still working on stabilising the region. We're struggling with Zookeeper, in charge of coordination between ClickHouse replicas, which is why write are a lot more affected than reads.\n\n \n\nWe've just tweaked a couple parameters that hope will aid the situation.","created_at":"2026-06-22T18:52:48Z","display_at":"2026-06-22T18:52:48Z","incident_id":"01KVR3WK2GK8ZXE6V17WPFHMR3","status":"identified","updated_at":"2026-06-22T18:52:48Z"},{"id":"01KVR77NXWB7TXJM8MD7VRR2SN","body":"We keep working on identifying the root cause. There's impact in both reads and writes, and in all clusters in the region as the incident is affecting a shared component. \n\n \n\nWe are storing the data that gets to the Events API to retry the inserts later.","created_at":"2026-06-22T17:49:44Z","display_at":"2026-06-22T17:49:44Z","incident_id":"01KVR3WK2GK8ZXE6V17WPFHMR3","status":"investigating","updated_at":"2026-06-22T17:49:44Z"},{"id":"01KVR3WK2G3Y4VR64VP3Q9XA7R","body":"There’s a performance degradation in the public cluster on AWS us-east.","created_at":"2026-06-22T16:51:15Z","display_at":"2026-06-22T16:51:15Z","incident_id":"01KVR3WK2GK8ZXE6V17WPFHMR3","status":"investigating","updated_at":"2026-06-22T16:51:15Z"}]},{"id":"01KV8AFPQD68TZR87Y809Y6F2G","name":"We are investigate a degradation in AWS US EAST public cluster","status":"resolved","created_at":"2026-06-16T13:38:42Z","updated_at":"2026-06-16T14:27:20Z","resolved_at":"2026-06-16T14:27:20Z","impact":"minor","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KV8D8S0AV2104N2FDDC2BCSB","body":"We've deployed a fix to improve the stability of the platform. Sorry for the inconvenience ","created_at":"2026-06-16T14:27:20Z","display_at":"2026-06-16T14:27:20Z","incident_id":"01KV8AFPQD68TZR87Y809Y6F2G","status":"resolved","updated_at":"2026-06-16T14:27:20Z"},{"id":"01KV8B6CEXP8J8NRBQGBHMYAES","body":"We deploying a fix to improve the stability of the platform","created_at":"2026-06-16T13:51:05Z","display_at":"2026-06-16T13:51:05Z","incident_id":"01KV8AFPQD68TZR87Y809Y6F2G","status":"monitoring","updated_at":"2026-06-16T13:51:05Z"},{"id":"01KV8AFPQDA4D5NPAPM2ENNA5B","body":"We are seeing a degradation of the performance in the public AWS US EAST cluster","created_at":"2026-06-16T13:38:42Z","display_at":"2026-06-16T13:38:42Z","incident_id":"01KV8AFPQD68TZR87Y809Y6F2G","status":"investigating","updated_at":"2026-06-16T13:38:42Z"}]},{"id":"01KV803C09HB4W1HJS5K2ZAERR","name":"GCP EU Public-a cluster is saturated","status":"resolved","created_at":"2026-06-16T10:37:12Z","updated_at":"2026-06-16T11:31:05Z","resolved_at":"2026-06-16T11:31:04Z","impact":"minor","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KV836147TCTEAR0T57V6HSCM","body":"Cluster operating normally now","created_at":"2026-06-16T11:31:05Z","display_at":"2026-06-16T11:31:04Z","incident_id":"01KV803C09HB4W1HJS5K2ZAERR","status":"resolved","updated_at":"2026-06-16T11:31:05Z"},{"id":"01KV82PZDF80JNSZ8SAVYJJ9XZ","body":"Cluster is recovering","created_at":"2026-06-16T11:22:51Z","display_at":"2026-06-16T11:22:51Z","incident_id":"01KV803C09HB4W1HJS5K2ZAERR","status":"monitoring","updated_at":"2026-06-16T11:22:51Z"},{"id":"01KV803C09Q71DYC4MF0WB8FMR","body":"Public cluster in gcp-europe-west3 is slightly degraded. We're working on restoring int.","created_at":"2026-06-16T10:37:12Z","display_at":"2026-06-16T10:37:12Z","incident_id":"01KV803C09HB4W1HJS5K2ZAERR","status":"identified","updated_at":"2026-06-16T10:37:12Z"}]},{"id":"01KSMPAQZQKESNQ4N0P2FQ1ZWM","name":"Degradation of the service in multiple regions","status":"resolved","created_at":"2026-05-27T12:25:11Z","updated_at":"2026-05-27T12:55:00Z","resolved_at":"2026-05-27T12:55:00Z","impact":"major","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KSMR1AHPP61GM15FCFW8PP2J","body":"The platform is stable again. We are going to review which has been the impact of the incident and contain the customers affected by it. \n\n","created_at":"2026-05-27T12:55:00Z","display_at":"2026-05-27T12:55:00Z","incident_id":"01KSMPAQZQKESNQ4N0P2FQ1ZWM","status":"resolved","updated_at":"2026-05-27T12:55:00Z"},{"id":"01KSMQA1FVJ2V5K204H71YEB1J","body":"We've identified a merge request that cause an expected moved of pods in Kubernetes causing a spike of 503. We are not seeing any more errors. We will keep monitoring everything","created_at":"2026-05-27T12:42:17Z","display_at":"2026-05-27T12:42:17Z","incident_id":"01KSMPAQZQKESNQ4N0P2FQ1ZWM","status":"monitoring","updated_at":"2026-05-27T12:42:17Z"},{"id":"01KSMPAQZQJ0Y4CX5A7W4562CX","body":"We are investigation an increase of errors in multiple regions","created_at":"2026-05-27T12:25:11Z","display_at":"2026-05-27T12:25:11Z","incident_id":"01KSMPAQZQKESNQ4N0P2FQ1ZWM","status":"investigating","updated_at":"2026-05-27T12:25:11Z"}]},{"id":"01KS844HP3Y2C9W7Q79A53ZZWW","name":"Increased latencies in aws-us-west-2","status":"resolved","created_at":"2026-05-22T15:16:21Z","updated_at":"2026-05-26T10:12:11Z","resolved_at":"2026-05-22T16:01:00Z","impact":"minor","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KSHWAFDC1K7E68RZXEY387D1","body":"The platform is back to normal and everything is stable","created_at":"2026-05-26T10:12:11Z","display_at":"2026-05-22T16:01:00Z","incident_id":"01KS844HP3Y2C9W7Q79A53ZZWW","status":"resolved","updated_at":"2026-05-26T10:12:38Z"},{"id":"01KS85KPGHVN3H1T331RP1PSV0","body":"We've identified the root cause and applied the fix, and latencies are back to normal now","created_at":"2026-05-22T15:42:06Z","display_at":"2026-05-22T15:42:06Z","incident_id":"01KS844HP3Y2C9W7Q79A53ZZWW","status":"monitoring","updated_at":"2026-05-22T15:42:06Z"},{"id":"01KS844HP3GT6ZRNPHH440KNES","body":"We're investigating an increase in latencies in the region","created_at":"2026-05-22T15:16:21Z","display_at":"2026-05-22T15:16:21Z","incident_id":"01KS844HP3Y2C9W7Q79A53ZZWW","status":"investigating","updated_at":"2026-05-22T15:16:21Z"}]},{"id":"01KR3J661QN0E2W0HQZ53517B0","name":"Connectivity issues affecting traffic in AWS us-east","status":"resolved","created_at":"2026-05-08T10:30:01Z","updated_at":"2026-05-08T16:16:25Z","resolved_at":"2026-05-08T16:16:25Z","impact":"minor","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KR460FVDZA5C14V70V56QYFG","body":"We've identified the root cause that was slowing down a small percentage of requests and have applied the solution.","created_at":"2026-05-08T16:16:25Z","display_at":"2026-05-08T16:16:25Z","incident_id":"01KR3J661QN0E2W0HQZ53517B0","status":"resolved","updated_at":"2026-05-08T16:16:25Z"},{"id":"01KR3J661QKP8JGG6TQX37P8W2","body":"We're currently investigating a connectivity issue affecting a subset of traffic in region AWS us-east. Some requests may experience increased latency or occasional delays.\n\nOur initial analysis indicates this is caused by network-level packet loss on the path between our load balancer infrastructure and the public internet. At this stage, we do not believe this is related to application code, recent changes, or backend service saturation.\n\nWe're continuing to investigate at the infrastructure and network layer, including host-level networking, NIC behavior, and potential AWS network/AZ conditions. We're also working on mitigation options to reduce the impact while we identify the root cause.\n\nWe'll share updates as soon as we have more information.","created_at":"2026-05-08T10:30:01Z","display_at":"2026-05-08T10:30:01Z","incident_id":"01KR3J661QN0E2W0HQZ53517B0","status":"investigating","updated_at":"2026-05-08T10:30:01Z"}]},{"id":"01KQWBH6PX3NHWYAN9WCWQQXAD","name":"Degraded ingestion in gcp-us-east4","status":"resolved","created_at":"2026-05-05T15:19:01Z","updated_at":"2026-05-05T18:13:26Z","resolved_at":"2026-05-05T18:13:26Z","impact":"major","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KQWNGJMVRCF1M9MJ1STE94MS","body":"We've caught up with ingestion.","created_at":"2026-05-05T18:13:26Z","display_at":"2026-05-05T18:13:26Z","incident_id":"01KQWBH6PX3NHWYAN9WCWQQXAD","status":"resolved","updated_at":"2026-05-05T18:13:26Z"},{"id":"01KQWM05FP534SW8JSPZY5HR7W","body":"There are still some delayed inserts being processed. We'll resolve the incident when all ingestion is caught up.","created_at":"2026-05-05T17:46:59Z","display_at":"2026-05-05T17:46:59Z","incident_id":"01KQWBH6PX3NHWYAN9WCWQQXAD","status":"monitoring","updated_at":"2026-05-05T17:46:59Z"},{"id":"01KQWCZS4FGEN0Q9X9B3TFMHVR","body":"Live ingestion is now much more stable, and shortly we'll start catching up with delayed inserts.","created_at":"2026-05-05T15:44:27Z","display_at":"2026-05-05T15:44:27Z","incident_id":"01KQWBH6PX3NHWYAN9WCWQQXAD","status":"monitoring","updated_at":"2026-05-05T15:44:36Z"},{"id":"01KQWCBAKZKMSFA7QVK2EHQNXH","body":"We keep working on stabilizing the situation. Ingestion is still delayed.","created_at":"2026-05-05T15:33:17Z","display_at":"2026-05-05T15:33:17Z","incident_id":"01KQWBH6PX3NHWYAN9WCWQQXAD","status":"investigating","updated_at":"2026-05-05T15:33:17Z"},{"id":"01KQWBH6PXHK85PPJ8AP7YD30N","body":"A component of the ingestion infra is degraded and ingestion is delayed in the region","created_at":"2026-05-05T15:19:01Z","display_at":"2026-05-05T15:19:01Z","incident_id":"01KQWBH6PX3NHWYAN9WCWQQXAD","status":"investigating","updated_at":"2026-05-05T15:19:01Z"}]},{"id":"01KQ9TN50VEE3P0KPV6EWGFMPH","name":"Job Queue Degradation","status":"resolved","created_at":"2026-04-28T10:37:44Z","updated_at":"2026-04-28T15:26:20Z","resolved_at":"2026-04-28T15:26:20Z","impact":"none","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KQAB5K6G994QYW4260EP4KWS","body":"The job queue processing issue has been fully resolved. All pending jobs are being processed and the system is operating normally.\n\nWe apologize for the disruption caused and thank you for your patience.","created_at":"2026-04-28T15:26:20Z","display_at":"2026-04-28T15:26:20Z","incident_id":"01KQ9TN50VEE3P0KPV6EWGFMPH","status":"resolved","updated_at":"2026-04-28T15:26:20Z"},{"id":"01KQ9XRZNEB0DSEM97FVVN79RG","body":"A fix has been applied to resolve the job queue processing issue. We are now monitoring the system to ensure stability and that all pending jobs are being processed correctly.\n\nNo action is needed on your end. We will confirm full resolution shortly.","created_at":"2026-04-28T11:32:16Z","display_at":"2026-04-28T11:32:16Z","incident_id":"01KQ9TN50VEE3P0KPV6EWGFMPH","status":"monitoring","updated_at":"2026-04-28T11:32:16Z"},{"id":"01KQ9TN50VT5565FQX31GDCPPZ","body":"We are currently investigating an issue affecting our job queue. Some jobs may be delayed or not processing as expected. Our team is actively looking into the root cause.\n\nWe apologize for the inconvenience.","created_at":"2026-04-28T10:37:44Z","display_at":"2026-04-28T10:37:44Z","incident_id":"01KQ9TN50VEE3P0KPV6EWGFMPH","status":"investigating","updated_at":"2026-04-28T10:37:44Z"}]},{"id":"01KPKHMAK45XJ189GBFRYTYSBB","name":"Several Tinybird services might be affected","status":"resolved","created_at":"2026-04-19T18:56:43Z","updated_at":"2026-04-19T20:07:42Z","resolved_at":"2026-04-19T20:07:42Z","impact":"minor","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KPKNPADSC8WMQXKYMNEA795B","body":"The credential rotation has been completed successfully. All services are operating normally and the impact caused by the incident has been resolved.\n\nWe appreciate your patience during this time. If you continue to experience any issues, please don't hesitate to reach out.","created_at":"2026-04-19T20:07:42Z","display_at":"2026-04-19T20:07:42Z","incident_id":"01KPKHMAK45XJ189GBFRYTYSBB","status":"resolved","updated_at":"2026-04-19T20:07:42Z"},{"id":"01KPKHMAK4Q085HWHWW41YNVJ0","body":"Due to the security incident on a third-party provider, we have initiated a credential rotation across our environment variables. Services depending on these credentials may be temporarily disrupted.\n\nETA for resolution: under investigation. Updates to follow.","created_at":"2026-04-19T18:56:43Z","display_at":"2026-04-19T18:56:43Z","incident_id":"01KPKHMAK45XJ189GBFRYTYSBB","status":"identified","updated_at":"2026-04-19T20:09:34Z"}]},{"id":"01KKEH7ZP1SXP7ENYYG5K39M2Z","name":"Load balance unresponsive in GCP US East","status":"resolved","created_at":"2026-03-11T13:26:41Z","updated_at":"2026-03-11T18:09:33Z","resolved_at":"2026-03-11T18:09:33Z","impact":"none","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KKF1DYD9BS0X1AZ7P1MFN9MV","body":"We were impacted by a spike of requests to the v0/events endpoint which bypass our protections in our load balancers causing a saturation of them which impacted on the rest of the requests. \n\nWe have added extra resources to be able to handle any news spikes and we started to work on better protections to avoid any new incident","created_at":"2026-03-11T18:09:33Z","display_at":"2026-03-11T18:09:33Z","incident_id":"01KKEH7ZP1SXP7ENYYG5K39M2Z","status":"resolved","updated_at":"2026-03-11T18:09:33Z"},{"id":"01KKEQW25Y349T9N1GS439XDF9","body":"Load balancing is experiencing intermittent errors - we continue to investigate","created_at":"2026-03-11T15:22:30Z","display_at":"2026-03-11T15:22:30Z","incident_id":"01KKEH7ZP1SXP7ENYYG5K39M2Z","status":"investigating","updated_at":"2026-03-11T15:22:30Z"},{"id":"01KKEH7ZP1X59MNA2ENEWP6W77","body":"Incident resolved and all workflows are working normally.","created_at":"2026-03-11T13:26:41Z","display_at":"2026-03-11T13:26:41Z","incident_id":"01KKEH7ZP1SXP7ENYYG5K39M2Z","status":"monitoring","updated_at":"2026-03-11T13:26:41Z"}]},{"id":"01KK911PP3WRY2YYC79X7XZ789","name":"Public cluster in GCP us east down","status":"resolved","created_at":"2026-03-09T10:07:25Z","updated_at":"2026-03-12T18:18:15Z","resolved_at":"2026-03-09T09:58:00Z","impact":"major","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KK9125WDPFM5G9TPYG93142R","body":"Everything should be working now","created_at":"2026-03-09T10:07:41Z","display_at":"2026-03-09T09:58:00Z","incident_id":"01KK911PP3WRY2YYC79X7XZ789","status":"resolved","updated_at":"2026-03-09T10:09:36Z"},{"id":"01KK911PP30VAF7M1Y1M66PFY6","body":"Ingestion and queries from GCP us east public cluster are failing.","created_at":"2026-03-09T10:07:25Z","display_at":"2026-03-09T09:52:00Z","incident_id":"01KK911PP3WRY2YYC79X7XZ789","status":"investigating","updated_at":"2026-03-09T10:09:28Z"}]},{"id":"01KK1NBR5MGVNJ4X4FRA0NFAXP","name":"UI is not accessible","status":"resolved","created_at":"2026-03-06T13:28:31Z","updated_at":"2026-03-12T18:18:15Z","resolved_at":"2026-03-06T14:28:34Z","impact":"major","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KK1RSQ5RXBVF60NEP2X8SQ4D","body":"UI is fully restored and operative","created_at":"2026-03-06T14:28:34Z","display_at":"2026-03-06T14:28:34Z","incident_id":"01KK1NBR5MGVNJ4X4FRA0NFAXP","status":"resolved","updated_at":"2026-03-06T14:28:34Z"},{"id":"01KK1R1H58HSHW6J1N7MYMT72F","body":"Access to the UI is working again","created_at":"2026-03-06T14:15:22Z","display_at":"2026-03-06T14:15:22Z","incident_id":"01KK1NBR5MGVNJ4X4FRA0NFAXP","status":"monitoring","updated_at":"2026-03-06T14:15:22Z"},{"id":"01KK1P9T4SKFCKSXW54SEXWNCB","body":"We're investigating a problem affecting the UI","created_at":"2026-03-06T13:44:56Z","display_at":"2026-03-06T13:44:56Z","incident_id":"01KK1NBR5MGVNJ4X4FRA0NFAXP","status":"investigating","updated_at":"2026-03-06T13:44:56Z"},{"id":"01KK1NT7WRJ0QJM7V1JNH27YRQ","body":"UI load is currently returning errors due to an identified problem being fixed","created_at":"2026-03-06T13:36:26Z","display_at":"2026-03-06T13:36:26Z","incident_id":"01KK1NBR5MGVNJ4X4FRA0NFAXP","status":"identified","updated_at":"2026-03-06T13:36:26Z"},{"id":"01KK1NKVWKCNNDWGEJY1DH6M1C","body":"A problem with an internal monitoring app is causing this, fix is underway.","created_at":"2026-03-06T13:32:57Z","display_at":"2026-03-06T13:32:57Z","incident_id":"01KK1NBR5MGVNJ4X4FRA0NFAXP","status":"identified","updated_at":"2026-03-06T13:32:57Z"},{"id":"01KK1NBR5M6DT7M04KAG6CQ29Z","body":"The [cloud.tinybird.co](http://cloud.tinybird.co \"cloud.tinybird.co\") is currently not accessible","created_at":"2026-03-06T13:28:31Z","display_at":"2026-03-06T13:28:31Z","incident_id":"01KK1NBR5MGVNJ4X4FRA0NFAXP","status":"investigating","updated_at":"2026-03-06T13:28:31Z"}]},{"id":"01KJTDA1K4M5FRSQ881TA44DVZ","name":"GCP us-east4 region degraded","status":"resolved","created_at":"2026-03-03T17:53:05Z","updated_at":"2026-03-12T18:18:15Z","resolved_at":"2026-03-03T19:10:22Z","impact":"minor","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KJTHQHV1Z8CAK6KMR9ST8QTM","body":"The region is recovered.","created_at":"2026-03-03T19:10:22Z","display_at":"2026-03-03T19:10:22Z","incident_id":"01KJTDA1K4M5FRSQ881TA44DVZ","status":"resolved","updated_at":"2026-03-03T19:10:22Z"},{"id":"01KJTDA1K4X5VSYJS9G109XM5N","body":"The region is degraded with some high latencies and rate limit possibly appled","created_at":"2026-03-03T17:53:05Z","display_at":"2026-03-03T17:53:05Z","incident_id":"01KJTDA1K4M5FRSQ881TA44DVZ","status":"monitoring","updated_at":"2026-03-03T17:53:05Z"}]},{"id":"01KJRK76XMGRN2S4WHKMY10NZJ","name":"Kafka service degradation","status":"resolved","created_at":"2026-03-03T00:57:55Z","updated_at":"2026-03-12T18:18:15Z","resolved_at":"2026-03-03T03:08:24Z","impact":"minor","page_id":"01HHF88M5X743YJG251VX5099P","incident_updates":[{"id":"01KJRTP4SS61F3HCW0J0AYH899","body":"All systems back to normal. No remaining lag to process","created_at":"2026-03-03T03:08:24Z","display_at":"2026-03-03T03:08:24Z","incident_id":"01KJRK76XMGRN2S4WHKMY10NZJ","status":"resolved","updated_at":"2026-03-03T03:08:24Z"},{"id":"01KJRMMAB4GJF155X512DCNTMK","body":"Root cause is now solved. We're still monitoring the region. Kafka service is back to normal.","created_at":"2026-03-03T01:22:33Z","display_at":"2026-03-03T01:22:33Z","incident_id":"01KJRK76XMGRN2S4WHKMY10NZJ","status":"monitoring","updated_at":"2026-03-03T01:22:33Z"},{"id":"01KJRKG9WN042WNFAJ7QN4NZ4F","body":"We're currently experiencing degradation with Kafka ingestion. We're looking for the root cause.","created_at":"2026-03-03T01:02:53Z","display_at":"2026-03-03T01:02:53Z","incident_id":"01KJRK76XMGRN2S4WHKMY10NZJ","status":"investigating","updated_at":"2026-03-03T01:02:53Z"},{"id":"01KJRK76XMZA2PKHDETPXXC77T","body":"We're currently experiencing degradation with Kafka ingestion. We're looking for the root cause.","created_at":"2026-03-03T00:57:55Z","display_at":"2026-03-03T00:57:55Z","incident_id":"01KJRK76XMGRN2S4WHKMY10NZJ","status":"investigating","updated_at":"2026-03-03T00:57:55Z"}]}]}