From 5a20e72cd27a3bd908ba02811e14f0f38d50c9d5 Mon Sep 17 00:00:00 2001 From: Tobias Schmidt Date: Mon, 15 Jun 2026 19:04:03 +0200 Subject: [PATCH 1/3] flag lukewarm cold runs ClickBench's "cold run" is only truly cold when the data must be read back from the storage device: the engine is restarted (or never persists between queries) AND the OS page cache is dropped before the first try. A system that instead runs an unseen query against a live engine with warm internal caches does a "lukewarm" run and is supposed to carry the "lukewarm-cold-run" tag. Several systems that do lukewarm runs were never tagged; this adds the tag to them. Newly flagged: - Persistent local daemons that are never restarted between queries, so only the page cache is dropped while engine-internal caches survive: impala, firebolt. - Managed/remote services that cannot be restarted or have their caches flushed at all: clickhouse-cloud, motherduck, databricks, snowflake, redshift(+serverless), bigquery, athena(+partitioned), aurora-mysql, aurora-postgresql, alloydb, hologres, supabase, tablespace, tembo-olap, chyt, bytehouse, crunchy-bridge-for-analytics, timescale-cloud, tinybird, singlestore, hydra. The tag is added to the displayed result files and, so future runs stay flagged, to each system's metadata source: template.json (impala, firebolt) and the inline JSON generators (clickhouse-cloud/collect-results.sh, motherduck/benchmark.py, databricks/benchmark.py). Co-Authored-By: Claude Opus 4.8 --- alloydb/results/20231125/gcp.128GB_tuned.json | 2 +- alloydb/results/20231125/gcp.64GB.json | 3 ++- alloydb/results/20231125/gcp.64GB_tuned.json | 2 +- athena-partitioned/results/20220701/serverless.json | 2 +- athena/results/20220701/serverless.json | 2 +- aurora-mysql/results/20220701/16acu.json | 3 ++- aurora-postgresql/results/20220701/16acu.json | 2 +- bigquery/results/20251028/result.json | 3 ++- bytehouse/results/20220727/l.json | 3 ++- bytehouse/results/20220727/m.json | 3 ++- bytehouse/results/20220727/s.json | 3 ++- bytehouse/results/20220727/xs.json | 3 ++- chyt/results/20240916/yt.360GB_YC.json | 2 +- chyt/results/20240916/yt.48GB_YC.json | 2 +- chyt/results/20240916/yt.720GB_YC.json | 3 ++- chyt/results/20240916/yt.96GB_YC.json | 2 +- clickhouse-cloud/collect-results.sh | 2 +- .../results/20240605/crunchy-bridge-analytics-256.json | 2 +- databricks/benchmark.py | 2 +- databricks/results/20251110/2x-small.json | 3 ++- databricks/results/20251113/2x-large.json | 3 ++- databricks/results/20251113/4x-large.json | 3 ++- databricks/results/20251113/large.json | 3 ++- databricks/results/20251113/medium.json | 3 ++- databricks/results/20251113/small.json | 3 ++- databricks/results/20251113/x-large.json | 3 ++- databricks/results/20251113/x-small.json | 3 ++- firebolt/results/20250607/medium.json | 3 ++- firebolt/results/20250607/small.json | 3 ++- firebolt/results/20260517/c6a.2xlarge.json | 2 +- firebolt/results/20260517/c6a.4xlarge.json | 2 +- firebolt/results/20260517/c6a.large.json | 2 +- firebolt/results/20260517/c6a.metal.json | 2 +- firebolt/results/20260517/c6a.xlarge.json | 2 +- firebolt/results/20260517/c7a.metal-48xl.json | 2 +- firebolt/results/20260517/c8g.4xlarge.json | 2 +- firebolt/results/20260517/c8g.metal-48xl.json | 2 +- firebolt/template.json | 3 ++- hologres/results/20250521/128core.json | 2 +- hologres/results/20250521/32core.json | 2 +- hologres/results/20250521/64core.json | 2 +- hydra/results/20230919/c6a.4xlarge.json | 3 ++- impala/results/20260517/c6a.2xlarge.json | 2 +- impala/results/20260517/c6a.4xlarge.json | 2 +- impala/results/20260517/c6a.metal.json | 2 +- impala/results/20260517/c6a.xlarge.json | 2 +- impala/results/20260517/c7a.metal-48xl.json | 2 +- impala/template.json | 3 ++- motherduck/benchmark.py | 2 +- motherduck/results/20250304/result_jumbo.json | 3 ++- motherduck/results/20250304/result_pulse.json | 3 ++- motherduck/results/20250304/result_standard.json | 3 ++- motherduck/results/20251019/pulse.json | 3 ++- motherduck/results/20251019/standard.json | 3 ++- motherduck/results/20251022/jumbo.json | 3 ++- motherduck/results/20251022/mega.json | 3 ++- redshift-serverless/results/20220729/serverless.json | 2 +- redshift/results/20220725/4x.ra3.16xlarge.json | 2 +- redshift/results/20220725/4x.ra3.4xlarge.json | 2 +- redshift/results/20220729/4x.ra3.xplus.json | 2 +- redshift/results/20230316/2x.dc2.8xlarge.json | 2 +- singlestore/results/20150405/historical-10m.json | 3 ++- singlestore/results/20220701/c6a.4xlarge.json | 2 +- singlestore/results/20220715/s2.json | 2 +- singlestore/results/20220715/s24.json | 2 +- snowflake/results/20220701/2xl.json | 2 +- snowflake/results/20220701/3xl.json | 2 +- snowflake/results/20220701/4xl.json | 2 +- snowflake/results/20220701/l.json | 2 +- snowflake/results/20220701/m.json | 2 +- snowflake/results/20220701/s.json | 2 +- snowflake/results/20220701/xl.json | 2 +- snowflake/results/20220701/xs.json | 2 +- snowflake/results/20260213/xs_interactive.json | 2 +- tablespace/results/20240225/large-1-tuned.json | 3 ++- tablespace/results/20240225/large-1.json | 2 +- timescale-cloud/results/20251213/16cpu.json | 2 +- timescale-cloud/results/20251213/4cpu.json | 2 +- timescale-cloud/results/20251213/8cpu.json | 2 +- tinybird/results/20241111/tinybird.json | 2 +- 80 files changed, 110 insertions(+), 80 deletions(-) diff --git a/alloydb/results/20231125/gcp.128GB_tuned.json b/alloydb/results/20231125/gcp.128GB_tuned.json index 6d75fd5ad6..fa33a7ab82 100644 --- a/alloydb/results/20231125/gcp.128GB_tuned.json +++ b/alloydb/results/20231125/gcp.128GB_tuned.json @@ -8,7 +8,7 @@ "hardware": "cpu", "tuned": "no", - "tags": ["C", "column-oriented", "PostgreSQL compatible", "managed", "gcp"], + "tags": ["C", "column-oriented", "PostgreSQL compatible", "managed", "gcp", "lukewarm-cold-run"], "load_time": 0, "data_size": 9941379875, diff --git a/alloydb/results/20231125/gcp.64GB.json b/alloydb/results/20231125/gcp.64GB.json index d0616f64cc..cb357752c1 100644 --- a/alloydb/results/20231125/gcp.64GB.json +++ b/alloydb/results/20231125/gcp.64GB.json @@ -12,7 +12,8 @@ "column-oriented", "PostgreSQL compatible", "managed", - "gcp" + "gcp", + "lukewarm-cold-run" ], "load_time": 1543, "data_size": 9941379875, diff --git a/alloydb/results/20231125/gcp.64GB_tuned.json b/alloydb/results/20231125/gcp.64GB_tuned.json index 3a45141443..47f2bd3f70 100644 --- a/alloydb/results/20231125/gcp.64GB_tuned.json +++ b/alloydb/results/20231125/gcp.64GB_tuned.json @@ -8,7 +8,7 @@ "tuned": "yes", "comment": "", - "tags": ["C", "column-oriented", "PostgreSQL compatible", "managed", "gcp"], + "tags": ["C", "column-oriented", "PostgreSQL compatible", "managed", "gcp", "lukewarm-cold-run"], "load_time": 0, "data_size": 9941379875, diff --git a/athena-partitioned/results/20220701/serverless.json b/athena-partitioned/results/20220701/serverless.json index b2f289cf58..9063bac5eb 100644 --- a/athena-partitioned/results/20220701/serverless.json +++ b/athena-partitioned/results/20220701/serverless.json @@ -8,7 +8,7 @@ "tuned": "no", "comment": "", - "tags": ["stateless", "managed", "Java", "column-oriented", "aws"], + "tags": ["stateless", "managed", "Java", "column-oriented", "aws", "lukewarm-cold-run"], "load_time": 0, "data_size": 13800000000, diff --git a/athena/results/20220701/serverless.json b/athena/results/20220701/serverless.json index ca453a0b1e..3fd73fa357 100644 --- a/athena/results/20220701/serverless.json +++ b/athena/results/20220701/serverless.json @@ -8,7 +8,7 @@ "tuned": "no", "comment": "", - "tags": ["stateless", "managed", "Java", "column-oriented", "aws", "historical"], + "tags": ["stateless", "managed", "Java", "column-oriented", "aws", "historical", "lukewarm-cold-run"], "load_time": 0, "data_size": 13800000000, diff --git a/aurora-mysql/results/20220701/16acu.json b/aurora-mysql/results/20220701/16acu.json index 42e1308ced..85d69dd6a7 100644 --- a/aurora-mysql/results/20220701/16acu.json +++ b/aurora-mysql/results/20220701/16acu.json @@ -12,7 +12,8 @@ "C++", "MySQL compatible", "row-oriented", - "aws" + "aws", + "lukewarm-cold-run" ], "load_time": 7687, "data_size": 89614492631, diff --git a/aurora-postgresql/results/20220701/16acu.json b/aurora-postgresql/results/20220701/16acu.json index 3115e29961..c5f3b868e2 100644 --- a/aurora-postgresql/results/20220701/16acu.json +++ b/aurora-postgresql/results/20220701/16acu.json @@ -8,7 +8,7 @@ "tuned": "no", "comment": "", - "tags": ["managed", "C", "PostgreSQL compatible", "row-oriented", "aws"], + "tags": ["managed", "C", "PostgreSQL compatible", "row-oriented", "aws", "lukewarm-cold-run"], "load_time": 2127, "data_size": 52183852646, diff --git a/bigquery/results/20251028/result.json b/bigquery/results/20251028/result.json index 67468dac45..36530a69a8 100644 --- a/bigquery/results/20251028/result.json +++ b/bigquery/results/20251028/result.json @@ -10,7 +10,8 @@ "serverless", "column-oriented", "gcp", - "managed" + "managed", + "lukewarm-cold-run" ], "load_time": 777, "data_size": 8760000000, diff --git a/bytehouse/results/20220727/l.json b/bytehouse/results/20220727/l.json index aab69a15bd..468ace812e 100644 --- a/bytehouse/results/20220727/l.json +++ b/bytehouse/results/20220727/l.json @@ -11,7 +11,8 @@ "column-oriented", "ClickHouse derivative", "C++", - "historical" + "historical", + "lukewarm-cold-run" ], "load_time": 745, "data_size": 27190000000, diff --git a/bytehouse/results/20220727/m.json b/bytehouse/results/20220727/m.json index 9dea02fb9d..3a92f7ca32 100644 --- a/bytehouse/results/20220727/m.json +++ b/bytehouse/results/20220727/m.json @@ -11,7 +11,8 @@ "column-oriented", "ClickHouse derivative", "C++", - "historical" + "historical", + "lukewarm-cold-run" ], "load_time": 745, "data_size": 27190000000, diff --git a/bytehouse/results/20220727/s.json b/bytehouse/results/20220727/s.json index dd6a868f50..97460378b5 100644 --- a/bytehouse/results/20220727/s.json +++ b/bytehouse/results/20220727/s.json @@ -11,7 +11,8 @@ "column-oriented", "ClickHouse derivative", "C++", - "historical" + "historical", + "lukewarm-cold-run" ], "load_time": 745, "data_size": 27190000000, diff --git a/bytehouse/results/20220727/xs.json b/bytehouse/results/20220727/xs.json index 2fc3736c89..11251922ab 100644 --- a/bytehouse/results/20220727/xs.json +++ b/bytehouse/results/20220727/xs.json @@ -11,7 +11,8 @@ "column-oriented", "ClickHouse derivative", "C++", - "historical" + "historical", + "lukewarm-cold-run" ], "load_time": 745, "data_size": 27190000000, diff --git a/chyt/results/20240916/yt.360GB_YC.json b/chyt/results/20240916/yt.360GB_YC.json index 7185b26fa3..a5a70946d9 100644 --- a/chyt/results/20240916/yt.360GB_YC.json +++ b/chyt/results/20240916/yt.360GB_YC.json @@ -9,7 +9,7 @@ "tuned": "no", "comment": "", - "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed"], + "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed", "lukewarm-cold-run"], "load_time": 0, "data_size": 11991598975, diff --git a/chyt/results/20240916/yt.48GB_YC.json b/chyt/results/20240916/yt.48GB_YC.json index 530e986468..d2d40fe67e 100644 --- a/chyt/results/20240916/yt.48GB_YC.json +++ b/chyt/results/20240916/yt.48GB_YC.json @@ -9,7 +9,7 @@ "tuned": "no", "comment": "", - "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed"], + "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed", "lukewarm-cold-run"], "load_time": 0, "data_size": 11991598975, diff --git a/chyt/results/20240916/yt.720GB_YC.json b/chyt/results/20240916/yt.720GB_YC.json index 404400193e..3678c065ae 100644 --- a/chyt/results/20240916/yt.720GB_YC.json +++ b/chyt/results/20240916/yt.720GB_YC.json @@ -10,7 +10,8 @@ "ClickHouse derivative", "managed", "YT", - "historical" + "historical", + "lukewarm-cold-run" ], "load_time": 0, "data_size": 11991598975, diff --git a/chyt/results/20240916/yt.96GB_YC.json b/chyt/results/20240916/yt.96GB_YC.json index 606a468ead..69ab2aa2ad 100644 --- a/chyt/results/20240916/yt.96GB_YC.json +++ b/chyt/results/20240916/yt.96GB_YC.json @@ -9,7 +9,7 @@ "tuned": "no", "comment": "", - "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed"], + "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed", "lukewarm-cold-run"], "load_time": 0, "data_size": 11991598975, diff --git a/clickhouse-cloud/collect-results.sh b/clickhouse-cloud/collect-results.sh index 0f645a78f4..7e9a18abe1 100755 --- a/clickhouse-cloud/collect-results.sh +++ b/clickhouse-cloud/collect-results.sh @@ -41,7 +41,7 @@ do "tuned": "no", "comment": "", - "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed", "'$PROVIDER'"], + "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed", "'$PROVIDER'", "lukewarm-cold-run"], "load_time": '$LOAD_TIME', "data_size": '$DATA_SIZE', diff --git a/crunchy-bridge-for-analytics/results/20240605/crunchy-bridge-analytics-256.json b/crunchy-bridge-for-analytics/results/20240605/crunchy-bridge-analytics-256.json index a65505eb6b..ce33ae634d 100644 --- a/crunchy-bridge-for-analytics/results/20240605/crunchy-bridge-analytics-256.json +++ b/crunchy-bridge-for-analytics/results/20240605/crunchy-bridge-analytics-256.json @@ -7,7 +7,7 @@ "hardware": "cpu", "tuned": "no", "comment":"", - "tags": ["column-oriented","DuckDB derivative", "PostgreSQL compatible", "managed"], + "tags": ["column-oriented","DuckDB derivative", "PostgreSQL compatible", "managed", "lukewarm-cold-run"], "load_time": 0, "data_size": 14779976446, diff --git a/databricks/benchmark.py b/databricks/benchmark.py index 497fee2228..0128780e86 100755 --- a/databricks/benchmark.py +++ b/databricks/benchmark.py @@ -351,7 +351,7 @@ def run_queries(): "cluster_size": "", "proprietary": "yes", "tuned": "no", - "tags": ["managed", "column-oriented"], + "tags": ["managed", "column-oriented", "lukewarm-cold-run"], } load_data(run_metadata) diff --git a/databricks/results/20251110/2x-small.json b/databricks/results/20251110/2x-small.json index 3742dbc951..10d38fc3f7 100644 --- a/databricks/results/20251110/2x-small.json +++ b/databricks/results/20251110/2x-small.json @@ -8,7 +8,8 @@ "tuned": "no", "tags": [ "managed", - "column-oriented" + "column-oriented", + "lukewarm-cold-run" ], "load_time": 90, "data_size": 10219802927, diff --git a/databricks/results/20251113/2x-large.json b/databricks/results/20251113/2x-large.json index ee56143485..59f71dcc6c 100644 --- a/databricks/results/20251113/2x-large.json +++ b/databricks/results/20251113/2x-large.json @@ -8,7 +8,8 @@ "tuned": "no", "tags": [ "managed", - "column-oriented" + "column-oriented", + "lukewarm-cold-run" ], "load_time": 36, "data_size": 10219802927, diff --git a/databricks/results/20251113/4x-large.json b/databricks/results/20251113/4x-large.json index f2e6d791c7..351001eeb8 100644 --- a/databricks/results/20251113/4x-large.json +++ b/databricks/results/20251113/4x-large.json @@ -8,7 +8,8 @@ "tuned": "no", "tags": [ "managed", - "column-oriented" + "column-oriented", + "lukewarm-cold-run" ], "load_time": 30, "data_size": 10219802927, diff --git a/databricks/results/20251113/large.json b/databricks/results/20251113/large.json index b4b5b59122..5365ad0174 100644 --- a/databricks/results/20251113/large.json +++ b/databricks/results/20251113/large.json @@ -8,7 +8,8 @@ "tuned": "no", "tags": [ "managed", - "column-oriented" + "column-oriented", + "lukewarm-cold-run" ], "load_time": 45, "data_size": 10219802927, diff --git a/databricks/results/20251113/medium.json b/databricks/results/20251113/medium.json index 1e6455562b..3344220f70 100644 --- a/databricks/results/20251113/medium.json +++ b/databricks/results/20251113/medium.json @@ -8,7 +8,8 @@ "tuned": "no", "tags": [ "managed", - "column-oriented" + "column-oriented", + "lukewarm-cold-run" ], "load_time": 53, "data_size": 10219802927, diff --git a/databricks/results/20251113/small.json b/databricks/results/20251113/small.json index 78c8f89cc1..4dbb5f6b08 100644 --- a/databricks/results/20251113/small.json +++ b/databricks/results/20251113/small.json @@ -8,7 +8,8 @@ "tuned": "no", "tags": [ "managed", - "column-oriented" + "column-oriented", + "lukewarm-cold-run" ], "load_time": 67, "data_size": 10219802927, diff --git a/databricks/results/20251113/x-large.json b/databricks/results/20251113/x-large.json index 7df5753ac5..d9d81431c8 100644 --- a/databricks/results/20251113/x-large.json +++ b/databricks/results/20251113/x-large.json @@ -8,7 +8,8 @@ "tuned": "no", "tags": [ "managed", - "column-oriented" + "column-oriented", + "lukewarm-cold-run" ], "load_time": 41, "data_size": 10219802927, diff --git a/databricks/results/20251113/x-small.json b/databricks/results/20251113/x-small.json index ace4e9f35d..29a03de473 100644 --- a/databricks/results/20251113/x-small.json +++ b/databricks/results/20251113/x-small.json @@ -8,7 +8,8 @@ "tuned": "no", "tags": [ "managed", - "column-oriented" + "column-oriented", + "lukewarm-cold-run" ], "load_time": 91, "data_size": 10219802927, diff --git a/firebolt/results/20250607/medium.json b/firebolt/results/20250607/medium.json index 0fe2660f42..1ac858fb69 100644 --- a/firebolt/results/20250607/medium.json +++ b/firebolt/results/20250607/medium.json @@ -8,7 +8,8 @@ "C++", "column-oriented", "ClickHouse derivative", - "historical" + "historical", + "lukewarm-cold-run" ], "load_time": 0, "data_size": 14730000000, diff --git a/firebolt/results/20250607/small.json b/firebolt/results/20250607/small.json index 619f5f8725..7951d51069 100644 --- a/firebolt/results/20250607/small.json +++ b/firebolt/results/20250607/small.json @@ -8,7 +8,8 @@ "C++", "column-oriented", "ClickHouse derivative", - "historical" + "historical", + "lukewarm-cold-run" ], "load_time": 0, "data_size": 14730000000, diff --git a/firebolt/results/20260517/c6a.2xlarge.json b/firebolt/results/20260517/c6a.2xlarge.json index e9691172fa..f930b56138 100644 --- a/firebolt/results/20260517/c6a.2xlarge.json +++ b/firebolt/results/20260517/c6a.2xlarge.json @@ -6,7 +6,7 @@ "proprietary": "yes", "hardware": "cpu", "tuned": "no", - "tags": ["C++","column-oriented","PostgreSQL compatible","ClickHouse derivative"], + "tags": ["C++","column-oriented","PostgreSQL compatible","ClickHouse derivative", "lukewarm-cold-run"], "load_time": 285, "data_size": 15827354954, "concurrent_qps": 0.375, diff --git a/firebolt/results/20260517/c6a.4xlarge.json b/firebolt/results/20260517/c6a.4xlarge.json index 32396ac781..73421ccecd 100644 --- a/firebolt/results/20260517/c6a.4xlarge.json +++ b/firebolt/results/20260517/c6a.4xlarge.json @@ -6,7 +6,7 @@ "proprietary": "yes", "hardware": "cpu", "tuned": "no", - "tags": ["C++","column-oriented","PostgreSQL compatible","ClickHouse derivative"], + "tags": ["C++","column-oriented","PostgreSQL compatible","ClickHouse derivative", "lukewarm-cold-run"], "load_time": 235, "data_size": 15825985134, "concurrent_qps": 1.908, diff --git a/firebolt/results/20260517/c6a.large.json b/firebolt/results/20260517/c6a.large.json index f355201696..796a4c49d2 100644 --- a/firebolt/results/20260517/c6a.large.json +++ b/firebolt/results/20260517/c6a.large.json @@ -6,7 +6,7 @@ "proprietary": "yes", "hardware": "cpu", "tuned": "no", - "tags": ["C++","column-oriented","PostgreSQL compatible","ClickHouse derivative"], + "tags": ["C++","column-oriented","PostgreSQL compatible","ClickHouse derivative", "lukewarm-cold-run"], "load_time": 488, "data_size": 15816235785, "concurrent_qps": 0.262, diff --git a/firebolt/results/20260517/c6a.metal.json b/firebolt/results/20260517/c6a.metal.json index fd35c3ac44..5284bd5f08 100644 --- a/firebolt/results/20260517/c6a.metal.json +++ b/firebolt/results/20260517/c6a.metal.json @@ -6,7 +6,7 @@ "proprietary": "yes", "hardware": "cpu", "tuned": "no", - "tags": ["C++","column-oriented","PostgreSQL compatible","ClickHouse derivative"], + "tags": ["C++","column-oriented","PostgreSQL compatible","ClickHouse derivative", "lukewarm-cold-run"], "load_time": 172, "data_size": 15823268468, "concurrent_qps": 9.84, diff --git a/firebolt/results/20260517/c6a.xlarge.json b/firebolt/results/20260517/c6a.xlarge.json index 9577dee3d7..eb43e71b6e 100644 --- a/firebolt/results/20260517/c6a.xlarge.json +++ b/firebolt/results/20260517/c6a.xlarge.json @@ -6,7 +6,7 @@ "proprietary": "yes", "hardware": "cpu", "tuned": "no", - "tags": ["C++","column-oriented","PostgreSQL compatible","ClickHouse derivative"], + "tags": ["C++","column-oriented","PostgreSQL compatible","ClickHouse derivative", "lukewarm-cold-run"], "load_time": 330, "data_size": 15827881790, "concurrent_qps": 0.2, diff --git a/firebolt/results/20260517/c7a.metal-48xl.json b/firebolt/results/20260517/c7a.metal-48xl.json index 60e661bf60..878ae13edb 100644 --- a/firebolt/results/20260517/c7a.metal-48xl.json +++ b/firebolt/results/20260517/c7a.metal-48xl.json @@ -6,7 +6,7 @@ "proprietary": "yes", "hardware": "cpu", "tuned": "no", - "tags": ["C++","column-oriented","PostgreSQL compatible","ClickHouse derivative"], + "tags": ["C++","column-oriented","PostgreSQL compatible","ClickHouse derivative", "lukewarm-cold-run"], "load_time": 197, "data_size": 15826419844, "concurrent_qps": 18.902, diff --git a/firebolt/results/20260517/c8g.4xlarge.json b/firebolt/results/20260517/c8g.4xlarge.json index 8b6f2d0233..f9ed42c3de 100644 --- a/firebolt/results/20260517/c8g.4xlarge.json +++ b/firebolt/results/20260517/c8g.4xlarge.json @@ -6,7 +6,7 @@ "proprietary": "yes", "hardware": "cpu", "tuned": "no", - "tags": ["C++","column-oriented","PostgreSQL compatible","ClickHouse derivative"], + "tags": ["C++","column-oriented","PostgreSQL compatible","ClickHouse derivative", "lukewarm-cold-run"], "load_time": 241, "data_size": 15827812385, "concurrent_qps": 3.773, diff --git a/firebolt/results/20260517/c8g.metal-48xl.json b/firebolt/results/20260517/c8g.metal-48xl.json index a02ea591ec..fd8a5f0d34 100644 --- a/firebolt/results/20260517/c8g.metal-48xl.json +++ b/firebolt/results/20260517/c8g.metal-48xl.json @@ -6,7 +6,7 @@ "proprietary": "yes", "hardware": "cpu", "tuned": "no", - "tags": ["C++","column-oriented","PostgreSQL compatible","ClickHouse derivative"], + "tags": ["C++","column-oriented","PostgreSQL compatible","ClickHouse derivative", "lukewarm-cold-run"], "load_time": 175, "data_size": 15825383440, "concurrent_qps": 19.565, diff --git a/firebolt/template.json b/firebolt/template.json index b48735638d..528313355c 100644 --- a/firebolt/template.json +++ b/firebolt/template.json @@ -7,6 +7,7 @@ "C++", "column-oriented", "PostgreSQL compatible", - "ClickHouse derivative" + "ClickHouse derivative", + "lukewarm-cold-run" ] } diff --git a/hologres/results/20250521/128core.json b/hologres/results/20250521/128core.json index c21281a5d2..1d5ba1f485 100644 --- a/hologres/results/20250521/128core.json +++ b/hologres/results/20250521/128core.json @@ -6,7 +6,7 @@ "proprietary": "yes", "hardware": "cpu", "tuned": "no", - "tags": ["managed", "PostgreSQL compatible", "column-oriented", "managed"], + "tags": ["managed", "PostgreSQL compatible", "column-oriented", "managed", "lukewarm-cold-run"], "load_time": 321, "data_size": 19923251230, "result": [ diff --git a/hologres/results/20250521/32core.json b/hologres/results/20250521/32core.json index d865b0b5f8..aedd403e4a 100644 --- a/hologres/results/20250521/32core.json +++ b/hologres/results/20250521/32core.json @@ -6,7 +6,7 @@ "proprietary": "yes", "hardware": "cpu", "tuned": "no", - "tags": ["managed", "PostgreSQL compatible", "column-oriented", "managed"], + "tags": ["managed", "PostgreSQL compatible", "column-oriented", "managed", "lukewarm-cold-run"], "load_time": 388, "data_size": 18778181002, "result": [ diff --git a/hologres/results/20250521/64core.json b/hologres/results/20250521/64core.json index c1c49189cc..47f3d57aa5 100644 --- a/hologres/results/20250521/64core.json +++ b/hologres/results/20250521/64core.json @@ -6,7 +6,7 @@ "proprietary": "yes", "hardware": "cpu", "tuned": "no", - "tags": ["managed", "PostgreSQL compatible", "column-oriented", "managed"], + "tags": ["managed", "PostgreSQL compatible", "column-oriented", "managed", "lukewarm-cold-run"], "load_time": 366, "data_size": 19708639814, "result": [ diff --git a/hydra/results/20230919/c6a.4xlarge.json b/hydra/results/20230919/c6a.4xlarge.json index 5391b81b0e..70bc42a0e6 100644 --- a/hydra/results/20230919/c6a.4xlarge.json +++ b/hydra/results/20230919/c6a.4xlarge.json @@ -6,7 +6,8 @@ "comment": "", "tags": [ "PostgreSQL compatible", - "column-oriented" + "column-oriented", + "lukewarm-cold-run" ], "load_time": 1222, "data_size": 18995669982, diff --git a/impala/results/20260517/c6a.2xlarge.json b/impala/results/20260517/c6a.2xlarge.json index 0117636514..acd2c48f26 100644 --- a/impala/results/20260517/c6a.2xlarge.json +++ b/impala/results/20260517/c6a.2xlarge.json @@ -6,7 +6,7 @@ "proprietary": "no", "hardware": "cpu", "tuned": "no", - "tags": ["C++","MPP","column-oriented","stateless"], + "tags": ["C++","MPP","column-oriented","stateless", "lukewarm-cold-run"], "load_time": 63, "data_size": 14779976446, "concurrent_qps": 0.23, diff --git a/impala/results/20260517/c6a.4xlarge.json b/impala/results/20260517/c6a.4xlarge.json index 993a11014e..33a56870a6 100644 --- a/impala/results/20260517/c6a.4xlarge.json +++ b/impala/results/20260517/c6a.4xlarge.json @@ -6,7 +6,7 @@ "proprietary": "no", "hardware": "cpu", "tuned": "no", - "tags": ["C++","MPP","column-oriented","stateless"], + "tags": ["C++","MPP","column-oriented","stateless", "lukewarm-cold-run"], "load_time": 55, "data_size": 14779976446, "concurrent_qps": 0.498, diff --git a/impala/results/20260517/c6a.metal.json b/impala/results/20260517/c6a.metal.json index 868c953f96..0b1d5ed8c0 100644 --- a/impala/results/20260517/c6a.metal.json +++ b/impala/results/20260517/c6a.metal.json @@ -6,7 +6,7 @@ "proprietary": "no", "hardware": "cpu", "tuned": "no", - "tags": ["C++","MPP","column-oriented","stateless"], + "tags": ["C++","MPP","column-oriented","stateless", "lukewarm-cold-run"], "load_time": 51, "data_size": 14779976446, "concurrent_qps": 0.933, diff --git a/impala/results/20260517/c6a.xlarge.json b/impala/results/20260517/c6a.xlarge.json index 3f4fb75c2f..a706d7df00 100644 --- a/impala/results/20260517/c6a.xlarge.json +++ b/impala/results/20260517/c6a.xlarge.json @@ -6,7 +6,7 @@ "proprietary": "no", "hardware": "cpu", "tuned": "no", - "tags": ["C++","MPP","column-oriented","stateless"], + "tags": ["C++","MPP","column-oriented","stateless", "lukewarm-cold-run"], "load_time": 94, "data_size": 14779976446, "concurrent_qps": 0.113, diff --git a/impala/results/20260517/c7a.metal-48xl.json b/impala/results/20260517/c7a.metal-48xl.json index bd3668e77e..7366a62c7f 100644 --- a/impala/results/20260517/c7a.metal-48xl.json +++ b/impala/results/20260517/c7a.metal-48xl.json @@ -6,7 +6,7 @@ "proprietary": "no", "hardware": "cpu", "tuned": "no", - "tags": ["C++","MPP","column-oriented","stateless"], + "tags": ["C++","MPP","column-oriented","stateless", "lukewarm-cold-run"], "load_time": 46, "data_size": 14779976446, "concurrent_qps": 0.988, diff --git a/impala/template.json b/impala/template.json index 870a0f6356..cbd3e3be27 100644 --- a/impala/template.json +++ b/impala/template.json @@ -7,6 +7,7 @@ "C++", "MPP", "column-oriented", - "stateless" + "stateless", + "lukewarm-cold-run" ] } diff --git a/motherduck/benchmark.py b/motherduck/benchmark.py index 9e2241e3d7..b98c71c775 100644 --- a/motherduck/benchmark.py +++ b/motherduck/benchmark.py @@ -126,7 +126,7 @@ def run_queries(): "proprietary": "yes", "hardware": "cpu", "tuned": "no", - "tags": ["C++", "column-oriented", "serverless", "managed"], + "tags": ["C++", "column-oriented", "serverless", "managed", "lukewarm-cold-run"], } load_data(run_metadata) diff --git a/motherduck/results/20250304/result_jumbo.json b/motherduck/results/20250304/result_jumbo.json index 492e66c750..92d8ca206c 100644 --- a/motherduck/results/20250304/result_jumbo.json +++ b/motherduck/results/20250304/result_jumbo.json @@ -8,7 +8,8 @@ "column-oriented", "DuckDB derivative", "serverless", - "historical" + "historical", + "lukewarm-cold-run" ], "load_time": 69, "data_size": 23620000000, diff --git a/motherduck/results/20250304/result_pulse.json b/motherduck/results/20250304/result_pulse.json index 7991ac7974..00551ef529 100644 --- a/motherduck/results/20250304/result_pulse.json +++ b/motherduck/results/20250304/result_pulse.json @@ -8,7 +8,8 @@ "column-oriented", "DuckDB derivative", "serverless", - "historical" + "historical", + "lukewarm-cold-run" ], "load_time": 756, "data_size": 23620000000, diff --git a/motherduck/results/20250304/result_standard.json b/motherduck/results/20250304/result_standard.json index 0aa73e2e97..2ce24cc24a 100644 --- a/motherduck/results/20250304/result_standard.json +++ b/motherduck/results/20250304/result_standard.json @@ -8,7 +8,8 @@ "column-oriented", "DuckDB derivative", "serverless", - "historical" + "historical", + "lukewarm-cold-run" ], "load_time": 77, "data_size": 23620000000, diff --git a/motherduck/results/20251019/pulse.json b/motherduck/results/20251019/pulse.json index cdf44bcfd3..6193a2c518 100644 --- a/motherduck/results/20251019/pulse.json +++ b/motherduck/results/20251019/pulse.json @@ -11,7 +11,8 @@ "column-oriented", "DuckDB derivative", "serverless", - "managed" + "managed", + "lukewarm-cold-run" ], "load_time": 633, "data_size": 23622320128, diff --git a/motherduck/results/20251019/standard.json b/motherduck/results/20251019/standard.json index f337350d24..c5adfe0e46 100644 --- a/motherduck/results/20251019/standard.json +++ b/motherduck/results/20251019/standard.json @@ -11,7 +11,8 @@ "column-oriented", "DuckDB derivative", "serverless", - "managed" + "managed", + "lukewarm-cold-run" ], "load_time": 181, "data_size": 23622320128, diff --git a/motherduck/results/20251022/jumbo.json b/motherduck/results/20251022/jumbo.json index 2d0a959556..173fcf3cd3 100644 --- a/motherduck/results/20251022/jumbo.json +++ b/motherduck/results/20251022/jumbo.json @@ -11,7 +11,8 @@ "column-oriented", "DuckDB derivative", "serverless", - "managed" + "managed", + "lukewarm-cold-run" ], "load_time": 76, "data_size": 23622320128, diff --git a/motherduck/results/20251022/mega.json b/motherduck/results/20251022/mega.json index d0973e58ad..202d25d755 100644 --- a/motherduck/results/20251022/mega.json +++ b/motherduck/results/20251022/mega.json @@ -11,7 +11,8 @@ "column-oriented", "DuckDB derivative", "serverless", - "managed" + "managed", + "lukewarm-cold-run" ], "load_time": 61, "data_size": 23622320128, diff --git a/redshift-serverless/results/20220729/serverless.json b/redshift-serverless/results/20220729/serverless.json index 5a238e8a62..355a562f04 100644 --- a/redshift-serverless/results/20220729/serverless.json +++ b/redshift-serverless/results/20220729/serverless.json @@ -8,7 +8,7 @@ "tuned": "no", "comment": "", - "tags": ["managed", "column-oriented", "aws"], + "tags": ["managed", "column-oriented", "aws", "lukewarm-cold-run"], "load_time": 1889, "data_size": 30300000000, diff --git a/redshift/results/20220725/4x.ra3.16xlarge.json b/redshift/results/20220725/4x.ra3.16xlarge.json index be8f54485f..2627110fce 100644 --- a/redshift/results/20220725/4x.ra3.16xlarge.json +++ b/redshift/results/20220725/4x.ra3.16xlarge.json @@ -8,7 +8,7 @@ "tuned": "no", "comment": "", - "tags": ["managed", "column-oriented", "aws"], + "tags": ["managed", "column-oriented", "aws", "lukewarm-cold-run"], "load_time": 1829, "data_size": 26330791936, diff --git a/redshift/results/20220725/4x.ra3.4xlarge.json b/redshift/results/20220725/4x.ra3.4xlarge.json index dab65ac2ee..3d6927ca0d 100644 --- a/redshift/results/20220725/4x.ra3.4xlarge.json +++ b/redshift/results/20220725/4x.ra3.4xlarge.json @@ -8,7 +8,7 @@ "tuned": "no", "comment": "", - "tags": ["managed", "column-oriented", "aws"], + "tags": ["managed", "column-oriented", "aws", "lukewarm-cold-run"], "load_time": 1904, "data_size": 22313697280, diff --git a/redshift/results/20220729/4x.ra3.xplus.json b/redshift/results/20220729/4x.ra3.xplus.json index 3c3efc84ac..fb4a48d1c1 100644 --- a/redshift/results/20220729/4x.ra3.xplus.json +++ b/redshift/results/20220729/4x.ra3.xplus.json @@ -8,7 +8,7 @@ "tuned": "no", "comment": "", - "tags": ["managed", "column-oriented", "aws"], + "tags": ["managed", "column-oriented", "aws", "lukewarm-cold-run"], "load_time": 2103, "data_size": 21060648960, diff --git a/redshift/results/20230316/2x.dc2.8xlarge.json b/redshift/results/20230316/2x.dc2.8xlarge.json index 61e9a8ea71..13ca10e45e 100644 --- a/redshift/results/20230316/2x.dc2.8xlarge.json +++ b/redshift/results/20230316/2x.dc2.8xlarge.json @@ -8,7 +8,7 @@ "tuned": "no", "comment": "", - "tags": ["managed", "column-oriented", "aws"], + "tags": ["managed", "column-oriented", "aws", "lukewarm-cold-run"], "load_time": 2485, "data_size": 27278704640, diff --git a/singlestore/results/20150405/historical-10m.json b/singlestore/results/20150405/historical-10m.json index 1f240958dd..b6174b76b2 100644 --- a/singlestore/results/20150405/historical-10m.json +++ b/singlestore/results/20150405/historical-10m.json @@ -10,7 +10,8 @@ "tags": [ "column-oriented", "MySQL compatible", - "historical" + "historical", + "lukewarm-cold-run" ], "load_time": 0, "data_size": 0, diff --git a/singlestore/results/20220701/c6a.4xlarge.json b/singlestore/results/20220701/c6a.4xlarge.json index efdae66bae..a53929795c 100644 --- a/singlestore/results/20220701/c6a.4xlarge.json +++ b/singlestore/results/20220701/c6a.4xlarge.json @@ -8,7 +8,7 @@ "tuned": "yes", "comment": "Previous name: MemSQL. Some queries did not run due to memory limits", - "tags": ["MySQL compatible", "column-oriented"], + "tags": ["MySQL compatible", "column-oriented", "lukewarm-cold-run"], "load_time": 690, "data_size": 29836263469, diff --git a/singlestore/results/20220715/s2.json b/singlestore/results/20220715/s2.json index facfc66777..6e49c2b580 100644 --- a/singlestore/results/20220715/s2.json +++ b/singlestore/results/20220715/s2.json @@ -8,7 +8,7 @@ "tuned": "no", "comment": "Previous name: MemSQL. Some queries did not run due to unsupported regex", - "tags": ["MySQL compatible", "column-oriented", "managed"], + "tags": ["MySQL compatible", "column-oriented", "managed", "lukewarm-cold-run"], "load_time": 1145, "data_size": 19219978649, diff --git a/singlestore/results/20220715/s24.json b/singlestore/results/20220715/s24.json index 4ac62a82d2..1e411a6cc2 100644 --- a/singlestore/results/20220715/s24.json +++ b/singlestore/results/20220715/s24.json @@ -8,7 +8,7 @@ "tuned": "no", "comment": "Previous name: MemSQL. Some queries did not run due to unsupported regex", - "tags": ["MySQL compatible", "column-oriented", "managed"], + "tags": ["MySQL compatible", "column-oriented", "managed", "lukewarm-cold-run"], "load_time": 1043, "data_size": 19219978649, diff --git a/snowflake/results/20220701/2xl.json b/snowflake/results/20220701/2xl.json index b2bda98c15..8dfa2d774e 100644 --- a/snowflake/results/20220701/2xl.json +++ b/snowflake/results/20220701/2xl.json @@ -8,7 +8,7 @@ "tuned": "no", "comment": "", - "tags": ["managed", "column-oriented"], + "tags": ["managed", "column-oriented", "lukewarm-cold-run"], "load_time": 2524, "data_size": 12300000000, diff --git a/snowflake/results/20220701/3xl.json b/snowflake/results/20220701/3xl.json index d9a6440a50..dcf477e2bf 100644 --- a/snowflake/results/20220701/3xl.json +++ b/snowflake/results/20220701/3xl.json @@ -8,7 +8,7 @@ "tuned": "no", "comment": "", - "tags": ["managed", "column-oriented"], + "tags": ["managed", "column-oriented", "lukewarm-cold-run"], "load_time": 2524, "data_size": 12300000000, diff --git a/snowflake/results/20220701/4xl.json b/snowflake/results/20220701/4xl.json index ebe7c2c4be..dedd735dc2 100644 --- a/snowflake/results/20220701/4xl.json +++ b/snowflake/results/20220701/4xl.json @@ -8,7 +8,7 @@ "tuned": "no", "comment": "", - "tags": ["managed", "column-oriented"], + "tags": ["managed", "column-oriented", "lukewarm-cold-run"], "load_time": 2524, "data_size": 12300000000, diff --git a/snowflake/results/20220701/l.json b/snowflake/results/20220701/l.json index 1fae27df66..5ccbe68bc3 100644 --- a/snowflake/results/20220701/l.json +++ b/snowflake/results/20220701/l.json @@ -8,7 +8,7 @@ "tuned": "no", "comment": "", - "tags": ["managed", "column-oriented"], + "tags": ["managed", "column-oriented", "lukewarm-cold-run"], "load_time": 2524, "data_size": 12300000000, diff --git a/snowflake/results/20220701/m.json b/snowflake/results/20220701/m.json index 6547344b12..9941441fec 100644 --- a/snowflake/results/20220701/m.json +++ b/snowflake/results/20220701/m.json @@ -8,7 +8,7 @@ "tuned": "no", "comment": "", - "tags": ["managed", "column-oriented"], + "tags": ["managed", "column-oriented", "lukewarm-cold-run"], "load_time": 2524, "data_size": 12300000000, diff --git a/snowflake/results/20220701/s.json b/snowflake/results/20220701/s.json index 957ab2adbd..ad5918d6ff 100644 --- a/snowflake/results/20220701/s.json +++ b/snowflake/results/20220701/s.json @@ -8,7 +8,7 @@ "hardware": "cpu", "tuned": "no", - "tags": ["managed", "column-oriented"], + "tags": ["managed", "column-oriented", "lukewarm-cold-run"], "load_time": 2524, "data_size": 12300000000, diff --git a/snowflake/results/20220701/xl.json b/snowflake/results/20220701/xl.json index 70d4349d1a..903de0595e 100644 --- a/snowflake/results/20220701/xl.json +++ b/snowflake/results/20220701/xl.json @@ -8,7 +8,7 @@ "hardware": "cpu", "tuned": "no", - "tags": ["managed", "column-oriented"], + "tags": ["managed", "column-oriented", "lukewarm-cold-run"], "load_time": 2524, "data_size": 12300000000, diff --git a/snowflake/results/20220701/xs.json b/snowflake/results/20220701/xs.json index f1a8df74c7..4b4762485f 100644 --- a/snowflake/results/20220701/xs.json +++ b/snowflake/results/20220701/xs.json @@ -8,7 +8,7 @@ "hardware": "cpu", "tuned": "no", - "tags": ["managed", "column-oriented"], + "tags": ["managed", "column-oriented", "lukewarm-cold-run"], "load_time": 2524, "data_size": 12300000000, diff --git a/snowflake/results/20260213/xs_interactive.json b/snowflake/results/20260213/xs_interactive.json index 76f4b514c8..c1e15735ed 100644 --- a/snowflake/results/20260213/xs_interactive.json +++ b/snowflake/results/20260213/xs_interactive.json @@ -8,7 +8,7 @@ "hardware": "cpu", "tuned": "no", - "tags": ["managed", "column-oriented"], + "tags": ["managed", "column-oriented", "lukewarm-cold-run"], "load_time": 67, "data_size": 13030859264, diff --git a/tablespace/results/20240225/large-1-tuned.json b/tablespace/results/20240225/large-1-tuned.json index 4e0955c89a..e8a8d45586 100644 --- a/tablespace/results/20240225/large-1-tuned.json +++ b/tablespace/results/20240225/large-1-tuned.json @@ -12,7 +12,8 @@ "column-oriented", "PostgreSQL compatible", "managed", - "historical" + "historical", + "lukewarm-cold-run" ], "load_time": 13711, "data_size": 135469176422, diff --git a/tablespace/results/20240225/large-1.json b/tablespace/results/20240225/large-1.json index fdfdf6b200..c0f032afdb 100644 --- a/tablespace/results/20240225/large-1.json +++ b/tablespace/results/20240225/large-1.json @@ -8,7 +8,7 @@ "tuned": "no", "comment": "", - "tags": ["Rust", "column-oriented", "PostgreSQL compatible", "managed"], + "tags": ["Rust", "column-oriented", "PostgreSQL compatible", "managed", "lukewarm-cold-run"], "load_time": 13711, "data_size": 135469176422, diff --git a/timescale-cloud/results/20251213/16cpu.json b/timescale-cloud/results/20251213/16cpu.json index 09033b3fcc..c14b10b644 100644 --- a/timescale-cloud/results/20251213/16cpu.json +++ b/timescale-cloud/results/20251213/16cpu.json @@ -3,7 +3,7 @@ "date": "2025-12-13", "machine": "64GiB", "cluster_size": 1, - "tags": ["C", "PostgreSQL compatible", "column-oriented", "time-series", "managed"], + "tags": ["C", "PostgreSQL compatible", "column-oriented", "time-series", "managed", "lukewarm-cold-run"], "proprietary": "yes", "hardware": "cpu", "tuned": "no", diff --git a/timescale-cloud/results/20251213/4cpu.json b/timescale-cloud/results/20251213/4cpu.json index 9b88a5e108..99427bca9d 100644 --- a/timescale-cloud/results/20251213/4cpu.json +++ b/timescale-cloud/results/20251213/4cpu.json @@ -3,7 +3,7 @@ "date": "2025-12-13", "machine": "16GiB", "cluster_size": 1, - "tags": ["C", "PostgreSQL compatible", "column-oriented", "time-series", "managed"], + "tags": ["C", "PostgreSQL compatible", "column-oriented", "time-series", "managed", "lukewarm-cold-run"], "proprietary": "yes", "hardware": "cpu", "tuned": "no", diff --git a/timescale-cloud/results/20251213/8cpu.json b/timescale-cloud/results/20251213/8cpu.json index 495008dfb0..fcce11445a 100644 --- a/timescale-cloud/results/20251213/8cpu.json +++ b/timescale-cloud/results/20251213/8cpu.json @@ -3,7 +3,7 @@ "date": "2025-12-13", "machine": "32GiB", "cluster_size": 1, - "tags": ["C", "PostgreSQL compatible", "column-oriented", "time-series", "managed"], + "tags": ["C", "PostgreSQL compatible", "column-oriented", "time-series", "managed", "lukewarm-cold-run"], "proprietary": "yes", "hardware": "cpu", "tuned": "no", diff --git a/tinybird/results/20241111/tinybird.json b/tinybird/results/20241111/tinybird.json index a2d5cc9340..7958f4ff08 100644 --- a/tinybird/results/20241111/tinybird.json +++ b/tinybird/results/20241111/tinybird.json @@ -8,7 +8,7 @@ "tuned": "no", "comment": "", - "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed"], + "tags": ["C++", "column-oriented", "ClickHouse derivative", "managed", "lukewarm-cold-run"], "load_time": 0, "data_size": null, From 7a67a4b3f6367b90193ca2cc6d0f7a419b4f8ed9 Mon Sep 17 00:00:00 2001 From: Tobias Schmidt Date: Mon, 15 Jun 2026 18:41:13 +0200 Subject: [PATCH 2/3] reweight the combined metric for lukewarm and inapplicable metrics The Combined score is a weighted geomean of load (10%), data size (10%), cold (20%) and hot (60%) ratios. That unfairly penalizes systems for metrics that don't apply to them, and lets lukewarm "cold" numbers (really warm queries) distort the cold component. Unify the per-metric exclusion rules in a single metricExcludes() helper (stateless from load, in-memory from cold/combined/load, lukewarm from cold, missing data size from size) and reuse it everywhere: - Cold Run metric: lukewarm systems are excluded from the ranking by default. - Combined per-query baseline: the cold-run minimum excludes lukewarm / in-memory systems, so their warm "cold" numbers can't depress the baseline and inflate every true-cold system's cold ratio. min load time / min data size likewise exclude systems that don't qualify. - Combined score: a metric that doesn't apply to a system is dropped and the remaining weights are renormalized, instead of feeding a bogus ratio. Lukewarm systems keep a cold component of 0 with its weight folded into hot (load 10% / size 10% / hot 80%); a stateless engine that still reports a load time (e.g. Polars (Parquet)) drops the load component; etc. The cold term is guarded so an all-lukewarm selection (empty cold baseline) can't poison the score with NaN. The Combined view still shows only the single overall score; the per-component breakdown is added in a follow-up commit. Co-Authored-By: Claude Opus 4.8 --- index.html | 94 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 75 insertions(+), 19 deletions(-) diff --git a/index.html b/index.html index 3f64e1bde9..9664448ce9 100644 --- a/index.html +++ b/index.html @@ -926,6 +926,33 @@

Detailed Comparison

return 60 + Math.abs(x % 240); } +/// Whether a system is excluded from a given metric. Single source of truth +/// shared by the top-level metric filter (render) and the per-run baseline +/// used inside the Combined metric (renderSummary) so the two never drift. +function metricExcludes(elem, metric) { + const tags = elem.tags || []; + switch (metric) { + case 'size': + /// Can't rank a system that didn't report a data size. + return !(elem.data_size > 1e9); + case 'load': + /// Needs a real load step: skip trivially-fast loads and + /// stateless / in-memory engines that don't persist data. + return !(elem.load_time >= 5) || tags.includes('stateless') || tags.includes('in-memory'); + case 'cold': + /// True cold-cache reads only: in-memory engines have nothing to + /// read from storage, and lukewarm runs never flush engine caches. + return tags.includes('in-memory') || tags.includes('lukewarm-cold-run'); + case 'combined': + /// Combined still scores lukewarm systems (with the cold component + /// dropped, see below), but in-memory engines are excluded. + return tags.includes('in-memory'); + case 'hot': + default: + return false; + } +} + function renderSummary(filtered_data) { let table = document.getElementById('summary'); clearElement(table); @@ -941,10 +968,18 @@

Detailed Comparison

const baseline_data = [...filtered_data[0].result.keys()].map(query_num => [...Array(3).keys()].map(run_num => - Math.min(...filtered_data.filter(elem => !elem.fake).map(elem => elem.result[query_num]?.[run_num]).filter(x => x != null)))); - - const min_load_time = Math.min(...filtered_data.map(elem => elem.load_time).filter(x => x && x > 5)); - const min_data_size = Math.min(...filtered_data.map(elem => elem.data_size).filter(x => x && x > 1e9)); + Math.min(...filtered_data.filter(elem => !elem.fake) + // Apply the same per-metric exclusions to the baseline that the + // standalone metric uses: run 0 is the cold run, runs 1/2 are + // hot. This keeps lukewarm/in-memory systems out of the cold + // baseline so they can't depress the per-query minimum and + // inflate every true-cold system's cold ratio in Combined. In + // the standalone Cold Run metric they're already filtered out. + .filter(elem => !metricExcludes(elem, run_num === 0 ? 'cold' : 'hot')) + .map(elem => elem.result[query_num]?.[run_num]).filter(x => x != null)))); + + const min_load_time = Math.min(...filtered_data.filter(elem => !metricExcludes(elem, 'load')).map(elem => elem.load_time)); + const min_data_size = Math.min(...filtered_data.filter(elem => !metricExcludes(elem, 'size')).map(elem => elem.data_size)); let summaries; if (selectors.metric == 'load') { @@ -957,11 +992,39 @@

Detailed Comparison

summaries = filtered_data.map(elem => relativeQueryTime(num_queries, baseline_data, elem, selectors.metric)); document.getElementById('time-or-size').innerText = 'time'; } else if (selectors.metric == 'combined') { - summaries = filtered_data.map(elem => Math.exp( - combined_load_time_share * Math.log(elem.load_time >= 5 ? (elem.load_time / min_load_time) : 1) + - combined_data_size_share * Math.log(elem.data_size >= 1e9 ? (elem.data_size / min_data_size) : 2) + - combined_cold_share * Math.log(relativeQueryTime(num_queries, baseline_data, elem, 'cold')) + - combined_hot_share * Math.log(relativeQueryTime(num_queries, baseline_data, elem, 'hot')))); + summaries = filtered_data.map(elem => { + // Exclude metrics that are not applicable to this system from the combined score, + // and reweight the rest so the final score is still on a similar scale. For example, + // lukewarm systems that don't have a true cold run get their cold weight folded + // into hot, so they can still be compared against true cold systems on the hot + // performance they do have without being penalized for the cold performance they + // can't have. + const exclude_cold = metricExcludes(elem, 'cold'); + const exclude_load = metricExcludes(elem, 'load'); + const exclude_size = metricExcludes(elem, 'size'); + + const hot_share = exclude_cold ? combined_cold_share + combined_hot_share : combined_hot_share; + let log_sum = hot_share * Math.log(relativeQueryTime(num_queries, baseline_data, elem, 'hot')); + + if (!exclude_cold) { + log_sum += combined_cold_share * Math.log(relativeQueryTime(num_queries, baseline_data, elem, 'cold')); + } + + if (!exclude_load) { + log_sum += combined_load_time_share * Math.log(elem.load_time / min_load_time); + } + + if (!exclude_size) { + log_sum += combined_data_size_share * Math.log(elem.data_size / min_data_size); + } + + if (exclude_load || exclude_size) { + const correction = 1 - (exclude_load ? combined_load_time_share : 0) - (exclude_size ? combined_data_size_share : 0); + log_sum = log_sum / correction; + } + + return Math.exp(log_sum); + }); document.getElementById('time-or-size').innerText = 'time and data size'; } @@ -1130,16 +1193,9 @@

Detailed Comparison

((selectors.hardware.cpu && (elem.hardware === "cpu" || !elem.hardware)) || (selectors.hardware.gpu && elem.hardware === "gpu")) ); - /// Filter out unreasonable entries - if (selectors.metric == 'size') { - filtered_data = filtered_data.filter(elem => elem.data_size); - } - if (selectors.metric == 'load') { - filtered_data = filtered_data.filter(elem => elem.load_time >= 5 && !elem.tags.includes('stateless')); - } - if (selectors.metric == 'cold' || selectors.metric == 'combined' || selectors.metric == 'load') { - filtered_data = filtered_data.filter(elem => !elem.tags.includes('in-memory')); - } + /// Filter out entries that can't be ranked under the selected metric + /// (see metricExcludes for the per-metric rules). + filtered_data = filtered_data.filter(elem => !metricExcludes(elem, selectors.metric)); let nothing_selected_elem = document.getElementById('nothing-selected'); if (filtered_data.length == 0) { From aad0b04dab8d57fe3df43f38524a411d97c3edab Mon Sep 17 00:00:00 2001 From: Tobias Schmidt Date: Mon, 15 Jun 2026 18:41:55 +0200 Subject: [PATCH 3/3] show individual hot/cold/load/storage ratios in the combined view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Combined metric collapses load, data size, cold and hot into a single weighted-geomean "×N" score, which hides how a system earned it. In the Combined view, expand the score cell to also list the four component ratios that feed it: hot, cold, load and storage, each as the relative "×N" ratio (a component that doesn't apply to the system, e.g. cold for a lukewarm engine or load for a stateless one, shows "n/a"). Each ratio is padded to a fixed width with non-breaking spaces so the columns line up in the monospace cell (6 fits "×12.34" for hot/cold/load, 5 fits "×1.23" for storage), and the overall score is shown in bold. Other metric views are unchanged. Co-Authored-By: Claude Opus 4.8 --- index.html | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/index.html b/index.html index 9664448ce9..38b419a949 100644 --- a/index.html +++ b/index.html @@ -1086,8 +1086,26 @@

Detailed Comparison

const text = selectors.metric == 'load' ? (elem.load_time ? `${Math.round(elem.load_time)}s (×${ratio.toFixed(2)})` : 'stateless') : selectors.metric == 'size' ? `${(elem.data_size / 1024 / 1024 / 1024).toFixed(2)} GiB (×${ratio.toFixed(2)})` + : selectors.metric == 'combined' ? (() => { + /// Break the combined score into the components it's built + /// from (hot, cold, load, storage) as the relative "×N" + /// ratios that feed the weighted score. Pad each to a fixed + /// width so the columns line up in the monospace cell. + const fmt = (v, n) => ((v == null || !isFinite(v)) ? 'n/a' : `×${v.toFixed(2)}`).padStart(n, ' '); + const hot = fmt(relativeQueryTime(num_queries, baseline_data, elem, 'hot'), 6); + const cold = fmt(metricExcludes(elem, 'cold') ? null : relativeQueryTime(num_queries, baseline_data, elem, 'cold'), 6); + const load = fmt(metricExcludes(elem, 'load') ? null : elem.load_time / min_load_time, 6); + const size = fmt(metricExcludes(elem, 'size') ? null : elem.data_size / min_data_size, 5); + return `  (hot ${hot}, cold ${cold}, load ${load}, storage ${size})`; + })() : `×${ratio.toFixed(2)}`; + // Combined shows a bold overall score followed by the (plain) breakdown. + if (selectors.metric == 'combined') { + let score = document.createElement('strong'); + score.appendChild(document.createTextNode(`×${ratio.toFixed(2)}`)); + td_number.appendChild(score); + } td_number.appendChild(document.createTextNode(text)); let td_bar = document.createElement('td');