Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions .mise/lib/jmx_exporter_compat.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,6 @@
import subprocess
import xml.etree.ElementTree as ET
from pathlib import Path
from typing import Optional


DEFAULT_JMX_EXPORTER_DIR = Path(
os.environ.get("JMX_EXPORTER_DIR", "/tmp/jmx-exporter-compat")
Expand All @@ -34,8 +32,8 @@

def run_cmd(
cmd: list[str],
cwd: Optional[Path] = None,
env: Optional[dict[str, str]] = None,
cwd: Path | None = None,
env: dict[str, str] | None = None,
) -> None:
subprocess.run(cmd, cwd=cwd, check=True, env=env)

Expand All @@ -59,10 +57,12 @@ def check_clean_worktree(jmx_exporter_dir: Path) -> None:
)


def get_prom_version(root_dir: Path = Path.cwd()) -> str:
def get_prom_version(root_dir: Path | None = None) -> str:
configured_version = DEFAULT_PROM_VERSION
if configured_version:
return configured_version
if root_dir is None:
root_dir = Path.cwd()
pom = ET.parse(root_dir / "pom.xml")
root = pom.getroot()
version = root.findtext("./{*}version")
Expand Down Expand Up @@ -96,7 +96,9 @@ def prepare_repo(
)


def install_local_artifacts(root_dir: Path = Path.cwd()) -> None:
def install_local_artifacts(root_dir: Path | None = None) -> None:
if root_dir is None:
root_dir = Path.cwd()
run_cmd(
[
"./mvnw",
Expand Down Expand Up @@ -141,7 +143,7 @@ def quick_test_images(

def run_maven_test(
jmx_exporter_dir: Path = DEFAULT_JMX_EXPORTER_DIR,
prom_version: Optional[str] = None,
prom_version: str | None = None,
) -> None:
if prom_version is None:
prom_version = get_prom_version()
Expand Down
16 changes: 9 additions & 7 deletions .mise/lib/micrometer_compat.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@
import subprocess
import xml.etree.ElementTree as ET
from pathlib import Path
from typing import Optional


DEFAULT_MICROMETER_DIR = Path(
os.environ.get("MICROMETER_DIR", "/tmp/micrometer-compat")
Expand All @@ -27,7 +25,7 @@
DEFAULT_PROM_VERSION = os.environ.get("PROM_VERSION")


def run_cmd(cmd: list[str], cwd: Optional[Path] = None) -> None:
def run_cmd(cmd: list[str], cwd: Path | None = None) -> None:
subprocess.run(cmd, cwd=cwd, check=True)


Expand All @@ -49,10 +47,12 @@ def check_clean_worktree(micrometer_dir: Path) -> None:
)


def get_prom_version(root_dir: Path = Path.cwd()) -> str:
def get_prom_version(root_dir: Path | None = None) -> str:
configured_version = DEFAULT_PROM_VERSION
if configured_version:
return configured_version
if root_dir is None:
root_dir = Path.cwd()
pom = ET.parse(root_dir / "pom.xml")
root = pom.getroot()
version = root.findtext("./{*}version")
Expand All @@ -64,7 +64,7 @@ def get_prom_version(root_dir: Path = Path.cwd()) -> str:


def write_init_script(
init_script: Path = DEFAULT_INIT_SCRIPT, prom_version: Optional[str] = None
init_script: Path = DEFAULT_INIT_SCRIPT, prom_version: str | None = None
) -> None:
if prom_version is None:
prom_version = get_prom_version()
Expand Down Expand Up @@ -120,7 +120,9 @@ def prepare_repo(
)


def install_local_artifacts(root_dir: Path = Path.cwd()) -> None:
def install_local_artifacts(root_dir: Path | None = None) -> None:
if root_dir is None:
root_dir = Path.cwd()
run_cmd(
[
"./mvnw",
Expand All @@ -144,7 +146,7 @@ def install_local_artifacts(root_dir: Path = Path.cwd()) -> None:


def run_gradle_test(
test_selector: Optional[str] = None,
test_selector: str | None = None,
micrometer_dir: Path = DEFAULT_MICROMETER_DIR,
init_script: Path = DEFAULT_INIT_SCRIPT,
) -> None:
Expand Down
80 changes: 39 additions & 41 deletions .mise/tasks/generate_benchmark_summary.py
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,13 @@

import argparse
import json
import math
import os
import shutil
import subprocess
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Dict, List, Optional, Tuple


def parse_args():
Expand Down Expand Up @@ -83,7 +84,7 @@ def parse_args():
return parser.parse_args()


def get_system_info() -> Dict[str, str]:
def get_system_info() -> dict[str, str]:
"""Capture system hardware information."""
import multiprocessing
import platform
Expand All @@ -92,7 +93,7 @@ def get_system_info() -> Dict[str, str]:

try:
info["cpu_cores"] = str(multiprocessing.cpu_count())
except Exception:
except NotImplementedError:
pass

try:
Expand All @@ -104,17 +105,16 @@ def get_system_info() -> Dict[str, str]:
except FileNotFoundError:
# macOS
try:
import subprocess

result = subprocess.run(
["sysctl", "-n", "machdep.cpu.brand_string"],
capture_output=True,
check=False,
text=True,
timeout=5,
)
if result.returncode == 0:
info["cpu_model"] = result.stdout.strip()
except Exception:
except (OSError, subprocess.SubprocessError):
pass

try:
Expand All @@ -127,26 +127,25 @@ def get_system_info() -> Dict[str, str]:
except FileNotFoundError:
# macOS
try:
import subprocess

result = subprocess.run(
["sysctl", "-n", "hw.memsize"],
capture_output=True,
check=False,
text=True,
timeout=5,
)
if result.returncode == 0:
bytes_mem = int(result.stdout.strip())
info["memory_gb"] = str(round(bytes_mem / 1024 / 1024 / 1024))
except Exception:
except (OSError, ValueError, subprocess.SubprocessError):
pass

info["os"] = f"{platform.system()} {platform.release()}"

return info


def read_system_info(path: Optional[str]) -> Dict[str, str]:
def read_system_info(path: str | None) -> dict[str, str]:
"""Read system info from JSON, or capture it from the current host."""
if not path:
return get_system_info()
Expand All @@ -163,7 +162,7 @@ def write_system_info(path: str) -> None:
f.write("\n")


def format_system_info(sysinfo: Optional[Dict[str, str]]) -> str:
def format_system_info(sysinfo: dict[str, str] | None) -> str:
"""Format captured system info for markdown."""
if not sysinfo:
return "unknown"
Expand All @@ -179,23 +178,22 @@ def format_system_info(sysinfo: Optional[Dict[str, str]]) -> str:
return ", ".join(parts) if parts else "unknown"


def get_commit_sha(provided_sha: Optional[str]) -> str:
def get_commit_sha(provided_sha: str | None) -> str:
"""Get commit SHA from argument, git, or return 'local'."""
if provided_sha:
return provided_sha

try:
import subprocess

result = subprocess.run(
["git", "rev-parse", "HEAD"],
capture_output=True,
check=False,
text=True,
timeout=5,
)
if result.returncode == 0:
return result.stdout.strip()
except Exception:
except (OSError, subprocess.SubprocessError):
pass

return "local"
Expand All @@ -221,7 +219,7 @@ def format_error(error) -> str:
"""Format error value, handling NaN."""
try:
error_val = float(error)
if error_val != error_val: # NaN check
if math.isnan(error_val):
return ""
elif error_val >= 1_000:
return f"± {error_val / 1_000:.2f}K"
Expand All @@ -244,26 +242,26 @@ def short_benchmark_name(name: str) -> str:
return name.replace("io.prometheus.metrics.benchmarks.", "")


def metric_score(result: Dict) -> Optional[float]:
def metric_score(result: dict) -> float | None:
"""Extract a benchmark score as a finite float."""
try:
score = float(result.get("primaryMetric", {}).get("score"))
if score == score:
if not math.isnan(score):
return score
except (ValueError, TypeError):
pass
return None


def score_interval(result: Dict) -> Optional[Tuple[float, float]]:
def score_interval(result: dict) -> tuple[float, float] | None:
"""Extract the JMH confidence interval for a benchmark result."""
metric = result.get("primaryMetric", {})
confidence = metric.get("scoreConfidence")
if isinstance(confidence, list) and len(confidence) == 2:
try:
low = float(confidence[0])
high = float(confidence[1])
if low == low and high == high:
if not math.isnan(low) and not math.isnan(high):
return min(low, high), max(low, high)
except (ValueError, TypeError):
pass
Expand All @@ -273,21 +271,21 @@ def score_interval(result: Dict) -> Optional[Tuple[float, float]]:
return None
try:
error = float(metric.get("scoreError"))
if error == error:
if not math.isnan(error):
return score - error, score + error
except (ValueError, TypeError):
pass
return None


def lower_is_better(result: Dict) -> bool:
def lower_is_better(result: dict) -> bool:
"""Return true for JMH modes where lower score is better."""
mode = str(result.get("mode", ""))
unit = str(result.get("primaryMetric", {}).get("scoreUnit", ""))
return mode in {"avgt", "sample", "ss"} or unit.endswith("/op")


def comparison_status(head: Dict, baseline: Dict) -> str:
def comparison_status(head: dict, baseline: dict) -> str:
"""Classify a benchmark comparison using confidence intervals."""
head_interval = score_interval(head)
baseline_interval = score_interval(baseline)
Expand Down Expand Up @@ -317,7 +315,7 @@ def comparison_status(head: Dict, baseline: Dict) -> str:
return "faster" if head_score > baseline_score else "slower"


def performance_change(head: Dict, baseline: Dict) -> Optional[float]:
def performance_change(head: dict, baseline: dict) -> float | None:
"""Return percent performance change, with positive meaning faster."""
head_score = metric_score(head)
baseline_score = metric_score(baseline)
Expand All @@ -328,24 +326,24 @@ def performance_change(head: Dict, baseline: Dict) -> Optional[float]:
return (head_score / float(baseline_score) - 1) * 100


def format_change(change: Optional[float]) -> str:
def format_change(change: float | None) -> str:
"""Format a percent performance change."""
if change is None:
return ""
return f"{change:+.1f}%"


def generate_comparison_section(
results: List,
baseline_results: List,
results: list,
baseline_results: list,
commit_sha: str,
baseline_sha: str,
repo: str,
baseline_repo: str,
comparison_note: Optional[str] = None,
system_info: Optional[Dict[str, str]] = None,
baseline_system_info: Optional[Dict[str, str]] = None,
) -> List[str]:
comparison_note: str | None = None,
system_info: dict[str, str] | None = None,
baseline_system_info: dict[str, str] | None = None,
) -> list[str]:
"""Generate a base-vs-head benchmark comparison section."""
by_name = {b.get("benchmark", ""): b for b in results if b.get("benchmark")}
baseline_by_name = {
Expand Down Expand Up @@ -408,15 +406,15 @@ def generate_comparison_section(


def generate_markdown(
results: List,
results: list,
commit_sha: str,
repo: str,
baseline_results: Optional[List] = None,
baseline_sha: Optional[str] = None,
baseline_repo: Optional[str] = None,
comparison_note: Optional[str] = None,
system_info: Optional[Dict[str, str]] = None,
baseline_system_info: Optional[Dict[str, str]] = None,
baseline_results: list | None = None,
baseline_sha: str | None = None,
baseline_repo: str | None = None,
comparison_note: str | None = None,
system_info: dict[str, str] | None = None,
baseline_system_info: dict[str, str] | None = None,
) -> str:
"""Generate markdown summary from JMH results."""
datetime_str = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
Expand Down Expand Up @@ -477,12 +475,12 @@ def generate_markdown(
)

# Group by benchmark class
benchmarks_by_class: Dict[str, List] = {}
benchmarks_by_class: dict[str, list] = {}
for b in results:
name = b.get("benchmark", "")
parts = name.rsplit(".", 1)
if len(parts) == 2:
class_name, method = parts
class_name, _method = parts
class_short = class_name.split(".")[-1]
else:
class_short = "Other"
Expand Down Expand Up @@ -563,7 +561,7 @@ def generate_markdown(

try:
error_val = float(error)
if error_val != error_val: # NaN
if math.isnan(error_val):
error_str = ""
else:
error_str = f"± {error_val:.3f}"
Expand Down
1 change: 0 additions & 1 deletion .mise/tasks/jmx-exporter/prepare.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@

import sys


sys.path.insert(0, ".mise/lib")


Expand Down
Loading