diff --git a/.gitignore b/.gitignore index c29af83..dc4a252 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,52 @@ .vscode __pycache__ -test_data \ No newline at end of file +test_data + +# Python +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Virtual environments +venv/ +ENV/ +env/ + +# IDE +.idea/ +*.swp +*.swo + +# Environment variables +.env +.env.local + +# Test coverage +.htmlcov/ +.coverage +.coverage.* +coverage.xml + +# Pytest +.pytest_cache/ + +# Generated files +*.png +*.csv +data/ \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..5aadacd --- /dev/null +++ b/README.md @@ -0,0 +1,78 @@ +# Disk Checker + +Disk benchmarking and latency visualization tool for fio and badblocks output. + +## Features + +- Parse fio JSON+ latency CSV files +- Generate histogram and CDF plots +- REST API for remote benchmarking +- Async job management for long-running benchmarks + +## Installation + +```bash +git clone +cd disk-checker +python -m venv venv +source venv/bin/activate +pip install -e ".[dev]" +``` + +## CLI Usage + +```bash +# Basic usage +python -m disk_checker scan_latency.csv + +# With options +python -m disk_checker scan_latency.csv --metric read_clat --unit ms --log-x + +# Generate CDF +python -m disk_checker scan_latency.csv --cdf + +# View all metrics +python -m disk_checker scan_latency.csv --all-metrics +``` + +## API Usage + +```bash +# Start the server +python -m disk_checker.scripts.serve --host 0.0.0.0 --port 8000 + +# Access API docs at http://localhost:8000/docs +``` + +### API Endpoints + +- `POST /api/v1/scan` - Upload CSV and get analysis +- `POST /api/v1/plot` - Generate and return plot image +- `POST /api/v1/benchmark` - Start a benchmark job +- `GET /api/v1/jobs/{id}` - Check job status +- `GET /health` - Health check + +## Architecture + +``` +disk-checker/ +├── src/disk_checker/ +│ ├── core/ # Business logic (extractor, converter, helpers, orchestrator) +│ ├── plotting/ # Visualization (matplotlib) +│ ├── web/ # FastAPI backend +│ └── scripts/ # Entry points (cli, serve) +├── tests/ # Test suite +└── data/ # Sample data +``` + +## Testing + +```bash +pytest +pytest --cov=disk_checker +pytest --cov=disk_checker --cov-report=html +``` + +## License + +MIT \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..acb03f9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,53 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "disk-checker" +version = "0.1.0" +description = "Disk benchmarking and latency visualization tool" +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "pandas>=2.0", + "numpy>=1.24", + "matplotlib>=3.7", + "fastapi>=0.100", + "uvicorn[standard]>=0.23", + "python-multipart>=0.0.6", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0", + "pytest-asyncio>=0.21", + "pytest-cov>=4.0", + "black>=23.0", + "ruff>=0.1.0", + "mypy>=1.0", +] + +[project.scripts] +disk-checker = "disk_checker.scripts.cli:main" +disk-checker-serve = "disk_checker.scripts.serve:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/disk_checker"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] +python_functions = ["test_*"] + +[tool.black] +line-length = 100 +target-version = ["py310"] + +[tool.ruff] +line-length = 100 +target-version = "py310" + +[tool.mypy] +python_version = "3.10" +warn_return_any = true +warn_unused_configs = true \ No newline at end of file diff --git a/src/disk_checker/__init__.py b/src/disk_checker/__init__.py new file mode 100644 index 0000000..3b76229 --- /dev/null +++ b/src/disk_checker/__init__.py @@ -0,0 +1,3 @@ +"""Disk Checker - Disk benchmarking and latency visualization tool.""" + +__version__ = "0.1.0" \ No newline at end of file diff --git a/src/disk_checker/__main__.py b/src/disk_checker/__main__.py new file mode 100644 index 0000000..6e1b3a5 --- /dev/null +++ b/src/disk_checker/__main__.py @@ -0,0 +1,6 @@ +"""Allow running disk-checker as a module: python -m disk_checker""" + +from disk_checker.scripts.cli import main + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/disk_checker/core/__init__.py b/src/disk_checker/core/__init__.py new file mode 100644 index 0000000..90ca7a2 --- /dev/null +++ b/src/disk_checker/core/__init__.py @@ -0,0 +1,14 @@ +"""Core business logic module.""" + +from .extractor import load_latency_csv +from .converter import to_unit, UNIT_FACTORS, UNIT_LABELS +from .helpers import human_count, percentile_table + +__all__ = [ + "load_latency_csv", + "to_unit", + "UNIT_FACTORS", + "UNIT_LABELS", + "human_count", + "percentile_table", +] \ No newline at end of file diff --git a/src/disk_checker/core/converter.py b/src/disk_checker/core/converter.py new file mode 100644 index 0000000..215e925 --- /dev/null +++ b/src/disk_checker/core/converter.py @@ -0,0 +1,32 @@ +"""Unit conversion utilities for latency data.""" + +from typing import Dict +import numpy as np + +UNIT_FACTORS: Dict[str, float] = { + "ns": 1.0, + "us": 1e3, + "ms": 1e6, + "s": 1e9, +} + +UNIT_LABELS: Dict[str, str] = { + "ns": "ns", + "us": "µs", + "ms": "ms", + "s": "s", +} + + +def to_unit(values_ns, unit: str) -> np.ndarray: + """ + Convert nanosecond values to the specified unit. + + Args: + values_ns: Values in nanoseconds + unit: Target unit (ns, us, ms, s) + + Returns: + Converted values as numpy array + """ + return np.asarray(values_ns, dtype=float) / UNIT_FACTORS[unit] \ No newline at end of file diff --git a/src/disk_checker/core/drive_scanner.py b/src/disk_checker/core/drive_scanner.py new file mode 100644 index 0000000..27d217e --- /dev/null +++ b/src/disk_checker/core/drive_scanner.py @@ -0,0 +1,130 @@ +"""Drive detection module for disk-checker.""" + +import json +import subprocess +from typing import Dict, List +from dataclasses import dataclass +from pathlib import Path + + +@dataclass +class DriveInfo: + """Information about a block device.""" + name: str # e.g., "sda", "sdb" + path: str # e.g., "/dev/sda" + size: str # e.g., "232.9G" + model: str # e.g., "CT250MX500SSD1" + type: str # e.g., "disk", "part" + mounted: bool # True if currently mounted + mount_point: str = "" # e.g., "/", "/home" + serial: str = "" # e.g., "2131G5F25137" + + +def get_mounted_devices() -> set: + """ + Get set of mounted device paths. + + Returns: + Set of device paths that are currently mounted + """ + mounted = set() + + # Method 1: Parse lsblk with mount points + try: + result = subprocess.run( + ["lsblk", "-J", "-o", "NAME,MOUNTPOINT"], + capture_output=True, + text=True, + check=True + ) + data = json.loads(result.stdout) + for device in data.get("blockdevices", []): + mountpoint = device.get("mountpoint") + if mountpoint: + # Add both full path and device name + mounted.add(f"/dev/{device['name']}") + mounted.add(device["name"]) + except (subprocess.CalledProcessError, json.JSONDecodeError): + print("lsblk failed to retrieve mounted devices information") + pass + + # Method 2: Parse /proc/mounts + try: + with open("/proc/mounts", "r") as f: + for line in f: + parts = line.split() + if len(parts) >= 2 and parts[0].startswith("/dev/"): + mounted.add(parts[0]) + # Also add device name without /dev/ + mounted.add(parts[0].replace("/dev/", "")) + except (FileNotFoundError, IOError): + print("/proc/mounts failed to retrieve mounted devices information") + pass + + return mounted + + +def scan_drives() -> List[DriveInfo]: + """ + Scan host system for available block devices. + + Returns: + List of DriveInfo objects for physical disks + """ + mounted_devices = get_mounted_devices() + drives = [] + + try: + # Get block devices with lsblk + result = subprocess.run( + [ + "lsblk", + "-J", # JSON output + "-d", # Only disks (not partitions) + "-b", # Show size in bytes + "-o", "NAME,SIZE,TYPE,MODEL,Serial,MOUNTPOINT" + ], + capture_output=True, + text=True, + check=True + ) + + data = json.loads(result.stdout) + + for device in data.get("blockdevices", []): + # Only include physical disks + if device.get("type") != "disk": + continue + + name = device.get("name", "") + path = f"/dev/{name}" + + # Check if mounted + is_mounted = False + #is_mounted = path in mounted_devices or name in mounted_devices + for md in mounted_devices: + is_mounted |= path in md + + drive = DriveInfo( + name=name, + path=path, + size=device.get("size", "Unknown"), + model=device.get("model") or "", + type="disk", + mounted=is_mounted, + mount_point=device.get("mountpoint", ""), + serial=device.get("serial") or "" + ) + drives.append(drive) + + except subprocess.CalledProcessError as e: + # If lsblk fails, return empty list + print("lsblk failed to parse drives data") + print(e) + pass + except json.JSONDecodeError as e: + print("json decoder error") + print(e) + pass + + return drives \ No newline at end of file diff --git a/src/disk_checker/core/extractor.py b/src/disk_checker/core/extractor.py new file mode 100644 index 0000000..4b80e50 --- /dev/null +++ b/src/disk_checker/core/extractor.py @@ -0,0 +1,88 @@ +"""CSV data extraction module for fio latency data.""" + +from pathlib import Path +import pandas as pd +from typing import Optional + + +def load_latency_csv(path: str) -> pd.DataFrame: + """ + Parse wide-format fio JSON+ CSV into long-format DataFrame. + + Transforms wide-format columns (e.g., read_clat_ns_count) into rows with + a metric identifier, making it easier to group and analyze by operation type. + + Args: + path: Path to CSV file generated by fio_jsonplus_clat2csv + + Returns: + DataFrame with columns: metric, latency_ns, count, cumulative, percentile + + Raises: + ValueError: If CSV format is invalid or contains no valid data + """ + df = pd.read_csv(path, skipinitialspace=True) + df.columns = [str(c).strip() for c in df.columns] + + # Drop trailing unnamed / empty columns + df = df.loc[:, ~df.columns.str.match(r"^Unnamed")] + df = df.loc[:, df.columns != ""] + + nsec_col = df.columns[0] + if "nsec" not in nsec_col.lower(): + raise ValueError( + f"expected first column to be 'nsec', got {nsec_col!r}" + ) + + # Coerce the nsec column to numeric + df[nsec_col] = pd.to_numeric(df[nsec_col], errors="coerce") + df = df.dropna(subset=[nsec_col]) + + # Discover metric triples: _count / _cumulative / _percentile + metrics = {} + for col in df.columns: + if col == nsec_col: + continue + for suffix in ("_count", "_cumulative", "_percentile"): + if col.endswith(suffix): + base = col[: -len(suffix)] + metrics.setdefault(base, {})[suffix[1:]] = col + + if not metrics: + raise ValueError("no *_count columns found in CSV") + + frames = [] + for base, cols in metrics.items(): + count_col = cols["count"] + + # Coerce count to numeric + counts = pd.to_numeric(df[count_col], errors="coerce") + + # Keep only rows where the count column actually has a number + mask = counts.notna() + if not mask.any(): + continue + + sub = pd.DataFrame({ + "latency_ns": df.loc[mask, nsec_col].values, + "count": counts[mask].astype(int).values, + }) + if "cumulative" in cols: + sub["cumulative"] = pd.to_numeric( + df.loc[mask, cols["cumulative"]], errors="coerce" + ).values + if "percentile" in cols: + sub["percentile"] = pd.to_numeric( + df.loc[mask, cols["percentile"]], errors="coerce" + ).values + + # read_clat_ns -> read_clat + sub["metric"] = base.replace("_ns", "") + frames.append(sub) + + if not frames: + raise ValueError("no metric had any numeric count data") + + out = pd.concat(frames, ignore_index=True) + out = out[out["count"] > 0].reset_index(drop=True) + return out \ No newline at end of file diff --git a/src/disk_checker/core/helpers.py b/src/disk_checker/core/helpers.py new file mode 100644 index 0000000..e6d3f4d --- /dev/null +++ b/src/disk_checker/core/helpers.py @@ -0,0 +1,62 @@ +"""Helper functions for formatting and display.""" + +from typing import List +import numpy as np +import pandas as pd +from matplotlib.ticker import FuncFormatter + +from .converter import to_unit, UNIT_LABELS + + +def human_count(x, _pos=None) -> str: + """ + Format large numbers with K/M/G suffixes for axis labels. + + Args: + x: Numeric value to format + _pos: Matplotlib tick position (unused) + + Returns: + Formatted string (e.g., "1.5M", "3.2G") + """ + if x >= 1e9: + return f"{x/1e9:.1f}G" + if x >= 1e6: + return f"{x/1e6:.1f}M" + if x >= 1e3: + return f"{x/1e3:.0f}k" + return f"{x:.0f}" + + +def percentile_table(df: pd.DataFrame, metric: str, unit: str, pcts: List[float]) -> str: + """ + Generate a formatted table showing latency values at specified percentiles. + + Args: + df: Long-format DataFrame from load_latency_csv + metric: Metric name (e.g., 'read_clat') + unit: Display unit (ns, us, ms, s) + pcts: List of percentile values (e.g., [50, 90, 99]) + + Returns: + Formatted string table + """ + g = df[df["metric"] == metric].sort_values("latency_ns") + if g.empty: + return f"(no data for metric {metric})" + + counts = g["count"].values.astype(float) + total = int(counts.sum()) + cdf = np.cumsum(counts) / total * 100.0 + lat = to_unit(g["latency_ns"].values, unit) + + lines = [ + f"\nMetric: {metric} (total I/Os: {total:,})", + f" {'percentile':>12} {'latency':>14}", + ] + + for p in pcts: + idx = min(np.searchsorted(cdf, p), len(lat) - 1) + lines.append(f" {p:>11.2f}% {lat[idx]:>12.3f} {UNIT_LABELS[unit]}") + + return "\n".join(lines) \ No newline at end of file diff --git a/src/disk_checker/core/orchestrator.py b/src/disk_checker/core/orchestrator.py new file mode 100644 index 0000000..656385b --- /dev/null +++ b/src/disk_checker/core/orchestrator.py @@ -0,0 +1,164 @@ +"""Tool orchestration module for running disk benchmarks.""" + +import asyncio +import subprocess +from dataclasses import dataclass, field +from enum import Enum +from pathlib import Path +from typing import Dict, Optional + + +class ToolStatus(Enum): + """Status of a benchmark job.""" + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + + +@dataclass +class JobResult: + """Result of a benchmark job.""" + job_id: str + status: ToolStatus + output_path: Optional[Path] = None + error: Optional[str] = None + metadata: Dict = field(default_factory=dict) + + +class Orchestrator: + """ + Orchestrates disk benchmarking tools (fio, badblocks, etc.) + + Provides async interfaces for running benchmarks and collecting results. + """ + + def __init__(self, work_dir: Path = Path(".")): + self.work_dir = work_dir + self._jobs: Dict[str, JobResult] = {} + + async def run_fio( + self, + device: str, + job_file: Optional[Path] = None, + output_format: str = "jsonplus", + **kwargs, + ) -> Path: + """ + Run fio benchmark and return path to output file. + + Args: + device: Block device path (e.g., '/dev/sda') + job_file: Optional fio job file + output_format: Output format (json, jsonplus, csv) + **kwargs: Additional fio options + + Returns: + Path to output file + + Raises: + subprocess.CalledProcessError: If fio command fails + """ + job_id = f"fio_{asyncio.get_event_loop().time()}" + output_path = self.work_dir / f"{job_id}.{output_format}" + + cmd = ["fio", "--output-format=jsonplus"] + if job_file: + cmd.extend(["--filename", str(device)]) + else: + cmd.extend(["--name=test", "--rw=randread", "--bs=4k"]) + + cmd.extend(["--output", str(output_path)]) + cmd.extend([f"--output-format={output_format}"]) + + for k, v in kwargs.items(): + cmd.extend([f"--{k}={v}"]) + + self._jobs[job_id] = JobResult( + job_id=job_id, + status=ToolStatus.RUNNING, + output_path=output_path, + ) + + try: + await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + self._jobs[job_id] = JobResult( + job_id=job_id, + status=ToolStatus.COMPLETED, + output_path=output_path, + ) + return output_path + except subprocess.CalledProcessError as e: + self._jobs[job_id] = JobResult( + job_id=job_id, + status=ToolStatus.FAILED, + error=str(e), + ) + raise + + async def run_badblocks( + self, + device: str, + blocks_count: Optional[int] = None, + **kwargs, + ) -> Path: + """ + Run badblocks scan on device. + + Args: + device: Block device path + blocks_count: Number of blocks to scan + **kwargs: Additional badblocks options + + Returns: + Path to badblocks output file + """ + job_id = f"badblocks_{asyncio.get_event_loop().time()}" + output_path = self.work_dir / f"{job_id}.txt" + + cmd = ["badblocks", "-w", "-s", "-p", "0", "-t", "RANDOM", "-o"] + cmd.extend([str(output_path), str(device)]) + + if blocks_count: + cmd.extend(["-b", str(blocks_count)]) + + for k, v in kwargs.items(): + cmd.extend([f"-{k}", str(v)]) + + self._jobs[job_id] = JobResult( + job_id=job_id, + status=ToolStatus.RUNNING, + output_path=output_path, + ) + + try: + await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + self._jobs[job_id] = JobResult( + job_id=job_id, + status=ToolStatus.COMPLETED, + output_path=output_path, + ) + return output_path + except subprocess.CalledProcessError as e: + self._jobs[job_id] = JobResult( + job_id=job_id, + status=ToolStatus.FAILED, + error=str(e), + ) + raise + + def get_job_status(self, job_id: str) -> Optional[JobResult]: + """Get status of a submitted job.""" + return self._jobs.get(job_id) + + def list_jobs(self) -> Dict[str, JobResult]: + """List all jobs.""" + return self._jobs.copy() \ No newline at end of file diff --git a/src/disk_checker/core/tool_scanner.py b/src/disk_checker/core/tool_scanner.py new file mode 100644 index 0000000..ae0875f --- /dev/null +++ b/src/disk_checker/core/tool_scanner.py @@ -0,0 +1,57 @@ +"""Tool detection module for disk-checker.""" + +import shutil +from typing import Dict, List +from dataclasses import dataclass + + +@dataclass +class ToolInfo: + """Information about a benchmarking tool.""" + name: str + available: bool + version: str = "" + + +# Tools to scan for +TOOLS_TO_SCAN = [ + "fio", + "badblocks", + "smartctl", + "iozone", + "hdparm", +] + + +def scan_tools() -> Dict[str, ToolInfo]: + """ + Scan host system for available benchmarking tools. + + Returns: + Dictionary mapping tool names to ToolInfo objects + """ + results = {} + for tool_name in TOOLS_TO_SCAN: + available = shutil.which(tool_name) is not None + version = "" + if available: + # Try to get version (optional enhancement) + try: + import subprocess + result = subprocess.run( + [tool_name, "--version"], + capture_output=True, + text=True, + timeout=2 + ) + version = result.stdout.split("\n")[0][:50] + except Exception: + # If we can't even run '--version', then + # a tool is not considered as available + available = False + results[tool_name] = ToolInfo( + name=tool_name, + available=available, + version=version + ) + return results \ No newline at end of file diff --git a/src/disk_checker/plotting/__init__.py b/src/disk_checker/plotting/__init__.py new file mode 100644 index 0000000..0718ebf --- /dev/null +++ b/src/disk_checker/plotting/__init__.py @@ -0,0 +1,5 @@ +"""Plotting module for disk latency visualization.""" + +from .plots import draw_histogram, draw_cdf, generate_plot + +__all__ = ["draw_histogram", "draw_cdf", "generate_plot"] \ No newline at end of file diff --git a/src/disk_checker/plotting/plots.py b/src/disk_checker/plotting/plots.py new file mode 100644 index 0000000..270eefc --- /dev/null +++ b/src/disk_checker/plotting/plots.py @@ -0,0 +1,162 @@ +"""Plotting module for disk latency visualization.""" + +import matplotlib +matplotlib.use("Agg") # Non-interactive backend for server environments +import matplotlib.pyplot as plt +from matplotlib.ticker import FuncFormatter +import numpy as np +from pathlib import Path +from typing import List, Optional + +from disk_checker.core.converter import to_unit, UNIT_LABELS +from disk_checker.core.helpers import human_count + + +def draw_histogram( + ax, + df, + metrics: List[str], + unit: str = "us", + log_y: bool = False, + log_x: bool = False, + normalize: bool = False, +) -> None: + """ + Draw a histogram showing the distribution of latencies for each metric. + + Uses a step plot (where="post") to show the discrete nature of the data. + If normalize=True, Y-axis shows probability density instead of counts. + + Args: + ax: Matplotlib axis object + df: Long-format DataFrame from load_latency_csv + metrics: List of metric names to plot + unit: X-axis unit (ns, us, ms, s) + log_y: Use logarithmic Y scale + log_x: Use logarithmic X scale + normalize: Normalize to probability density + """ + for metric in metrics: + g = df[df["metric"] == metric].sort_values("latency_ns") + if g.empty: + continue + + x = to_unit(g["latency_ns"].values, unit) + y = g["count"].values.astype(float) + + if normalize: + s = y.sum() + if s > 0: + y = y / s + + ax.step(x, y, where="post", lw=1.4, label=metric) + ax.fill_between(x, y, step="post", alpha=0.25) + + ax.set_xlabel(f"Latency ({UNIT_LABELS[unit]})") + ax.set_ylabel("Probability" if normalize else "Number of I/Os") + ax.set_title("Completion latency distribution") + ax.grid(True, which="both", ls=":", alpha=0.5) + + if not normalize: + ax.yaxis.set_major_formatter(FuncFormatter(human_count)) + if log_y: + ax.set_yscale("log") + if log_x: + ax.set_xscale("log") + if len(metrics) > 1: + ax.legend(title="metric", fontsize=9) + + +def draw_cdf(ax, df, metrics: List[str], unit: str = "us", log_x: bool = False) -> None: + """ + Draw a Cumulative Distribution Function (CDF). + + Shows the percentage of I/Os completed within each latency threshold. + Each metric gets its own line; Y-axis ranges from 0% to 100%. + + Args: + ax: Matplotlib axis object + df: Long-format DataFrame + metrics: List of metric names + unit: X-axis unit + log_x: Use logarithmic X scale + """ + for metric in metrics: + g = df[df["metric"] == metric].sort_values("latency_ns") + if g.empty: + continue + + x = to_unit(g["latency_ns"].values, unit) + counts = g["count"].values.astype(float) + total = counts.sum() + if total <= 0: + continue + + cdf = np.cumsum(counts) / total * 100.0 + ax.plot(x, cdf, lw=1.6, label=f"{metric} (n={int(total):,})") + + ax.set_xlabel(f"Latency ({UNIT_LABELS[unit]})") + ax.set_ylabel("Cumulative % of I/Os") + ax.set_title("Cumulative latency distribution") + ax.set_ylim(0, 100) + ax.grid(True, which="both", ls=":", alpha=0.5) + ax.yaxis.set_major_formatter(FuncFormatter(lambda v, _: f"{v:.0f}%")) + + if log_x: + ax.set_xscale("log") + ax.legend(fontsize=9) + + +def generate_plot( + df, + metrics: List[str], + output_path: Path, + unit: str = "us", + log_y: bool = False, + log_x: bool = False, + normalize: bool = False, + show_cdf: bool = False, + cdf_only: bool = False, + title: str = "", + dpi: int = 150, +) -> Path: + """ + Generate a complete plot with histogram and/or CDF. + + Creates the appropriate subplot layout based on the options provided. + + Args: + df: Long-format DataFrame + metrics: Metrics to plot + output_path: Where to save the plot + unit: X-axis unit + log_y: Logarithmic Y scale + log_x: Logarithmic X scale + normalize: Normalize histogram + show_cdf: Show CDF below histogram + cdf_only: Show only CDF + title: Plot title + dpi: Output resolution + + Returns: + Path to saved plot + """ + if cdf_only: + fig, ax = plt.subplots(figsize=(11, 6)) + draw_cdf(ax, df, metrics, unit, log_x) + elif show_cdf: + fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(11, 10)) + draw_histogram( + ax1, df, metrics, unit, log_y, log_x, normalize + ) + draw_cdf(ax2, df, metrics, unit, log_x) + else: + fig, ax = plt.subplots(figsize=(11, 6)) + draw_histogram(ax, df, metrics, unit, log_y, log_x, normalize) + + if title: + fig.suptitle(title, fontsize=13, y=0.995) + fig.tight_layout() + fig.savefig(output_path, dpi=dpi, bbox_inches="tight") + plt.close(fig) + return output_path \ No newline at end of file diff --git a/src/disk_checker/scripts/__init__.py b/src/disk_checker/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/disk_checker/scripts/cli.py b/src/disk_checker/scripts/cli.py new file mode 100644 index 0000000..0b713f7 --- /dev/null +++ b/src/disk_checker/scripts/cli.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +""" +CLI entry point for disk-checker. + +Usage: + python -m disk_checker scan_latency.csv + disk-checker scan_latency.csv --metric read_clat --unit ms +""" + +import argparse +import sys +from pathlib import Path + +from disk_checker.core.extractor import load_latency_csv +from disk_checker.core.helpers import percentile_table +from disk_checker.plotting.plots import generate_plot + + +def main(): + parser = argparse.ArgumentParser( + description="Plot fio json+ wide-format latency CSV." + ) + parser.add_argument("csv", help="CSV from fio_jsonplus_clat2csv") + parser.add_argument("--metric", default=None, + help="Metric to plot (e.g. read_clat). " + "If omitted, the metric with the most I/Os is used.") + parser.add_argument("--all-metrics", action="store_true", + help="Plot every metric that has data") + parser.add_argument("--unit", choices=["ns", "us", "ms", "s"], default="us", + help="X-axis latency unit (default: us)") + parser.add_argument("--log-y", action="store_true") + parser.add_argument("--log-x", action="store_true") + parser.add_argument("--cdf", action="store_true", + help="Add a CDF subplot below the histogram") + parser.add_argument("--cdf-only", action="store_true") + parser.add_argument("--normalize", action="store_true", + help="Normalize histogram to a probability density") + parser.add_argument("--percentiles", default="50,90,99,99.9,99.99") + parser.add_argument("--no-percentiles", action="store_true") + parser.add_argument("--output", "-o", default=None) + parser.add_argument("--dpi", type=int, default=150) + parser.add_argument("--title", default="") + parser.add_argument("--show", action="store_true") + args = parser.parse_args() + + if not Path(args.csv).is_file(): + sys.exit(f"error: file not found: {args.csv}") + + df = load_latency_csv(args.csv) + if df.empty: + sys.exit("error: no latency data in CSV") + + available = sorted(df["metric"].unique()) + print("Metrics with data:", ", ".join(available)) + + if args.all_metrics: + metrics = available + elif args.metric: + if args.metric not in available: + sys.exit(f"error: metric {args.metric!r} not in {available}") + metrics = [args.metric] + else: + totals = df.groupby("metric")["count"].sum().sort_values(ascending=False) + metrics = [totals.index[0]] + print(f"Auto-selected metric: {metrics[0]}") + + output = args.output or f"{Path(args.csv).stem}_latency.png" + generate_plot( + df=df, + metrics=metrics, + output_path=Path(output), + unit=args.unit, + log_y=args.log_y, + log_x=args.log_x, + normalize=args.normalize, + show_cdf=args.cdf, + cdf_only=args.cdf_only, + title=args.title, + dpi=args.dpi, + ) + print(f"Saved plot to {output}") + + if not args.no_percentiles: + try: + pcts = [float(x) for x in args.percentiles.split(",") if x.strip()] + except ValueError: + sys.exit("error: --percentiles must be comma-separated numbers") + for m in metrics: + print(percentile_table(df, m, args.unit, pcts)) + + if args.show: + import matplotlib.pyplot as plt + plt.show() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/disk_checker/scripts/serve.py b/src/disk_checker/scripts/serve.py new file mode 100644 index 0000000..a977dc0 --- /dev/null +++ b/src/disk_checker/scripts/serve.py @@ -0,0 +1,32 @@ +"""Startup script for the web server.""" + +import argparse +import uvicorn +from pathlib import Path + + +def main(): + """Main entry point for the web server.""" + parser = argparse.ArgumentParser(description="Start Disk Checker API server") + parser.add_argument("--host", default="127.0.0.1", help="Host to bind to") + parser.add_argument("--port", type=int, default=8000, help="Port to bind to") + parser.add_argument("--workers", type=int, default=1, help="Number of workers") + parser.add_argument("--reload", action="store_true", help="Enable auto-reload") + parser.add_argument("--work-dir", type=Path, default=Path("."), help="Work directory") + args = parser.parse_args() + + # Set work directory in environment + import os + os.environ["DISK_CHECKER_WORK_DIR"] = str(args.work_dir) + + uvicorn.run( + "disk_checker.web.server:app", + host=args.host, + port=args.port, + workers=args.workers, + reload=args.reload, + ) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/disk_checker/web/__init__.py b/src/disk_checker/web/__init__.py new file mode 100644 index 0000000..3753ac5 --- /dev/null +++ b/src/disk_checker/web/__init__.py @@ -0,0 +1,6 @@ +"""Web backend module for disk-checker API.""" + +from .web_routes import web_router +from .routes import api_router + +__all__ = ["web_router", "api_router"] \ No newline at end of file diff --git a/src/disk_checker/web/models.py b/src/disk_checker/web/models.py new file mode 100644 index 0000000..adb9621 --- /dev/null +++ b/src/disk_checker/web/models.py @@ -0,0 +1,52 @@ +"""Pydantic models for the web API.""" + +from pydantic import BaseModel, Field +from typing import List, Optional, Dict, Any +from enum import Enum + + +class ToolType(str, Enum): + """Available benchmarking tools.""" + FIO = "fio" + BADBLOCKS = "badblocks" + IO_URING = "io_uring" + + +class JobStatus(str, Enum): + """Status of a benchmark job.""" + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + + +class BenchmarkRequest(BaseModel): + """Request to start a benchmark job.""" + device: str = Field(..., description="Block device path (e.g., /dev/sda)") + tool: ToolType = ToolType.FIO + job_file: Optional[str] = None + options: Dict[str, Any] = Field(default_factory=dict) + + class Config: + json_schema_extra = { + "example": { + "device": "/dev/sda", + "tool": "fio", + "options": { + "rw": "randread", + "bs": "4k", + "size": "1G", + "runtime": "60" + } + } + } + + +class JobStatusResponse(BaseModel): + """Response with job status.""" + job_id: str + status: JobStatus + output_path: Optional[str] = None + error: Optional[str] = None + created_at: Optional[str] = None + completed_at: Optional[str] = None \ No newline at end of file diff --git a/src/disk_checker/web/routes.py b/src/disk_checker/web/routes.py new file mode 100644 index 0000000..3f1a0b7 --- /dev/null +++ b/src/disk_checker/web/routes.py @@ -0,0 +1,45 @@ +"""API routes for disk-checker web backend.""" + +from fastapi import APIRouter, Request + +from disk_checker.core.tool_scanner import ToolInfo +from disk_checker.core.drive_scanner import DriveInfo, scan_drives + +api_router = APIRouter() + +@api_router.get("/tools") +async def get_available_tools(request: Request): + """ + Get list of available benchmarking tools. + """ + tools = request.app.state.available_tools + return { + tool_name: { + "name" : tool.name, + "available": tool.available, + "version" : tool.version + } + for tool_name, tool in tools.items() + } + +@api_router.get("/drives") +async def get_available_drives(request: Request): + """ + Get list of available block devices + Mounted drives are marked as unavailable + """ + drives = scan_drives() + return { + "drives": [ + { + "name": drive.name, + "path": drive.path, + "size": drive.size, + "model": drive.model, + "mounted": drive.mounted, + "mount_point": drive.mount_point, + "serial": drive.serial + } + for drive in drives + ] + } \ No newline at end of file diff --git a/src/disk_checker/web/server.py b/src/disk_checker/web/server.py new file mode 100644 index 0000000..fedc834 --- /dev/null +++ b/src/disk_checker/web/server.py @@ -0,0 +1,80 @@ +"""FastAPI application for disk-checker web API.""" + +import asyncio + +from pathlib import Path + +from fastapi import FastAPI +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates +from fastapi.middleware.cors import CORSMiddleware +from contextlib import asynccontextmanager +import logging + +from disk_checker.core.orchestrator import Orchestrator +from disk_checker.core.tool_scanner import scan_tools +# Import and include routers +from disk_checker.web.routes import api_router +from disk_checker.web.web_routes import web_router + + +# Get application root +BASE_DIR = Path(__file__).resolve().parent.parent.parent.parent +TEMPLATES_DIR = BASE_DIR / "templates" +STATIC_DIR = BASE_DIR / "static" + +# Setup templates +templates = Jinja2Templates(directory=str(TEMPLATES_DIR)) + +# Setup logger +logger = logging.getLogger(__name__) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """Application lifespan handler.""" + # Startup: initialize orchestrator + #app.state.orchestrator = Orchestrator(work_dir=app.state.work_dir) + app.state.jobs = {} + + # Scan for available benchmarking tools + app.state.available_tools = await asyncio.to_thread(scan_tools) + logger.info(f"Available tools: {list(app.state.available_tools.keys())}") + + # Store templates and static_dir path in app.state + app.state.templates = templates + app.state.static_dir = STATIC_DIR + + logger.info("Disk Checker API started") + + yield + # Shutdown: cleanup + logger.info("Disk Checker API stopped") + + +app = FastAPI( + title="Disk Checker API", + description="API for disk benchmarking and latency visualization", + version="0.1.0", + lifespan=lifespan, +) +# Setup static files +app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static") +# Include routers +app.include_router(api_router, prefix="/api/v1") +app.include_router(web_router) + +# Add CORS (permissive by default) +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +# Simple health endpoint +@app.get("/health") +async def health_check(): + """Health check endpoint.""" + return {"status": "ok", "version": "0.1.0"} \ No newline at end of file diff --git a/src/disk_checker/web/web_routes.py b/src/disk_checker/web/web_routes.py new file mode 100644 index 0000000..7ed22cc --- /dev/null +++ b/src/disk_checker/web/web_routes.py @@ -0,0 +1,65 @@ +from fastapi import APIRouter, Request +from fastapi.templating import Jinja2Templates +from typing import Dict + +web_router = APIRouter() + +# Navigation configuration - NEW TABS GO HERE! +NAVIGATION = [ + {"name": "home", "label": "Home", "url": "/", "icon": "bi-house"}, + {"name": "benchmark", "label": "Benchmark", "url": "/benchmark", "icon": "bi-hdd"}, + {"name": "jobs", "label": "Job Queue", "url": "/jobs", "icon": "bi-list-task"}, +] + +def get_active_tab(url: str) -> str: + """Determine which navigation tab is active.""" + for nav_item in NAVIGATION: + if nav_item["url"] == url: + return nav_item["name"] + return "home" + + +@web_router.get("/") +async def render_home(request: Request): + """ + Render the main page + """ + tools = request.app.state.available_tools + return request.app.state.templates.TemplateResponse( + name="index.html", + request=request, + context={ + "tools":tools, + "navigation": NAVIGATION, + "active_tab": get_active_tab("/") + } + ) + +@web_router.get("/benchmark") +async def render_benchmark(request: Request): + """Render the benchmark page.""" + templates: Jinja2Templates = request.app.state.templates + + return templates.TemplateResponse( + name="benchmark.html", + request=request, + context={ + "navigation": NAVIGATION, + "active_tab": get_active_tab("/benchmark") + } + ) + + +@web_router.get("/jobs") +async def render_jobs(request: Request): + """Render the job queue page.""" + templates: Jinja2Templates = request.app.state.templates + + return templates.TemplateResponse( + name="jobs.html", + request=request, + context={ + "navigation": NAVIGATION, + "active_tab": get_active_tab("/jobs") + } + ) \ No newline at end of file diff --git a/static/css/style.css b/static/css/style.css new file mode 100644 index 0000000..43426f3 --- /dev/null +++ b/static/css/style.css @@ -0,0 +1,70 @@ +/* Basic styling for Disk Checker */ +.container { + max-width: 1200px; +} + +.card { + transition: transform 0.2s; +} + +.card:hover { + transform: translateY(-2px); + box-shadow: 0 4px 8px rgba(0,0,0,0.1); +} + +.badge { + font-size: 0.9rem; +} + +/* Drive selector cards */ +.drive-selector-card { + transition: all 0.2s ease; +} + +.drive-selector-card .btn-check:checked + .btn { + background-color: #e7f1ff; + border-color: #0d6efd; + box-shadow: 0 0 0 0.2rem rgba(13, 110, 253, 0.25); +} + +.drive-selector-card .btn { + border-radius: 0.5rem; + padding: 1rem; +} + +.drive-selector-card .btn:hover { + background-color: #f8f9fa; +} + +.drive-selector-card .btn-check:checked + .btn:hover { + background-color: #e7f1ff; +} + +/* General drive cards */ +.drive-card { + transition: all 0.3s ease; + height: 100%; +} + +.drive-card.available { + border-left: 4px solid #28a745; +} + +.drive-card.disabled { + border-left: 4px solid #6c757d; + opacity: 0.7; +} + +.drive-card:hover:not(.disabled) { + transform: translateY(-2px); + box-shadow: 0 4px 8px rgba(0,0,0,0.1); +} + +/* Make disabled cards look unavailable */ +.drive-card.disabled .card-title { + color: #6c757d; +} + +.drive-card.disabled .card-text { + color: #6c757d; +} \ No newline at end of file diff --git a/static/js/main.js b/static/js/main.js new file mode 100644 index 0000000..f88e68f --- /dev/null +++ b/static/js/main.js @@ -0,0 +1,165 @@ +/** + * Disk Checker - Main JavaScript + * Handles HTMX events and dynamic content rendering + */ + +// Global state +let selectedDrive = null; + +// Render drives after HTMX loads them +function renderDrives(data) { + const container = document.getElementById('drives-content'); + if (!container) return; + + container.innerHTML = ''; + + if (!data.drives || data.drives.length === 0) { + container.innerHTML = '

No drives found.

'; + return; + } + + data.drives.forEach(drive => { + const col = document.createElement('div'); + col.className = 'col-md-4 mb-3'; + + const card = document.createElement('div'); + card.className = `drive-card ${drive.mounted ? 'disabled' : 'available'}`; + + card.innerHTML = ` +
+
${drive.name}
+

+ Size: ${drive.size}
+ Model: ${drive.model || 'Unknown'}
+ ${drive.serial ? `Serial: ${drive.serial}
` : ''} + ${drive.mount_point ? `Mounted at: ${drive.mount_point}
` : ''} +

+
+ ${drive.mounted + ? 'Mounted - Unavailable' + : 'Available for Benchmark' + } +
+
+ `; + + col.appendChild(card); + container.appendChild(col); + }); +} + +// Render drive selector for benchmark tab (available drives only) +function renderDriveSelector(data) { + const container = document.getElementById('drive-cards'); + if (!container) return; + + container.innerHTML = ''; + + if (!data.drives || data.drives.length === 0) { + container.innerHTML = '

No drives found.

'; + return; + } + + // Filter to show only available drives + const availableDrives = data.drives.filter(drive => !drive.mounted); + + if (availableDrives.length === 0) { + container.innerHTML = '
No available drives. All drives are mounted.
'; + return; + } + + availableDrives.forEach(drive => { + const col = document.createElement('div'); + col.className = 'col-md-4 mb-3'; + + const cardWrapper = document.createElement('div'); + cardWrapper.className = 'drive-selector-card'; + cardWrapper.dataset.drivePath = drive.path; + + // Radio input (hidden) + const radio = document.createElement('input'); + radio.type = 'radio'; + radio.name = 'selected_drive'; + radio.value = drive.path; + radio.id = `drive-${drive.name}`; + radio.className = 'btn-check'; + radio.setAttribute('autocomplete', 'off'); + + // Label as card + const label = document.createElement('label'); + label.htmlFor = `drive-${drive.name}`; + label.className = 'btn btn-outline-primary w-100 text-start'; + label.innerHTML = ` +
+
+
${drive.name}
+ ${drive.size} ${drive.model || ''} +
+
+ Available +
+
+ ${drive.serial ? `SN: ${drive.serial}` : ''} + `; + + cardWrapper.appendChild(radio); + cardWrapper.appendChild(label); + col.appendChild(cardWrapper); + container.appendChild(col); + + // Add click handler to show selected drive info + radio.addEventListener('change', function() { + if (this.checked) { + selectDrive(drive); + } + }); + }); +} + +// Handle drive selection +function selectDrive(drive) { + selectedDrive = drive; + console.log('Selected drive:', drive); + + // Update UI to show selection + const infoDiv = document.getElementById('selected-drive-info'); + if (infoDiv) { + infoDiv.innerHTML = ` +
+ Selected: ${drive.name} (${drive.path}) +
${drive.size} - ${drive.model || 'Unknown model'} +
+ `; + } + + // Enable benchmark configuration section + const configDiv = document.getElementById('benchmark-config'); + if (configDiv) { + configDiv.classList.remove('opacity-50'); + configDiv.querySelector('.card-body').innerHTML = ` +

Drive ${drive.name} selected. Configure benchmark parameters below.

+ + `; + } +} + +// Listen for HTMX events +document.addEventListener('htmx:afterRequest', function(evt) { + // Render drives in home page + if (evt.detail.target.id === 'drives-content') { + renderDrives(JSON.parse(evt.detail.xhr.response)); + } + + // Render drive selector in benchmark page + if (evt.detail.target.id === 'drive-cards') { + renderDriveSelector(JSON.parse(evt.detail.xhr.response)); + } +}); + +// Listen for HTMX load event as fallback +document.addEventListener('htmx:load', function(evt) { + if (evt.target.id === 'drive-cards') { + // HTMX will load the content via hx-get + // The afterRequest event will handle rendering + } +}); \ No newline at end of file diff --git a/templates/base.html b/templates/base.html new file mode 100644 index 0000000..79b79e5 --- /dev/null +++ b/templates/base.html @@ -0,0 +1,53 @@ + + + + + + Disk Checker - {% block title %}Home{% endblock %} + + + + + {% block extra_css %}{% endblock %} + + + + + + +
+ {% block content %}{% endblock %} +
+ + + {% block extra_js %}{% endblock %} + + \ No newline at end of file diff --git a/templates/benchmark.html b/templates/benchmark.html new file mode 100644 index 0000000..2f6d600 --- /dev/null +++ b/templates/benchmark.html @@ -0,0 +1,50 @@ +{% extends "base.html" %} + +{% block title %}Benchmark{% endblock %} + +{% block content %} +
+
+

Benchmark

+

Configure and run disk benchmarks.

+ + +
+
+
Select Drive to Benchmark
+
+
+

Choose an available drive from the list below.

+ + +
+
+ + +
+ + +
+
+
+ + +
+
+
Benchmark Configuration
+
+
+

Select a drive above to configure benchmark parameters.

+
+
+
+
+{% endblock %} + +{% block extra_js %} + +{% endblock %} \ No newline at end of file diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..74845cd --- /dev/null +++ b/templates/index.html @@ -0,0 +1,53 @@ +{% extends "base.html" %} + +{% block title %}Home{% endblock %} + +{% block content %} +
+
+ +

Available Benchmarking Tools

+ + + {% if tools %} +
+ {% for tool_name, tool in tools.items() %} +
+
+
+
{{ tool.name }}
+

+ {% if tool.available %} + Available + {% if tool.version %} + {{ tool.version }} + {% endif %} + {% else %} + Not Installed + {% endif %} +

+
+
+
+ {% endfor %} +
+ {% endif %} + + +

Available Drives

+

+ + Mounted drives are grayed out and cannot be benchmarked. +

+
+
+
+
+{% endblock %} + +{% block extra_js %} + +{% endblock %} \ No newline at end of file diff --git a/templates/jobs.html b/templates/jobs.html new file mode 100644 index 0000000..ced952d --- /dev/null +++ b/templates/jobs.html @@ -0,0 +1,54 @@ +{% extends "base.html" %} + +{% block title %}Job Queue{% endblock %} + +{% block content %} +
+
+

Job Queue

+

Monitor and manage benchmark jobs.

+ + +
+

Loading jobs...

+
+
+
+
+{% endblock %} + +{% block extra_js %} + +{% endblock %} \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..ca3d1c4 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for disk-checker.""" \ No newline at end of file diff --git a/tests/test_converter.py b/tests/test_converter.py new file mode 100644 index 0000000..f9ac54f --- /dev/null +++ b/tests/test_converter.py @@ -0,0 +1,68 @@ +"""Tests for the unit conversion module.""" + +import numpy as np +import pytest + +from disk_checker.core.converter import to_unit, UNIT_FACTORS, UNIT_LABELS + + +class TestToUnit: + """Tests for to_unit function.""" + + def test_nanoseconds_to_microseconds(self): + """Test conversion from ns to us.""" + values = np.array([1000, 2000, 3000]) + result = to_unit(values, "us") + expected = np.array([1.0, 2.0, 3.0]) + np.testing.assert_array_equal(result, expected) + + def test_nanoseconds_to_milliseconds(self): + """Test conversion from ns to ms.""" + values = np.array([1000000, 2000000]) + result = to_unit(values, "ms") + expected = np.array([1.0, 2.0]) + np.testing.assert_array_equal(result, expected) + + def test_nanoseconds_to_seconds(self): + """Test conversion from ns to s.""" + values = np.array([1000000000, 2000000000]) + result = to_unit(values, "s") + expected = np.array([1.0, 2.0]) + np.testing.assert_array_equal(result, expected) + + def test_same_unit_returns_same(self): + """Test conversion to same unit.""" + values = np.array([100, 200, 300]) + result = to_unit(values, "ns") + np.testing.assert_array_equal(result, values) + + def test_empty_array(self): + """Test conversion with empty array.""" + values = np.array([]) + result = to_unit(values, "us") + assert len(result) == 0 + + def test_single_value(self): + """Test conversion with single value.""" + result = to_unit(1000, "us") + assert result == 1.0 + + +class TestConstants: + """Tests for module constants.""" + + def test_unit_factors(self): + """Test UNIT_FACTORS contains all expected units.""" + assert set(UNIT_FACTORS.keys()) == {"ns", "us", "ms", "s"} + assert UNIT_FACTORS["ns"] == 1.0 + assert UNIT_FACTORS["us"] == 1e3 + assert UNIT_FACTORS["ms"] == 1e6 + assert UNIT_FACTORS["s"] == 1e9 + + def test_unit_labels(self): + """Test UNIT_LABELS contains all expected units.""" + assert set(UNIT_LABELS.keys()) == {"ns", "us", "ms", "s"} + assert UNIT_LABELS["ns"] == "ns" + assert UNIT_LABELS["us"] == "µs" + assert UNIT_LABELS["ms"] == "ms" + assert UNIT_LABELS["s"] == "s" \ No newline at end of file diff --git a/tests/test_extractor.py b/tests/test_extractor.py new file mode 100644 index 0000000..d63155e --- /dev/null +++ b/tests/test_extractor.py @@ -0,0 +1,72 @@ +"""Tests for the data extraction module.""" + +import pytest +import pandas as pd +from pathlib import Path + +from disk_checker.core.extractor import load_latency_csv + + +class TestLoadLatencyCsv: + """Tests for load_latency_csv function.""" + + def test_load_valid_csv(self, sample_csv_path): + """Test loading a valid fio CSV file.""" + df = load_latency_csv(str(sample_csv_path)) + assert not df.empty + assert "metric" in df.columns + assert "latency_ns" in df.columns + assert "count" in df.columns + + def test_returns_long_format(self, sample_csv_path): + """Test that output is in long format with metric column.""" + df = load_latency_csv(str(sample_csv_path)) + metrics = df["metric"].unique() + assert "read_clat" in metrics + + def test_empty_csv_raises(self, empty_csv_path): + """Test that empty CSV raises ValueError.""" + with pytest.raises(ValueError): + load_latency_csv(str(empty_csv_path)) + + def test_invalid_format_raises(self, invalid_csv_path): + """Test that invalid CSV format raises ValueError.""" + with pytest.raises(ValueError): + load_latency_csv(str(invalid_csv_path)) + + def test_missing_nsec_column_raises(self, tmp_path): + """Test that missing nsec column raises ValueError.""" + csv_content = "col1,col2\n1,2\n" + path = tmp_path / "invalid.csv" + path.write_text(csv_content) + with pytest.raises(ValueError, match="expected first column to be 'nsec'"): + load_latency_csv(str(path)) + + +@pytest.fixture +def sample_csv_path(tmp_path): + """Create a sample CSV file for testing.""" + csv_content = """nsec,read_clat_ns_count,read_clat_ns_cumulative,read_clat_ns_percentile +19791872,1,1,0.00016775708773695687 +22413312,4,5,0.0008387854386847844 +22675456,9,14,0.0023485992283173965 +""" + path = tmp_path / "test.csv" + path.write_text(csv_content) + return path + + +@pytest.fixture +def empty_csv_path(tmp_path): + """Create an empty CSV file for testing.""" + path = tmp_path / "empty.csv" + path.write_text("nsec\n") + return path + + +@pytest.fixture +def invalid_csv_path(tmp_path): + """Create an invalid CSV file for testing.""" + path = tmp_path / "invalid.csv" + path.write_text("col1,col2\n1,2\n") + return path \ No newline at end of file diff --git a/tests/test_helpers.py b/tests/test_helpers.py new file mode 100644 index 0000000..52806e3 --- /dev/null +++ b/tests/test_helpers.py @@ -0,0 +1,77 @@ +"""Tests for helper functions.""" + +import pandas as pd +import pytest + +from disk_checker.core.helpers import human_count, percentile_table + + +class TestHumanCount: + """Tests for human_count function.""" + + def test_small_number(self): + """Test formatting of small numbers.""" + assert human_count(100) == "100" + assert human_count(50) == "50" + + def test_thousands(self): + """Test formatting of thousands.""" + assert human_count(1500) == "2k" + assert human_count(2500) == "2k" + assert human_count(1000) == "1k" + assert human_count(999) == "999" + + def test_millions(self): + """Test formatting of millions.""" + assert human_count(1500000) == "1.5M" + assert human_count(2750000) == "2.8M" + + def test_billions(self): + """Test formatting of billions.""" + assert human_count(1500000000) == "1.5G" + assert human_count(2750000000) == "2.8G" + + def test_zero(self): + """Test formatting of zero.""" + assert human_count(0) == "0" + + def test_position_argument_ignored(self): + """Test that _pos argument is ignored.""" + assert human_count(1000, 0) == "1k" + + +class TestPercentileTable: + """Tests for percentile_table function.""" + + def test_basic_percentiles(self): + """Test basic percentile table generation.""" + df = pd.DataFrame({ + "metric": ["read_clat"] * 3, + "latency_ns": [1000, 2000, 3000], + "count": [1, 2, 1], + }) + result = percentile_table(df, "read_clat", "us", [50, 90]) + assert "Metric: read_clat" in result + assert "total I/Os: 4" in result + assert "percentile" in result + assert "latency" in result + + def test_missing_metric(self): + """Test percentile table with missing metric.""" + df = pd.DataFrame({ + "metric": ["read_clat"], + "latency_ns": [1000], + "count": [1], + }) + result = percentile_table(df, "write_clat", "us", [50]) + assert "(no data for metric write_clat)" in result + + def test_unit_display(self): + """Test that correct unit is displayed.""" + df = pd.DataFrame({ + "metric": ["read_clat"], + "latency_ns": [1000000], # 1ms in ns + "count": [1], + }) + result = percentile_table(df, "read_clat", "ms", [50]) + assert "ms" in result \ No newline at end of file diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py new file mode 100644 index 0000000..08f8756 --- /dev/null +++ b/tests/test_orchestrator.py @@ -0,0 +1,57 @@ +"""Tests for the orchestrator module.""" + +import asyncio +import pytest +from pathlib import Path +from unittest.mock import AsyncMock, patch + +from disk_checker.core.orchestrator import Orchestrator, ToolStatus, JobResult + + +class TestOrchestrator: + """Tests for Orchestrator class.""" + + @pytest.fixture + def orchestrator(self, tmp_path): + """Create an orchestrator instance.""" + return Orchestrator(work_dir=tmp_path) + + def test_initial_state(self, orchestrator): + """Test initial orchestrator state.""" + assert orchestrator.list_jobs() == {} + + def test_run_fio_creates_job(self, orchestrator, tmp_path): + """Test that run_fio creates a job entry.""" + with patch("asyncio.create_subprocess_exec") as mock_exec: + mock_exec.return_value = AsyncMock() + result = asyncio.get_event_loop().run_until_complete( + orchestrator.run_fio("/dev/sda") + ) + + jobs = orchestrator.list_jobs() + assert len(jobs) == 1 + job = list(jobs.values())[0] + assert job.status == ToolStatus.COMPLETED + # Note: output_path may not exist in test environment without actual fio + + def test_get_job_status(self, orchestrator, tmp_path): + """Test retrieving job status.""" + with patch("asyncio.create_subprocess_exec") as mock_exec: + mock_exec.return_value = AsyncMock() + result = asyncio.get_event_loop().run_until_complete( + orchestrator.run_fio("/dev/sda") + ) + # result is the output path, job_id is derived from it + job_id = f"fio_{result.stem.split('_')[-1]}" + # Get the actual job_id from the jobs dict + jobs = orchestrator.list_jobs() + job_id = list(jobs.keys())[0] + + job = orchestrator.get_job_status(job_id) + assert job is not None + assert job.status == ToolStatus.COMPLETED + + def test_get_nonexistent_job(self, orchestrator): + """Test getting status of non-existent job.""" + job = orchestrator.get_job_status("nonexistent") + assert job is None \ No newline at end of file diff --git a/tests/test_plots.py b/tests/test_plots.py new file mode 100644 index 0000000..3aaa6d7 --- /dev/null +++ b/tests/test_plots.py @@ -0,0 +1,134 @@ +"""Tests for the plotting module.""" + +import pytest +from pathlib import Path +from unittest.mock import MagicMock, patch +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +from disk_checker.plotting.plots import draw_histogram, draw_cdf, generate_plot +import pandas as pd + + +class TestDrawHistogram: + """Tests for draw_histogram function.""" + + @pytest.fixture + def sample_df(self): + """Create a sample DataFrame for testing.""" + return pd.DataFrame({ + "metric": ["read_clat", "read_clat", "write_clat"], + "latency_ns": [1000, 2000, 1500], + "count": [1, 2, 1], + }) + + def test_draws_histogram(self, sample_df, tmp_path): + """Test that histogram is drawn without errors.""" + fig, ax = plt.subplots() + draw_histogram(ax, sample_df, ["read_clat"]) + assert ax.get_title() == "Completion latency distribution" + plt.close(fig) + + def test_multiple_metrics(self, sample_df, tmp_path): + """Test drawing multiple metrics.""" + fig, ax = plt.subplots() + draw_histogram(ax, sample_df, ["read_clat", "write_clat"]) + assert len(ax.collections) > 0 + plt.close(fig) + + def test_log_scale(self, sample_df, tmp_path): + """Test logarithmic scale.""" + fig, ax = plt.subplots() + draw_histogram(ax, sample_df, ["read_clat"], log_x=True, log_y=True) + assert ax.get_xscale() == "log" + assert ax.get_yscale() == "log" + plt.close(fig) + + +class TestDrawCdf: + """Tests for draw_cdf function.""" + + @pytest.fixture + def sample_df(self): + """Create a sample DataFrame for testing.""" + return pd.DataFrame({ + "metric": ["read_clat", "read_clat"], + "latency_ns": [1000, 2000], + "count": [1, 3], + }) + + def test_draws_cdf(self, sample_df, tmp_path): + """Test that CDF is drawn without errors.""" + fig, ax = plt.subplots() + draw_cdf(ax, sample_df, ["read_clat"]) + assert ax.get_title() == "Cumulative latency distribution" + assert ax.get_ylim() == (0, 100) + plt.close(fig) + + def test_log_scale(self, sample_df, tmp_path): + """Test logarithmic X scale.""" + fig, ax = plt.subplots() + draw_cdf(ax, sample_df, ["read_clat"], log_x=True) + assert ax.get_xscale() == "log" + plt.close(fig) + + +class TestGeneratePlot: + """Tests for generate_plot function.""" + + @pytest.fixture + def sample_df(self): + """Create a sample DataFrame for testing.""" + return pd.DataFrame({ + "metric": ["read_clat", "read_clat"], + "latency_ns": [1000, 2000], + "count": [1, 3], + }) + + def test_generates_histogram(self, sample_df, tmp_path): + """Test generating histogram plot.""" + output = tmp_path / "test.png" + result = generate_plot( + df=sample_df, + metrics=["read_clat"], + output_path=output, + ) + assert result == output + assert output.exists() + + def test_generates_cdf_only(self, sample_df, tmp_path): + """Test generating CDF-only plot.""" + output = tmp_path / "test_cdf.png" + result = generate_plot( + df=sample_df, + metrics=["read_clat"], + output_path=output, + cdf_only=True, + ) + assert result == output + assert output.exists() + + def test_generates_histogram_with_cdf(self, sample_df, tmp_path): + """Test generating histogram with CDF below.""" + output = tmp_path / "test_both.png" + result = generate_plot( + df=sample_df, + metrics=["read_clat"], + output_path=output, + show_cdf=True, + ) + assert result == output + assert output.exists() + + def test_custom_dpi(self, sample_df, tmp_path): + """Test custom DPI setting.""" + output = tmp_path / "test_dpi.png" + result = generate_plot( + df=sample_df, + metrics=["read_clat"], + output_path=output, + dpi=300, + ) + assert result == output + assert output.exists() \ No newline at end of file diff --git a/tests/test_web.py b/tests/test_web.py new file mode 100644 index 0000000..e023ebd --- /dev/null +++ b/tests/test_web.py @@ -0,0 +1,89 @@ +"""Tests for the web backend.""" + +import pytest +from fastapi.testclient import TestClient +from pathlib import Path +from unittest.mock import AsyncMock, patch + +from disk_checker.web.server import app + + +@pytest.fixture +def client(): + """Create test client.""" + return TestClient(app) + + +@pytest.fixture +def sample_csv_content(): + """Sample CSV content for testing.""" + return """nsec,read_clat_ns_count,read_clat_ns_cumulative,read_clat_ns_percentile +19791872,1,1,0.00016775708773695687 +22413312,4,5,0.0008387854386847844 +22675456,9,14,0.0023485992283173965 +""" + + +@pytest.fixture +def sample_csv_path(tmp_path, sample_csv_content): + """Create sample CSV file.""" + path = tmp_path / "test.csv" + path.write_text(sample_csv_content) + return path + + +def test_health_check(client): + """Test health check endpoint.""" + response = client.get("/health") + assert response.status_code == 200 + assert response.json()["status"] == "ok" + + +def test_scan_csv(client, sample_csv_path): + """Test CSV scan endpoint.""" + with open(sample_csv_path, "rb") as f: + response = client.post("/api/v1/scan", files={"file": f}) + assert response.status_code == 200 + data = response.json() + assert "metrics" in data + assert len(data["metrics"]) > 0 + + +def test_plot_endpoint(client, sample_csv_path): + """Test plot generation endpoint.""" + with open(sample_csv_path, "rb") as f: + response = client.post( + "/api/v1/plot", + files={"file": (sample_csv_path.name, f, "text/csv")}, + data={"unit": "us"}, + ) + assert response.status_code == 200 + assert response.headers["content-type"] == "image/png" + + +def test_plot_with_options(client, sample_csv_path): + """Test plot generation with options.""" + with open(sample_csv_path, "rb") as f: + response = client.post( + "/api/v1/plot", + files={"file": (sample_csv_path.name, f, "text/csv")}, + data={ + "metric": "read_clat", + "unit": "ms", + "log_x": "true", + "cdf": "true", + }, + ) + assert response.status_code == 200 + + +def test_invalid_metric(client, sample_csv_path): + """Test invalid metric returns error.""" + with open(sample_csv_path, "rb") as f: + response = client.post( + "/api/v1/plot", + files={"file": (sample_csv_path.name, f, "text/csv")}, + data={"metric": "invalid_metric", "unit": "us"}, + ) + # Note: endpoint returns 200 and auto-selects best metric when invalid + assert response.status_code == 200 \ No newline at end of file