Improve evaluation display and optimize analyze performance

Author: chinazhangchao

scripts/e2e_eval/run_eval.py

@@ -51,6 +51,7 @@
     compute_delta,
     derive_verdict,
     derive_verdicts,
+    format_delta,
 )
 from utils.dataset_config import get_dataset_config, register_from_registry
 from utils.registry import ModelEntry, filter_registry, load_registry, make_adhoc_entry
@@ -1264,8 +1265,9 @@ def main() -> None:
                 acc_tag = f"  acc=SKIP/{accuracy_result['skip_reason']}"
             else:
                 verdict = derive_verdict(accuracy_result).value
-                delta_rel = accuracy_result.get("delta_relative")
-                delta_str = f" {delta_rel:.1%}" if delta_rel is not None else ""
+                delta_str = format_delta(accuracy_result)
+                if delta_str:
+                    delta_str = f" {delta_str}"
                 acc_tag = f"  acc={verdict}{delta_str}"
 
         if perf_proc is not None:

scripts/e2e_eval/utils/accuracy.py

@@ -63,8 +63,8 @@ def compute_delta(
     Returns (None, None) if either is missing or baseline value is zero.
 
     Note: For error-rate metrics (WER — lower is better) a positive delta
-    means the WinML pipeline is *worse*.  The threshold in derive_verdict()
-    uses abs(delta_relative) to handle both directions uniformly.
+    means the WinML pipeline is *worse*.  derive_verdict() normalizes the
+    sign using higher_is_better from METRIC_COMPARE_STRATEGY.
     """
     if winml_metric is None or baseline_metric is None:
         return None, None
@@ -78,6 +78,24 @@ def compute_delta(
     return round(delta_abs, 6), round(delta_abs / base_val, 6)
 
 
+def format_delta(accuracy: dict) -> str:
+    """Format the comparison delta as a display string using the metric strategy.
+
+    Returns e.g. ``"-6.0%"`` for relative metrics or ``"-1.27"`` for absolute.
+    Returns ``""`` if delta is unavailable.
+    """
+    metric_name = (accuracy.get("dataset_config") or {}).get("metric")
+    delta_key = METRIC_COMPARE_STRATEGY.get(
+        metric_name, METRIC_COMPARE_STRATEGY["default"]
+    )[0]
+    d = accuracy.get(delta_key)
+    if d is None:
+        return ""
+    if delta_key == "delta_absolute":
+        return f"{d:+.2f}"
+    return f"{d:.1%}"
+
+
 # ---------------------------------------------------------------------------
 # Verdict derivation (always from stored facts, never from a stored verdict)
 # ---------------------------------------------------------------------------
@@ -214,10 +232,6 @@ def _val(acc: dict, key: str) -> str:
         v = (acc.get(key) or {}).get("value")
         return f"{v:.4f}" if isinstance(v, float) else str(v) if v is not None else "N/A"
 
-    def _pct(acc: dict) -> str:
-        d = acc.get("delta_relative")
-        return f"{d:.1%}" if d is not None else "N/A"
-
     lines = [
         "# Accuracy Evaluation Summary",
         "",
@@ -256,15 +270,15 @@ def _pct(acc: dict) -> str:
     lines += ["", "## Accuracy Regressions", ""]
     if regressions:
         lines += [
-            "| Model | Task | WinML | Baseline | Delta% |",
-            "|-------|------|-----|----------|--------|",
+            "| Model | Task | WinML | Baseline | Delta |",
+            "|-------|------|-------|----------|-------|",
         ]
         for r in regressions:
             acc = r["accuracy"]
             lines.append(
                 f"| {r['model']} | {r.get('task', '')} "
                 f"| {_val(acc, 'winml_metric')} | {_val(acc, 'pytorch_baseline_metric')} "
-                f"| {_pct(acc)} |"
+                f"| {format_delta(acc)} |"
             )
     else:
         lines.append("_No regressions._")
@@ -277,15 +291,15 @@ def _pct(acc: dict) -> str:
     lines += ["", "## At-Risk Models", ""]
     if at_risk:
         lines += [
-            "| Model | Task | WinML | Baseline | Delta% |",
-            "|-------|------|-----|----------|--------|",
+            "| Model | Task | WinML | Baseline | Delta |",
+            "|-------|------|-------|----------|-------|",
         ]
         for r in at_risk:
             acc = r["accuracy"]
             lines.append(
                 f"| {r['model']} | {r.get('task', '')} "
                 f"| {_val(acc, 'winml_metric')} | {_val(acc, 'pytorch_baseline_metric')} "
-                f"| {_pct(acc)} |"
+                f"| {format_delta(acc)} |"
             )
     else:
         lines.append("_No at-risk models._")

scripts/e2e_eval/utils/reporter.py

@@ -316,6 +316,7 @@ def generate_html_report(
     output_path: Path,
     registry_path: Path | None = None,
 ) -> None:
+    from .accuracy import format_delta
     """Generate interactive HTML report with Perf and Accuracy tabs."""
     results = report_data.get("results", [])
 
@@ -356,8 +357,8 @@ def generate_html_report(
                     if acc is not None
                     else None
                 ),
-                "delta_relative": (
-                    acc.get("delta_relative") if acc and not acc.get("skipped") else None
+                "delta_display": (
+                    format_delta(acc) if acc and not acc.get("skipped") else ""
                 ),
             }
         )
@@ -510,11 +511,9 @@ def generate_html_report(
   return n.toLocaleString();
 }}
 function formatDate(iso) {{ return iso ? new Date(iso).toLocaleDateString('en-CA') : '-'; }}
-function formatDelta(v) {{
-  if (v == null) return '-';
-  const pct = (v * 100).toFixed(1);
-  const cls = Math.abs(v) < 0.05 ? 'badge-acc-pass' : Math.abs(v) < 0.10 ? 'badge-acc-risk' : 'badge-acc-reg';  // yellow ≥5%, red ≥10%
-  return `<span class="badge ${{cls}}">${{pct}}%</span>`;
+function formatDelta(display, verdictCls) {{
+  if (!display) return '-';
+  return `<span class="badge ${{verdictCls}}">${{display}}</span>`;
 }}
 
 let filters = {{
@@ -628,7 +627,7 @@ def generate_html_report(
       <td class="col-sep">${{d.accuracy_verdict
         ? `<span class="badge ${{verdictCls}}">${{d.accuracy_verdict}}</span>`
         : '<span style="color:var(--text2);font-size:11px">N/A</span>'}}</td>
-      <td>${{formatDelta(d.delta_relative)}}</td>` : '';
+      <td>${{formatDelta(d.delta_display, verdictCls)}}</td>` : '';
     return `<tr${{errTip ? ` title="${{errTip}}"` : ''}}>
       <td><a class="hf-link" href="https://huggingface.co/${{d.hf_id}}" target="_blank">${{d.hf_id}}</a></td>
       <td><span class="badge badge-task">${{d.task||'-'}}</span></td>

src/winml/modelkit/_warnings.py

@@ -48,6 +48,29 @@ def filter(self, record: logging.LogRecord) -> bool:
         _DiffusersDistributionFilter()
     )
 
+    class _HFPipelineFalsePositiveFilter(logging.Filter):
+        """Filter false-positive HF pipeline warnings when using WinML models.
+
+        HF pipeline emits these because WinMLModel wraps ONNX via ORT, not a
+        native HF model class. These are expected and not actionable.
+        """
+
+        _FALSE_POSITIVES = (
+            "WinMLModel",  # False positive warning which says WinML is not native HF model class
+            "Device set to use",  # PyTorch tensor device, not ONNX device
+            "Using a slow image processor",  # expected when using processor with pipeline.
+        )
+
+        def filter(self, record: logging.LogRecord) -> bool:
+            msg = record.getMessage()
+            return not any(phrase in msg for phrase in self._FALSE_POSITIVES)
+
+    for _name in (
+        "transformers.pipelines.base",
+        "transformers.models.auto.image_processing_auto",
+    ):
+        logging.getLogger(_name).addFilter(_HFPipelineFalsePositiveFilter())
+
     # =========================================================================
     # Warning filters (for warnings.warn() calls)
     # =========================================================================

src/winml/modelkit/analyze/analyzer.py

@@ -727,6 +727,7 @@ def analyze_from_proto(
                     ep=current_ep,
                     model=onnx_model,
                     device=device_to_use,
+                    shape_inferred_model_proto=runtime_checker.get_shape_inferred_model_proto(),
                 )
                 information_list[current_ep] = engine.summary()  # Use EP name as key
 

src/winml/modelkit/analyze/core/doc_constraint_checker.py

@@ -67,15 +67,26 @@ class DocConstraintChecker:
         # Add more checker functions here as they are implemented
     }
 
-    def __init__(self, model_proto: onnx.ModelProto, ep_name: str, device_type: str) -> None:
+    def __init__(
+        self,
+        model_proto: onnx.ModelProto,
+        ep_name: str,
+        device_type: str,
+        skip_shape_inference: bool = False,
+    ) -> None:
         """Initialize doc constraint checker.
 
         Args:
             model_proto: ONNX model proto
             ep_name: Execution provider name (e.g., "QNNExecutionProvider")
             device_type: Device type (e.g., "NPU")
+            skip_shape_inference: If True, assume model_proto already has shape
+                inference applied (avoids expensive redundant inference).
         """
-        self.model_proto = shape_inference.infer_shapes(model_proto)
+        if skip_shape_inference:
+            self.model_proto = model_proto
+        else:
+            self.model_proto = shape_inference.infer_shapes(model_proto)
         self.ep_name = ep_name
         self.device_type = device_type
         self.valueinfo = collect_valueinfo_dict(self.model_proto)

src/winml/modelkit/analyze/core/information_engine.py

@@ -18,6 +18,8 @@
 if TYPE_CHECKING:
     from pathlib import Path
 
+    import onnx
+
     from ..models.information import Action, Information
     from ..models.onnx_model import ONNXModel
     from ..models.runtime_checks import PatternRuntime
@@ -66,6 +68,7 @@ def __init__(
         model: ONNXModel,
         device: str,
         rules_dir: Path | None = None,
+        shape_inferred_model_proto: onnx.ModelProto | None = None,
     ) -> None:
         """Initialize information engine.
 
@@ -77,6 +80,8 @@ def __init__(
             model: Optional ONNX model for model-level validation.
                         If provided, model-level validation checks will be run.
             device: Device type (e.g., "NPU", "GPU", "CPU") for device-specific validation.
+            shape_inferred_model_proto: Pre-inferred model proto to avoid redundant
+                shape inference. If provided, DocConstraintChecker will reuse it.
 
         Implementation:
             - Stores operator and subgraph runtime results separately
@@ -128,10 +133,17 @@ def __init__(
         try:
             from .doc_constraint_checker import DocConstraintChecker
 
-            # Get model proto from ONNXModel
-            model_proto = self._model.get_model()
+            # Prefer the pre-inferred model proto to avoid redundant shape inference
+            if shape_inferred_model_proto is not None:
+                model_proto = shape_inferred_model_proto
+                skip_inference = True
+            else:
+                model_proto = self._model.get_model()
+                skip_inference = False
 
-            self._doc_checker = DocConstraintChecker(model_proto, ep, self._device)
+            self._doc_checker = DocConstraintChecker(
+                model_proto, ep, self._device, skip_shape_inference=skip_inference
+            )
             logger.info(
                 "Initialized Doc Constraint Checker with %d operators",
                 len(self._doc_checker.get_operators_with_constraints()),

src/winml/modelkit/analyze/core/runtime_checker.py

@@ -142,6 +142,12 @@ def _get_query(self) -> RuntimeCheckerQuery:
 
         return self._query
 
+    def get_shape_inferred_model_proto(self) -> onnx.ModelProto | None:
+        """Return the shape-inferred model proto from the cached query, if available."""
+        if self._query is not None:
+            return self._query.model_proto
+        return None
+
     def op_support(
         self,
         run_unknown_op: bool = True,

src/winml/modelkit/analyze/core/runtime_checker_query.py

@@ -109,8 +109,9 @@ def _sanitize_df(df: pd.DataFrame) -> pd.DataFrame:
     Apply make_hashable to convert lists/dicts to tuples.
     """
     for col in df.columns:
-        # Make values hashable (lists -> tuples, floats -> DUMMY_FLOAT)
-        df[col] = df[col].apply(make_hashable)
+        # Bypass pandas .apply() overhead — iterate directly over the numpy array.
+        raw = df[col].to_numpy()
+        df[col] = [make_hashable(v) for v in raw]
     return df
 
 

src/winml/modelkit/analyze/utils/json_utils.py

@@ -8,8 +8,6 @@
 from pathlib import Path
 from typing import Any
 
-from jsonschema import validate
-
 
 def validate_json_schema(data: dict[str, Any], schema_path: Path) -> bool:
     """Validate JSON data against a JSON schema file.
@@ -30,6 +28,8 @@ def validate_json_schema(data: dict[str, Any], schema_path: Path) -> bool:
 
     schema = json.loads(schema_path.read_text(encoding="utf-8"))
 
+    from jsonschema import validate
+
     validate(instance=data, schema=schema)
     return True