{ "applies_to_current_product": true, "campaign_ids": { "esp32s3": "cd90a70b75fc2ae7720d3acd", "host": "c9b2488e707cce1ef4198bd9" }, "compiler_binding": [ { "campaign": "host", "is_current": true, "measured_by_identity": "605919e39bc98523fa816c347cea35d33159de0f81a60e496d91a66d04a1b330", "note": "Measured by the compiler in the tree.", "repair_task": null }, { "campaign": "baseline-reproduction", "is_current": true, "measured_by_identity": "605919e39bc98523fa816c347cea35d33159de0f81a60e496d91a66d04a1b330", "note": "Measured on the admitted board by the compiler in the tree.", "repair_task": null }, { "campaign": "esp32s3", "is_current": true, "measured_by_identity": "605919e39bc98523fa816c347cea35d33159de0f81a60e496d91a66d04a1b330", "note": "Measured on the admitted board by the compiler in the tree.", "repair_task": null } ], "exclusions": [ "Sensor capture, feature extraction, and energy are outside every campaign here.", "A quality number measured on a repository-owned corpus validates the pipeline and is not a public workload score; the two keyword-spotting lanes name their dataset bindings explicitly.", "No family other than the control lane carries an industry standard control, so no comparative claim is made for them.", "A cell whose disposition is `not-directly-comparable` may not be compared with a cell from another definition." ], "families": [ { "benchmark_id": "anomaly-detection-autoencoder", "campaigns": [ "host", "esp32s3" ], "cells": [ { "backend": "esp32s3-pie", "campaign": "esp32s3", "cell_id": "anomaly-detection-autoencoder/esp32s3-pie/w8", "disposition": "not-directly-comparable", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "host", "metric_id": "detection_auc", "sample_count": 256, "value": 1.0 }, { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 100, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 3.256875e-05 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 3.317916666666667e-05 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 100, "value": 72988.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 100, "value": 8043.0 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 100, "value": 336.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "esp32s3", "cell_id": "anomaly-detection-autoencoder/esp32s3-portable/w8", "disposition": "not-directly-comparable", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "host", "metric_id": "detection_auc", "sample_count": 256, "value": 1.0 }, { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 100, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 7.863958333333333e-05 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 7.890416666666666e-05 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 100, "value": 72700.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 100, "value": 6727.0 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 100, "value": 48.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "anomaly-detection-autoencoder/generated-host-c/w8", "disposition": "not-directly-comparable", "evidence_tier": "compiled_c", "metrics": [ { "evidence_tier": "compiled_c", "metric_id": "detection_auc", "sample_count": 256, "value": 1.0 }, { "evidence_tier": "compiled_c", "metric_id": "latency_p50_seconds", "sample_count": 256, "value": 5e-07 }, { "evidence_tier": "compiled_c", "metric_id": "latency_p99_seconds", "sample_count": 256, "value": 6.25e-07 }, { "evidence_tier": "compiled_c", "metric_id": "workspace_bytes", "sample_count": 256, "value": 48.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "anomaly-detection-autoencoder/microquant-numpy/w8", "disposition": "not-directly-comparable", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "detection_auc", "sample_count": 256, "value": 1.0 }, { "evidence_tier": "host", "metric_id": "latency_p50_seconds", "sample_count": 1, "value": 1.3526992006518412e-06 }, { "evidence_tier": "host", "metric_id": "latency_p99_seconds", "sample_count": 1, "value": 1.3526992006518412e-06 }, { "evidence_tier": "host", "metric_id": "workspace_bytes", "sample_count": 256, "value": 48.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "anomaly-detection-autoencoder/source-framework-host/w8", "disposition": "not-directly-comparable", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "detection_auc", "sample_count": 256, "value": 1.0 } ], "scheme": "w8" } ], "control": { "measured": false, "statement": "No industry standard control was measured beside this family. Its lane is `framework-validation`, which admits no vendor runtime, and the published comparison against a control is `keyword-spotting-dense-int4-runtime-control`, `keyword-spotting-dense-runtime-control`, `keyword-spotting-runtime-control`, `wake-vision-runtime-control`. These cells describe the pipeline on this workload and are not a comparative result." }, "lane": "framework-validation", "state": "active", "title": "Anomaly detection: owned dense autoencoder", "workload_class": "anomaly-detection" }, { "benchmark_id": "customer-template-pooling", "campaigns": [ "host" ], "cells": [ { "backend": "portable", "campaign": "host", "cell_id": "customer-template-pooling/generated-host-c/w8", "disposition": "not-directly-comparable", "evidence_tier": "compiled_c", "metrics": [ { "evidence_tier": "compiled_c", "metric_id": "top1_accuracy", "sample_count": 128, "value": 0.9921875 }, { "evidence_tier": "compiled_c", "metric_id": "workspace_bytes", "sample_count": 128, "value": 400.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "customer-template-pooling/microquant-numpy/w8", "disposition": "not-directly-comparable", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 128, "value": 0.9921875 }, { "evidence_tier": "host", "metric_id": "workspace_bytes", "sample_count": 128, "value": 400.0 } ], "scheme": "w8" } ], "control": { "measured": false, "statement": "No industry standard control was measured beside this family. Its lane is `framework-validation`, which admits no vendor runtime, and the published comparison against a control is `keyword-spotting-dense-int4-runtime-control`, `keyword-spotting-dense-runtime-control`, `keyword-spotting-runtime-control`, `wake-vision-runtime-control`. These cells describe the pipeline on this workload and are not a comparative result." }, "lane": "framework-validation", "state": "active", "title": "Customer template: a package a customer copies and re-points", "workload_class": "customer-supplied" }, { "benchmark_id": "graph-stress-multi-interface", "campaigns": [ "host" ], "cells": [ { "backend": "portable", "campaign": "host", "cell_id": "graph-stress-multi-interface/generated-host-c/w8", "disposition": "not-directly-comparable", "evidence_tier": "compiled_c", "metrics": [ { "evidence_tier": "compiled_c", "metric_id": "latency_p50_seconds", "sample_count": 128, "value": 2.5e-07 }, { "evidence_tier": "compiled_c", "metric_id": "workspace_bytes", "sample_count": 128, "value": 48.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "graph-stress-multi-interface/microquant-numpy/w8", "disposition": "not-directly-comparable", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "latency_p50_seconds", "sample_count": 1, "value": 1.261390934814699e-06 }, { "evidence_tier": "host", "metric_id": "workspace_bytes", "sample_count": 128, "value": 48.0 } ], "scheme": "w8" } ], "control": { "measured": false, "statement": "No industry standard control was measured beside this family. Its lane is `framework-validation`, which admits no vendor runtime, and the published comparison against a control is `keyword-spotting-dense-int4-runtime-control`, `keyword-spotting-dense-runtime-control`, `keyword-spotting-runtime-control`, `wake-vision-runtime-control`. These cells describe the pipeline on this workload and are not a comparative result." }, "lane": "framework-validation", "state": "active", "title": "Graph stress: multiple inputs, multiple outputs, overlapping lifetimes", "workload_class": "graph-stress" }, { "benchmark_id": "image-classification-residual", "campaigns": [ "host" ], "cells": [ { "backend": "portable", "campaign": "host", "cell_id": "image-classification-residual/generated-host-c/w8", "disposition": "not-directly-comparable", "evidence_tier": "compiled_c", "metrics": [ { "evidence_tier": "compiled_c", "metric_id": "latency_p50_seconds", "sample_count": 256, "value": 0.00018625 }, { "evidence_tier": "compiled_c", "metric_id": "latency_p99_seconds", "sample_count": 256, "value": 0.000410375 }, { "evidence_tier": "compiled_c", "metric_id": "top1_accuracy", "sample_count": 256, "value": 1.0 }, { "evidence_tier": "compiled_c", "metric_id": "workspace_bytes", "sample_count": 256, "value": 9104.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "image-classification-residual/microquant-numpy/w8", "disposition": "not-directly-comparable", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "latency_p50_seconds", "sample_count": 1, "value": 0.00023436409401256242 }, { "evidence_tier": "host", "metric_id": "latency_p99_seconds", "sample_count": 1, "value": 0.00023436409401256242 }, { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 256, "value": 1.0 }, { "evidence_tier": "host", "metric_id": "workspace_bytes", "sample_count": 256, "value": 9104.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "image-classification-residual/source-framework-host/w8", "disposition": "not-directly-comparable", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 256, "value": 1.0 } ], "scheme": "w8" } ], "control": { "measured": false, "statement": "No industry standard control was measured beside this family. Its lane is `framework-validation`, which admits no vendor runtime, and the published comparison against a control is `keyword-spotting-dense-int4-runtime-control`, `keyword-spotting-dense-runtime-control`, `keyword-spotting-runtime-control`, `wake-vision-runtime-control`. These cells describe the pipeline on this workload and are not a comparative result." }, "lane": "framework-validation", "state": "active", "title": "Image classification: owned residual CNN", "workload_class": "image-classification" }, { "benchmark_id": "keyword-spotting-compression-quality", "campaigns": [ "host", "esp32s3" ], "cells": [ { "backend": "auto", "campaign": "esp32s3", "cell_id": "keyword-spotting-compression-quality/esp32s3-mixed/w8", "disposition": "shipping-accelerated", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 100, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.0112975125 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.011303591666666666 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 100, "value": 89420.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 100, "value": 42188.0 }, { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.9177914110429448 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 100, "value": 16320.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "esp32s3", "cell_id": "keyword-spotting-compression-quality/esp32s3-portable/w8", "disposition": "portable-reference", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 100, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.15081610625 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.15082929583333332 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 100, "value": 89100.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 100, "value": 38034.0 }, { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.9177914110429448 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 100, "value": 16000.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "keyword-spotting-compression-quality/generated-host-c/w8", "disposition": "portable-reference", "evidence_tier": "compiled_c", "metrics": [ { "evidence_tier": "compiled_c", "metric_id": "latency_p50_seconds", "sample_count": 4890, "value": 0.00029425 }, { "evidence_tier": "compiled_c", "metric_id": "latency_p99_seconds", "sample_count": 4890, "value": 0.000305562 }, { "evidence_tier": "compiled_c", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.9177914110429448 }, { "evidence_tier": "compiled_c", "metric_id": "workspace_bytes", "sample_count": 4890, "value": 16000.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "keyword-spotting-compression-quality/microquant-numpy/w8", "disposition": "portable-reference", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "latency_p50_seconds", "sample_count": 1, "value": 0.002281178987741181 }, { "evidence_tier": "host", "metric_id": "latency_p99_seconds", "sample_count": 1, "value": 0.002281178987741181 }, { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.9177914110429448 }, { "evidence_tier": "host", "metric_id": "workspace_bytes", "sample_count": 4890, "value": 16000.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "keyword-spotting-compression-quality/source-framework-host/w8", "disposition": "not-directly-comparable", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.9186094069529652 } ], "scheme": "w8" } ], "control": { "measured": false, "statement": "No industry standard control was measured beside this family. Its lane is `raw-source-compression-quality`, which admits no vendor runtime, and the published comparison against a control is `keyword-spotting-dense-int4-runtime-control`, `keyword-spotting-dense-runtime-control`, `keyword-spotting-runtime-control`, `wake-vision-runtime-control`. These cells describe the pipeline on this workload and are not a comparative result." }, "lane": "raw-source-compression-quality", "state": "active", "title": "Keyword spotting from float source: the public float DS-CNN compiled by MicroQuant", "workload_class": "keyword-spotting" }, { "benchmark_id": "keyword-spotting-dense-int4-runtime-control", "campaigns": [ "host", "esp32s3" ], "cells": [ { "backend": "esp-nn-control", "campaign": "esp32s3", "cell_id": "keyword-spotting-dense-int4-runtime-control/esp-nn-control/w8", "disposition": "not-directly-comparable", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 4890, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.03875146041666666 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.03875207916666667 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 4890, "value": 270396.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 4890, "value": 340010.0 }, { "evidence_tier": "physical_hil", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.8012269938650307 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 4890, "value": 196608.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "esp32s3", "cell_id": "keyword-spotting-dense-int4-runtime-control/esp32s3-portable/w4", "disposition": "portable-reference", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 100, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.055106575000000005 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.055107629166666665 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 100, "value": 73756.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 100, "value": 264043.0 }, { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.8010224948875255 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 100, "value": 896.0 } ], "scheme": "w4" }, { "backend": "tflm-portable-control", "campaign": "esp32s3", "cell_id": "keyword-spotting-dense-int4-runtime-control/tflm-portable-control/w8", "disposition": "not-directly-comparable", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 4890, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.038766241666666666 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.03876765833333334 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 4890, "value": 270380.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 4890, "value": 314849.0 }, { "evidence_tier": "physical_hil", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.8012269938650307 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 4890, "value": 196608.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "keyword-spotting-dense-int4-runtime-control/generated-host-c/w4", "disposition": "portable-reference", "evidence_tier": "compiled_c", "metrics": [ { "evidence_tier": "compiled_c", "metric_id": "latency_p50_seconds", "sample_count": 4890, "value": 0.000316187 }, { "evidence_tier": "compiled_c", "metric_id": "latency_p99_seconds", "sample_count": 4890, "value": 0.000327562 }, { "evidence_tier": "compiled_c", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.8010224948875255 }, { "evidence_tier": "compiled_c", "metric_id": "workspace_bytes", "sample_count": 4890, "value": 896.0 } ], "scheme": "w4" }, { "backend": "portable", "campaign": "host", "cell_id": "keyword-spotting-dense-int4-runtime-control/microquant-numpy/w4", "disposition": "portable-reference", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "latency_p50_seconds", "sample_count": 1, "value": 0.00016072465931512948 }, { "evidence_tier": "host", "metric_id": "latency_p99_seconds", "sample_count": 1, "value": 0.00016072465931512948 }, { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.8010224948875255 }, { "evidence_tier": "host", "metric_id": "workspace_bytes", "sample_count": 4890, "value": 896.0 } ], "scheme": "w4" } ], "control": { "measured": true, "statement": "This is the control lane. It binds one already-integer model and measures the vendor runtimes and MicroQuant on it in the same campaign, on the same board, over the same endpoints." }, "lane": "same-model-runtime-control", "state": "active", "title": "Keyword spotting dense int4 runtime control: the same four-layer network at four-bit weights, four runtimes, one board", "workload_class": "keyword-spotting" }, { "benchmark_id": "keyword-spotting-dense-recompression", "campaigns": [ "host", "esp32s3" ], "cells": [ { "backend": "portable", "campaign": "esp32s3", "cell_id": "keyword-spotting-dense-recompression/esp32s3-portable/w4", "disposition": "portable-reference", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 100, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.055112372916666666 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.0551133375 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 100, "value": 73756.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 100, "value": 264043.0 }, { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.805521472392638 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 100, "value": 896.0 } ], "scheme": "w4" }, { "backend": "portable", "campaign": "host", "cell_id": "keyword-spotting-dense-recompression/generated-host-c/w4", "disposition": "portable-reference", "evidence_tier": "compiled_c", "metrics": [ { "evidence_tier": "compiled_c", "metric_id": "latency_p50_seconds", "sample_count": 4890, "value": 0.0003155 }, { "evidence_tier": "compiled_c", "metric_id": "latency_p99_seconds", "sample_count": 4890, "value": 0.00032875 }, { "evidence_tier": "compiled_c", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.805521472392638 }, { "evidence_tier": "compiled_c", "metric_id": "workspace_bytes", "sample_count": 4890, "value": 896.0 } ], "scheme": "w4" }, { "backend": "portable", "campaign": "host", "cell_id": "keyword-spotting-dense-recompression/microquant-numpy/w4", "disposition": "portable-reference", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "latency_p50_seconds", "sample_count": 1, "value": 0.00016468787484408065 }, { "evidence_tier": "host", "metric_id": "latency_p99_seconds", "sample_count": 1, "value": 0.00016468787484408065 }, { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.805521472392638 }, { "evidence_tier": "host", "metric_id": "workspace_bytes", "sample_count": 4890, "value": 896.0 } ], "scheme": "w4" } ], "control": { "measured": false, "statement": "No industry standard control was measured beside this family. Its lane is `container-recompression-quality`, which admits no vendor runtime, and the published comparison against a control is `keyword-spotting-dense-int4-runtime-control`, `keyword-spotting-dense-runtime-control`, `keyword-spotting-runtime-control`, `wake-vision-runtime-control`. These cells describe the pipeline on this workload and are not a comparative result." }, "lane": "container-recompression-quality", "state": "active", "title": "Keyword spotting dense recompression: the published int8 network re-encoded to four-bit weights by MicroQuant", "workload_class": "keyword-spotting" }, { "benchmark_id": "keyword-spotting-dense-runtime-control", "campaigns": [ "host", "esp32s3" ], "cells": [ { "backend": "esp-nn-control", "campaign": "esp32s3", "cell_id": "keyword-spotting-dense-runtime-control/esp-nn-control/w8", "disposition": "not-directly-comparable", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 4890, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.05820070625 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.058204770833333336 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 4890, "value": 106556.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 4890, "value": 587314.0 }, { "evidence_tier": "physical_hil", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.8235173824130879 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 4890, "value": 32768.0 } ], "scheme": "w8" }, { "backend": "esp32s3-pie", "campaign": "esp32s3", "cell_id": "keyword-spotting-dense-runtime-control/esp32s3-pie/w8", "disposition": "shipping-accelerated", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 100, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.03450053958333334 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.03450244166666667 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 100, "value": 74460.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 100, "value": 525545.0 }, { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.8239263803680982 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 100, "value": 1600.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "esp32s3", "cell_id": "keyword-spotting-dense-runtime-control/esp32s3-portable/w8", "disposition": "portable-reference", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 100, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.03553859166666667 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.03554467916666667 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 100, "value": 73756.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 100, "value": 511123.0 }, { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.8239263803680982 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 100, "value": 896.0 } ], "scheme": "w8" }, { "backend": "tflm-portable-control", "campaign": "esp32s3", "cell_id": "keyword-spotting-dense-runtime-control/tflm-portable-control/w8", "disposition": "not-directly-comparable", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 4890, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.03531223541666667 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.03531306666666666 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 4890, "value": 106540.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 4890, "value": 562153.0 }, { "evidence_tier": "physical_hil", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.8235173824130879 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 4890, "value": 32768.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "keyword-spotting-dense-runtime-control/generated-host-c/w8", "disposition": "portable-reference", "evidence_tier": "compiled_c", "metrics": [ { "evidence_tier": "compiled_c", "metric_id": "latency_p50_seconds", "sample_count": 4890, "value": 2.8312e-05 }, { "evidence_tier": "compiled_c", "metric_id": "latency_p99_seconds", "sample_count": 4890, "value": 3.7062e-05 }, { "evidence_tier": "compiled_c", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.8239263803680982 }, { "evidence_tier": "compiled_c", "metric_id": "workspace_bytes", "sample_count": 4890, "value": 896.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "keyword-spotting-dense-runtime-control/microquant-numpy/w8", "disposition": "portable-reference", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "latency_p50_seconds", "sample_count": 1, "value": 0.00016127690858091465 }, { "evidence_tier": "host", "metric_id": "latency_p99_seconds", "sample_count": 1, "value": 0.00016127690858091465 }, { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.8239263803680982 }, { "evidence_tier": "host", "metric_id": "workspace_bytes", "sample_count": 4890, "value": 896.0 } ], "scheme": "w8" } ], "control": { "measured": true, "statement": "This is the control lane. It binds one already-integer model and measures the vendor runtimes and MicroQuant on it in the same campaign, on the same board, over the same endpoints." }, "lane": "same-model-runtime-control", "state": "active", "title": "Keyword spotting dense runtime control: one int8 four-layer network, four runtimes, one board", "workload_class": "keyword-spotting" }, { "benchmark_id": "keyword-spotting-dscnn", "campaigns": [ "host", "esp32s3" ], "cells": [ { "backend": "auto", "campaign": "esp32s3", "cell_id": "keyword-spotting-dscnn/esp32s3-mixed/w8", "disposition": "not-directly-comparable", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 100, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.001374825 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.0013750166666666667 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 100, "value": 80444.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 100, "value": 16430.0 }, { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 256, "value": 1.0 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 100, "value": 7584.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "esp32s3", "cell_id": "keyword-spotting-dscnn/esp32s3-portable/w8", "disposition": "not-directly-comparable", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 100, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.007488308333333334 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.0074925375 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 100, "value": 80172.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 100, "value": 12026.0 }, { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 256, "value": 1.0 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 100, "value": 7312.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "keyword-spotting-dscnn/generated-host-c/w8", "disposition": "not-directly-comparable", "evidence_tier": "compiled_c", "metrics": [ { "evidence_tier": "compiled_c", "metric_id": "latency_p50_seconds", "sample_count": 256, "value": 2.03125e-05 }, { "evidence_tier": "compiled_c", "metric_id": "latency_p99_seconds", "sample_count": 256, "value": 6.25e-05 }, { "evidence_tier": "compiled_c", "metric_id": "top1_accuracy", "sample_count": 256, "value": 1.0 }, { "evidence_tier": "compiled_c", "metric_id": "workspace_bytes", "sample_count": 256, "value": 7312.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "keyword-spotting-dscnn/microquant-numpy/w8", "disposition": "not-directly-comparable", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "latency_p50_seconds", "sample_count": 1, "value": 0.0001764628909768362 }, { "evidence_tier": "host", "metric_id": "latency_p99_seconds", "sample_count": 1, "value": 0.0001764628909768362 }, { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 256, "value": 1.0 }, { "evidence_tier": "host", "metric_id": "workspace_bytes", "sample_count": 256, "value": 7312.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "keyword-spotting-dscnn/source-framework-host/w8", "disposition": "not-directly-comparable", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 256, "value": 1.0 } ], "scheme": "w8" } ], "control": { "measured": false, "statement": "No industry standard control was measured beside this family. Its lane is `framework-validation`, which admits no vendor runtime, and the published comparison against a control is `keyword-spotting-dense-int4-runtime-control`, `keyword-spotting-dense-runtime-control`, `keyword-spotting-runtime-control`, `wake-vision-runtime-control`. These cells describe the pipeline on this workload and are not a comparative result." }, "lane": "framework-validation", "state": "active", "title": "Keyword spotting: owned depthwise-separable CNN", "workload_class": "keyword-spotting" }, { "benchmark_id": "keyword-spotting-runtime-control", "campaigns": [ "host", "esp32s3" ], "cells": [ { "backend": "esp-nn-control", "campaign": "esp32s3", "cell_id": "keyword-spotting-runtime-control/esp-nn-control/w8", "disposition": "not-directly-comparable", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 4890, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.017782445833333334 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.017801283333333334 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 4890, "value": 123180.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 4890, "value": 138978.0 }, { "evidence_tier": "physical_hil", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.9179959100204499 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 4890, "value": 49152.0 } ], "scheme": "w8" }, { "backend": "auto", "campaign": "esp32s3", "cell_id": "keyword-spotting-runtime-control/esp32s3-mixed/w8", "disposition": "shipping-accelerated", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 100, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.0112952125 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.0113012875 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 100, "value": 89420.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 100, "value": 42171.0 }, { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.9186094069529652 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 100, "value": 16320.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "esp32s3", "cell_id": "keyword-spotting-runtime-control/esp32s3-portable/w8", "disposition": "portable-reference", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 100, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.15086200208333334 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.15087275416666668 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 100, "value": 89100.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 100, "value": 38004.0 }, { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.9186094069529652 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 100, "value": 16000.0 } ], "scheme": "w8" }, { "backend": "tflm-portable-control", "campaign": "esp32s3", "cell_id": "keyword-spotting-runtime-control/tflm-portable-control/w8", "disposition": "not-directly-comparable", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 4890, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.17097874583333333 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.17098909583333333 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 4890, "value": 106780.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 4890, "value": 113817.0 }, { "evidence_tier": "physical_hil", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.9179959100204499 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 4890, "value": 32768.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "keyword-spotting-runtime-control/generated-host-c/w8", "disposition": "portable-reference", "evidence_tier": "compiled_c", "metrics": [ { "evidence_tier": "compiled_c", "metric_id": "latency_p50_seconds", "sample_count": 4890, "value": 0.000295062 }, { "evidence_tier": "compiled_c", "metric_id": "latency_p99_seconds", "sample_count": 4890, "value": 0.000308562 }, { "evidence_tier": "compiled_c", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.9186094069529652 }, { "evidence_tier": "compiled_c", "metric_id": "workspace_bytes", "sample_count": 4890, "value": 16000.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "keyword-spotting-runtime-control/microquant-numpy/w8", "disposition": "portable-reference", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "latency_p50_seconds", "sample_count": 1, "value": 0.0023223582566392004 }, { "evidence_tier": "host", "metric_id": "latency_p99_seconds", "sample_count": 1, "value": 0.0023223582566392004 }, { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 4890, "value": 0.9186094069529652 }, { "evidence_tier": "host", "metric_id": "workspace_bytes", "sample_count": 4890, "value": 16000.0 } ], "scheme": "w8" } ], "control": { "measured": true, "statement": "This is the control lane. It binds one already-integer model and measures the vendor runtimes and MicroQuant on it in the same campaign, on the same board, over the same endpoints." }, "lane": "same-model-runtime-control", "state": "active", "title": "Keyword spotting runtime control: one int8 DS-CNN, four runtimes, one board", "workload_class": "keyword-spotting" }, { "benchmark_id": "operator-geometry-conv-row-64", "campaigns": [ "host" ], "cells": [ { "backend": "portable", "campaign": "host", "cell_id": "operator-geometry-conv-row-64/generated-host-c/w8", "disposition": "not-directly-comparable", "evidence_tier": "compiled_c", "metrics": [ { "evidence_tier": "compiled_c", "metric_id": "latency_p50_seconds", "sample_count": 64, "value": 2.5e-07 }, { "evidence_tier": "compiled_c", "metric_id": "latency_p99_seconds", "sample_count": 64, "value": 2.5e-07 }, { "evidence_tier": "compiled_c", "metric_id": "workspace_bytes", "sample_count": 64, "value": 0.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "operator-geometry-conv-row-64/microquant-numpy/w8", "disposition": "not-directly-comparable", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "latency_p50_seconds", "sample_count": 1, "value": 5.587235136772506e-06 }, { "evidence_tier": "host", "metric_id": "latency_p99_seconds", "sample_count": 1, "value": 5.587235136772506e-06 }, { "evidence_tier": "host", "metric_id": "workspace_bytes", "sample_count": 64, "value": 0.0 } ], "scheme": "w8" } ], "control": { "measured": false, "statement": "No industry standard control was measured beside this family. Its lane is `framework-validation`, which admits no vendor runtime, and the published comparison against a control is `keyword-spotting-dense-int4-runtime-control`, `keyword-spotting-dense-runtime-control`, `keyword-spotting-runtime-control`, `wake-vision-runtime-control`. These cells describe the pipeline on this workload and are not a comparative result." }, "lane": "framework-validation", "state": "active", "title": "Operator microbenchmark: conv-row-64", "workload_class": "operator-microbenchmark" }, { "benchmark_id": "operator-geometry-conv-row-9", "campaigns": [ "host" ], "cells": [ { "backend": "portable", "campaign": "host", "cell_id": "operator-geometry-conv-row-9/generated-host-c/w8", "disposition": "not-directly-comparable", "evidence_tier": "compiled_c", "metrics": [ { "evidence_tier": "compiled_c", "metric_id": "latency_p50_seconds", "sample_count": 64, "value": 1.5e-06 }, { "evidence_tier": "compiled_c", "metric_id": "latency_p99_seconds", "sample_count": 64, "value": 2.125e-06 }, { "evidence_tier": "compiled_c", "metric_id": "workspace_bytes", "sample_count": 64, "value": 0.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "operator-geometry-conv-row-9/microquant-numpy/w8", "disposition": "not-directly-comparable", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "latency_p50_seconds", "sample_count": 1, "value": 1.0779953299788758e-05 }, { "evidence_tier": "host", "metric_id": "latency_p99_seconds", "sample_count": 1, "value": 1.0779953299788758e-05 }, { "evidence_tier": "host", "metric_id": "workspace_bytes", "sample_count": 64, "value": 0.0 } ], "scheme": "w8" } ], "control": { "measured": false, "statement": "No industry standard control was measured beside this family. Its lane is `framework-validation`, which admits no vendor runtime, and the published comparison against a control is `keyword-spotting-dense-int4-runtime-control`, `keyword-spotting-dense-runtime-control`, `keyword-spotting-runtime-control`, `wake-vision-runtime-control`. These cells describe the pipeline on this workload and are not a comparative result." }, "lane": "framework-validation", "state": "active", "title": "Operator microbenchmark: conv-row-9", "workload_class": "operator-microbenchmark" }, { "benchmark_id": "operator-geometry-dense-row-64", "campaigns": [ "host" ], "cells": [ { "backend": "portable", "campaign": "host", "cell_id": "operator-geometry-dense-row-64/generated-host-c/w8", "disposition": "not-directly-comparable", "evidence_tier": "compiled_c", "metrics": [ { "evidence_tier": "compiled_c", "metric_id": "latency_p50_seconds", "sample_count": 64, "value": 0.0 }, { "evidence_tier": "compiled_c", "metric_id": "latency_p99_seconds", "sample_count": 64, "value": 1.25e-07 }, { "evidence_tier": "compiled_c", "metric_id": "workspace_bytes", "sample_count": 64, "value": 0.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "operator-geometry-dense-row-64/microquant-numpy/w8", "disposition": "not-directly-comparable", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "latency_p50_seconds", "sample_count": 1, "value": 1.115889972425066e-06 }, { "evidence_tier": "host", "metric_id": "latency_p99_seconds", "sample_count": 1, "value": 1.115889972425066e-06 }, { "evidence_tier": "host", "metric_id": "workspace_bytes", "sample_count": 64, "value": 0.0 } ], "scheme": "w8" } ], "control": { "measured": false, "statement": "No industry standard control was measured beside this family. Its lane is `framework-validation`, which admits no vendor runtime, and the published comparison against a control is `keyword-spotting-dense-int4-runtime-control`, `keyword-spotting-dense-runtime-control`, `keyword-spotting-runtime-control`, `wake-vision-runtime-control`. These cells describe the pipeline on this workload and are not a comparative result." }, "lane": "framework-validation", "state": "active", "title": "Operator microbenchmark: dense-row-64", "workload_class": "operator-microbenchmark" }, { "benchmark_id": "operator-geometry-dense-row-65", "campaigns": [ "host" ], "cells": [ { "backend": "portable", "campaign": "host", "cell_id": "operator-geometry-dense-row-65/generated-host-c/w8", "disposition": "not-directly-comparable", "evidence_tier": "compiled_c", "metrics": [ { "evidence_tier": "compiled_c", "metric_id": "latency_p50_seconds", "sample_count": 64, "value": 0.0 }, { "evidence_tier": "compiled_c", "metric_id": "latency_p99_seconds", "sample_count": 64, "value": 1.25e-07 }, { "evidence_tier": "compiled_c", "metric_id": "workspace_bytes", "sample_count": 64, "value": 0.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "operator-geometry-dense-row-65/microquant-numpy/w8", "disposition": "not-directly-comparable", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "latency_p50_seconds", "sample_count": 1, "value": 6.976562872296199e-06 }, { "evidence_tier": "host", "metric_id": "latency_p99_seconds", "sample_count": 1, "value": 6.976562872296199e-06 }, { "evidence_tier": "host", "metric_id": "workspace_bytes", "sample_count": 64, "value": 0.0 } ], "scheme": "w8" } ], "control": { "measured": false, "statement": "No industry standard control was measured beside this family. Its lane is `framework-validation`, which admits no vendor runtime, and the published comparison against a control is `keyword-spotting-dense-int4-runtime-control`, `keyword-spotting-dense-runtime-control`, `keyword-spotting-runtime-control`, `wake-vision-runtime-control`. These cells describe the pipeline on this workload and are not a comparative result." }, "lane": "framework-validation", "state": "active", "title": "Operator microbenchmark: dense-row-65", "workload_class": "operator-microbenchmark" }, { "benchmark_id": "streaming-wake-word-debounced", "campaigns": [ "host" ], "cells": [ { "backend": "portable", "campaign": "host", "cell_id": "streaming-wake-word-debounced/generated-host-c/w8", "disposition": "not-directly-comparable", "evidence_tier": "compiled_c", "metrics": [ { "evidence_tier": "compiled_c", "metric_id": "latency_p50_seconds", "sample_count": 256, "value": 1.75e-05 }, { "evidence_tier": "compiled_c", "metric_id": "top1_accuracy", "sample_count": 256, "value": 0.99609375 }, { "evidence_tier": "compiled_c", "metric_id": "workspace_bytes", "sample_count": 256, "value": 7312.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "streaming-wake-word-debounced/microquant-numpy/w8", "disposition": "not-directly-comparable", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "latency_p50_seconds", "sample_count": 1, "value": 0.0001800198556338728 }, { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 256, "value": 0.99609375 }, { "evidence_tier": "host", "metric_id": "workspace_bytes", "sample_count": 256, "value": 7312.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "streaming-wake-word-debounced/source-framework-host/w8", "disposition": "not-directly-comparable", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "top1_accuracy", "sample_count": 256, "value": 0.99609375 } ], "scheme": "w8" } ], "control": { "measured": false, "statement": "No industry standard control was measured beside this family. Its lane is `framework-validation`, which admits no vendor runtime, and the published comparison against a control is `keyword-spotting-dense-int4-runtime-control`, `keyword-spotting-dense-runtime-control`, `keyword-spotting-runtime-control`, `wake-vision-runtime-control`. These cells describe the pipeline on this workload and are not a comparative result." }, "lane": "framework-validation", "state": "active", "title": "Streaming wake word: stateless inference under a debounced application rule", "workload_class": "streaming-wake-word" }, { "benchmark_id": "visual-wake-words-mobile", "campaigns": [ "host", "esp32s3" ], "cells": [ { "backend": "auto", "campaign": "esp32s3", "cell_id": "visual-wake-words-mobile/esp32s3-mixed/w8", "disposition": "not-directly-comparable", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "host", "metric_id": "binary_accuracy", "sample_count": 256, "value": 1.0 }, { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 100, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.0026574041666666665 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.0026645333333333333 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 100, "value": 77276.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 100, "value": 15269.0 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 100, "value": 3904.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "esp32s3", "cell_id": "visual-wake-words-mobile/esp32s3-portable/w8", "disposition": "not-directly-comparable", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "host", "metric_id": "binary_accuracy", "sample_count": 256, "value": 1.0 }, { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 100, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.0038675541666666666 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.0038731666666666667 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 100, "value": 76988.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 100, "value": 8779.0 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 100, "value": 3616.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "visual-wake-words-mobile/generated-host-c/w8", "disposition": "not-directly-comparable", "evidence_tier": "compiled_c", "metrics": [ { "evidence_tier": "compiled_c", "metric_id": "binary_accuracy", "sample_count": 256, "value": 1.0 }, { "evidence_tier": "compiled_c", "metric_id": "latency_p50_seconds", "sample_count": 256, "value": 1.575e-05 }, { "evidence_tier": "compiled_c", "metric_id": "latency_p99_seconds", "sample_count": 256, "value": 2.65e-05 }, { "evidence_tier": "compiled_c", "metric_id": "workspace_bytes", "sample_count": 256, "value": 3616.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "visual-wake-words-mobile/microquant-numpy/w8", "disposition": "not-directly-comparable", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "binary_accuracy", "sample_count": 256, "value": 1.0 }, { "evidence_tier": "host", "metric_id": "latency_p50_seconds", "sample_count": 1, "value": 7.878955057094572e-05 }, { "evidence_tier": "host", "metric_id": "latency_p99_seconds", "sample_count": 1, "value": 7.878955057094572e-05 }, { "evidence_tier": "host", "metric_id": "workspace_bytes", "sample_count": 256, "value": 3616.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "visual-wake-words-mobile/source-framework-host/w8", "disposition": "not-directly-comparable", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "binary_accuracy", "sample_count": 256, "value": 1.0 } ], "scheme": "w8" } ], "control": { "measured": false, "statement": "No industry standard control was measured beside this family. Its lane is `framework-validation`, which admits no vendor runtime, and the published comparison against a control is `keyword-spotting-dense-int4-runtime-control`, `keyword-spotting-dense-runtime-control`, `keyword-spotting-runtime-control`, `wake-vision-runtime-control`. These cells describe the pipeline on this workload and are not a comparative result." }, "lane": "framework-validation", "state": "active", "title": "Visual wake words: owned depthwise/pointwise mobile network", "workload_class": "visual-wake-words" }, { "benchmark_id": "wake-vision-runtime-control", "campaigns": [ "host", "esp32s3" ], "cells": [ { "backend": "esp-nn-control", "campaign": "esp32s3", "cell_id": "wake-vision-runtime-control/esp-nn-control/w8", "disposition": "not-directly-comparable", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "physical_hil", "metric_id": "binary_accuracy", "sample_count": 1000, "value": 0.765 }, { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 1000, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.07280208333333334 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.07283597916666666 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 1000, "value": 232252.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 1000, "value": 418334.0 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 1000, "value": 131072.0 } ], "scheme": "w8" }, { "backend": "auto", "campaign": "esp32s3", "cell_id": "wake-vision-runtime-control/esp32s3-mixed/w8", "disposition": "shipping-accelerated", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "host", "metric_id": "binary_accuracy", "sample_count": 1000, "value": 0.768 }, { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 100, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.04649309791666667 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.046493925 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 100, "value": 156732.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 100, "value": 65953.0 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 100, "value": 56480.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "esp32s3", "cell_id": "wake-vision-runtime-control/esp32s3-portable/w8", "disposition": "portable-reference", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "host", "metric_id": "binary_accuracy", "sample_count": 1000, "value": 0.768 }, { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 100, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.27888635625 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.278958 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 100, "value": 155548.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 100, "value": 63321.0 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 100, "value": 55296.0 } ], "scheme": "w8" }, { "backend": "tflm-portable-control", "campaign": "esp32s3", "cell_id": "wake-vision-runtime-control/tflm-portable-control/w8", "disposition": "not-directly-comparable", "evidence_tier": "physical_hil", "metrics": [ { "evidence_tier": "physical_hil", "metric_id": "binary_accuracy", "sample_count": 1000, "value": 0.765 }, { "evidence_tier": "physical_hil", "metric_id": "heap_growth_bytes", "sample_count": 1000, "value": 0.0 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p50_seconds", "sample_count": 100, "value": 0.48077039375 }, { "evidence_tier": "physical_hil", "metric_id": "latency_p99_seconds", "sample_count": 100, "value": 0.4807724875 }, { "evidence_tier": "physical_hil", "metric_id": "peak_internal_sram_bytes", "sample_count": 1000, "value": 232236.0 }, { "evidence_tier": "physical_hil", "metric_id": "product_flash_bytes", "sample_count": 1000, "value": 393173.0 }, { "evidence_tier": "physical_hil", "metric_id": "workspace_bytes", "sample_count": 1000, "value": 131072.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "wake-vision-runtime-control/generated-host-c/w8", "disposition": "portable-reference", "evidence_tier": "compiled_c", "metrics": [ { "evidence_tier": "compiled_c", "metric_id": "binary_accuracy", "sample_count": 1000, "value": 0.768 }, { "evidence_tier": "compiled_c", "metric_id": "latency_p50_seconds", "sample_count": 1000, "value": 0.000482687 }, { "evidence_tier": "compiled_c", "metric_id": "latency_p99_seconds", "sample_count": 1000, "value": 0.000504875 }, { "evidence_tier": "compiled_c", "metric_id": "workspace_bytes", "sample_count": 1000, "value": 55296.0 } ], "scheme": "w8" }, { "backend": "portable", "campaign": "host", "cell_id": "wake-vision-runtime-control/microquant-numpy/w8", "disposition": "portable-reference", "evidence_tier": "host", "metrics": [ { "evidence_tier": "host", "metric_id": "binary_accuracy", "sample_count": 1000, "value": 0.768 }, { "evidence_tier": "host", "metric_id": "latency_p50_seconds", "sample_count": 1, "value": 0.003949471625033766 }, { "evidence_tier": "host", "metric_id": "latency_p99_seconds", "sample_count": 1, "value": 0.003949471625033766 }, { "evidence_tier": "host", "metric_id": "workspace_bytes", "sample_count": 1000, "value": 55296.0 } ], "scheme": "w8" } ], "control": { "measured": true, "statement": "This is the control lane. It binds one already-integer model and measures the vendor runtimes and MicroQuant on it in the same campaign, on the same board, over the same endpoints." }, "lane": "same-model-runtime-control", "state": "active", "title": "Wake Vision runtime control: one int8 MobileNetV1, four runtimes, one board", "workload_class": "visual-wake-words" } ], "headline_comparison": "kws-benchmark.json", "implementation_line": "1.2", "published_note": "Every cell here was produced from a committed definition in one of two sealed campaigns: a host campaign executed as compiled C, and an ESP32-S3 campaign executed on one admitted board. It is development evidence, not a customer acceptance result. Every campaign was measured by the compiler that is in the tree now, and `compiler_binding` names it.", "schema": "microquant-public-current-source-benchmark-suite-v1", "status": "measured-current-source-result" }