Upload folder using huggingface_hub

README.md

@@ -9,7 +9,7 @@ emoji: 🛍️
 
 # PEFT Shop
 
-A Gradio app to browse PEFT methods like an online store: filter by capabilities (merging, multi-adapter support, quantization backends, targetable layer types, …) and check benchmark results — star ratings for the benchmark-specific metrics (test accuracy and forgetting for MetaMathQA, DINO similarity and drift for image generation) as well as peak memory, checkpoint size, and train time, switchable between the benchmarks of the method comparison suite. Methods can be added to a cart 🛒, which shows usage code snippets and a feature comparison table for the collected methods — and checkout is, of course, free.
+A Gradio app to browse PEFT methods like an online store: filter by capabilities (merging, multi-adapter support, quantization backends, targetable layer types, …) or by minimum customer rating ("★★★☆☆ & up"), and check benchmark results — star ratings for the benchmark-specific metrics (test accuracy and forgetting for MetaMathQA, DINO similarity and drift for image generation) as well as peak memory, checkpoint size, and train time, switchable between the benchmarks of the method comparison suite. Methods can be added to a cart 🛒, which shows usage code snippets and a feature comparison table for the collected methods — and checkout is, of course, free.
 
 ## Running
 

app.py

@@ -14,8 +14,8 @@
 """The PEFT shop: a Gradio app to browse PEFT methods like an online store.
 
 Users can filter methods by their capabilities (merging, multi-adapter support, quantization backends, targetable
-layer types, ...), check benchmark results (switchable between the benchmarks of the method comparison suite, e.g.
-MetaMathQA or image generation), and jump to the PEFT docs. The app has two tabs: one to browse the shop, one for the
+layer types, ...) and by minimum star ratings, check benchmark results (switchable between the benchmarks of the
+method comparison suite, e.g. MetaMathQA or image generation), and jump to the PEFT docs. The app has two tabs: one to browse the shop, one for the
 cart, which shows usage code snippets and a feature comparison table for the collected methods. In keeping with the
 shop theme, every method has a (crossed-out) price tag, benchmark results double as customer star ratings, and
 checkout is free.
@@ -509,8 +509,8 @@ def _stars_span(n_stars: int, title: str) -> str:
     return f'<span class="stars" title="{esc(title)}">{"★" * n_stars}{"☆" * (5 - n_stars)}</span>'
 
 
-def benchmark_stars(spec: BenchmarkSpec, field: str, value: float, title: str, lower_is_better: bool = False) -> str:
-    """A benchmark metric as a customer star rating, based on quantiles among the benchmarked PEFT methods.
+def star_rating(spec: BenchmarkSpec, field: str, value: float, lower_is_better: bool = False) -> int:
+    """The customer star rating of one benchmark metric value, based on quantiles among the benchmarked PEFT methods.
 
     The best 20% of the methods get five stars, the next 20% four, and so on; even the worst method keeps one star.
     Quantiles are used instead of e.g. min-max scaling so that a single outlier cannot compress everyone else's
@@ -519,8 +519,35 @@ def benchmark_stars(spec: BenchmarkSpec, field: str, value: float, title: str, l
     """
     values = [info["benchmarks"][spec.key][field] for info in METHODS.values() if spec.key in info["benchmarks"]]
     rank = sum(other < value if lower_is_better else other > value for other in values)
-    n_stars = 5 - int(5 * rank / len(values))
-    return _stars_span(n_stars, title)
+    return 5 - int(5 * rank / len(values))
+
+
+def benchmark_stars(spec: BenchmarkSpec, field: str, value: float, title: str, lower_is_better: bool = False) -> str:
+    """A benchmark metric as a customer star rating (see star_rating), rendered as a hoverable span."""
+    return _stars_span(star_rating(spec, field, value, lower_is_better), title)
+
+
+def rated_metrics(spec: BenchmarkSpec) -> list[tuple[str, str, bool]]:
+    """(field, label, lower_is_better) of every star-rated metric of a benchmark, in card-row order.
+
+    This is the single source of truth for the rated rows: the cards, the rating filters, and their labels all
+    derive from it.
+    """
+    rows = [(metric.field, metric.label, not metric.higher_is_better) for metric in spec.metrics]
+    rows += [
+        ("peak_memory_bytes", "max memory allocated", True),
+        ("adapter_file_size_bytes", "checkpoint size", True),
+        ("train_time_sec", "train time", True),
+    ]
+    return rows
+
+
+# The five clickable stars of a minimum-rating filter (a gr.Radio restyled into a star bar via the .star-filter
+# CSS). Every choice renders as one star, the value is the minimum rating ("n stars & up"). The default of 1 filters
+# nothing, as even the worst-rated method keeps one star.
+RATING_CHOICES = [("★", n) for n in range(1, 6)]
+# Number of rating filter slots; for benchmarks with fewer rated metrics, the surplus slots are hidden.
+N_RATING_SLOTS = max(len(rated_metrics(spec)) for spec in BENCHMARKS)
 
 
 def badge(value: bool | None, label: str, title: str | None = None) -> str:
@@ -559,55 +586,30 @@ def render_card(method: str, spec: BenchmarkSpec) -> str:
             else ""
         )
 
-        # per row: field in the benchmark entry, display label, formatted value, description for the hover text,
-        # whether lower values are better, and an optional reference note
-        bench_rows = [
-            (
-                metric.field,
-                metric.label,
-                fmt_metric(metric, bench[metric.field]),
-                f"{metric.label} on the PEFT {spec.label} benchmark",
-                not metric.higher_is_better,
-                reference if metric is score else "",
-            )
-            for metric in spec.metrics
-        ]
-        bench_rows += [
-            (
-                "peak_memory_bytes",
-                "max memory allocated",
-                fmt_bytes(bench["peak_memory_bytes"]),
-                f"Peak accelerator memory while training on the PEFT {spec.label} benchmark",
-                True,
-                "",
-            ),
-            (
-                "adapter_file_size_bytes",
-                "checkpoint size",
-                fmt_megabytes(bench["adapter_file_size_bytes"]),
-                f"Size of the saved checkpoint on the PEFT {spec.label} benchmark",
-                True,
-                "",
-            ),
-            (
-                "train_time_sec",
-                "train time",
-                fmt_minutes(bench["train_time_sec"]),
-                f"Training time on the PEFT {spec.label} benchmark",
-                True,
-                "",
-            ),
-        ]
+        # per-field value formatting and hover-text description; the rows themselves come from rated_metrics
+        formatters = {metric.field: (lambda v, metric=metric: fmt_metric(metric, v)) for metric in spec.metrics}
+        formatters |= {
+            "peak_memory_bytes": fmt_bytes,
+            "adapter_file_size_bytes": fmt_megabytes,
+            "train_time_sec": fmt_minutes,
+        }
+        descriptions = {metric.field: f"{metric.label} on the PEFT {spec.label} benchmark" for metric in spec.metrics}
+        descriptions |= {
+            "peak_memory_bytes": f"Peak accelerator memory while training on the PEFT {spec.label} benchmark",
+            "adapter_file_size_bytes": f"Size of the saved checkpoint on the PEFT {spec.label} benchmark",
+            "train_time_sec": f"Training time on the PEFT {spec.label} benchmark",
+        }
         # the hover text sits on the whole row (and, redundantly, on the stars span inside it), so hovering the
         # metric name or the value explains the metric as well
         row_html = []
-        for field, label, value, description, lower_is_better, ref in bench_rows:
+        for field, label, lower_is_better in rated_metrics(spec):
             direction = "lower is better" if lower_is_better else "higher is better"
-            title = f"{description}; {direction}{ref}. Stars rank the method among the other PEFT methods."
+            ref = reference if field == score.field else ""
+            title = f"{descriptions[field]}; {direction}{ref}. Stars rank the method among the other PEFT methods."
             stars = benchmark_stars(spec, field, bench[field], title, lower_is_better=lower_is_better)
             row_html.append(
                 f'<div class="bench-row" title="{esc(title)}"><span>{stars} {esc(label)}:</span>'
-                f"<strong>{esc(value)}</strong></div>"
+                f"<strong>{esc(formatters[field](bench[field]))}</strong></div>"
             )
         bench_html = f"""
         <div class="bench" title="Best of {bench["num_runs"]} run(s): {esc(bench["experiment_name"])}">
@@ -660,6 +662,7 @@ def matches_filters(
     quant: list[str],
     benchmarked_only: bool,
     bench_key: str,
+    min_stars: tuple[int, ...],
 ) -> bool:
     """Filter semantics ("e-commerce" style):
 
@@ -669,25 +672,39 @@ def matches_filters(
     - across filter groups: AND
     - values reported as "unknown" by the capability script never match a positive filter: users filtering for a
       feature should only see methods where support is established.
+    - min_stars holds the minimum-rating filters, one per rated metric (in rated_metrics order, 1 = no minimum);
+      they apply to the selected benchmark, so methods without results on it cannot match.
     """
     info = METHODS[method]
     if search:
         haystack = f"{method} {info['config_class']} {info['description']}".lower()
         if search.lower() not in haystack:
             return False
+
     if categories and feature(method, "category")["value"] not in categories:
         return False
+
     if any(capability_value(method, key) is not True for key in capabilities):
         return False
+
     if layers:
         supported = layer_types(method)
         if supported is None or any(not supported.get(layer) for layer in layers):
             return False
+
     if quant and not any(supports_quant(method, backend) for backend in quant):
         return False
-    if benchmarked_only and bench_key not in info["benchmarks"]:
-        return False
-    return True
+
+    if any(minimum > 1 for minimum in min_stars):
+        bench = info["benchmarks"].get(bench_key)
+        if bench is None:
+            return False
+        spec = BENCHMARKS_BY_KEY[bench_key]
+        for (field, _, lower_is_better), minimum in zip(rated_metrics(spec), min_stars):
+            if star_rating(spec, field, bench[field], lower_is_better) < minimum:
+                return False
+
+    return not (benchmarked_only and bench_key not in info["benchmarks"])
 
 
 def sort_key(sort_by: str, bench_key: str):
@@ -714,19 +731,24 @@ ADD_TO_CART_LABEL = "🛒 Add to cart"
 IN_CART_LABEL = "✅ In cart"
 
 
-def update_cards(search, categories, capabilities, layers, quant, benchmarked_only, bench_key, sort_by, cart):
+def update_cards(
+    search, categories, capabilities, layers, quant, benchmarked_only, bench_key, sort_by, cart, *min_stars
+):
     """Assign the filtered, sorted methods to the fixed pool of card slots.
 
-    Returns the count markdown followed by (visibility, card HTML, method name, add button) for every slot; the
-    button label shows whether the slot's method is already in the cart. Slots beyond the number of matching methods
-    are hidden and get an empty method name, which add_to_cart treats as a no-op.
+    The trailing arguments are the values of the rating filter slots (in rated_metrics order). Returns the count
+    markdown followed by (visibility, card HTML, method name, add button) for every slot; the button label shows
+    whether the slot's method is already in the cart. Slots beyond the number of matching methods are hidden and get
+    an empty method name, which add_to_cart treats as a no-op.
     """
     spec = BENCHMARKS_BY_KEY[bench_key]
     cart = cart or []
     selected = [
         method
         for method in METHODS
-        if matches_filters(method, search, categories, capabilities, layers, quant, benchmarked_only, bench_key)
+        if matches_filters(
+            method, search, categories, capabilities, layers, quant, benchmarked_only, bench_key, min_stars
+        )
     ]
     selected.sort(key=sort_key(sort_by, bench_key))
     count = f"**{len(selected)} of {len(METHODS)} items** — all free, all in stock"
@@ -751,7 +773,17 @@ def reset_filters():
 
     Programmatically resetting the components triggers their change listeners, which re-render the cards.
     """
-    return "", [], [], [], [], False, BENCHMARKS[0].key, "name"
+    return ("", [], [], [], [], False, BENCHMARKS[0].key, "name") + (1,) * N_RATING_SLOTS
+
+
+def update_rating_filters(bench_key):
+    """Relabel the rating filter slots to the selected benchmark's rated metrics; surplus slots are hidden and
+    reset, so that a stale minimum cannot keep filtering invisibly."""
+    rows = rated_metrics(BENCHMARKS_BY_KEY[bench_key])
+    return [
+        gr.update(label=rows[i][1], visible=True) if i < len(rows) else gr.update(value=1, visible=False)
+        for i in range(N_RATING_SLOTS)
+    ]
 
 
 # ---------------------------------------------------------------------------------------------------------------
@@ -1019,6 +1051,18 @@ CSS = """
    hidden, independently of whether Gradio applies the visible=False update to the column. */
 .method-card:not(:has(.card)) { display: none !important; }
 
+/* The minimum-rating filters: a gr.Radio restyled into a clickable star bar. The radio circles are hidden and every
+   choice renders as one star; the stars up to the selected one stay gold, the ones after it are dimmed -- clicking
+   the n-th star therefore reads as "n stars & up". */
+.star-filter .wrap { display: flex; flex-direction: row; flex-wrap: nowrap; gap: 0.15rem; }
+.star-filter label { background: none !important; border: none !important; box-shadow: none !important;
+  padding: 0 !important; margin: 0 !important; cursor: pointer; }
+.star-filter label span { font-size: 1.5rem; line-height: 1; color: #f5b50a; padding: 0;
+  display: inline-block; transition: transform 0.1s ease; }
+.star-filter label:hover span { transform: scale(1.2); }
+.star-filter input[type="radio"] { display: none; }
+.star-filter label:has(input:checked) ~ label span { color: #9ca3af; opacity: 0.45; }
+
 /* Enlarge the tab labels (Browse methods / Cart) so they are hard to miss. Tab buttons carry the ARIA role "tab";
    the .tab-nav fallback covers Gradio versions that don't set it. */
 button[role="tab"], .tab-nav button { font-size: 1.3rem !important; font-weight: 600 !important;
@@ -1065,6 +1109,25 @@ def build_demo() -> gr.Blocks:
                         quant = gr.CheckboxGroup(
                             choices=sorted(quant_names), label="Quantization (any selected backend)"
                         )
+                        # one minimum-rating filter per star-rated card row, like a shop's "customer rating" filter;
+                        # the labels follow the selected benchmark (see update_rating_filters)
+                        with gr.Accordion("⭐ Minimum customer rating", open=False):
+                            gr.Markdown(
+                                "<small>Click the lowest acceptable rating ('n stars & up'). Ratings refer to the "
+                                "selected benchmark; with a minimum above one star, methods without results on it "
+                                "are filtered out.</small>"
+                            )
+                            default_rows = rated_metrics(BENCHMARKS[0])
+                            rating_filters = [
+                                gr.Radio(
+                                    choices=RATING_CHOICES,
+                                    value=1,
+                                    label=default_rows[i][1] if i < len(default_rows) else "",
+                                    visible=i < len(default_rows),
+                                    elem_classes="star-filter",
+                                )
+                                for i in range(N_RATING_SLOTS)
+                            ]
                         benchmarked_only = gr.Checkbox(label="Only methods with results on the selected benchmark")
                     with gr.Column(scale=3):
                         with gr.Row():
@@ -1116,10 +1179,12 @@ def build_demo() -> gr.Blocks:
             "each method shows its best run on the selected benchmark.</small>"
         )
 
-        filter_inputs = [search, categories, capabilities, layers, quant, benchmarked_only, benchmark, sort_by]
-        # the cart is an extra input to the card rendering (for the "in cart" button labels) but deliberately not
-        # part of filter_inputs: resetting the filters must not clear the cart
-        card_inputs = filter_inputs + [cart_select]
+        basic_filters = [search, categories, capabilities, layers, quant, benchmarked_only, benchmark, sort_by]
+        filter_inputs = basic_filters + rating_filters
+        # The cart is an extra input to the card rendering (for the "in cart" button labels) but deliberately not
+        # part of filter_inputs: resetting the filters must not clear the cart. It sits between the basic filters
+        # and the rating filters so that update_cards can take the latter as its variadic tail.
+        card_inputs = basic_filters + [cart_select] + rating_filters
         slot_outputs = [count_md]
         for slot_column, slot_html, slot_method, slot_button in slots:
             slot_outputs.extend([slot_column, slot_html, slot_method, slot_button])
@@ -1142,9 +1207,13 @@ def build_demo() -> gr.Blocks:
                     trigger_mode="always_last",
                 )
         demo.load(update_cards, inputs=card_inputs, outputs=slot_outputs, show_progress="hidden")
+        # the rating filters are labeled after the selected benchmark's metrics
+        benchmark.change(update_rating_filters, inputs=benchmark, outputs=rating_filters, show_progress="hidden")
 
         for _, _, slot_method, slot_button in slots:
-            slot_button.click(add_to_cart, inputs=[cart_select, slot_method], outputs=cart_select, show_progress="hidden")
+            slot_button.click(
+                add_to_cart, inputs=[cart_select, slot_method], outputs=cart_select, show_progress="hidden"
+            )
 
         # Updating cart_select programmatically (from the cards' add buttons or "clear cart") also triggers these
         # listeners, so the code snippet, the comparison table, the pay receipt, the cart tab label, and the
@@ -1152,7 +1221,7 @@ def build_demo() -> gr.Blocks:
         cart_outputs = [cart_code, compare_html, receipt_html, cart_tab]
         cart_select.change(update_cart, inputs=cart_select, outputs=cart_outputs, show_progress="hidden")
         cart_select.change(update_cards, inputs=card_inputs, outputs=slot_outputs, show_progress="hidden")
-        clear_button.click(lambda: [], outputs=cart_select, show_progress="hidden")
+        clear_button.click(list, outputs=cart_select, show_progress="hidden")
         # the receipt is kept up to date by update_cart, so paying only needs to open the popover, which is a purely
         # client-side affair (fn=None + js); same for writing to the clipboard, which only the browser can do
         pay_button.click(None, js="() => document.getElementById('pay-receipt')?.togglePopover(true)")

requirements.txt

@@ -1 +1 @@
-gradio>=5.0
+gradio>=6.2