Revert MCP workarounds.. unfixable: @gradio-app fix MCP schema $ref - $defs nested incorrectly!

app.py

@@ -1,739 +1,692 @@
-#!/usr/bin/env python3
-"""SD Dataset Automaker - HF Space (CPU) - Anime character dataset generator for LoRA/fine-tuning."""
-
-import warnings
-warnings.filterwarnings('ignore', category=FutureWarning)  # Suppress torch.cuda.amp spam
-warnings.filterwarnings('ignore', category=UserWarning, message='.*trust_repo.*')
-
-import os, re, shutil, zipfile, random, tempfile, argparse, sys
-from urllib.parse import quote_plus
-from collections import Counter
-from pathlib import Path
-
-from tqdm import tqdm
-import gradio as gr
-from bs4 import BeautifulSoup
-import requests as req_lib
-import time
-import numpy as np
-from PIL import Image
-import torch
-import torch.nn as nn
-from torchvision import models, transforms
-from sklearn.metrics.pairwise import pairwise_distances
-import onnxruntime as rt
-import pandas as pd
-import huggingface_hub
-
-# =============================================================================
-# CONFIG
-# =============================================================================
-EXTS = ('.jpg', '.jpeg', '.png')
-MODEL_DIR = Path(__file__).parent.resolve()  # Ensure absolute path
-YOLO_PATH = MODEL_DIR / "yolov5s_anime.pt"
-SIM_PATH = MODEL_DIR / "similarity.pt"
-EXAMPLES = [str(MODEL_DIR / f"from_url_spike_spiegel{i}.jpg") for i in range(1, 4)]  # absolute paths for gr.Examples
-WD_REPO = "SmilingWolf/wd-swinv2-tagger-v3"
-TAG_THRESH, BLACKLIST = 0.35, ["bangs", "breasts", "multicolored hair", "gradient hair", "two-tone hair", "virtual youtuber"]
-FACE_CONF, FACE_IOU, MIN_FACE, CROP_PAD = 0.5, 0.5, 35, 0.2
-SIM_THRESH, BATCH_SZ, FACE_SZ = 32, 16, 224
-
-# =============================================================================
-# UTILS
-# =============================================================================
-sanitize = lambda s: re.sub(r'[^\w.-]', '', s.replace(" ", "_"))
-get_imgs = lambda d: sorted([os.path.join(r,f) for r,_,fs in os.walk(d) for f in fs if f.lower().endswith(EXTS)])
-valid_img = lambda p: (lambda i: i.load() or True)(Image.open(p)) if os.path.exists(p) else False
-
-# HTTP session - mode depends on environment
-# CLI (local Windows): cloudscraper bypasses Cloudflare
-# HF Spaces: plain requests (cloudscraper fingerprint gets blocked on datacenter IPs)
-def init_session(use_cloudscraper=False):
-    global SESSION, HTTP_CLIENT
-    if use_cloudscraper:
-        try:
-            import cloudscraper
-            SESSION = cloudscraper.create_scraper()
-            HTTP_CLIENT = "cloudscraper"
-            return
-        except ImportError:
-            pass  # fallback to requests
-    SESSION = req_lib.Session()
-    SESSION.headers.update({
-        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
-        'Accept-Language': 'en-US,en;q=0.5',
-        'Accept-Encoding': 'gzip, deflate, br',
-        'DNT': '1',
-        'Connection': 'keep-alive',
-        'Upgrade-Insecure-Requests': '1',
-        'Sec-Fetch-Dest': 'document',
-        'Sec-Fetch-Mode': 'navigate',
-        'Sec-Fetch-Site': 'none',
-        'Sec-Fetch-User': '?1',
-        'Cache-Control': 'max-age=0',
-        'Referer': 'https://fancaps.net/',
-    })
-    HTTP_CLIENT = "requests/browser-headers"
-
-# Default: plain requests (for HF Spaces import)
-SESSION = None
-HTTP_CLIENT = None
-init_session(use_cloudscraper=False)
-
-# =============================================================================
-# SCRAPING
-# =============================================================================
-def search_fancaps(prompt, anime=True, movies=False, tv=False, log_fn=None):
-    L = log_fn or print
-    url = f"https://fancaps.net/search.php?q={quote_plus(prompt)}&submit=Submit"
-    if anime: url += "&animeCB=Anime"
-    if movies: url += "&MoviesCB=Movies"
-    if tv: url += "&TVCB=TV"
-    L(f"  URL: {url}")
-    try:
-        resp = SESSION.get(url, timeout=30)
-        L(f"  Status: {resp.status_code}, Size: {len(resp.content)} bytes")
-        # Log key headers for debugging
-        cf_ray = resp.headers.get('cf-ray', 'none')
-        server = resp.headers.get('server', 'unknown')
-        L(f"  Server: {server}, CF-Ray: {cf_ray}")
-        if resp.status_code != 200:
-            L(f"  ERROR: HTTP {resp.status_code}")
-            # Show snippet of response to understand the block reason
-            content_snippet = resp.text[:500].replace('\n', ' ').strip()
-            L(f"  Response snippet: {content_snippet[:200]}...")
-            return {}
-        soup = BeautifulSoup(resp.content, "html.parser")
-        # Debug: check if we got Cloudflare challenge
-        title = soup.title.string if soup.title else "No title"
-        L(f"  Page title: {title[:50]}")
-        if "cloudflare" in title.lower() or "challenge" in title.lower():
-            L("  ERROR: Cloudflare challenge detected!")
-            return {}
-    except Exception as e:
-        L(f"  ERROR: {type(e).__name__}: {e}")
-        return {}
-    results, cnt = {}, 1
-    divs = soup.find_all("div", class_="single_post_content")
-    L(f"  Found {len(divs)} content divs")
-    for div in divs:
-        if not div.find('h2'): continue
-        for h2 in div.find_all('h2'):
-            title = h2.get_text(strip=True).lower()
-            cat = 'Movies' if 'movie' in title else 'TV' if 'tv' in title else 'Anime' if 'anime' in title else None
-            if not cat: continue
-            table = h2.find_next('table')
-            if not table: continue
-            results.setdefault(cat, [])
-            for h4 in table.find_all('h4'):
-                a = h4.find('a')
-                if a and a.get('href'):
-                    results[cat].append((a.get_text(strip=True), a['href'], cnt)); cnt += 1
-        break
-    L(f"  Parsed results: {sum(len(v) for v in results.values())} items in {list(results.keys())}")
-    return results
-
-def get_episodes(url, log_fn=None):
-    L = log_fn or (lambda x: None)
-    links, page = [], 1
-    while True:
-        try:
-            resp = SESSION.get(f"{url}&page={page}", timeout=20)
-            L(f"    get_episodes page {page}: status={resp.status_code}")
-            if resp.status_code != 200:
-                L(f"    ERROR: {resp.text[:150]}...")
-                break
-            soup = BeautifulSoup(resp.content, "html.parser")
-        except Exception as e:
-            L(f"    get_episodes ERROR: {type(e).__name__}: {e}")
-            break
-        btns = soup.find_all('a', class_='btn btn-block')
-        if not btns:
-            L(f"    No episode buttons on page {page}")
-            break
-        links.extend([("https://fancaps.net" + b['href'] if b['href'].startswith('/') else b['href']) for b in btns if b.get('href')])
-        L(f"    Found {len(btns)} episodes on page {page}, total: {len(links)}")
-        page += 1
-    return links or [url]
-
-def get_frame_names(url, log_fn=None):
-    L = log_fn or (lambda x: None)
-    names, page = [], 1
-    while True:
-        try:
-            resp = SESSION.get(f"{url}&page={page}", timeout=20)
-            if resp.status_code != 200:
-                L(f"    get_frames page {page}: status={resp.status_code}")
-                break
-            soup = BeautifulSoup(resp.content, "html.parser")
-        except Exception as e:
-            L(f"    get_frames ERROR: {type(e).__name__}: {e}")
-            break
-        imgs = soup.find_all('img', class_='imageFade')
-        if not imgs:
-            if page == 1: L(f"    No images found on first page!")
-            break
-        names.extend([s.split('/')[-1] for i in imgs if (s := i.get('src')) and s.split('/')[-1] not in names])
-        pager = soup.select_one('ul.pagination li:last-child a')
-        if not pager or pager.get('href') in ['#', None]: break
-        page += 1
-    L(f"    Total frame names: {len(names)}")
-    return names
-
-def download(url, folder, name, timeout=10, retries=3):
-    """Download single image with retry - returns (success, status_code)."""
-    fp = os.path.join(folder, name)
-    if os.path.exists(fp): return True, 200
-    for attempt in range(retries):
-        try:
-            r = SESSION.get(url, stream=True, timeout=timeout)
-            if r.status_code == 200:
-                with open(fp, 'wb') as f:
-                    for chunk in r.iter_content(16384):
-                        if chunk: f.write(chunk)
-                return True, 200
-            if r.status_code == 429:  # Rate limit - don't retry immediately
-                return False, 429
-            # Other errors - retry
-        except:
-            pass
-        if attempt < retries - 1:
-            time.sleep(1)
-    return False, None
-
-def scrape(name, link, save_dir, max_imgs, progress=None, log_fn=None):
-    L = log_fn or print
-    url, folder = "https://fancaps.net" + link, os.path.join(save_dir, sanitize(name))
-    os.makedirs(folder, exist_ok=True)
-    section = 'movie' if '/movies/' in link else 'anime' if '/anime/' in link else 'tv'
-    L(f"  [2/8] Scraping: {url}")
-    L(f"    Section: {section}, max: {max_imgs}")
-    consecutive_429 = 0
-    max_429 = 3  # Abort after 3 consecutive 429s
-
-    if section == 'movie':
-        names = get_frame_names(url, log_fn=L)
-        L(f"    Movie frames: {len(names)}")
-        sampled = random.sample(names, min(max_imgs, len(names))) if names else []
-        downloaded = 0
-        for i, n in enumerate(sampled):
-            if consecutive_429 >= max_429:
-                L(f"    Aborting: {consecutive_429} consecutive 429s")
-                break
-            if i > 0: time.sleep(random.uniform(0.3, 0.8))  # Faster delay
-            try:
-                if progress and len(sampled) > 0: progress((i+1)/len(sampled), desc=f"Downloading {name[:20]}")
-            except: pass
-            success, status = download(f"https://cdni.fancaps.net/file/fancaps-{section}images/{n}", folder, n)
-            if success:
-                downloaded += 1
-                consecutive_429 = 0
-            elif status == 429:
-                consecutive_429 += 1
-                cooldown = 30 * consecutive_429
-                L(f"    429 rate limit ({consecutive_429}/{max_429}), cooling {cooldown}s...")
-                time.sleep(cooldown)
-            else:
-                consecutive_429 = 0
-        L(f"    Downloaded: {downloaded}/{len(sampled)}")
-    else:
-        L(f"    Fetching episodes...")
-        eps = get_episodes(url, log_fn=L)
-        L(f"    Episodes: {len(eps)}")
-        total = 0
-        per_ep = max(1, max_imgs // len(eps)) if eps else max_imgs
-        for i, ep in enumerate(eps):
-            if total >= max_imgs or consecutive_429 >= max_429: break
-            names = get_frame_names(ep, log_fn=L)
-            if not names: continue
-            ep_dir = os.path.join(folder, f"Ep{i+1}")
-            os.makedirs(ep_dir, exist_ok=True)
-            sampled = random.sample(names, min(per_ep, len(names), max_imgs - total))
-            for j, n in enumerate(sampled):
-                if consecutive_429 >= max_429: break
-                if j > 0: time.sleep(random.uniform(0.3, 0.8))  # Faster delay
-                try:
-                    if progress and max_imgs > 0: progress(total/max_imgs, desc=f"Ep{i+1}")
-                except: pass  # Gradio progress can fail in some contexts
-                success, status = download(f"https://cdni.fancaps.net/file/fancaps-{section}images/{n}", ep_dir, n)
-                if success:
-                    total += 1
-                    consecutive_429 = 0
-                elif status == 429:
-                    consecutive_429 += 1
-                    cooldown = 30 * consecutive_429
-                    L(f"    429 rate limit ({consecutive_429}/{max_429}), cooling {cooldown}s...")
-                    time.sleep(cooldown)
-                else:
-                    consecutive_429 = 0
-        L(f"    Total downloaded: {total}")
-
-# =============================================================================
-# ML MODELS (cached)
-# =============================================================================
-_models = {}
-
-def get_yolo():
-    if 'yolo' not in _models:
-        _models['yolo'] = torch.hub.load('ultralytics/yolov5', 'custom', path=str(YOLO_PATH), force_reload=False, verbose=False)
-        _models['yolo'].conf, _models['yolo'].iou = FACE_CONF, FACE_IOU
-    return _models['yolo']
-
-def get_sim():
-    if 'sim' not in _models:
-        class SiameseNetwork(nn.Module):
-            def __init__(self):
-                super().__init__()
-                self.base_model = models.efficientnet_b0(weights=models.EfficientNet_B0_Weights.DEFAULT)
-            def forward(self, x):
-                return self.base_model(x)  # 1000-class output (trained this way)
-        m = SiameseNetwork()
-        # Keep on CPU for consistent distance values across devices
-        m.load_state_dict(torch.load(str(SIM_PATH), map_location="cpu", weights_only=True))
-        m.eval()
-        _models['sim'] = m
-    return _models['sim']
-
-def get_tagger():
-    if 'tag' not in _models:
-        mp, cp = huggingface_hub.hf_hub_download(WD_REPO, "model.onnx"), huggingface_hub.hf_hub_download(WD_REPO, "selected_tags.csv")
-        tags = [str(x).replace('_', ' ') for x in pd.read_csv(cp)['name'].tolist()]
-        sess = rt.InferenceSession(mp, providers=['CPUExecutionProvider'])
-        _models['tag'] = (sess, tags, sess.get_inputs()[0].shape[1])
-    return _models['tag']
-
-# =============================================================================
-# PROCESSING
-# =============================================================================
-def dedup(paths, thresh=0.98):
-    if not paths: return [], []
-    m = models.resnet50(weights=models.ResNet50_Weights.IMAGENET1K_V1); m.fc = nn.Identity(); m.eval()
-    tf = transforms.Compose([transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize([.485,.456,.406],[.229,.224,.225])])
-    emb, valid = [], []
-    with torch.no_grad():
-        for i in range(0, len(paths), 32):
-            batch = [(tf(Image.open(p).convert('RGB')), p) for p in paths[i:i+32] if valid_img(p)]
-            if batch:
-                x = torch.stack([b[0] for b in batch])
-                emb.append(m(x).numpy()); valid.extend([b[1] for b in batch])
-    del m
-    if not emb: return [], []
-    emb = np.vstack(emb); emb /= np.linalg.norm(emb, axis=1, keepdims=True).clip(1e-8)
-    sim = emb @ emb.T; np.fill_diagonal(sim, 0)
-    keep, drop = [], set()
-    for i in range(len(valid)):
-        if i not in drop: keep.append(valid[i]); drop.update(j for j in np.where(sim[i] > thresh)[0] if j > i)
-    return keep, [valid[i] for i in drop]
-
-def detect_faces(paths, out_dir):
-    yolo = get_yolo(); os.makedirs(out_dir, exist_ok=True); cnt = 0
-    for p in paths:
-        try:
-            img = Image.open(p).convert('RGB'); w, h = img.size
-            for j, det in enumerate(yolo(img, size=640).xyxy[0].cpu().numpy()):
-                x1, y1, x2, y2, conf, _ = det
-                bw, bh = x2-x1, y2-y1
-                x1, y1, x2, y2 = max(0, x1-bw*CROP_PAD), max(0, y1-bh*CROP_PAD), min(w, x2+bw*CROP_PAD), min(h, y2+bh*CROP_PAD)
-                if min(x2-x1, y2-y1) >= MIN_FACE:
-                    img.crop((int(x1), int(y1), int(x2), int(y2))).save(os.path.join(out_dir, f"{Path(p).stem}-{j+1}-{conf:.2f}.jpg"), quality=95)
-                    cnt += 1
-        except: pass
-    return cnt
-
-def face_emb(paths):
-    if not paths: return np.array([]), []
-    m = get_sim()  # Always on CPU for consistent distances
-    def pad(img):
-        t, w, h = FACE_SZ, img.size[0], img.size[1]; r = w/h
-        nw, nh = (t, int(t/r)) if r > 1 else (int(t*r), t)
-        img = img.resize((nw, nh), Image.BICUBIC)
-        out = Image.new('RGB', (t, t), (0,0,0)); out.paste(img, ((t-nw)//2, (t-nh)//2)); return out
-    tf = transforms.Compose([lambda x: pad(x.convert('RGB') if x.mode == 'RGBA' else x), transforms.ToTensor()])
-    emb, valid = [], []
-    with torch.no_grad():
-        for i in range(0, len(paths), BATCH_SZ):
-            batch = [(tf(Image.open(p)), p) for p in paths[i:i+BATCH_SZ] if valid_img(p)]
-            if batch:
-                x = torch.stack([b[0] for b in batch])  # CPU tensor
-                emb.append(m(x).numpy())
-                valid.extend([b[1] for b in batch])
-    return (np.vstack(emb), valid) if emb else (np.array([]), [])
-
-def tag(path, act_tag="", char_tag=""):
-    sess, tags, sz = get_tagger()
-    img = Image.open(path).convert('RGB'); w, h = img.size
-    s = min(sz/w, sz/h); nw, nh = int(w*s), int(h*s)
-    pad = Image.new('RGB', (sz, sz), (255,255,255)); pad.paste(img.resize((nw, nh), Image.BICUBIC), ((sz-nw)//2, (sz-nh)//2))
-    probs = sess.run(None, {sess.get_inputs()[0].name: np.expand_dims(np.array(pad).astype(np.float32)[:,:,::-1], 0)})[0][0]
-    found = [tags[i] for i, p in enumerate(probs) if p > TAG_THRESH and tags[i] not in BLACKLIST]
-    # Prepend activation tag and character tag if provided
-    prefix = []
-    if act_tag: prefix.append(act_tag); found = [t for t in found if t != act_tag]
-    if char_tag: prefix.append(char_tag.replace("_", " ")); found = [t for t in found if t != char_tag.replace("_", " ")]
-    return prefix + found
-
-# =============================================================================
-# PIPELINE
-# =============================================================================
-def parse_direct_url(url):
-    """Parse direct fancaps URL, extract show name and relative link. Returns (name, link) or (None, None)."""
-    # Match patterns: showimages.php?ID-Name, MovieImages.php?movieid=ID&name=Name, episodeimages.php?ID-Name
-    patterns = [
-        r'fancaps\.net/anime/showimages\.php\?(\d+)-([^&/]+)',      # anime show
-        r'fancaps\.net/tv/showimages\.php\?(\d+)-([^&/]+)',         # tv show
-        r'fancaps\.net/movies/MovieImages\.php\?.*?movieid=(\d+)',  # movie
-    ]
-    for pat in patterns:
-        m = re.search(pat, url)
-        if m:
-            if 'anime' in url: section = 'anime'
-            elif 'movies' in url: section = 'movies'
-            else: section = 'tv'
-            # Extract name from URL (replace underscores with spaces)
-            name = m.group(2).replace('_', ' ') if len(m.groups()) > 1 else f"Show_{m.group(1)}"
-            # Build relative link (what scrape() expects)
-            if section == 'movies':
-                link = f"/movies/MovieImages.php?movieid={m.group(1)}"
-            else:
-                link = f"/{section}/showimages.php?{m.group(1)}-{m.group(2) if len(m.groups()) > 1 else ''}"
-            return name, link
-    return None, None
-
-def run(query, char, examples, max_img, thresh, act_tag, anime, movies, tv, progress=None, cli_mode=False):
-    log = []
-    def L(m): log.append(m); print(m)
-    def prog(val, desc=""):
-        if progress and not cli_mode: progress(val, desc=desc)
-
-    work = tempfile.mkdtemp(prefix="ds_")
-    dirs = {k: os.path.join(work, f"{i}_{k}") for i, k in enumerate(['scrapped','filtered','faces','ex_faces','similar','results'], 1)}
-    for d in dirs.values(): os.makedirs(d, exist_ok=True)
-    final_zip = None  # Track ZIP for cleanup
-
-    try:
-        L(f"HTTP client: {HTTP_CLIENT}")
-        t0 = time.time()
-
-        # Check if query is a direct fancaps URL (bypasses search, works on HF Spaces)
-        if 'fancaps.net' in query and ('showimages.php' in query or 'MovieImages.php' in query):
-            L(f"[1/8] Direct URL mode")
-            name, link = parse_direct_url(query)
-            if not link:
-                return None, "\n".join(log) + "\n\nCouldn't parse URL!"
-            item = (name, link, 1)
-            L(f"  Parsed: {name}")
-        else:
-            L(f"[1/8] Search: {query}")
-            prog(0.05, desc="Searching...")
-            res = search_fancaps(query, anime, movies, tv, log_fn=L)
-            if not res:
-                return None, "\n".join(log) + "\n\nSearch blocked! Use direct fancaps URL."
-            item = next((items[0] for items in res.values() if items), None)
-            if not item: return None, "No results!"
-
-        show_name = item[0]
-        if not char: char = sanitize(show_name)
-        t1 = time.time(); L(f"  Found: {show_name} ({t1-t0:.0f}s)"); prog(0.1, desc="Downloading...")
-
-        # [2/8] Scrape
-        scrape(item[0], item[1], dirs['scrapped'], max_img, progress if not cli_mode else None, log_fn=L)
-        imgs = get_imgs(dirs['scrapped'])
-        t2 = time.time(); L(f"[2/8] Downloaded: {len(imgs)} ({t2-t1:.0f}s)")
-        if not imgs: return None, "No images downloaded!"
-
-        # [3/8] Dedup
-        prog(0.3, desc="Dedup...")
-        imgs = [p for p in imgs if valid_img(p)]
-        kept, rm = dedup(imgs)
-        for p in kept: shutil.copy(p, os.path.join(dirs['filtered'], os.path.basename(p)))
-        t3 = time.time(); L(f"[3/8] Dedup: {len(kept)} kept, -{len(rm)} ({t3-t2:.0f}s)")
-
-        # [4/8] Detect faces
-        prog(0.4, desc="Faces...")
-        n = detect_faces(get_imgs(dirs['filtered']), dirs['faces'])
-        t4 = time.time(); L(f"[4/8] Faces: {n} ({t4-t3:.0f}s)")
-        if n == 0: return None, "No faces detected!"
-
-        # [5/8] Process examples
-        prog(0.5, desc="Examples...")
-        ex_paths = [p for p in (examples or []) if p and os.path.exists(p)]
-        if not ex_paths: ex_paths = [p for p in EXAMPLES if os.path.exists(p)]
-        if not ex_paths: return None, "No example images!"
-        n_ex = detect_faces(ex_paths, dirs['ex_faces'])
-        t5 = time.time(); L(f"[5/8] Examples: {len(ex_paths)} imgs -> {n_ex} faces ({t5-t4:.0f}s)")
-        if n_ex == 0: return None, "No faces in examples!"
-
-        # [6/8] Match
-        prog(0.6, desc="Matching...")
-        f_emb, f_valid = face_emb(get_imgs(dirs['faces']))
-        e_emb, _ = face_emb(get_imgs(dirs['ex_faces']))
-        dists = pairwise_distances(f_emb, e_emb, metric='euclidean').min(axis=1)
-        similar_idx = np.where(dists < thresh)[0]
-        similar = [f_valid[i] for i in similar_idx]
-        similar_dists = dists[similar_idx]
-        t6 = time.time()
-        L(f"[6/8] Matches: {len(similar)} (thresh={thresh}) ({t6-t5:.0f}s)")
-        if len(similar_dists) > 0:
-            L(f"  Distances: min={similar_dists.min():.1f}, max={similar_dists.max():.1f}, mean={similar_dists.mean():.1f}")
-        if not similar: return None, f"No matches! Try threshold > {thresh}"
-
-        # [7/8] Get originals
-        prog(0.7, desc="Collect...")
-        origs = set()
-        orig_to_dist = {}
-        for i, fp in enumerate(similar):
-            parts = os.path.basename(fp).rsplit('-', 2)
-            base = parts[0] if len(parts) >= 3 else Path(fp).stem
-            for ext in EXTS:
-                op = os.path.join(dirs['filtered'], base + ext)
-                if os.path.exists(op):
-                    origs.add(op)
-                    orig_to_dist[os.path.basename(op)] = similar_dists[i]
-                    break
-        res_dir = os.path.join(work, f"results_{sanitize(char)}")
-        os.makedirs(res_dir, exist_ok=True)
-        for p in origs: shutil.copy(p, os.path.join(res_dir, os.path.basename(p)))
-        t7 = time.time(); L(f"[7/8] Collected: {len(origs)} ({t7-t6:.0f}s)")
-
-        # [8/8] Tag
-        prog(0.8, desc="Tagging...")
-        char_tag = char if char != sanitize(show_name) else ""
-        for p in get_imgs(res_dir):
-            tags = tag(p, act_tag, char_tag)
-            with open(os.path.splitext(p)[0] + ".txt", 'w') as f: f.write(", ".join(tags))
-        t8 = time.time(); L(f"[8/8] Tagged: {len(origs)} ({t8-t7:.0f}s)")
-
-        # Log each image with distance
-        L(f"\nResults (distance to ref):")
-        for name, d in sorted(orig_to_dist.items(), key=lambda x: x[1]):
-            L(f"  {name}: {d:.1f}")
-
-        # Zip
-        prog(0.95, desc="Zipping...")
-        zp = os.path.join(work, f"{sanitize(char)}_dataset.zip")
-        with zipfile.ZipFile(zp, 'w', zipfile.ZIP_DEFLATED) as z:
-            for p in get_imgs(res_dir) + [os.path.splitext(p)[0]+".txt" for p in get_imgs(res_dir)]:
-                if os.path.exists(p): z.write(p, os.path.basename(p))
-        # Copy ZIP to persistent temp location (Gradio needs file to exist after return)
-        final_zip = tempfile.NamedTemporaryFile(delete=False, suffix=".zip", prefix=f"{sanitize(char)}_").name
-        shutil.copy(zp, final_zip)
-        L(f"\nDone! {len(origs)} images, total {t8-t0:.0f}s"); prog(1.0, desc="Complete!")
-        return final_zip, "\n".join(log)
-    except Exception as e:
-        import traceback
-        return None, "\n".join(log) + f"\n\nERROR: {e}\n{traceback.format_exc()}"
-    finally:
-        # Clean up work directory (ZIP already copied out)
-        if os.path.exists(work):
-            shutil.rmtree(work, ignore_errors=True)
-
-# =============================================================================
-# UI
-# =============================================================================
-css = """
-.gradio-container { padding-top: 10px !important; }
-.compact-group { margin-bottom: 8px !important; }
-"""
-
-with gr.Blocks(title="SD Dataset Automaker: Fancaps → Face Crop (YOLO) → Similarity (Siamese) → WD Tagger → ZIP") as demo:
-    gr.Markdown("### SD Dataset Automaker: Fancaps → Face Crop (YOLO) → Similarity (Siamese) → WD Tagger → ZIP")
-
-    with gr.Row():
-        with gr.Column(scale=3):
-            # Compact input group
-            with gr.Group():
-                with gr.Row():
-                    query = gr.Textbox(
-                        label="Anime / Movie / Fancaps URL",
-                        placeholder="'Cowboy Bebop' or paste URL",
-                        scale=3
-                    )
-                    char = gr.Textbox(label="Character (optional, appends to tags)", placeholder="spike_spiegel", scale=2)
-
-            with gr.Row():
-                ref_imgs = gr.Gallery(
-                    label="Reference Face Image(s)",
-                    columns=4,
-                    height=100,
-                    interactive=True,
-                    object_fit="scale-down",
-                    scale=3,
-                )
-                run_btn = gr.Button("Generate Dataset", variant="primary", size="lg", scale=1)
-
-            # Hidden File input for MCP compatibility (Gallery $ref schema bug persists in Gradio 6.0.1)
-            ref_files = gr.File(
-                label="Reference Images (MCP)",
-                file_count="multiple",
-                file_types=["image"],
-                visible=False,
-            )
-
-            # gr.Examples + gr.Gallery works in Gradio 5.46.0+ (PR #11787)
-            gr.Examples(
-                examples=[
-                    ["https://fancaps.net/anime/showimages.php?3092-Cowboy_Bebop", "spike_spiegel", EXAMPLES],
-                ],
-                inputs=[query, char, ref_imgs],
-                label="Example (click to load)",
-            )
-
-            # Advanced settings in accordion
-            with gr.Accordion("Advanced Settings", open=False):
-                with gr.Row():
-                    max_img = gr.Slider(50, 500, 200, step=50, label="Max Downloads (frames to scrape)")
-                    thresh = gr.Slider(20, 60, 32, step=1, label="Face Similarity (lower=stricter)")
-                with gr.Row():
-                    act_tag = gr.Textbox(label="Trigger Word (prepends to captions)", placeholder="e.g. sks_style", scale=2)
-                    anime_cb = gr.Checkbox(label="Anime", value=True, scale=1)
-                    movies_cb = gr.Checkbox(label="Movies", scale=1)
-                    tv_cb = gr.Checkbox(label="TV", scale=1)
-
-        with gr.Column(scale=1):
-            out_file = gr.File(label="Download ZIP", interactive=False)
-            with gr.Accordion("Log", open=True):
-                out_log = gr.Textbox(label="", lines=12, max_lines=50, show_label=False, autoscroll=False)
-            gr.Markdown("*CPU: ~5-10 min/run*")
-
-    def process(q, c, imgs, files, mi, th, at, an, mo, tv, prog=gr.Progress()):
-        if not q:
-            gr.Warning("Enter anime name or URL")
-            return None, ""
-        # Collect paths from Gallery (imgs) or File input (files)
-        paths = []
-        for item in (imgs or []):
-            p = item[0] if isinstance(item, (list, tuple)) else item
-            if p and os.path.exists(p): paths.append(p)
-        if not paths and files:
-            for f in (files if isinstance(files, list) else [files]):
-                fp = f.name if hasattr(f, 'name') else str(f)
-                if fp and os.path.exists(fp): paths.append(fp)
-        if not paths:
-            gr.Warning("Upload reference images or click 'Load Example'")
-            return None, ""
-        if 'fancaps.net' in q:
-            gr.Info("Direct URL detected")
-        else:
-            gr.Info(f"Searching: {q}")
-        zp, log = run(q, c, paths, mi, th, at, an, mo, tv, prog)
-        if zp:
-            gr.Info("Done!")
-        return zp, log
-
-    run_btn.click(process, [query, char, ref_imgs, ref_files, max_img, thresh, act_tag, anime_cb, movies_cb, tv_cb], [out_file, out_log])
-
-# MCP-only function (no Gallery - avoids $ref schema bug)
-def create_dataset(url: str, character: str, reference_images, max_images: int = 200, similarity_threshold: int = 32, trigger_tag: str = ""):
-    """Create anime character dataset for LoRA/SD training.
-
-    Args:
-        url: Fancaps.net URL (e.g. https://fancaps.net/anime/showimages.php?3092-Cowboy_Bebop)
-        character: Character name for tagging (e.g. spike_spiegel)
-        reference_images: 1-5 face reference images of target character
-        max_images: Maximum frames to download (50-500)
-        similarity_threshold: Face matching threshold, lower=stricter (25-40, default 32)
-        trigger_tag: Optional trigger word to prepend to captions
-
-    Returns:
-        Tuple of (zip_file_path, log_text)
-    """
-    paths = []
-    if reference_images:
-        for f in (reference_images if isinstance(reference_images, list) else [reference_images]):
-            fp = f.name if hasattr(f, 'name') else str(f)
-            if fp and os.path.exists(fp): paths.append(fp)
-    if not paths:
-        return None, "ERROR: No reference images provided. Upload 1-5 face images."
-    if not url or 'fancaps.net' not in url:
-        return None, "ERROR: Invalid URL. Provide a fancaps.net URL."
-    zp, log = run(url, character, paths, max_images, similarity_threshold, trigger_tag, True, False, False, None)
-    return zp, log
-
-# Create MCP interface (File input only - no Gallery)
-mcp_interface = gr.Interface(
-    fn=create_dataset,
-    inputs=[
-        gr.Textbox(label="Fancaps URL"),
-        gr.Textbox(label="Character Name"),
-        gr.File(label="Reference Images", file_count="multiple", file_types=["image"]),
-        gr.Slider(50, 500, 200, step=50, label="Max Images"),
-        gr.Slider(25, 40, 32, step=1, label="Similarity Threshold"),
-        gr.Textbox(label="Trigger Tag"),
-    ],
-    outputs=[gr.File(label="Dataset ZIP"), gr.Textbox(label="Log")],
-    api_name="create_dataset",
-    title="SD Dataset Automaker (MCP)",
-)
-
-def run_cli():
-    """CLI mode with cloudscraper for Cloudflare bypass"""
-    # Use cloudscraper for CLI (bypasses Cloudflare on local/residential IPs)
-    init_session(use_cloudscraper=True)
-
-    parser = argparse.ArgumentParser(description="SD Dataset Automaker - Anime character dataset generator")
-    parser.add_argument("--title", "-t", required=True, help="Anime name or fancaps.net URL")
-    parser.add_argument("--image", "-i", nargs="+", required=True, help="Reference face images (1-5)")
-    parser.add_argument("--char", "-c", default="", help="Character name (optional, appends to tags)")
-    parser.add_argument("--max", "-m", type=int, default=200, help="Max frames to scrape (default: 200)")
-    parser.add_argument("--thresh", type=float, default=32.0, help="Face similarity threshold, lower=stricter (default: 32)")
-    parser.add_argument("--tag", default="", help="Trigger word to prepend to captions")
-    parser.add_argument("--anime", action="store_true", default=True, help="Search anime (default)")
-    parser.add_argument("--movies", action="store_true", help="Search movies")
-    parser.add_argument("--tv", action="store_true", help="Search TV")
-    parser.add_argument("--output", "-o", default=".", help="Output directory (default: current)")
-    args = parser.parse_args()
-
-    # Validate images
-    ref_imgs = [p for p in args.image if os.path.exists(p)]
-    if not ref_imgs:
-        print(f"ERROR: No valid reference images found: {args.image}")
-        sys.exit(1)
-
-    print(f"SD Dataset Automaker - CLI Mode")
-    print(f"  Title: {args.title}")
-    print(f"  Refs:  {len(ref_imgs)} images")
-    print(f"  Char:  {args.char or '(auto from title)'}")
-    print()
-
-    zp, log = run(
-        query=args.title,
-        char=args.char,
-        examples=ref_imgs,
-        max_img=args.max,
-        thresh=args.thresh,
-        act_tag=args.tag,
-        anime=args.anime,
-        movies=args.movies,
-        tv=args.tv,
-        cli_mode=True
-    )
-
-    if zp:
-        # Copy to output dir
-        out_path = os.path.join(args.output, os.path.basename(zp))
-        shutil.copy(zp, out_path)
-        print(f"\nSaved: {out_path}")
-    else:
-        print(f"\nFailed!")
-        sys.exit(1)
-
-if __name__ == "__main__":
-    # CLI mode if args provided, else Gradio UI
-    if len(sys.argv) > 1:
-        run_cli()
-    else:
-        # Gradio UI mode - combine main UI and MCP interface
-        allowed_dir = os.path.dirname(os.path.abspath(__file__))
-        app = gr.TabbedInterface(
-            [demo, mcp_interface],
-            ["Dataset Maker", "MCP API"],
-            title="SD Dataset Automaker"
-        )
-        app.launch(
-            server_name="0.0.0.0",
-            server_port=7860,
-            mcp_server=True,
-            show_error=True,
-            allowed_paths=[allowed_dir],
-        )
+#!/usr/bin/env python3
+"""SD Dataset Automaker - HF Space (CPU) - Anime character dataset generator for LoRA/fine-tuning."""
+
+import warnings
+warnings.filterwarnings('ignore', category=FutureWarning)  # Suppress torch.cuda.amp spam
+warnings.filterwarnings('ignore', category=UserWarning, message='.*trust_repo.*')
+
+import os, re, shutil, zipfile, random, tempfile, argparse, sys
+from urllib.parse import quote_plus
+from collections import Counter
+from pathlib import Path
+
+from tqdm import tqdm
+import gradio as gr
+from bs4 import BeautifulSoup
+import requests as req_lib
+import time
+import numpy as np
+from PIL import Image
+import torch
+import torch.nn as nn
+from torchvision import models, transforms
+from sklearn.metrics.pairwise import pairwise_distances
+import onnxruntime as rt
+import pandas as pd
+import huggingface_hub
+
+# =============================================================================
+# CONFIG
+# =============================================================================
+EXTS = ('.jpg', '.jpeg', '.png')
+MODEL_DIR = Path(__file__).parent.resolve()  # Ensure absolute path
+YOLO_PATH = MODEL_DIR / "yolov5s_anime.pt"
+SIM_PATH = MODEL_DIR / "similarity.pt"
+EXAMPLES = [str(MODEL_DIR / f"from_url_spike_spiegel{i}.jpg") for i in range(1, 4)]  # absolute paths for gr.Examples
+WD_REPO = "SmilingWolf/wd-swinv2-tagger-v3"
+TAG_THRESH, BLACKLIST = 0.35, ["bangs", "breasts", "multicolored hair", "gradient hair", "two-tone hair", "virtual youtuber"]
+FACE_CONF, FACE_IOU, MIN_FACE, CROP_PAD = 0.5, 0.5, 35, 0.2
+SIM_THRESH, BATCH_SZ, FACE_SZ = 32, 16, 224
+
+# =============================================================================
+# UTILS
+# =============================================================================
+sanitize = lambda s: re.sub(r'[^\w.-]', '', s.replace(" ", "_"))
+get_imgs = lambda d: sorted([os.path.join(r,f) for r,_,fs in os.walk(d) for f in fs if f.lower().endswith(EXTS)])
+valid_img = lambda p: (lambda i: i.load() or True)(Image.open(p)) if os.path.exists(p) else False
+
+# HTTP session - mode depends on environment
+# CLI (local Windows): cloudscraper bypasses Cloudflare
+# HF Spaces: plain requests (cloudscraper fingerprint gets blocked on datacenter IPs)
+def init_session(use_cloudscraper=False):
+    global SESSION, HTTP_CLIENT
+    if use_cloudscraper:
+        try:
+            import cloudscraper
+            SESSION = cloudscraper.create_scraper()
+            HTTP_CLIENT = "cloudscraper"
+            return
+        except ImportError:
+            pass  # fallback to requests
+    SESSION = req_lib.Session()
+    SESSION.headers.update({
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.5',
+        'Accept-Encoding': 'gzip, deflate, br',
+        'DNT': '1',
+        'Connection': 'keep-alive',
+        'Upgrade-Insecure-Requests': '1',
+        'Sec-Fetch-Dest': 'document',
+        'Sec-Fetch-Mode': 'navigate',
+        'Sec-Fetch-Site': 'none',
+        'Sec-Fetch-User': '?1',
+        'Cache-Control': 'max-age=0',
+        'Referer': 'https://fancaps.net/',
+    })
+    HTTP_CLIENT = "requests/browser-headers"
+
+# Default: plain requests (for HF Spaces import)
+SESSION = None
+HTTP_CLIENT = None
+init_session(use_cloudscraper=False)
+
+# =============================================================================
+# SCRAPING
+# =============================================================================
+def search_fancaps(prompt, anime=True, movies=False, tv=False, log_fn=None):
+    L = log_fn or print
+    url = f"https://fancaps.net/search.php?q={quote_plus(prompt)}&submit=Submit"
+    if anime: url += "&animeCB=Anime"
+    if movies: url += "&MoviesCB=Movies"
+    if tv: url += "&TVCB=TV"
+    L(f"  URL: {url}")
+    try:
+        resp = SESSION.get(url, timeout=30)
+        L(f"  Status: {resp.status_code}, Size: {len(resp.content)} bytes")
+        # Log key headers for debugging
+        cf_ray = resp.headers.get('cf-ray', 'none')
+        server = resp.headers.get('server', 'unknown')
+        L(f"  Server: {server}, CF-Ray: {cf_ray}")
+        if resp.status_code != 200:
+            L(f"  ERROR: HTTP {resp.status_code}")
+            # Show snippet of response to understand the block reason
+            content_snippet = resp.text[:500].replace('\n', ' ').strip()
+            L(f"  Response snippet: {content_snippet[:200]}...")
+            return {}
+        soup = BeautifulSoup(resp.content, "html.parser")
+        # Debug: check if we got Cloudflare challenge
+        title = soup.title.string if soup.title else "No title"
+        L(f"  Page title: {title[:50]}")
+        if "cloudflare" in title.lower() or "challenge" in title.lower():
+            L("  ERROR: Cloudflare challenge detected!")
+            return {}
+    except Exception as e:
+        L(f"  ERROR: {type(e).__name__}: {e}")
+        return {}
+    results, cnt = {}, 1
+    divs = soup.find_all("div", class_="single_post_content")
+    L(f"  Found {len(divs)} content divs")
+    for div in divs:
+        if not div.find('h2'): continue
+        for h2 in div.find_all('h2'):
+            title = h2.get_text(strip=True).lower()
+            cat = 'Movies' if 'movie' in title else 'TV' if 'tv' in title else 'Anime' if 'anime' in title else None
+            if not cat: continue
+            table = h2.find_next('table')
+            if not table: continue
+            results.setdefault(cat, [])
+            for h4 in table.find_all('h4'):
+                a = h4.find('a')
+                if a and a.get('href'):
+                    results[cat].append((a.get_text(strip=True), a['href'], cnt)); cnt += 1
+        break
+    L(f"  Parsed results: {sum(len(v) for v in results.values())} items in {list(results.keys())}")
+    return results
+
+def get_episodes(url, log_fn=None):
+    L = log_fn or (lambda x: None)
+    links, page = [], 1
+    while True:
+        try:
+            resp = SESSION.get(f"{url}&page={page}", timeout=20)
+            L(f"    get_episodes page {page}: status={resp.status_code}")
+            if resp.status_code != 200:
+                L(f"    ERROR: {resp.text[:150]}...")
+                break
+            soup = BeautifulSoup(resp.content, "html.parser")
+        except Exception as e:
+            L(f"    get_episodes ERROR: {type(e).__name__}: {e}")
+            break
+        btns = soup.find_all('a', class_='btn btn-block')
+        if not btns:
+            L(f"    No episode buttons on page {page}")
+            break
+        links.extend([("https://fancaps.net" + b['href'] if b['href'].startswith('/') else b['href']) for b in btns if b.get('href')])
+        L(f"    Found {len(btns)} episodes on page {page}, total: {len(links)}")
+        page += 1
+    return links or [url]
+
+def get_frame_names(url, log_fn=None):
+    L = log_fn or (lambda x: None)
+    names, page = [], 1
+    while True:
+        try:
+            resp = SESSION.get(f"{url}&page={page}", timeout=20)
+            if resp.status_code != 200:
+                L(f"    get_frames page {page}: status={resp.status_code}")
+                break
+            soup = BeautifulSoup(resp.content, "html.parser")
+        except Exception as e:
+            L(f"    get_frames ERROR: {type(e).__name__}: {e}")
+            break
+        imgs = soup.find_all('img', class_='imageFade')
+        if not imgs:
+            if page == 1: L(f"    No images found on first page!")
+            break
+        names.extend([s.split('/')[-1] for i in imgs if (s := i.get('src')) and s.split('/')[-1] not in names])
+        pager = soup.select_one('ul.pagination li:last-child a')
+        if not pager or pager.get('href') in ['#', None]: break
+        page += 1
+    L(f"    Total frame names: {len(names)}")
+    return names
+
+def download(url, folder, name, timeout=10, retries=3):
+    """Download single image with retry - returns (success, status_code)."""
+    fp = os.path.join(folder, name)
+    if os.path.exists(fp): return True, 200
+    for attempt in range(retries):
+        try:
+            r = SESSION.get(url, stream=True, timeout=timeout)
+            if r.status_code == 200:
+                with open(fp, 'wb') as f:
+                    for chunk in r.iter_content(16384):
+                        if chunk: f.write(chunk)
+                return True, 200
+            if r.status_code == 429:  # Rate limit - don't retry immediately
+                return False, 429
+            # Other errors - retry
+        except:
+            pass
+        if attempt < retries - 1:
+            time.sleep(1)
+    return False, None
+
+def scrape(name, link, save_dir, max_imgs, progress=None, log_fn=None):
+    L = log_fn or print
+    url, folder = "https://fancaps.net" + link, os.path.join(save_dir, sanitize(name))
+    os.makedirs(folder, exist_ok=True)
+    section = 'movie' if '/movies/' in link else 'anime' if '/anime/' in link else 'tv'
+    L(f"  [2/8] Scraping: {url}")
+    L(f"    Section: {section}, max: {max_imgs}")
+    consecutive_429 = 0
+    max_429 = 3  # Abort after 3 consecutive 429s
+
+    if section == 'movie':
+        names = get_frame_names(url, log_fn=L)
+        L(f"    Movie frames: {len(names)}")
+        sampled = random.sample(names, min(max_imgs, len(names))) if names else []
+        downloaded = 0
+        for i, n in enumerate(sampled):
+            if consecutive_429 >= max_429:
+                L(f"    Aborting: {consecutive_429} consecutive 429s")
+                break
+            if i > 0: time.sleep(random.uniform(0.3, 0.8))  # Faster delay
+            try:
+                if progress and len(sampled) > 0: progress((i+1)/len(sampled), desc=f"Downloading {name[:20]}")
+            except: pass
+            success, status = download(f"https://cdni.fancaps.net/file/fancaps-{section}images/{n}", folder, n)
+            if success:
+                downloaded += 1
+                consecutive_429 = 0
+            elif status == 429:
+                consecutive_429 += 1
+                cooldown = 30 * consecutive_429
+                L(f"    429 rate limit ({consecutive_429}/{max_429}), cooling {cooldown}s...")
+                time.sleep(cooldown)
+            else:
+                consecutive_429 = 0
+        L(f"    Downloaded: {downloaded}/{len(sampled)}")
+    else:
+        L(f"    Fetching episodes...")
+        eps = get_episodes(url, log_fn=L)
+        L(f"    Episodes: {len(eps)}")
+        total = 0
+        per_ep = max(1, max_imgs // len(eps)) if eps else max_imgs
+        for i, ep in enumerate(eps):
+            if total >= max_imgs or consecutive_429 >= max_429: break
+            names = get_frame_names(ep, log_fn=L)
+            if not names: continue
+            ep_dir = os.path.join(folder, f"Ep{i+1}")
+            os.makedirs(ep_dir, exist_ok=True)
+            sampled = random.sample(names, min(per_ep, len(names), max_imgs - total))
+            for j, n in enumerate(sampled):
+                if consecutive_429 >= max_429: break
+                if j > 0: time.sleep(random.uniform(0.3, 0.8))  # Faster delay
+                try:
+                    if progress and max_imgs > 0: progress(total/max_imgs, desc=f"Ep{i+1}")
+                except: pass  # Gradio progress can fail in some contexts
+                success, status = download(f"https://cdni.fancaps.net/file/fancaps-{section}images/{n}", ep_dir, n)
+                if success:
+                    total += 1
+                    consecutive_429 = 0
+                elif status == 429:
+                    consecutive_429 += 1
+                    cooldown = 30 * consecutive_429
+                    L(f"    429 rate limit ({consecutive_429}/{max_429}), cooling {cooldown}s...")
+                    time.sleep(cooldown)
+                else:
+                    consecutive_429 = 0
+        L(f"    Total downloaded: {total}")
+
+# =============================================================================
+# ML MODELS (cached)
+# =============================================================================
+_models = {}
+
+def get_yolo():
+    if 'yolo' not in _models:
+        _models['yolo'] = torch.hub.load('ultralytics/yolov5', 'custom', path=str(YOLO_PATH), force_reload=False, verbose=False)
+        _models['yolo'].conf, _models['yolo'].iou = FACE_CONF, FACE_IOU
+    return _models['yolo']
+
+def get_sim():
+    if 'sim' not in _models:
+        class SiameseNetwork(nn.Module):
+            def __init__(self):
+                super().__init__()
+                self.base_model = models.efficientnet_b0(weights=models.EfficientNet_B0_Weights.DEFAULT)
+            def forward(self, x):
+                return self.base_model(x)  # 1000-class output (trained this way)
+        m = SiameseNetwork()
+        # Keep on CPU for consistent distance values across devices
+        m.load_state_dict(torch.load(str(SIM_PATH), map_location="cpu", weights_only=True))
+        m.eval()
+        _models['sim'] = m
+    return _models['sim']
+
+def get_tagger():
+    if 'tag' not in _models:
+        mp, cp = huggingface_hub.hf_hub_download(WD_REPO, "model.onnx"), huggingface_hub.hf_hub_download(WD_REPO, "selected_tags.csv")
+        tags = [str(x).replace('_', ' ') for x in pd.read_csv(cp)['name'].tolist()]
+        sess = rt.InferenceSession(mp, providers=['CPUExecutionProvider'])
+        _models['tag'] = (sess, tags, sess.get_inputs()[0].shape[1])
+    return _models['tag']
+
+# =============================================================================
+# PROCESSING
+# =============================================================================
+def dedup(paths, thresh=0.98):
+    if not paths: return [], []
+    m = models.resnet50(weights=models.ResNet50_Weights.IMAGENET1K_V1); m.fc = nn.Identity(); m.eval()
+    tf = transforms.Compose([transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize([.485,.456,.406],[.229,.224,.225])])
+    emb, valid = [], []
+    with torch.no_grad():
+        for i in range(0, len(paths), 32):
+            batch = [(tf(Image.open(p).convert('RGB')), p) for p in paths[i:i+32] if valid_img(p)]
+            if batch:
+                x = torch.stack([b[0] for b in batch])
+                emb.append(m(x).numpy()); valid.extend([b[1] for b in batch])
+    del m
+    if not emb: return [], []
+    emb = np.vstack(emb); emb /= np.linalg.norm(emb, axis=1, keepdims=True).clip(1e-8)
+    sim = emb @ emb.T; np.fill_diagonal(sim, 0)
+    keep, drop = [], set()
+    for i in range(len(valid)):
+        if i not in drop: keep.append(valid[i]); drop.update(j for j in np.where(sim[i] > thresh)[0] if j > i)
+    return keep, [valid[i] for i in drop]
+
+def detect_faces(paths, out_dir):
+    yolo = get_yolo(); os.makedirs(out_dir, exist_ok=True); cnt = 0
+    for p in paths:
+        try:
+            img = Image.open(p).convert('RGB'); w, h = img.size
+            for j, det in enumerate(yolo(img, size=640).xyxy[0].cpu().numpy()):
+                x1, y1, x2, y2, conf, _ = det
+                bw, bh = x2-x1, y2-y1
+                x1, y1, x2, y2 = max(0, x1-bw*CROP_PAD), max(0, y1-bh*CROP_PAD), min(w, x2+bw*CROP_PAD), min(h, y2+bh*CROP_PAD)
+                if min(x2-x1, y2-y1) >= MIN_FACE:
+                    img.crop((int(x1), int(y1), int(x2), int(y2))).save(os.path.join(out_dir, f"{Path(p).stem}-{j+1}-{conf:.2f}.jpg"), quality=95)
+                    cnt += 1
+        except: pass
+    return cnt
+
+def face_emb(paths):
+    if not paths: return np.array([]), []
+    m = get_sim()  # Always on CPU for consistent distances
+    def pad(img):
+        t, w, h = FACE_SZ, img.size[0], img.size[1]; r = w/h
+        nw, nh = (t, int(t/r)) if r > 1 else (int(t*r), t)
+        img = img.resize((nw, nh), Image.BICUBIC)
+        out = Image.new('RGB', (t, t), (0,0,0)); out.paste(img, ((t-nw)//2, (t-nh)//2)); return out
+    tf = transforms.Compose([lambda x: pad(x.convert('RGB') if x.mode == 'RGBA' else x), transforms.ToTensor()])
+    emb, valid = [], []
+    with torch.no_grad():
+        for i in range(0, len(paths), BATCH_SZ):
+            batch = [(tf(Image.open(p)), p) for p in paths[i:i+BATCH_SZ] if valid_img(p)]
+            if batch:
+                x = torch.stack([b[0] for b in batch])  # CPU tensor
+                emb.append(m(x).numpy())
+                valid.extend([b[1] for b in batch])
+    return (np.vstack(emb), valid) if emb else (np.array([]), [])
+
+def tag(path, act_tag="", char_tag=""):
+    sess, tags, sz = get_tagger()
+    img = Image.open(path).convert('RGB'); w, h = img.size
+    s = min(sz/w, sz/h); nw, nh = int(w*s), int(h*s)
+    pad = Image.new('RGB', (sz, sz), (255,255,255)); pad.paste(img.resize((nw, nh), Image.BICUBIC), ((sz-nw)//2, (sz-nh)//2))
+    probs = sess.run(None, {sess.get_inputs()[0].name: np.expand_dims(np.array(pad).astype(np.float32)[:,:,::-1], 0)})[0][0]
+    found = [tags[i] for i, p in enumerate(probs) if p > TAG_THRESH and tags[i] not in BLACKLIST]
+    # Prepend activation tag and character tag if provided
+    prefix = []
+    if act_tag: prefix.append(act_tag); found = [t for t in found if t != act_tag]
+    if char_tag: prefix.append(char_tag.replace("_", " ")); found = [t for t in found if t != char_tag.replace("_", " ")]
+    return prefix + found
+
+# =============================================================================
+# PIPELINE
+# =============================================================================
+def parse_direct_url(url):
+    """Parse direct fancaps URL, extract show name and relative link. Returns (name, link) or (None, None)."""
+    # Match patterns: showimages.php?ID-Name, MovieImages.php?movieid=ID&name=Name, episodeimages.php?ID-Name
+    patterns = [
+        r'fancaps\.net/anime/showimages\.php\?(\d+)-([^&/]+)',      # anime show
+        r'fancaps\.net/tv/showimages\.php\?(\d+)-([^&/]+)',         # tv show
+        r'fancaps\.net/movies/MovieImages\.php\?.*?movieid=(\d+)',  # movie
+    ]
+    for pat in patterns:
+        m = re.search(pat, url)
+        if m:
+            if 'anime' in url: section = 'anime'
+            elif 'movies' in url: section = 'movies'
+            else: section = 'tv'
+            # Extract name from URL (replace underscores with spaces)
+            name = m.group(2).replace('_', ' ') if len(m.groups()) > 1 else f"Show_{m.group(1)}"
+            # Build relative link (what scrape() expects)
+            if section == 'movies':
+                link = f"/movies/MovieImages.php?movieid={m.group(1)}"
+            else:
+                link = f"/{section}/showimages.php?{m.group(1)}-{m.group(2) if len(m.groups()) > 1 else ''}"
+            return name, link
+    return None, None
+
+def run(query, char, examples, max_img, thresh, act_tag, anime, movies, tv, progress=None, cli_mode=False):
+    log = []
+    def L(m): log.append(m); print(m)
+    def prog(val, desc=""):
+        if progress and not cli_mode: progress(val, desc=desc)
+
+    work = tempfile.mkdtemp(prefix="ds_")
+    dirs = {k: os.path.join(work, f"{i}_{k}") for i, k in enumerate(['scrapped','filtered','faces','ex_faces','similar','results'], 1)}
+    for d in dirs.values(): os.makedirs(d, exist_ok=True)
+    final_zip = None  # Track ZIP for cleanup
+
+    try:
+        L(f"HTTP client: {HTTP_CLIENT}")
+        t0 = time.time()
+
+        # Check if query is a direct fancaps URL (bypasses search, works on HF Spaces)
+        if 'fancaps.net' in query and ('showimages.php' in query or 'MovieImages.php' in query):
+            L(f"[1/8] Direct URL mode")
+            name, link = parse_direct_url(query)
+            if not link:
+                return None, "\n".join(log) + "\n\nCouldn't parse URL!"
+            item = (name, link, 1)
+            L(f"  Parsed: {name}")
+        else:
+            L(f"[1/8] Search: {query}")
+            prog(0.05, desc="Searching...")
+            res = search_fancaps(query, anime, movies, tv, log_fn=L)
+            if not res:
+                return None, "\n".join(log) + "\n\nSearch blocked! Use direct fancaps URL."
+            item = next((items[0] for items in res.values() if items), None)
+            if not item: return None, "No results!"
+
+        show_name = item[0]
+        if not char: char = sanitize(show_name)
+        t1 = time.time(); L(f"  Found: {show_name} ({t1-t0:.0f}s)"); prog(0.1, desc="Downloading...")
+
+        # [2/8] Scrape
+        scrape(item[0], item[1], dirs['scrapped'], max_img, progress if not cli_mode else None, log_fn=L)
+        imgs = get_imgs(dirs['scrapped'])
+        t2 = time.time(); L(f"[2/8] Downloaded: {len(imgs)} ({t2-t1:.0f}s)")
+        if not imgs: return None, "No images downloaded!"
+
+        # [3/8] Dedup
+        prog(0.3, desc="Dedup...")
+        imgs = [p for p in imgs if valid_img(p)]
+        kept, rm = dedup(imgs)
+        for p in kept: shutil.copy(p, os.path.join(dirs['filtered'], os.path.basename(p)))
+        t3 = time.time(); L(f"[3/8] Dedup: {len(kept)} kept, -{len(rm)} ({t3-t2:.0f}s)")
+
+        # [4/8] Detect faces
+        prog(0.4, desc="Faces...")
+        n = detect_faces(get_imgs(dirs['filtered']), dirs['faces'])
+        t4 = time.time(); L(f"[4/8] Faces: {n} ({t4-t3:.0f}s)")
+        if n == 0: return None, "No faces detected!"
+
+        # [5/8] Process examples
+        prog(0.5, desc="Examples...")
+        ex_paths = [p for p in (examples or []) if p and os.path.exists(p)]
+        if not ex_paths: ex_paths = [p for p in EXAMPLES if os.path.exists(p)]
+        if not ex_paths: return None, "No example images!"
+        n_ex = detect_faces(ex_paths, dirs['ex_faces'])
+        t5 = time.time(); L(f"[5/8] Examples: {len(ex_paths)} imgs -> {n_ex} faces ({t5-t4:.0f}s)")
+        if n_ex == 0: return None, "No faces in examples!"
+
+        # [6/8] Match
+        prog(0.6, desc="Matching...")
+        f_emb, f_valid = face_emb(get_imgs(dirs['faces']))
+        e_emb, _ = face_emb(get_imgs(dirs['ex_faces']))
+        dists = pairwise_distances(f_emb, e_emb, metric='euclidean').min(axis=1)
+        similar_idx = np.where(dists < thresh)[0]
+        similar = [f_valid[i] for i in similar_idx]
+        similar_dists = dists[similar_idx]
+        t6 = time.time()
+        L(f"[6/8] Matches: {len(similar)} (thresh={thresh}) ({t6-t5:.0f}s)")
+        if len(similar_dists) > 0:
+            L(f"  Distances: min={similar_dists.min():.1f}, max={similar_dists.max():.1f}, mean={similar_dists.mean():.1f}")
+        if not similar: return None, f"No matches! Try threshold > {thresh}"
+
+        # [7/8] Get originals
+        prog(0.7, desc="Collect...")
+        origs = set()
+        orig_to_dist = {}
+        for i, fp in enumerate(similar):
+            parts = os.path.basename(fp).rsplit('-', 2)
+            base = parts[0] if len(parts) >= 3 else Path(fp).stem
+            for ext in EXTS:
+                op = os.path.join(dirs['filtered'], base + ext)
+                if os.path.exists(op):
+                    origs.add(op)
+                    orig_to_dist[os.path.basename(op)] = similar_dists[i]
+                    break
+        res_dir = os.path.join(work, f"results_{sanitize(char)}")
+        os.makedirs(res_dir, exist_ok=True)
+        for p in origs: shutil.copy(p, os.path.join(res_dir, os.path.basename(p)))
+        t7 = time.time(); L(f"[7/8] Collected: {len(origs)} ({t7-t6:.0f}s)")
+
+        # [8/8] Tag
+        prog(0.8, desc="Tagging...")
+        char_tag = char if char != sanitize(show_name) else ""
+        for p in get_imgs(res_dir):
+            tags = tag(p, act_tag, char_tag)
+            with open(os.path.splitext(p)[0] + ".txt", 'w') as f: f.write(", ".join(tags))
+        t8 = time.time(); L(f"[8/8] Tagged: {len(origs)} ({t8-t7:.0f}s)")
+
+        # Log each image with distance
+        L(f"\nResults (distance to ref):")
+        for name, d in sorted(orig_to_dist.items(), key=lambda x: x[1]):
+            L(f"  {name}: {d:.1f}")
+
+        # Zip
+        prog(0.95, desc="Zipping...")
+        zp = os.path.join(work, f"{sanitize(char)}_dataset.zip")
+        with zipfile.ZipFile(zp, 'w', zipfile.ZIP_DEFLATED) as z:
+            for p in get_imgs(res_dir) + [os.path.splitext(p)[0]+".txt" for p in get_imgs(res_dir)]:
+                if os.path.exists(p): z.write(p, os.path.basename(p))
+        # Copy ZIP to persistent temp location (Gradio needs file to exist after return)
+        final_zip = tempfile.NamedTemporaryFile(delete=False, suffix=".zip", prefix=f"{sanitize(char)}_").name
+        shutil.copy(zp, final_zip)
+        L(f"\nDone! {len(origs)} images, total {t8-t0:.0f}s"); prog(1.0, desc="Complete!")
+        return final_zip, "\n".join(log)
+    except Exception as e:
+        import traceback
+        return None, "\n".join(log) + f"\n\nERROR: {e}\n{traceback.format_exc()}"
+    finally:
+        # Clean up work directory (ZIP already copied out)
+        if os.path.exists(work):
+            shutil.rmtree(work, ignore_errors=True)
+
+# =============================================================================
+# UI
+# =============================================================================
+css = """
+.gradio-container { padding-top: 10px !important; }
+.compact-group { margin-bottom: 8px !important; }
+"""
+
+with gr.Blocks(title="SD Dataset Automaker: Fancaps → Face Crop (YOLO) → Similarity (Siamese) → WD Tagger → ZIP") as demo:
+    gr.Markdown("### SD Dataset Automaker: Fancaps → Face Crop (YOLO) → Similarity (Siamese) → WD Tagger → ZIP")
+
+    with gr.Row():
+        with gr.Column(scale=3):
+            # Compact input group
+            with gr.Group():
+                with gr.Row():
+                    query = gr.Textbox(
+                        label="Anime / Movie / Fancaps URL",
+                        placeholder="'Cowboy Bebop' or paste URL",
+                        scale=3
+                    )
+                    char = gr.Textbox(label="Character (optional, appends to tags)", placeholder="spike_spiegel", scale=2)
+
+            with gr.Row():
+                ref_imgs = gr.Gallery(
+                    label="Reference Face Image(s)",
+                    columns=4,
+                    height=100,
+                    interactive=True,
+                    object_fit="scale-down",
+                    scale=3,
+                )
+                run_btn = gr.Button("Generate Dataset", variant="primary", size="lg", scale=1)
+
+            # Hidden File input for MCP compatibility (Gallery $ref schema bug persists in Gradio 6.0.1)
+            ref_files = gr.File(
+                label="Reference Images (MCP)",
+                file_count="multiple",
+                file_types=["image"],
+                visible=False,
+            )
+
+            # gr.Examples + gr.Gallery works in Gradio 5.46.0+ (PR #11787)
+            gr.Examples(
+                examples=[
+                    ["https://fancaps.net/anime/showimages.php?3092-Cowboy_Bebop", "spike_spiegel", EXAMPLES],
+                ],
+                inputs=[query, char, ref_imgs],
+                label="Example (click to load)",
+            )
+
+            # Advanced settings in accordion
+            with gr.Accordion("Advanced Settings", open=False):
+                with gr.Row():
+                    max_img = gr.Slider(50, 500, 200, step=50, label="Max Downloads (frames to scrape)")
+                    thresh = gr.Slider(20, 60, 32, step=1, label="Face Similarity (lower=stricter)")
+                with gr.Row():
+                    act_tag = gr.Textbox(label="Trigger Word (prepends to captions)", placeholder="e.g. sks_style", scale=2)
+                    anime_cb = gr.Checkbox(label="Anime", value=True, scale=1)
+                    movies_cb = gr.Checkbox(label="Movies", scale=1)
+                    tv_cb = gr.Checkbox(label="TV", scale=1)
+
+        with gr.Column(scale=1):
+            out_file = gr.File(label="Download ZIP", interactive=False)
+            with gr.Accordion("Log", open=True):
+                out_log = gr.Textbox(label="", lines=12, max_lines=50, show_label=False, autoscroll=False)
+            gr.Markdown("*CPU: ~5-10 min/run*")
+
+    def process(q, c, imgs, files, mi, th, at, an, mo, tv, prog=gr.Progress()):
+        if not q:
+            gr.Warning("Enter anime name or URL")
+            return None, ""
+        # Collect paths from Gallery (imgs) or File input (files)
+        paths = []
+        for item in (imgs or []):
+            p = item[0] if isinstance(item, (list, tuple)) else item
+            if p and os.path.exists(p): paths.append(p)
+        if not paths and files:
+            for f in (files if isinstance(files, list) else [files]):
+                fp = f.name if hasattr(f, 'name') else str(f)
+                if fp and os.path.exists(fp): paths.append(fp)
+        if not paths:
+            gr.Warning("Upload reference images or click 'Load Example'")
+            return None, ""
+        if 'fancaps.net' in q:
+            gr.Info("Direct URL detected")
+        else:
+            gr.Info(f"Searching: {q}")
+        zp, log = run(q, c, paths, mi, th, at, an, mo, tv, prog)
+        if zp:
+            gr.Info("Done!")
+        return zp, log
+
+    run_btn.click(process, [query, char, ref_imgs, ref_files, max_img, thresh, act_tag, anime_cb, movies_cb, tv_cb], [out_file, out_log])
+
+def run_cli():
+    """CLI mode with cloudscraper for Cloudflare bypass"""
+    # Use cloudscraper for CLI (bypasses Cloudflare on local/residential IPs)
+    init_session(use_cloudscraper=True)
+
+    parser = argparse.ArgumentParser(description="SD Dataset Automaker - Anime character dataset generator")
+    parser.add_argument("--title", "-t", required=True, help="Anime name or fancaps.net URL")
+    parser.add_argument("--image", "-i", nargs="+", required=True, help="Reference face images (1-5)")
+    parser.add_argument("--char", "-c", default="", help="Character name (optional, appends to tags)")
+    parser.add_argument("--max", "-m", type=int, default=200, help="Max frames to scrape (default: 200)")
+    parser.add_argument("--thresh", type=float, default=32.0, help="Face similarity threshold, lower=stricter (default: 32)")
+    parser.add_argument("--tag", default="", help="Trigger word to prepend to captions")
+    parser.add_argument("--anime", action="store_true", default=True, help="Search anime (default)")
+    parser.add_argument("--movies", action="store_true", help="Search movies")
+    parser.add_argument("--tv", action="store_true", help="Search TV")
+    parser.add_argument("--output", "-o", default=".", help="Output directory (default: current)")
+    args = parser.parse_args()
+
+    # Validate images
+    ref_imgs = [p for p in args.image if os.path.exists(p)]
+    if not ref_imgs:
+        print(f"ERROR: No valid reference images found: {args.image}")
+        sys.exit(1)
+
+    print(f"SD Dataset Automaker - CLI Mode")
+    print(f"  Title: {args.title}")
+    print(f"  Refs:  {len(ref_imgs)} images")
+    print(f"  Char:  {args.char or '(auto from title)'}")
+    print()
+
+    zp, log = run(
+        query=args.title,
+        char=args.char,
+        examples=ref_imgs,
+        max_img=args.max,
+        thresh=args.thresh,
+        act_tag=args.tag,
+        anime=args.anime,
+        movies=args.movies,
+        tv=args.tv,
+        cli_mode=True
+    )
+
+    if zp:
+        # Copy to output dir
+        out_path = os.path.join(args.output, os.path.basename(zp))
+        shutil.copy(zp, out_path)
+        print(f"\nSaved: {out_path}")
+    else:
+        print(f"\nFailed!")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    # CLI mode if args provided, else Gradio UI
+    if len(sys.argv) > 1:
+        run_cli()
+    else:
+        # Gradio UI mode
+        allowed_dir = os.path.dirname(os.path.abspath(__file__))
+        demo.launch(
+            server_name="0.0.0.0",
+            server_port=7860,
+            mcp_server=True,
+            show_error=True,
+            allowed_paths=[allowed_dir],
+            css=css,
+        )