#!/usr/bin/env python3
"""Download images for batch2 + batch3 products."""
import json, os, re, urllib.request, time, sys

base = os.path.dirname(os.path.abspath(__file__))
out_root = os.path.join(base, "products")
os.makedirs(out_root, exist_ok=True)

def slugify(name):
    s = re.sub(r'[^a-zA-Z0-9]+', '-', name).strip('-').lower()
    return s[:60]

def load_jsonl(path):
    items = []
    with open(path) as f:
        for line in f:
            line = line.strip()
            if line:
                items.append(json.loads(line))
    return items

existing = load_jsonl(os.path.join(base, "products.jsonl"))
offset = len(existing)

batches = []
for fname in ["products-batch2.jsonl", "products-batch3.jsonl"]:
    path = os.path.join(base, fname)
    if os.path.exists(path):
        batches.extend(load_jsonl(path))

print(f"Existing: {offset} products, new: {len(batches)} to download")

ok, fail = 0, 0
for i, p in enumerate(batches, 1):
    idx = offset + i
    dirname = f"{idx:03d}-{slugify(p['brand'])}-{slugify(p['name'])}"
    pdir = os.path.join(out_root, dirname)
    os.makedirs(pdir, exist_ok=True)

    info = {"name": p["name"], "brand": p["brand"], "price_myr": p["price"], "source_code": p["code"]}
    if "category" in p:
        info["category"] = p["category"]
    with open(os.path.join(pdir, "info.json"), "w") as f:
        json.dump(info, f, indent=2)

    img_ok = 0
    for j, url in enumerate(p["imgs"], 1):
        dest = os.path.join(pdir, f"image-{j}.jpg")
        if os.path.exists(dest) and os.path.getsize(dest) > 1000:
            img_ok += 1
            continue
        try:
            req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"})
            with urllib.request.urlopen(req, timeout=15) as r, open(dest, "wb") as out:
                out.write(r.read())
            img_ok += 1
        except Exception as e:
            print(f"  FAIL {url}: {e}")
            fail += 1
        time.sleep(0.2)

    ok += img_ok
    print(f"[{i}/{len(batches)}] {p['brand']} - {p['name'][:40]} -> {img_ok}/{len(p['imgs'])} imgs")

print(f"\nDone: {ok} images downloaded, {fail} failed")