script.py

#!/usr/bin/env python3
"""IOL-AI 2026 submission + environment probe.

Order of operations (all wall-clock guarded, submission.csv always valid):
  1. blank submission.csv immediately
  2. environment diagnostics -> stdout (visible in eval logs)
  3. try vLLM path: pip install bundled wheels into ./pyenv_vllm (isolated,
     base env untouched) and run solver_vllm.py in a subprocess
  4. fallback: pip install newer transformers into ./pyenv_tf and run
     solver_tf.py (base torch 2.4) in a subprocess

No test-set content is printed — only counts/shapes.
"""
import csv
import json
import os
import shutil
import socket
import subprocess
import sys
import time

START = time.time()
HERE = os.path.dirname(os.path.abspath(__file__)) or "."
os.chdir(HERE)
os.environ.setdefault("HF_HUB_OFFLINE", "1")
os.environ.setdefault("TRANSFORMERS_OFFLINE", "1")
os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")

TEST_CSV = os.environ.get("IOL_TEST_CSV", "/tmp/data/test.csv")
OUT_CSV = "submission.csv"
TIME_LIMIT = float(os.environ.get("IOL_TIME_LIMIT", 30 * 60))
SAFETY_S = 100  # leave this much for the platform to grab the csv


def log(msg):
    print(f"[probe +{time.time()-START:6.1f}s] {msg}", flush=True)


def remaining():
    return TIME_LIMIT - SAFETY_S - (time.time() - START)


def sh(cmd, timeout=60):
    try:
        r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
        return (r.stdout + r.stderr).strip()
    except Exception as e:
        return f"<failed: {e}>"


def find_test_csv():
    """The platform snapshots the whole dataset repo into /tmp/data — the csv
    may not be exactly /tmp/data/test.csv. Find it."""
    import glob
    cands = [TEST_CSV] + sorted(glob.glob("/tmp/data/**/*.csv", recursive=True))
    for c in cands:
        if not os.path.isfile(c):
            continue
        try:
            with open(c, newline="", encoding="utf-8") as f:
                rows = list(csv.DictReader(f))
            if rows and "id" in rows[0] and "query" in rows[0]:
                log(f"test csv: {c} ({len(rows)} rows)")
                return c, rows
            log(f"csv {c} rejected (cols={list(rows[0].keys()) if rows else 'empty'})")
        except Exception as e:
            log(f"csv {c} unreadable: {e}")
    log("NO usable test csv found under /tmp/data; listing:")
    for root, _, files in os.walk("/tmp/data"):
        for fn in files[:50]:
            log(f"  {os.path.join(root, fn)}")
    return None, []


def write_submission(rows, phase=0, note=""):
    """Refresh submission.csv, preserving any real answers already present, and
    stamp diagnostic phase markers into the (never auto-scored) explanation
    column of still-unanswered rows: phase p -> first 9*p rows, so the reported
    explanation_rate ~= p/10 tells us how far the run got."""
    existing = {}
    try:
        with open(OUT_CSV, newline="", encoding="utf-8") as f:
            for r in csv.DictReader(f):
                existing[r["id"]] = r
    except Exception:
        pass
    with open(OUT_CSV, "w", newline="", encoding="utf-8") as f:
        w = csv.DictWriter(f, fieldnames=["id", "pred", "explanation"])
        w.writeheader()
        for i, r in enumerate(rows):
            rid = r.get("id", "")
            old = existing.get(rid, {})
            pred = old.get("pred") or "[]"
            expl = old.get("explanation", "")
            answered = any(str(x).strip() for x in json.loads(pred or "[]")) if pred else False
            if not answered and i < 9 * phase:
                expl = f"[diag] {note}"
            w.writerow({"id": rid, "pred": pred, "explanation": expl})
    log(f"submission refreshed ({len(rows)} rows, phase={phase} {note!r})")


FOUND_CSV = None


def blank_submission():
    global FOUND_CSV
    FOUND_CSV, rows = find_test_csv()
    write_submission(rows, 1, "script started, test data located")
    return rows


def diagnostics():
    from importlib.metadata import version

    log(f"python {sys.version.split()[0]} exe={sys.executable} cwd={os.getcwd()}")
    log(f"cpu={os.cpu_count()} | disk_free={shutil.disk_usage('.').free/1e9:.1f}GB | "
        f"tmp_free={shutil.disk_usage('/tmp').free/1e9:.1f}GB")
    log("mem: " + sh("free -g | sed -n 2p"))
    log("gpu: " + sh("nvidia-smi --query-gpu=name,memory.total,driver_version,compute_cap --format=csv,noheader"))
    for mod in ["torch", "transformers", "accelerate", "bitsandbytes", "autoawq",
                "vllm", "xformers", "triton", "numpy", "pandas"]:
        try:
            log(f"pkg {mod}=={version(mod)}")
        except Exception:
            log(f"pkg {mod}: absent")
    log("repo files: " + ", ".join(sorted(os.listdir("."))[:40]))
    if os.path.isdir("wheels"):
        n = os.listdir("wheels")
        log(f"wheels/: {len(n)} files, "
            f"{sum(os.path.getsize(os.path.join('wheels', f)) for f in n)/1e9:.2f}GB")
    try:
        with open(TEST_CSV, newline="", encoding="utf-8") as f:
            rows = list(csv.DictReader(f))
        tt = {}
        for r in rows:
            tt[r.get("task_type", "?")] = tt.get(r.get("task_type", "?"), 0) + 1
        log(f"test.csv: {len(rows)} rows, cols={list(rows[0].keys()) if rows else []}, "
            f"task_types={tt}")
    except Exception as e:
        log(f"test.csv stats failed: {e}")
    for host in ["pypi.org", "files.pythonhosted.org", "huggingface.co"]:
        t = time.time()
        try:
            socket.create_connection((host, 443), timeout=4).close()
            log(f"net {host}:443 OK ({time.time()-t:.2f}s)")
        except Exception as e:
            log(f"net {host}:443 BLOCKED ({type(e).__name__})")
    log("env: " + sh("env | grep -iE 'cuda|hf_|vllm|torch' | grep -viE 'token|secret|key' | head -8"))


def target_install(target, spec, timeout=600):
    os.makedirs(target, exist_ok=True)
    t = time.time()
    out = sh(f"{sys.executable} -m pip install --no-index --find-links wheels "
             f"--disable-pip-version-check --no-cache-dir --target {target} "
             f"{spec} 2>&1 | tail -4", timeout=timeout)
    ok = "Successfully installed" in out
    log(f"target-install {spec} -> {target}: {'OK' if ok else 'FAIL'} "
        f"({time.time()-t:.0f}s) :: {out[-300:]}")
    return ok


def run_solver(solver, pyenv, budget_s):
    env = os.environ.copy()
    if pyenv:
        env["PYTHONPATH"] = os.path.abspath(pyenv)
    env["IOL_TEST_CSV"] = FOUND_CSV or TEST_CSV
    env["IOL_BUDGET_S"] = str(max(int(budget_s), 60))
    log(f"launching {solver} (budget {budget_s:.0f}s, PYTHONPATH={pyenv or '-'})")
    try:
        p = subprocess.run([sys.executable, solver], env=env, timeout=budget_s + 30)
        log(f"{solver} exited rc={p.returncode}")
        return p.returncode == 0
    except subprocess.TimeoutExpired:
        log(f"{solver} hit its budget; whatever it wrote incrementally stands")
        return False
    except Exception as e:
        log(f"{solver} failed: {e!r}")
        return False


def submission_filled():
    try:
        with open(OUT_CSV, newline="", encoding="utf-8") as f:
            rows = list(csv.DictReader(f))
        filled = sum(1 for r in rows
                     if any(str(x).strip() for x in json.loads(r["pred"] or "[]")))
        log(f"submission.csv: {filled}/{len(rows)} rows have answers")
        return len(rows) > 0 and filled >= len(rows) * 0.8
    except Exception as e:
        log(f"submission check failed: {e}")
        return False


def main():
    rows = blank_submission()
    diagnostics()
    if rows:
        write_submission(rows, 2, "diagnostics done")

    # ---- vLLM path ----
    vllm_ready = False
    if os.path.isdir("wheels") and remaining() > 12 * 60:
        # transformers must stay <5 for vllm 0.8.5's tokenizer interface
        vllm_ready = target_install("pyenv_vllm", "vllm==0.8.5.post1 transformers==4.51.3",
                                    timeout=600)
    if vllm_ready and rows:
        write_submission(rows, 3, "vllm wheels installed, launching engine")
        run_solver("solver_vllm.py", "pyenv_vllm", remaining() - 120)
        if submission_filled():
            log("vLLM path succeeded; done")
            return

    # ---- transformers fallback ----
    if rows:
        write_submission(rows, 4, "vllm path failed, trying transformers fallback")
    pyenv = None
    if os.path.isdir("wheels") and target_install("pyenv_tf", "transformers==4.51.3",
                                                  timeout=300):
        pyenv = "pyenv_tf"
    run_solver("solver_tf.py", pyenv, max(remaining(), 120))
    submission_filled()


if __name__ == "__main__":
    try:
        main()
    except Exception:
        import traceback
        traceback.print_exc()
        if not os.path.exists(OUT_CSV):
            blank_submission()
    log("probe finished")