QuentinJG's picture
4b-v6-cross-best: weights, readout, decision config and source
2b5c0a6 verified
Raw History Blame Contribute Delete
6.82 kB
"""Decision Index kit engines (github.com/apolinario/decision-index) over DecisionModel.
python -m decision_index pipeline --engine decision_model.engine:DecisionModelEngine \\
--option model=<repo or directory> --out runs/<name>
python -m decision_index vision pipeline --dir <vision suite> \\
--engine decision_model.engine:DecisionModelVisionEngine \\
--option model=<repo or directory> --out runs/<name>-vision
Options: model, revision, device, batching ("padded" or "sequential"), max_length (tokens),
warmup (true), and for vision max_pixels (default the board's cap, 1,638,400). A request
that does not fit max_length, or a question the model cannot answer, is Unsupported (counted
as unanswered by the kit); nothing is truncated.
"""
from __future__ import annotations
import importlib.metadata as md
import io
import time
from decision_index.engines.base import Engine, Unsupported
from PIL import Image
from .model import (
BOARD_MAX_IMAGE_PIXELS,
DEFAULT_MAX_LENGTH,
DecisionModel,
InputTooLong,
UnsupportedQuestion,
answer,
fit_size,
render,
)
def _versions() -> dict:
out = {}
for dist in ("torch", "transformers", "flash-linear-attention", "causal-conv1d"):
try:
out[dist] = md.version(dist)
except md.PackageNotFoundError:
out[dist] = None
return out
class DecisionModelEngine(Engine):
name = "decision-model"
latency = ("In-process request wall time: rendering, tokenization and the forward passes "
"of the request's questions on one GPU, until the answers are on the host; "
"excludes loading and warm-up.")
def __init__(self, model: str, revision: str | None = None, device: str | None = None,
batching: str = "padded", max_length: int = DEFAULT_MAX_LENGTH,
warmup: bool = True, max_pixels: int | None = None, **options):
super().__init__(**options)
t = time.perf_counter()
self.model = DecisionModel.from_pretrained(
str(model), revision=revision, device=device, batching=batching,
max_length=int(max_length), max_image_pixels=max_pixels)
self.load_seconds = time.perf_counter() - t
self.warmup_seconds = self.model.warmup() if warmup else None
self.provenance = {"kind": "decision-model", "model": str(model), "revision": revision,
"base_model": self.model.config.get("base_model"),
"attention_mode": self.model.attention_mode,
"temperature": self.model.temperature, "batching": batching,
"max_length": int(max_length),
"max_image_pixels": self.model.max_image_pixels}
def warmup(self): # the kit's own warm-up: done in __init__
pass
def __call__(self, state, questions):
try:
out = self.model.decide(state, questions)
except (InputTooLong, UnsupportedQuestion) as exc:
raise Unsupported(str(exc)) from exc
return {"model": out["model"], "answers": out["answers"]}, None
def synchronize(self):
import torch
if self.model.device_.type == "cuda":
torch.cuda.synchronize(self.model.device_)
def runtime(self):
import torch
dev = self.model.device_
return {"device": torch.cuda.get_device_name(dev) if dev.type == "cuda" else str(dev),
**_versions(), "attention_check": self.model.attention_check,
"load_seconds": self.load_seconds, "warmup_seconds": self.warmup_seconds}
class DecisionModelVisionEngine(DecisionModelEngine):
"""The vision suite: each row's choice question over its images, rendered as the vision
protocol (render style "vision"), images at native size up to max_pixels."""
name = "decision-model-vision"
latency = ("In-process wall time of a chunk of rows (forward passes planned by length) "
"divided by its rows; excludes loading and warm-up.")
def __init__(self, model: str, image_root: str | None = None,
max_pixels: int = BOARD_MAX_IMAGE_PIXELS, **options):
super().__init__(model, max_pixels=int(max_pixels), **options)
self.image_root = image_root
def _row(self, payload):
from decision_index.vision.images import load
images = [Image.open(io.BytesIO(load(ref, self.image_root)))
for ref in payload.get("images") or ()]
for im in images:
im.load()
rendered = {qid: render(payload.get("state"), q, images, "vision")
for qid, q in payload["questions"].items()}
pixels = [w * h for w, h in (fit_size(im.width, im.height, self.model.max_image_pixels,
self.model.min_image_pixels) for im in images)]
return rendered, pixels
def _answers(self, payload, rendered, probs):
answers = {qid: answer(payload["questions"][qid], keys, p)
for (qid, (_, keys)), p in zip(rendered.items(), probs, strict=True)}
return {"model": self.model.name, "answers": answers}
def batch(self, payloads):
out: list = [None] * len(payloads)
rows = {}
for i, p in enumerate(payloads):
try:
rows[i] = self._row(p)
except UnsupportedQuestion as exc:
out[i] = Unsupported(str(exc))
except Exception as exc: # recorded for this row only
out[i] = exc
flat = [(i, d) for i, (rendered, _) in rows.items() for d, _ in rendered.values()]
try:
probs = self.model.predict([d for _, d in flat])
k = 0
for i, (rendered, pixels) in rows.items():
n = len(rendered)
out[i] = (self._answers(payloads[i], rendered, probs[k:k + n]),
{"image_pixels": pixels})
k += n
except Exception: # isolate the row that fails; the others still get answers
for i, (rendered, pixels) in rows.items():
try:
probs = self.model.predict([d for d, _ in rendered.values()])
out[i] = (self._answers(payloads[i], rendered, probs),
{"image_pixels": pixels})
except InputTooLong as exc:
out[i] = Unsupported(str(exc))
except Exception as exc:
out[i] = exc
return out
def __call__(self, state, questions, images=(), prompt=None):
(r,) = self.batch([{"state": state, "questions": questions, "images": images}])
if isinstance(r, Exception):
raise r
return r