QuentinJG's picture
4b-v6-cross-best: weights, readout, decision config and source
2b5c0a6 verified
Raw History Blame Contribute Delete
7.28 kB
"""HTTP server over DecisionModel: the model is loaded and warmed up once at startup (kernels
compiled and tuned), then requests run one at a time on it.
python -m decision_model.server --model <repo or directory> [--port 8000]
POST /v1/systemone {"model", "state", "questions", "images"?}: the Decision Index text
protocol (the kit's `http` engine), rendered as the text engine renders.
POST /v1/decisions the same body: the Decision Index vision protocol (the kit's `decide`
engine with protocol=decisions), rendered as the vision engine renders.
GET /health, /v1/models
Answers: {"model", "answers": {id: {"type": "choice", "choice", "probabilities"} | {"type":
"noul", "noul"} | {"type": "score", "score", "probabilities"}}, "usage": {"input_tokens",
"output_tokens": 0}}. Capacity refusals use the kit's markers (HTTP 413 "maximum context
length", 422 "options per choice"), so the kit records them as unsupported, not as errors.
Images are base64 data URLs (or bare base64); the server never opens a path from a request.
`server-timing` gives prepare (tokenization, images) and forward (GPU) milliseconds.
A bearer token is required when DECISION_MODEL_API_KEY is set.
"""
from __future__ import annotations
import argparse
import base64
import binascii
import hmac
import os
import threading
import time
from contextlib import asynccontextmanager
from typing import Any
from fastapi import Depends, FastAPI, Header, HTTPException, Request
from fastapi.responses import JSONResponse
from pydantic import BaseModel, ConfigDict, Field
from .model import (
BOARD_MAX_IMAGE_PIXELS,
DecisionModel,
InputTooLong,
UnsupportedQuestion,
)
MAX_IMAGES = 16
MAX_IMAGE_BYTES = 16_000_000
class Body(BaseModel):
"""A request: the kit's /v1/systemone body, images optional (extra fields ignored)."""
model_config = ConfigDict(extra="allow")
model: str = "default"
state: Any = None
questions: dict[str, dict[str, Any]] = Field(min_length=1)
images: list[str] = Field(default_factory=list, max_length=MAX_IMAGES)
class _State:
model: DecisionModel | None = None
lock = threading.Lock()
options: dict[str, Any] = {}
warmup_seconds: float | None = None
state = _State()
def _image_bytes(value: str) -> bytes:
data = value.split(",", 1)[1] if value.startswith("data:") else value
try:
raw = base64.b64decode(data, validate=True)
except (binascii.Error, ValueError) as exc:
raise ValueError("images must be base64 data URLs") from exc
if len(raw) > MAX_IMAGE_BYTES:
raise ValueError(f"an image is over {MAX_IMAGE_BYTES} bytes")
return raw
def create_app(**options):
"""options: model (repo or directory), revision, device, max_image_pixels, batching,
warmup (bool)."""
state.options = options
@asynccontextmanager
async def lifespan(app):
from starlette.concurrency import run_in_threadpool
def load():
model = DecisionModel.from_pretrained(
options["model"], revision=options.get("revision"),
device=options.get("device"), batching=options.get("batching", "padded"),
max_image_pixels=options.get("max_image_pixels") or BOARD_MAX_IMAGE_PIXELS)
if options.get("warmup", True):
state.warmup_seconds = model.warmup()
return model
state.model = await run_in_threadpool(load)
yield
state.model = None
app = FastAPI(title="decision-model", lifespan=lifespan)
def authenticate(authorization: str | None = Header(default=None)) -> None:
key = os.environ.get("DECISION_MODEL_API_KEY")
if key and not hmac.compare_digest((authorization or "").encode(),
f"Bearer {key}".encode()):
raise HTTPException(401, "missing or invalid API key")
def run(body: Body, style: str) -> JSONResponse:
model = state.model
if model is None:
raise HTTPException(503, "the model is loading")
try:
images = [_image_bytes(x) for x in body.images]
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
with state.lock: # one request at a time on the GPU; the others wait their turn
t = time.perf_counter()
try:
out = model.decide(body.state, body.questions, images, style)
except InputTooLong as exc:
raise HTTPException(413, f"input exceeds the maximum context length of "
f"{model.max_length} tokens: {exc}") from exc
except UnsupportedQuestion as exc:
raise HTTPException(422, str(exc)) from exc
timing = dict(model.last_timing)
total = time.perf_counter() - t
header = (f"prepare;dur={1000 * timing['prepare_s']:.1f}, "
f"forward;dur={1000 * timing['forward_s']:.1f}, total;dur={1000 * total:.1f}")
return JSONResponse(out, headers={"server-timing": header})
@app.post("/v1/systemone", dependencies=[Depends(authenticate)])
def systemone(body: Body) -> JSONResponse:
return run(body, "text")
@app.post("/v1/decisions", dependencies=[Depends(authenticate)])
def decisions(body: Body) -> JSONResponse:
return run(body, "vision")
@app.get("/health")
def health() -> dict[str, Any]:
m = state.model
return {"status": "ready" if m is not None else "loading",
"model": None if m is None else m.name,
"warmup_seconds": state.warmup_seconds,
"attention": None if m is None else m.attention_check}
@app.get("/v1/models", dependencies=[Depends(authenticate)])
def models() -> dict[str, Any]:
m = state.model
return {"models": [] if m is None else [{"name": m.name}]}
@app.exception_handler(ValueError)
async def value_error(request: Request, exc: ValueError) -> JSONResponse:
return JSONResponse({"detail": str(exc)}, status_code=422)
return app
def main() -> None:
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
p.add_argument("--model", default=os.environ.get("DECISION_MODEL"), required=
"DECISION_MODEL" not in os.environ, help="Hub repository or directory")
p.add_argument("--revision", default=os.environ.get("DECISION_MODEL_REVISION"))
p.add_argument("--device", default=os.environ.get("DECISION_MODEL_DEVICE"))
p.add_argument("--max-image-pixels", type=int, default=BOARD_MAX_IMAGE_PIXELS)
p.add_argument("--batching", choices=("padded", "sequential"), default="padded")
p.add_argument("--no-warmup", action="store_true")
p.add_argument("--host", default="127.0.0.1")
p.add_argument("--port", type=int, default=8000)
a = p.parse_args()
import uvicorn
app = create_app(model=a.model, revision=a.revision, device=a.device,
max_image_pixels=a.max_image_pixels, batching=a.batching,
warmup=not a.no_warmup)
uvicorn.run(app, host=a.host, port=a.port, workers=1)
if __name__ == "__main__":
main()