Download source/src/decision_model/server.py from illuin/IVE-4B: direct link, hf CLI and curl.
- Browser
- Download file 7.28 kB
-
https://huggingface.co/illuin/IVE-4B/resolve/main/source/src/decision_model/server.py
- Command line
-
hf download hf://illuin/IVE-4B/source/src/decision_model/server.py
-
curl -L -o server.py https://huggingface.co/illuin/IVE-4B/resolve/main/source/src/decision_model/server.py
7.28 kB
| """HTTP server over DecisionModel: the model is loaded and warmed up once at startup (kernels | |
| compiled and tuned), then requests run one at a time on it. | |
| python -m decision_model.server --model <repo or directory> [--port 8000] | |
| POST /v1/systemone {"model", "state", "questions", "images"?}: the Decision Index text | |
| protocol (the kit's `http` engine), rendered as the text engine renders. | |
| POST /v1/decisions the same body: the Decision Index vision protocol (the kit's `decide` | |
| engine with protocol=decisions), rendered as the vision engine renders. | |
| GET /health, /v1/models | |
| Answers: {"model", "answers": {id: {"type": "choice", "choice", "probabilities"} | {"type": | |
| "noul", "noul"} | {"type": "score", "score", "probabilities"}}, "usage": {"input_tokens", | |
| "output_tokens": 0}}. Capacity refusals use the kit's markers (HTTP 413 "maximum context | |
| length", 422 "options per choice"), so the kit records them as unsupported, not as errors. | |
| Images are base64 data URLs (or bare base64); the server never opens a path from a request. | |
| `server-timing` gives prepare (tokenization, images) and forward (GPU) milliseconds. | |
| A bearer token is required when DECISION_MODEL_API_KEY is set. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import base64 | |
| import binascii | |
| import hmac | |
| import os | |
| import threading | |
| import time | |
| from contextlib import asynccontextmanager | |
| from typing import Any | |
| from fastapi import Depends, FastAPI, Header, HTTPException, Request | |
| from fastapi.responses import JSONResponse | |
| from pydantic import BaseModel, ConfigDict, Field | |
| from .model import ( | |
| BOARD_MAX_IMAGE_PIXELS, | |
| DecisionModel, | |
| InputTooLong, | |
| UnsupportedQuestion, | |
| ) | |
| MAX_IMAGES = 16 | |
| MAX_IMAGE_BYTES = 16_000_000 | |
| class Body(BaseModel): | |
| """A request: the kit's /v1/systemone body, images optional (extra fields ignored).""" | |
| model_config = ConfigDict(extra="allow") | |
| model: str = "default" | |
| state: Any = None | |
| questions: dict[str, dict[str, Any]] = Field(min_length=1) | |
| images: list[str] = Field(default_factory=list, max_length=MAX_IMAGES) | |
| class _State: | |
| model: DecisionModel | None = None | |
| lock = threading.Lock() | |
| options: dict[str, Any] = {} | |
| warmup_seconds: float | None = None | |
| state = _State() | |
| def _image_bytes(value: str) -> bytes: | |
| data = value.split(",", 1)[1] if value.startswith("data:") else value | |
| try: | |
| raw = base64.b64decode(data, validate=True) | |
| except (binascii.Error, ValueError) as exc: | |
| raise ValueError("images must be base64 data URLs") from exc | |
| if len(raw) > MAX_IMAGE_BYTES: | |
| raise ValueError(f"an image is over {MAX_IMAGE_BYTES} bytes") | |
| return raw | |
| def create_app(**options): | |
| """options: model (repo or directory), revision, device, max_image_pixels, batching, | |
| warmup (bool).""" | |
| state.options = options | |
| async def lifespan(app): | |
| from starlette.concurrency import run_in_threadpool | |
| def load(): | |
| model = DecisionModel.from_pretrained( | |
| options["model"], revision=options.get("revision"), | |
| device=options.get("device"), batching=options.get("batching", "padded"), | |
| max_image_pixels=options.get("max_image_pixels") or BOARD_MAX_IMAGE_PIXELS) | |
| if options.get("warmup", True): | |
| state.warmup_seconds = model.warmup() | |
| return model | |
| state.model = await run_in_threadpool(load) | |
| yield | |
| state.model = None | |
| app = FastAPI(title="decision-model", lifespan=lifespan) | |
| def authenticate(authorization: str | None = Header(default=None)) -> None: | |
| key = os.environ.get("DECISION_MODEL_API_KEY") | |
| if key and not hmac.compare_digest((authorization or "").encode(), | |
| f"Bearer {key}".encode()): | |
| raise HTTPException(401, "missing or invalid API key") | |
| def run(body: Body, style: str) -> JSONResponse: | |
| model = state.model | |
| if model is None: | |
| raise HTTPException(503, "the model is loading") | |
| try: | |
| images = [_image_bytes(x) for x in body.images] | |
| except ValueError as exc: | |
| raise HTTPException(422, str(exc)) from exc | |
| with state.lock: # one request at a time on the GPU; the others wait their turn | |
| t = time.perf_counter() | |
| try: | |
| out = model.decide(body.state, body.questions, images, style) | |
| except InputTooLong as exc: | |
| raise HTTPException(413, f"input exceeds the maximum context length of " | |
| f"{model.max_length} tokens: {exc}") from exc | |
| except UnsupportedQuestion as exc: | |
| raise HTTPException(422, str(exc)) from exc | |
| timing = dict(model.last_timing) | |
| total = time.perf_counter() - t | |
| header = (f"prepare;dur={1000 * timing['prepare_s']:.1f}, " | |
| f"forward;dur={1000 * timing['forward_s']:.1f}, total;dur={1000 * total:.1f}") | |
| return JSONResponse(out, headers={"server-timing": header}) | |
| def systemone(body: Body) -> JSONResponse: | |
| return run(body, "text") | |
| def decisions(body: Body) -> JSONResponse: | |
| return run(body, "vision") | |
| def health() -> dict[str, Any]: | |
| m = state.model | |
| return {"status": "ready" if m is not None else "loading", | |
| "model": None if m is None else m.name, | |
| "warmup_seconds": state.warmup_seconds, | |
| "attention": None if m is None else m.attention_check} | |
| def models() -> dict[str, Any]: | |
| m = state.model | |
| return {"models": [] if m is None else [{"name": m.name}]} | |
| async def value_error(request: Request, exc: ValueError) -> JSONResponse: | |
| return JSONResponse({"detail": str(exc)}, status_code=422) | |
| return app | |
| def main() -> None: | |
| p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawTextHelpFormatter) | |
| p.add_argument("--model", default=os.environ.get("DECISION_MODEL"), required= | |
| "DECISION_MODEL" not in os.environ, help="Hub repository or directory") | |
| p.add_argument("--revision", default=os.environ.get("DECISION_MODEL_REVISION")) | |
| p.add_argument("--device", default=os.environ.get("DECISION_MODEL_DEVICE")) | |
| p.add_argument("--max-image-pixels", type=int, default=BOARD_MAX_IMAGE_PIXELS) | |
| p.add_argument("--batching", choices=("padded", "sequential"), default="padded") | |
| p.add_argument("--no-warmup", action="store_true") | |
| p.add_argument("--host", default="127.0.0.1") | |
| p.add_argument("--port", type=int, default=8000) | |
| a = p.parse_args() | |
| import uvicorn | |
| app = create_app(model=a.model, revision=a.revision, device=a.device, | |
| max_image_pixels=a.max_image_pixels, batching=a.batching, | |
| warmup=not a.no_warmup) | |
| uvicorn.run(app, host=a.host, port=a.port, workers=1) | |
| if __name__ == "__main__": | |
| main() | |