Bereitstellen eines 1-Bit-Bonsai-27B-Modells mit PrismML llama.cpp und OpenAI-kompatiblen lokalen Inferenz-Workflows

import os
import sys
import time
import json
import shutil
import subprocess
import multiprocessing
WORK_DIR      = "/content material"
REPO_URL      = "https://github.com/PrismML-Eng/llama.cpp"
REPO_DIR      = os.path.be a part of(WORK_DIR, "llama.cpp")
BUILD_DIR     = os.path.be a part of(REPO_DIR, "construct")
BIN_DIR       = os.path.be a part of(BUILD_DIR, "bin")
HF_REPO       = "prism-ml/Bonsai-27B-gguf"
MODEL_FILE    = "Bonsai-27B-Q1_0.gguf"
MODEL_PATH    = os.path.be a part of(WORK_DIR, MODEL_FILE)
SERVER_HOST   = "127.0.0.1"
SERVER_PORT   = 8080
SERVER_URL    = f"http://{SERVER_HOST}:{SERVER_PORT}"
GEN_PARAMS    = {"temperature": 0.7, "top_p": 0.95, "top_k": 20}
CTX_SIZE      = 8192
N_GPU_LAYERS  = 99
USE_KV_Q4     = False
def sh(cmd, examine=True, **kw):
   """Run a shell command, streaming output to the pocket book."""
   print(f"n$ {cmd}")
   return subprocess.run(cmd, shell=True, examine=examine, **kw)
print("=" * 70)
print("(1/7) Checking surroundings")
print("=" * 70)
gpu = subprocess.run("nvidia-smi --query-gpu=identify,reminiscence.whole --format=csv,noheader",
                    shell=True, capture_output=True, textual content=True)
if gpu.returncode != 0:
   sys.exit("No GPU detected. In Colab: Runtime -> Change runtime sort -> GPU (T4).")
print(f"GPU detected: {gpu.stdout.strip()}")
print("Bonsai-27B wants solely ~5.2 GB peak at 4K context — any Colab GPU works.")
sh("pip -q set up huggingface_hub requests")

print("=" * 70)
print("(2/7) Constructing PrismML llama.cpp fork with CUDA (cached after 1st run)")
print("=" * 70)
if not os.path.isdir(REPO_DIR):
   sh(f"git clone --depth 1 {REPO_URL} {REPO_DIR}")
else:
   print("Repo already cloned — skipping.")
cli_bin    = os.path.be a part of(BIN_DIR, "llama-cli")
server_bin = os.path.be a part of(BIN_DIR, "llama-server")
bench_bin  = os.path.be a part of(BIN_DIR, "llama-bench")
if not (os.path.exists(cli_bin) and os.path.exists(server_bin)):
   jobs = multiprocessing.cpu_count()
   sh(f"cmake -S {REPO_DIR} -B {BUILD_DIR} -DGGML_CUDA=ON -DCMAKE_BUILD_TYPE=Launch")
   sh(f"cmake --build {BUILD_DIR} -j{jobs} --target llama-cli llama-server llama-bench")
else:
   print("Binaries already constructed — skipping.")

print("=" * 70)
print("(3/7) Downloading weights from Hugging Face")
print("=" * 70)
from huggingface_hub import hf_hub_download
if not os.path.exists(MODEL_PATH):
   downloaded = hf_hub_download(repo_id=HF_REPO, filename=MODEL_FILE,
                                local_dir=WORK_DIR)
   print(f"Downloaded to: {downloaded}")
else:
   print("Mannequin already on disk — skipping.")
print(f"Mannequin dimension on disk: {os.path.getsize(MODEL_PATH) / 1e9:.2f} GB")

print("=" * 70)
print("(4/7) Smoke check with llama-cli")
print("=" * 70)
sh(
   f'{cli_bin} -m {MODEL_PATH} '
   f'-p "Clarify in two sentences why 1-bit quantization saves reminiscence." '
   f'-n 128 -ngl {N_GPU_LAYERS} '
   f'--temp {GEN_PARAMS("temperature")} '
   f'--top-p {GEN_PARAMS("top_p")} --top-k {GEN_PARAMS("top_k")} '
   f'-no-cnv 2>/dev/null',
   examine=False,
)
print("=" * 70)
print("(5/7) Beginning llama-server (OpenAI-compatible API)")
print("=" * 70)
import requests
kv_flags = "-ctk q4_0 -ctv q4_0" if USE_KV_Q4 else ""
server_cmd = (
   f"{server_bin} -m {MODEL_PATH} "
   f"--host {SERVER_HOST} --port {SERVER_PORT} "
   f"-ngl {N_GPU_LAYERS} -c {CTX_SIZE} {kv_flags}"
)
print(f"$ {server_cmd}  (background)")
server_log = open(os.path.be a part of(WORK_DIR, "server.log"), "w")
server_proc = subprocess.Popen(server_cmd, shell=True,
                              stdout=server_log, stderr=server_log)
for _ in vary(120):
   attempt:
       if requests.get(f"{SERVER_URL}/well being", timeout=2).status_code == 200:
           print("Server is up.")
           break
   besides requests.exceptions.RequestException:
       move
   time.sleep(2)
else:
   server_proc.kill()
   sys.exit("Server failed to start out — examine /content material/server.log")

print("=" * 70)
print("(6/7) Speaking to Bonsai-27B through the OpenAI-compatible API")
print("=" * 70)
def chat(messages, stream=False, max_tokens=512, **overrides):
   """Minimal OpenAI-compatible chat consumer for the native llama-server."""
   payload = {
       "mannequin": "bonsai-27b",
       "messages": messages,
       "max_tokens": max_tokens,
       "stream": stream,
       **GEN_PARAMS,
       **overrides,
   }
   if not stream:
       r = requests.submit(f"{SERVER_URL}/v1/chat/completions", json=payload)
       r.raise_for_status()
       return r.json()("selections")(0)("message")("content material")
   r = requests.submit(f"{SERVER_URL}/v1/chat/completions", json=payload, stream=True)
   r.raise_for_status()
   full = ()
   for line in r.iter_lines():
       if not line or not line.startswith(b"information: "):
           proceed
       chunk = line(len(b"information: "):)
       if chunk == b"(DONE)":
           break
       delta = json.hundreds(chunk)("selections")(0)("delta").get("content material", "")
       full.append(delta)
       print(delta, finish="", flush=True)
   print()
   return "".be a part of(full)
SYSTEM = {"function": "system", "content material": "You're a useful assistant"}
print("n--- 6a: fundamental completion ---")
reply = chat((SYSTEM, {"function": "consumer",
                       "content material": "What's the capital of France? One sentence."}))
print(reply)
print("n--- 6b: math reasoning, streamed token-by-token ---")
chat((SYSTEM, {"function": "consumer",
              "content material": "A prepare travels 120 km at 80 km/h, then 90 km at "
                         "60 km/h. What's its common velocity for the entire "
                         "journey? Present your reasoning briefly."}),
    stream=True, max_tokens=700)
print("n--- 6c: multi-turn chat ---")
historical past = (SYSTEM)
for user_msg in ("My identify is Priya and I like graph algorithms.",
                "Counsel one mission thought that mixes my curiosity with LLMs.",
                "What was my identify once more?"):
   historical past.append({"function": "consumer", "content material": user_msg})
   reply = chat(historical past, max_tokens=300)
   historical past.append({"function": "assistant", "content material": reply})
   print(f"nUSER: {user_msg}nBONSAI: {reply}")
print("n--- 6d: code era ---")
print(chat((SYSTEM, {"function": "consumer",
                    "content material": "Write a Python operate that returns the n-th "
                               "Fibonacci quantity utilizing memoization. Code solely."}),
          max_tokens=400))

print("=" * 70)
print("(7/7) Non-compulsory extras")
print("=" * 70)
RUN_BENCHMARK = False
if RUN_BENCHMARK:
   sh(f"{bench_bin} -m {MODEL_PATH} -ngl {N_GPU_LAYERS}", examine=False)
print("""
NOTES & NEXT STEPS
------------------
* Lengthy context: the mannequin helps as much as 262K tokens. On Colab, increase
 CTX_SIZE and set USE_KV_Q4 = True (4-bit KV cache) — with it, 100K-token
 contexts slot in roughly 6.8 GB peak, properly inside a T4's 16 GB.
* Speculative decoding: the repo ships a DSpark drafter (Q4_1, ~1.79 GB)
 that provides a lossless ~1.37x decode speedup on CUDA. See the PrismML
 llama.cpp fork's README for the serving flags, and obtain the drafter
 pack from the identical HF repo if you wish to attempt it.
* Imaginative and prescient: an non-obligatory ~0.63 GB mmproj pack provides picture enter; it's only
 loaded when a picture arrives, so text-only serving by no means pays for it.
* High quality vs dimension: if you'd like extra headroom, the ternary sibling
 (prism-ml/Ternary-Bonsai-27B-gguf, ~5.9 GB, ~95% of FP16) is a drop-in
 swap — simply change HF_REPO / MODEL_FILE above.
* Shutting down: run  server_proc.kill()  in a later cell to free the GPU.
""")
print("Finished. The server remains to be operating — name chat((...)) from new cells!")