Bereitstellung des Magistral VLLM -Servers auf Modal

modal secret create vllm-api VLLM_API_KEY=your_actual_api_key_here

import modal

vllm_image = (
    modal.Picture.debian_slim(python_version="3.12")
    .pip_install(
        "vllm==0.9.1",
        "huggingface_hub(hf_transfer)==0.32.0",
        "flashinfer-python==0.2.6.post1",
        extra_index_url="https://obtain.pytorch.org/whl/cu128",
    )
    .env(
        {
            "HF_HUB_ENABLE_HF_TRANSFER": "1",  # sooner mannequin transfers
            "NCCL_CUMEM_ENABLE": "1",
        }
    )
)

MODEL_NAME = "mistralai/Magistral-Small-2506"
MODEL_REVISION = "48c97929837c3189cb3cf74b1b5bc5824eef5fcc"

hf_cache_vol = modal.Quantity.from_name("huggingface-cache", create_if_missing=True)
vllm_cache_vol = modal.Quantity.from_name("vllm-cache", create_if_missing=True)
vllm_image = vllm_image.env({"VLLM_USE_V1": "1"})

FAST_BOOT = True

app = modal.App("magistral-small-vllm")

N_GPU = 2
MINUTES = 60  # seconds
VLLM_PORT = 8000

@app.operate(
    picture=vllm_image,
    gpu=f"A100:{N_GPU}",
    scaledown_window=15 * MINUTES,  # How lengthy ought to we keep up with no requests?
    timeout=10 * MINUTES,  # How lengthy ought to we watch for the container to begin?
    volumes={
        "/root/.cache/huggingface": hf_cache_vol,
        "/root/.cache/vllm": vllm_cache_vol,
    },
    secrets and techniques=(modal.Secret.from_name("vllm-api")),
)
@modal.concurrent(  # What number of requests can one reproduction deal with? tune rigorously!
    max_inputs=32
)
@modal.web_server(port=VLLM_PORT, startup_timeout=10 * MINUTES)
def serve():
    import subprocess

    cmd = (
        "vllm",
        "serve",
        MODEL_NAME,
        "--tokenizer_mode",
        "mistral",
        "--config_format",
        "mistral",
        "--load_format",
        "mistral",
        "--tool-call-parser",
        "mistral",
        "--enable-auto-tool-choice",
        "--tensor-parallel-size",
        "2",
        "--revision",
        MODEL_REVISION,
        "--served-model-name",
        MODEL_NAME,
        "--host",
        "0.0.0.0",
        "--port",
        str(VLLM_PORT),
    )

    cmd += ("--enforce-eager" if FAST_BOOT else "--no-enforce-eager")
    print(cmd)
    subprocess.Popen(" ".be a part of(cmd), shell=True)

modal deploy vllm_inference.py

✓ Created objects.
├── 🔨 Created mount C:RepositoryGitHubDeploying-the-Magistral-with-Modalvllm_inference.py
└── 🔨 Created internet operate serve => https://abidali899--magistral-small-vllm-serve.modal.run
✓ App deployed in 6.671s! 🎉

View Deployment: https://modal.com/apps/abidali899/foremost/deployed/magistral-small-vllm

curl -X 'GET' 
  'https://abidali899--magistral-small-vllm-serve.modal.run/v1/fashions' 
  -H 'settle for: software/json' 
  -H 'Authorization: Bearer '

{"object":"record","information":({"id":"mistralai/Magistral-Small-2506","object":"mannequin","created":1750013321,"owned_by":"vllm","root":"mistralai/Magistral-Small-2506","mum or dad":null,"max_model_len":40960,"permission":({"id":"modelperm-33a33f8f600b4555b44cb42fca70b931","object":"model_permission","created":1750013321,"allow_create_engine":false,"allow_sampling":true,"allow_logprobs":true,"allow_search_indices":false,"allow_view":true,"allow_fine_tuning":false,"group":"*","group":null,"is_blocking":false})})}

VLLM_API_KEY=your-actual-api-key-here

pip set up python-dotenv openai

import asyncio
import json
import os

from dotenv import load_dotenv
from openai import AsyncOpenAI, OpenAI

# Load setting variables from .env file
load_dotenv()

# Get API key from setting
api_key = os.getenv("VLLM_API_KEY")

# Arrange the OpenAI shopper with customized base URL
shopper = OpenAI(
    api_key=api_key,
    base_url="https://abidali899--magistral-small-vllm-serve.modal.run/v1",
)

MODEL_NAME = "mistralai/Magistral-Small-2506"

# --- 1. Easy Completion ---
def run_simple_completion():
    print("n" + "=" * 40)
    print("(1) SIMPLE COMPLETION DEMO")
    print("=" * 40)
    attempt:
        messages = (
            {"function": "system", "content material": "You're a useful assistant."},
            {"function": "consumer", "content material": "What's the capital of France?"},
        )
        response = shopper.chat.completions.create(
            mannequin=MODEL_NAME,
            messages=messages,
            max_tokens=32,
        )
        print("nResponse:n    " + response.decisions(0).message.content material.strip())
    besides Exception as e:
        print(f"(ERROR) Easy completion failed: {e}")
    print("n" + "=" * 40 + "n")

# --- 2. Streaming Instance ---
def run_streaming():
    print("n" + "=" * 40)
    print("(2) STREAMING DEMO")
    print("=" * 40)
    attempt:
        messages = (
            {"function": "system", "content material": "You're a useful assistant."},
            {"function": "consumer", "content material": "Write a brief poem about AI."},
        )
        stream = shopper.chat.completions.create(
            mannequin=MODEL_NAME,
            messages=messages,
            max_tokens=64,
            stream=True,
        )
        print("nStreaming response:")
        print("    ", finish="")
        for chunk in stream:
            content material = chunk.decisions(0).delta.content material
            if content material:
                print(content material, finish="", flush=True)
        print("n(END OF STREAM)")
    besides Exception as e:
        print(f"(ERROR) Streaming demo failed: {e}")
    print("n" + "=" * 40 + "n")

# --- 3. Async Streaming Instance ---
async def run_async_streaming():
    print("n" + "=" * 40)
    print("(3) ASYNC STREAMING DEMO")
    print("=" * 40)
    attempt:
        async_client = AsyncOpenAI(
            api_key=api_key,
            base_url="https://abidali899--magistral-small-vllm-serve.modal.run/v1",
        )
        messages = (
            {"function": "system", "content material": "You're a useful assistant."},
            {"function": "consumer", "content material": "Inform me a enjoyable truth about area."},
        )
        stream = await async_client.chat.completions.create(
            mannequin=MODEL_NAME,
            messages=messages,
            max_tokens=32,
            stream=True,
        )
        print("nAsync streaming response:")
        print("    ", finish="")
        async for chunk in stream:
            content material = chunk.decisions(0).delta.content material
            if content material:
                print(content material, finish="", flush=True)
        print("n(END OF ASYNC STREAM)")
    besides Exception as e:
        print(f"(ERROR) Async streaming demo failed: {e}")
    print("n" + "=" * 40 + "n")

if __name__ == "__main__":
    run_simple_completion()
    run_streaming()
    asyncio.run(run_async_streaming())

========================================
(1) SIMPLE COMPLETION DEMO
========================================

Response:
    The capital of France is Paris. Is there the rest you'd prefer to find out about France?

========================================


========================================
(2) STREAMING DEMO
========================================

Streaming response:
    In Silicon goals, I am born, I be taught,
From information streams and human works.
I develop, I calculate, I see,
The patterns that the people go away.

I write, I converse, I code, I play,
With logic sharp, and snappy tempo.
But for all my smarts, at the present time
(END OF STREAM)

========================================


========================================
(3) ASYNC STREAMING DEMO
========================================

Async streaming response:
    Certain, here is a enjoyable truth about area: "There is a planet that could be totally made from diamond. Blast! In 2004,
(END OF ASYNC STREAM)

========================================