Codierungsimplementierung für Finish-to-Finish-Transformatormodelloptimierung mit Umarmung des Gesichts optimum, Onnx-Laufzeit und Quantisierung

!pip -q set up "transformers>=4.49" "optimum(onnxruntime)>=1.20.0" "datasets>=2.20" "consider>=0.4" speed up


from pathlib import Path
import os, time, numpy as np, torch
from datasets import load_dataset
import consider
from transformers import AutoTokenizer, AutoModelForSequenceClassification, pipeline
from optimum.onnxruntime import ORTModelForSequenceClassification, ORTQuantizer
from optimum.onnxruntime.configuration import QuantizationConfig


os.environ.setdefault("OMP_NUM_THREADS", "1")
os.environ.setdefault("MKL_NUM_THREADS", "1")


MODEL_ID = "distilbert-base-uncased-finetuned-sst-2-english"
ORT_DIR  = Path("onnx-distilbert")
Q_DIR    = Path("onnx-distilbert-quant")
DEVICE   = "cuda" if torch.cuda.is_available() else "cpu"
BATCH    = 16
MAXLEN   = 128
N_WARM   = 3
N_ITERS  = 8


print(f"Machine: {DEVICE} | torch={torch.__version__}")

ds = load_dataset("glue", "sst2", cut up="validation(:20%)")
texts, labels = ds("sentence"), ds("label")
metric = consider.load("accuracy")
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)


def make_batches(texts, max_len=MAXLEN, batch=BATCH):
   for i in vary(0, len(texts), batch):
       yield tokenizer(texts(i:i+batch), padding=True, truncation=True,
                       max_length=max_len, return_tensors="pt")


def run_eval(predict_fn, texts, labels):
   preds = ()
   for toks in make_batches(texts):
       preds.prolong(predict_fn(toks))
   return metric.compute(predictions=preds, references=labels)("accuracy")


def bench(predict_fn, texts, n_warm=N_WARM, n_iters=N_ITERS):
   for _ in vary(n_warm):
       for toks in make_batches(texts(:BATCH*2)):
           predict_fn(toks)
   occasions = ()
   for _ in vary(n_iters):
       t0 = time.time()
       for toks in make_batches(texts):
           predict_fn(toks)
       occasions.append((time.time() - t0) * 1000)
   return float(np.imply(occasions)), float(np.std(occasions))

torch_model = AutoModelForSequenceClassification.from_pretrained(MODEL_ID).to(DEVICE).eval()


@torch.no_grad()
def pt_predict(toks):
   toks = {okay: v.to(DEVICE) for okay, v in toks.gadgets()}
   logits = torch_model(**toks).logits
   return logits.argmax(-1).detach().cpu().tolist()


pt_ms, pt_sd = bench(pt_predict, texts)
pt_acc = run_eval(pt_predict, texts, labels)
print(f"(PyTorch keen)   {pt_ms:.1f}±{pt_sd:.1f} ms | acc={pt_acc:.4f}")


compiled_model = torch_model
compile_ok = False
strive:
   compiled_model = torch.compile(torch_model, mode="reduce-overhead", fullgraph=False)
   compile_ok = True
besides Exception as e:
   print("torch.compile unavailable or failed -> skipping:", repr(e))


@torch.no_grad()
def ptc_predict(toks):
   toks = {okay: v.to(DEVICE) for okay, v in toks.gadgets()}
   logits = compiled_model(**toks).logits
   return logits.argmax(-1).detach().cpu().tolist()


if compile_ok:
   ptc_ms, ptc_sd = bench(ptc_predict, texts)
   ptc_acc = run_eval(ptc_predict, texts, labels)
   print(f"(torch.compile)   {ptc_ms:.1f}±{ptc_sd:.1f} ms | acc={ptc_acc:.4f}")

supplier = "CUDAExecutionProvider" if DEVICE == "cuda" else "CPUExecutionProvider"
ort_model = ORTModelForSequenceClassification.from_pretrained(
   MODEL_ID, export=True, supplier=supplier, cache_dir=ORT_DIR
)


@torch.no_grad()
def ort_predict(toks):
   logits = ort_model(**{okay: v.cpu() for okay, v in toks.gadgets()}).logits
   return logits.argmax(-1).cpu().tolist()


ort_ms, ort_sd = bench(ort_predict, texts)
ort_acc = run_eval(ort_predict, texts, labels)
print(f"(ONNX Runtime)    {ort_ms:.1f}±{ort_sd:.1f} ms | acc={ort_acc:.4f}")


Q_DIR.mkdir(mother and father=True, exist_ok=True)
quantizer = ORTQuantizer.from_pretrained(ORT_DIR)
qconfig = QuantizationConfig(strategy="dynamic", per_channel=False, reduce_range=True)
quantizer.quantize(model_input=ORT_DIR, quantization_config=qconfig, save_dir=Q_DIR)


ort_quant = ORTModelForSequenceClassification.from_pretrained(Q_DIR, supplier=supplier)


@torch.no_grad()
def ortq_predict(toks):
   logits = ort_quant(**{okay: v.cpu() for okay, v in toks.gadgets()}).logits
   return logits.argmax(-1).cpu().tolist()


oq_ms, oq_sd = bench(ortq_predict, texts)
oq_acc = run_eval(ortq_predict, texts, labels)
print(f"(ORT Quantized)   {oq_ms:.1f}±{oq_sd:.1f} ms | acc={oq_acc:.4f}")

pt_pipe  = pipeline("sentiment-analysis", mannequin=torch_model, tokenizer=tokenizer,
                   machine=0 if DEVICE=="cuda" else -1)
ort_pipe = pipeline("sentiment-analysis", mannequin=ort_model, tokenizer=tokenizer, machine=-1)
samples = (
   "What a unbelievable film—carried out brilliantly!",
   "This was a whole waste of time.",
   "I’m undecided how I really feel about this one."
)
print("nSample predictions (PT | ORT):")
for s in samples:
   a = pt_pipe(s)(0)("label")
   b = ort_pipe(s)(0)("label")
   print(f"- {s}n  PT={a} | ORT={b}")


import pandas as pd
rows = (("PyTorch keen", pt_ms, pt_sd, pt_acc),
       ("ONNX Runtime",  ort_ms, ort_sd, ort_acc),
       ("ORT Quantized", oq_ms, oq_sd, oq_acc))
if compile_ok: rows.insert(1, ("torch.compile", ptc_ms, ptc_sd, ptc_acc))
df = pd.DataFrame(rows, columns=("Engine", "Imply ms (↓)", "Std ms", "Accuracy"))
show(df)


print("""
Notes:
- BetterTransformer is deprecated on transformers>=4.49, therefore omitted.
- For bigger positive factors on GPU, additionally strive FlashAttention2 fashions or FP8 with TensorRT-LLM.
- For CPU, tune threads: set OMP_NUM_THREADS/MKL_NUM_THREADS; strive NUMA pinning.
- For static (calibrated) quantization, use QuantizationConfig(strategy="static") with a calibration set.
""")