Eine Codierungsimplementierung zum Parsen, Analysieren, Visualisieren und Feinabstimmen von Agent Reasoning Traces mithilfe des Lambda/Hermes-Agent-Reasoning-Traces-Datensatzes

!pip -q set up -U datasets pandas matplotlib seaborn transformers speed up trl


import json, re, random, textwrap
from collections import Counter, defaultdict
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from datasets import load_dataset, concatenate_datasets


random.seed(0)


CONFIG = "kimi"
ds = load_dataset("lambda/hermes-agent-reasoning-traces", CONFIG, break up="prepare")
print(ds)
print("Config:", CONFIG, "| Fields:", ds.column_names)
print("Classes:", sorted(set(ds("class"))))


COMPARE_BOTH = False
if COMPARE_BOTH:
   ds_kimi = load_dataset("lambda/hermes-agent-reasoning-traces", "kimi", break up="prepare")
   ds_glm  = load_dataset("lambda/hermes-agent-reasoning-traces", "glm-5.1", break up="prepare")
   ds_kimi = ds_kimi.add_column("supply", ("kimi") * len(ds_kimi))
   ds_glm  = ds_glm.add_column("supply", ("glm-5.1") * len(ds_glm))
   ds = concatenate_datasets((ds_kimi, ds_glm)).shuffle(seed=0)
   print("Mixed:", ds, "→ counts:", Counter(ds("supply")))


pattern = ds(0)
print("n=== Pattern 0 ===")
print("id        :", pattern("id"))
print("class  :", pattern("class"), "/", pattern("subcategory"))
print("process      :", pattern("process"))
print("turns     :", len(pattern("conversations")))
print("system(0) :", pattern("conversations")(0)("worth")(:220), "...n")

THINK_RE     = re.compile(r"<assume>(.*?)</assume>", re.DOTALL)
TOOL_CALL_RE = re.compile(r"<tool_call>s*({.*?})s*</tool_call>", re.DOTALL)
TOOL_RESP_RE = re.compile(r"<tool_response>s*(.*?)s*</tool_response>", re.DOTALL)


def parse_assistant(worth: str) -> dict:
   ideas = (t.strip() for t in THINK_RE.findall(worth))
   calls = ()
   for uncooked in TOOL_CALL_RE.findall(worth):
       strive:
           calls.append(json.hundreds(uncooked))
       besides json.JSONDecodeError:
           calls.append({"identify": "<malformed>", "arguments": {}})
   last = TOOL_CALL_RE.sub("", THINK_RE.sub("", worth)).strip()
   return {"ideas": ideas, "tool_calls": calls, "last": last}


def parse_tool(worth: str):
   uncooked = TOOL_RESP_RE.search(worth)
   if not uncooked: return {"uncooked": worth}
   physique = uncooked.group(1)
   strive:    return json.hundreds(physique)
   besides: return {"uncooked": physique}


first_gpt = subsequent(t for t in pattern("conversations") if t("from") == "gpt")
p = parse_assistant(first_gpt("worth"))
print("Thought preview :", (p("ideas")(0)(:160) + "...") if p("ideas") else "(none)")
print("Instrument calls       :", ((c.get("identify"), record(c.get("arguments", {}).keys())) for c in p("tool_calls")))

N = 3000
sub = ds.choose(vary(min(N, len(ds))))


tool_calls         = Counter()
parallel_widths    = Counter()
thoughts_per_turn  = ()
calls_per_traj     = ()
errors_per_traj    = ()
turns_per_traj     = ()
cat_counts         = Counter()


for ex in sub:
   cat_counts(ex("class")) += 1
   n_calls = n_err = 0
   turns_per_traj.append(len(ex("conversations")))
   for t in ex("conversations"):
       if t("from") == "gpt":
           p = parse_assistant(t("worth"))
           thoughts_per_turn.append(len(p("ideas")))
           if p("tool_calls"):
               parallel_widths(len(p("tool_calls"))) += 1
               for c in p("tool_calls"):
                   tool_calls(c.get("identify", "<unknown>")) += 1
               n_calls += len(p("tool_calls"))
       elif t("from") == "software":
           r = parse_tool(t("worth"))
           blob = json.dumps(r).decrease()
           if "error" in blob or '"exit_code": 1' in blob or "traceback" in blob:
               n_err += 1
   calls_per_traj.append(n_calls)
   errors_per_traj.append(n_err)


print(f"nScanned {len(sub)} trajectories")
print(f"Avg turns/traj      : {np.imply(turns_per_traj):.1f}")
print(f"Avg software calls/traj : {np.imply(calls_per_traj):.1f}")
print(f"% with >=1 error    : {100*np.imply((e>0 for e in errors_per_traj)):.1f}%")
print(f"% parallel turns    : {100*sum(v for okay,v in parallel_widths.gadgets() if okay>1)/max(1,sum(parallel_widths.values())):.1f}%")
print("High 10 instruments        :", tool_calls.most_common(10))


fig, axes = plt.subplots(2, 2, figsize=(13, 9))


high = tool_calls.most_common(15)
axes(0,0).barh((t for t,_ in high)(::-1), (c for _,c in high)(::-1), colour="teal")
axes(0,0).set_title("High 15 instruments by name quantity")
axes(0,0).set_xlabel("calls")


ks = sorted(parallel_widths)
axes(0,1).bar((str(okay) for okay in ks), (parallel_widths(okay) for okay in ks), colour="coral")
axes(0,1).set_title("Instrument-calls per assistant flip (parallel width)")
axes(0,1).set_xlabel("# software calls in a single flip"); axes(0,1).set_ylabel("depend")
axes(0,1).set_yscale("log")


axes(1,0).hist(turns_per_traj, bins=40, colour="steelblue")
axes(1,0).set_title("Dialog size"); axes(1,0).set_xlabel("turns")


cats, vals = zip(*cat_counts.most_common())
axes(1,1).pie(vals, labels=cats, autopct="%1.0f%%", startangle=90)
axes(1,1).set_title("Class distribution")


plt.tight_layout(); plt.present()

def render_trace(ex, max_chars=350):
   print(f"n{'='*72}nTASK ({ex('class')} / {ex('subcategory')}): {ex('process')}n{'='*72}")
   for t in ex("conversations"):
       position = t("from")
       if position == "system":
           proceed
       if position == "human":
           print(f"n(USER)n{textwrap.shorten(t('worth'), 600)}")
       elif position == "gpt":
           p = parse_assistant(t("worth"))
           for th in p("ideas"):
               print(f"n(THINK)n{textwrap.shorten(th, max_chars)}")
           for c in p("tool_calls"):
               args = json.dumps(c.get("arguments", {}))(:200)
               print(f"(CALL) {c.get('identify')}({args})")
           if p("last"):
               print(f"n(ANSWER)n{textwrap.shorten(p('last'), max_chars)}")
       elif position == "software":
           print(f"(TOOL_RESPONSE) {textwrap.shorten(t('worth'), 220)}")
   print("="*72)


idx = int(np.argmin(np.abs(np.array(turns_per_traj) - 10)))
render_trace(sub(idx))


def get_tool_schemas(ex):
   strive:    return json.hundreds(ex("instruments"))
   besides: return ()


schemas = get_tool_schemas(pattern)
print(f"nSample 0 has {len(schemas)} instruments obtainable")
for s in schemas(:3):
   fn = s.get("perform", {})
   print(" -", fn.get("identify"), "—", (fn.get("description") or "")(:80))


ROLE_MAP = {"system": "system", "human": "person", "gpt": "assistant", "software": "software"}


def to_openai_messages(conv):
   return ({"position": ROLE_MAP(t("from")), "content material": t("worth")} for t in conv)


example_msgs = to_openai_messages(pattern("conversations"))
print("nFirst 2 OpenAI messages:")
for m in example_msgs(:2):
   print(" ", m("position"), "→", m("content material")(:120).change("n", " "), "...")

from transformers import AutoTokenizer
TOK_ID = "Qwen/Qwen2.5-0.5B-Instruct"
tok = AutoTokenizer.from_pretrained(TOK_ID)


def build_masked(conv, tokenizer, max_len=2048):
   msgs = to_openai_messages(conv)
   for m in msgs:
       if m("position") == "software":
           m("position") = "person"
           m("content material") = "(TOOL OUTPUT)n" + m("content material")
   input_ids, labels = (), ()
   for m in msgs:
       textual content = tokenizer.apply_chat_template((m), tokenize=False, add_generation_prompt=False)
       ids = tokenizer.encode(textual content, add_special_tokens=False)
       input_ids.lengthen(ids)
       labels.lengthen(ids if m("position") == "assistant" else (-100) * len(ids))
   return input_ids(:max_len), labels(:max_len)


ids, lbls = build_masked(pattern("conversations"), tok)
trainable = sum(1 for x in lbls if x != -100)
print(f"nTokenized instance: {len(ids)} tokens, {trainable} trainable ({100*trainable/len(ids):.1f}%)")


think_lens, call_lens, ans_lens = (), (), ()
for ex in sub.choose(vary(min(500, len(sub)))):
   for t in ex("conversations"):
       if t("from") != "gpt": proceed
       p = parse_assistant(t("worth"))
       for th in p("ideas"): think_lens.append(len(th))
       for c in p("tool_calls"): call_lens.append(len(json.dumps(c)))
       if p("last"): ans_lens.append(len(p("last")))


plt.determine(figsize=(10,4))
plt.hist((think_lens, call_lens, ans_lens), bins=40, log=True,
        label=("<assume>", "<tool_call>", "last reply"), stacked=False)
plt.legend(); plt.xlabel("characters"); plt.title("Size distributions (log y)")
plt.tight_layout(); plt.present()


class TraceReplayer:
   def __init__(self, ex):
       self.ex = ex
       self.steps = ()
       pending = None
       for t in ex("conversations"):
           if t("from") == "gpt":
               if pending: self.steps.append(pending)
               pending = {"assume": parse_assistant(t("worth")), "responses": ()}
           elif t("from") == "software" and pending:
               pending("responses").append(parse_tool(t("worth")))
       if pending: self.steps.append(pending)
   def __len__(self): return len(self.steps)
   def play(self, i):
       s = self.steps(i)
       print(f"n── Step {i+1}/{len(self)} ──")
       for th in s("assume")("ideas"):
           print(f"💭 {textwrap.shorten(th, 280)}")
       for c in s("assume")("tool_calls"):
           print(f"⚙️  {c.get('identify')}({json.dumps(c.get('arguments', {}))(:140)})")
       for r in s("responses"):
           print(f"📥 {textwrap.shorten(json.dumps(r), 200)}")
       if s("assume")("last"):
           print(f"💬 {textwrap.shorten(s('assume')('last'), 200)}")


rp = TraceReplayer(pattern)
for i in vary(min(3, len(rp))):
   rp.play(i)


TRAIN = False
if TRAIN:
   import torch
   from transformers import AutoModelForCausalLM
   from trl import SFTTrainer, SFTConfig


   train_subset = ds.choose(vary(200))


   def to_text(batch):
       msgs = to_openai_messages(batch("conversations"))
       for m in msgs:
           if m("position") == "software":
               m("position") = "person"; m("content material") = "(TOOL)n" + m("content material")
       batch("textual content") = tok.apply_chat_template(msgs, tokenize=False, add_generation_prompt=False)
       return batch


   train_subset = train_subset.map(to_text)


   mannequin = AutoModelForCausalLM.from_pretrained(
       TOK_ID,
       torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
       device_map="auto" if torch.cuda.is_available() else None,
   )


   cfg = SFTConfig(
       output_dir="hermes-sft-demo",
       per_device_train_batch_size=1,
       gradient_accumulation_steps=4,
       max_steps=20,
       learning_rate=2e-5,
       logging_steps=2,
       max_seq_length=1024,
       dataset_text_field="textual content",
       report_to="none",
       fp16=torch.cuda.is_available(),
   )
   SFTTrainer(mannequin=mannequin, args=cfg, train_dataset=train_subset, processing_class=tok).prepare()
   print("Wonderful-tune demo completed.")


print("n✅ Tutorial full. You now have parsers, analytics, plots, a replayer, "
     "tokenized + label-masked SFT examples, and an optionally available coaching hook.")