BG   = "#fafaf8"
DARK = "#1a1a1a"

# Colour ramp: blue for widespread tokens, crimson for uncommon
TOKEN_COLORS = ("#1a5276", "#2471a3", "#5dade2", "#e67e22", "#c0392b", "#7d2a2a")

steps = np.arange(N_STEPS)

fig = plt.determine(figsize=(16, 11), facecolor=BG)
fig.suptitle(
    "SGD vs. Adam on Uncommon Tokens -- Frequency Bias and Variance Normalization",
    fontsize=14, fontweight="daring", colour=DARK, y=0.99
)

gs = gridspec.GridSpec(2, 3, determine=fig, hspace=0.45, wspace=0.35)

# ── 1. SGD weight trajectories ────────────────────────────────
ax1 = fig.add_subplot(gs(0, :2))
ax1.set_facecolor(BG)
ax1.axhline(1.0, colour=DARK, lw=1, ls="--", alpha=0.3, label="True weight = 1.0")

for i, (token, colour) in enumerate(zip(TOKENS, TOKEN_COLORS)):
    ax1.plot(steps, sgd_history(:, i), colour=colour, lw=1.8,
             label=f"{token} (freq={FREQ(i):.3f})")

ax1.set_title("SGD -- Weight TrajectoriesnRare tokens barely transfer from zero", fontsize=11, colour=DARK)
ax1.set_xlabel("Coaching Step", fontsize=9)
ax1.set_ylabel("Realized Weight", fontsize=9)
ax1.legend(fontsize=8, loc="proper")
ax1.set_ylim(-0.3, 1.6)
ax1.spines(("prime", "proper")).set_visible(False)

# Annotate failure zone
ax1.annotate(
    "Uncommon tokens stucknnear zero",
    xy=(N_STEPS * 0.95, sgd_history(-1, 5)),
    xytext=(N_STEPS * 0.65, -0.15),
    fontsize=8.5, colour="#c0392b",
    arrowprops=dict(arrowstyle="->", colour="#c0392b", lw=1.2),
    bbox=dict(boxstyle="spherical,pad=0.3", facecolor="#fff0f0", edgecolor="#c0392b", alpha=0.85)
)

# ── 2. Ultimate weight error bar chart ───────────────────────────
ax2 = fig.add_subplot(gs(0, 2))
ax2.set_facecolor(BG)

x      = np.arange(6)
w_sgd  = sgd_final
w_adam = adam_final
width  = 0.35

bars_sgd  = ax2.bar(x - width/2, np.abs(w_sgd  - TRUE_W), width, colour="#c0392b", alpha=0.85, label="SGD error")
bars_adam = ax2.bar(x + width/2, np.abs(w_adam - TRUE_W), width, colour="#2980b9", alpha=0.85, label="Adam error")

ax2.set_xticks(x)
ax2.set_xticklabels((t(:8) for t in TOKENS), rotation=30, ha="proper", fontsize=8)
ax2.set_ylabel("|discovered w − true w|", fontsize=9)
ax2.set_title("Ultimate Weight Errorn(decrease = higher)", fontsize=11, colour=DARK)
ax2.legend(fontsize=8)
ax2.spines(("prime", "proper")).set_visible(False)

# ── 3. Adam weight trajectories ───────────────────────────────
ax3 = fig.add_subplot(gs(1, :2))
ax3.set_facecolor(BG)
ax3.axhline(1.0, colour=DARK, lw=1, ls="--", alpha=0.3, label="True weight = 1.0")

for i, (token, colour) in enumerate(zip(TOKENS, TOKEN_COLORS)):
    ax3.plot(steps, adam_history(:, i), colour=colour, lw=1.8,
             label=f"{token} (freq={FREQ(i):.3f})")

ax3.set_title("Adam -- Weight TrajectoriesnRare tokens converge by way of variance normalization", fontsize=11, colour=DARK)
ax3.set_xlabel("Coaching Step", fontsize=9)
ax3.set_ylabel("Realized Weight", fontsize=9)
ax3.legend(fontsize=8, loc="proper")
ax3.set_ylim(-0.3, 1.6)
ax3.spines(("prime", "proper")).set_visible(False)

ax3.annotate(
    "Uncommon tokens convergendespite sparse gradients",
    xy=(N_STEPS * 0.95, adam_history(-1, 5)),
    xytext=(N_STEPS * 0.60, 0.3),
    fontsize=8.5, colour="#27ae60",
    arrowprops=dict(arrowstyle="->", colour="#27ae60", lw=1.2),
    bbox=dict(boxstyle="spherical,pad=0.3", facecolor="#f0fff4", edgecolor="#27ae60", alpha=0.85)
)

# ── 4. Efficient LR vs frequency ─────────────────────────────
ax4 = fig.add_subplot(gs(1, 2))
ax4.set_facecolor(BG)

ax4.scatter(FREQ, effective_lr, c=TOKEN_COLORS, s=120, zorder=5, edgecolors="white", lw=1.5)
for i, token in enumerate(TOKENS):
    ax4.annotate(token, (FREQ(i), effective_lr(i)),
                 textcoords="offset factors", xytext=(6, 4), fontsize=7.5, colour=TOKEN_COLORS(i))

ax4.axhline(LR, colour=DARK, lw=1, ls="--", alpha=0.4)
ax4.textual content(0.5, LR * 1.05, f"Nominal LR = {LR}", fontsize=8, colour=DARK, alpha=0.6)

ax4.set_xscale("log")
ax4.set_yscale("log")
ax4.set_xlabel("Token Frequency (log scale)", fontsize=9)
ax4.set_ylabel("Adam Efficient LR  lr/√v̂  (log scale)", fontsize=9)
ax4.set_title("Adam's Automated EqualizernRare tokens get amplified LR", fontsize=11, colour=DARK)
ax4.spines(("prime", "proper")).set_visible(False)

plt.savefig("sgd_vs_adam.png", dpi=150, bbox_inches="tight", facecolor=BG)
plt.present()

Vor einigen Wochen habe ich veröffentlicht ein Beitrag auf LinkedIn.

Der Beitrag basierte auf der folgenden Abbildung und verglich die Vorhersagen zweier Modelle: Lineare Regression und CatBoost.

Der Kern des Beitrags warfare, dass ein Gradientenverstärkungsmodell wie CatBoost eine „vernünftigere“ Interpretation der Beziehung zwischen einem Prädiktor und der Zielvariablen (nämlich Hauszustand und Hauspreis) zu liefern scheint.

Tatsächlich sind viele Beziehungen in der Natur nichtlinear.

Der Beitrag erhielt einige Einwände, unter denen der folgende Kommentar durch die große Anzahl von Likes hervorstach:

Dies löste eine Diskussion aus und ich fand den folgenden Kommentar (vom selben Autor verfasst …)