key = f"loss@{tok.name}"
+frame[key] = losses
+cache.put(cfg_hash, frame)
+ def val_loss(frame, tokenizer):
+ return frame[f"loss@{tokenizer}"]
+ @@ grad/model.py L64 @@ +- attn = MHA(d, heads=d // 64) ++ attn = MHA(d, heads=d // 96, ++ qk_norm=True) +@@ grad/train.py L21 @@ +- warmup = 1000 ++ warmup = 2500+
\subsection{Width vs depth} + +At fixed compute the widest configuration wins: +\gradcite{run-8f21} reaches $L_{val}=2.94$, inside the +band registered as \gradexp{exp-7c41}. + +Depth beyond 12 layers does not help. % uncited + +\begin{figure} + \includegraphics{figures/loss_band.png} + \caption{Nine configurations against the band.} +\end{figure}+
| job | +what | +device | +progress | +eta | +cost | +state | +
|---|---|---|---|---|---|---|
| job-221 | sweep w1024 d12 · bpe-50k | cuda:0 · 4090 | +6m 10s | $0.61 | +RUNNING | +|
| job-222 | sweep w512 d16 · bpe-50k | cuda:1 · queued | ++ | — | ~$0.58 | +WAITING GATE | +
| job-220 | evolve gen 12 eval | cuda:0 | +done | $0.34 | +DONE | +|
| job-219 | sweep w256 d8 · bpe-32k | cuda:1 | +— | $0.12 | +FAILED · KeyError | +
import torch, math +from grad.scaling import sweep, val_loss + +CFG = dict(widths=[256, 512, 1024], depths=[8, 12, 16], tokens=1e9) +df = sweep(**CFG, cache=True) # 9 configs, ledger-backed +print(f"{len(df)} rows · {df.tokens.sum():.2e} tokens")+
9 rows · 9.00e+09 tokens+
df.sort_values("val_loss").head(4)+
| run | +width | +depth | +params | +val_loss | +band | +
|---|---|---|---|---|---|
| run-8f21 | 1024 | 12 | 1.31B | 2.94 | +IN BAND | +
| run-8f19 | 512 | 16 | 1.29B | 3.01 | +IN BAND | +
| run-8f24 | 1024 | 16 | 1.74B | 3.28 | +HIGH · UNJUDGED | +
| run-8f17 | 256 | 8 | 0.41B | 3.55 | +OUT OF BAND | +
plot_band(df, expectation="exp-7c41", save="figures/loss_band.png")+
val_loss(df, at_tokens=1e9, tokenizer="bpe-32k")+
Traceback (most recent call last)
+ File "grad/scaling.py", line 148, in val_loss
+ return frame[f"loss@{tokenizer}"]
+KeyError: 'loss@bpe-32k' — the sweep cached 'bpe-50k'
+