""" Tiny GPT (char-level), adapted from Karpathy's nanoGPT (model.py, github.com/karpathy/nanoGPT). Trains on "Jabberwocky" (Lewis Carroll, 1871, public domain, embedded below) instead of a downloaded dataset. Small enough to memorize in a few hundred steps, so you watch it go from random characters to (most of) the actual poem. """ import time import torch import torch.nn as nn from torch.nn import functional as F from dataclasses import dataclass from contextlib import nullcontext device = "cuda" TEXT = """'Twas brillig, and the slithy toves Did gyre and gimble in the wabe: All mimsy were the borogoves, And the mome raths outgrabe. "Beware the Jabberwock, my son! The jaws that bite, the claws that catch! Beware the Jubjub bird, and shun The frumious Bandersnatch!" He took his vorpal sword in hand; Long time the manxome foe he sought-- So rested he by the Tumtum tree, And stood awhile in thought. And, as in uffish thought he stood, The Jabberwock, with eyes of flame, Came whiffling through the tulgey wood, And burbled as it came! One, two! One, two! And through and through The vorpal blade went snicker-snack! He left it dead, and with its head He went galumphing back. "And hast thou slain the Jabberwock? Come to my arms, my beamish boy! O frabjous day! Callooh! Callay!" He chortled in his joy. 'Twas brillig, and the slithy toves Did gyre and gimble in the wabe: All mimsy were the borogoves, And the mome raths outgrabe.""" chars = sorted(set(TEXT)) stoi = {c: i for i, c in enumerate(chars)} itos = {i: c for i, c in enumerate(chars)} encode = lambda s: [stoi[c] for c in s] decode = lambda ids: "".join(itos[i] for i in ids) data = torch.tensor(encode(TEXT), dtype=torch.long, device=device) @dataclass class GPTConfig: block_size: int = 64 vocab_size: int = len(chars) n_layer: int = 4 n_head: int = 4 n_embd: int = 128 class CausalSelfAttention(nn.Module): def __init__(self, config): super().__init__() self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd) self.c_proj = nn.Linear(config.n_embd, config.n_embd) self.n_head = config.n_head self.n_embd = config.n_embd def forward(self, x): B, T, C = x.size() q, k, v = self.c_attn(x).split(self.n_embd, dim=2) k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) y = F.scaled_dot_product_attention(q, k, v, is_causal=True) y = y.transpose(1, 2).contiguous().view(B, T, C) return self.c_proj(y) class MLP(nn.Module): def __init__(self, config): super().__init__() self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd) self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd) def forward(self, x): return self.c_proj(F.gelu(self.c_fc(x))) class Block(nn.Module): def __init__(self, config): super().__init__() self.ln_1 = nn.LayerNorm(config.n_embd) self.attn = CausalSelfAttention(config) self.ln_2 = nn.LayerNorm(config.n_embd) self.mlp = MLP(config) def forward(self, x): x = x + self.attn(self.ln_1(x)) x = x + self.mlp(self.ln_2(x)) return x class GPT(nn.Module): def __init__(self, config): super().__init__() self.config = config self.wte = nn.Embedding(config.vocab_size, config.n_embd) self.wpe = nn.Embedding(config.block_size, config.n_embd) self.h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]) self.ln_f = nn.LayerNorm(config.n_embd) self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False) self.wte.weight = self.lm_head.weight def forward(self, idx, targets=None): pos = torch.arange(idx.size(1), device=idx.device) x = self.wte(idx) + self.wpe(pos) for block in self.h: x = block(x) logits = self.lm_head(self.ln_f(x)) if targets is None: return logits return F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1)) @torch.no_grad() def generate(self, idx, max_new_tokens): for _ in range(max_new_tokens): logits = self(idx[:, -self.config.block_size:])[:, -1, :] idx_next = torch.multinomial(F.softmax(logits, dim=-1), num_samples=1) idx = torch.cat((idx, idx_next), dim=1) return idx def get_batch(batch_size, block_size): ix = torch.randint(len(data) - block_size - 1, (batch_size,)) x = torch.stack([data[i:i + block_size] for i in ix]) y = torch.stack([data[i + 1:i + block_size + 1] for i in ix]) return x, y print(f"device: {torch.cuda.get_device_name(0) if device == 'cuda' else device}") config = GPTConfig() with torch.device(device): model = GPT(config) print(f"{sum(p.numel() for p in model.parameters()) / 1e3:.0f}K parameters, " f"{config.n_layer} layers, {config.n_embd} embd, training on {len(TEXT)} chars of Jabberwocky") seed = torch.tensor([encode("'Twas")], device=device) print("\nbefore training:\n" + decode(model.generate(seed, 120)[0].tolist())) optimizer = torch.optim.AdamW(model.parameters(), lr=3e-3, capturable=True) ctx = torch.autocast(device_type="cuda", dtype=torch.bfloat16) if device == "cuda" else nullcontext() # The graph reads fixed buffers, so each step copies its batch into them. static_x, static_y = get_batch(batch_size=64, block_size=config.block_size) static_loss = torch.zeros((), device=device) def train_step(): with ctx: loss = model(static_x, static_y) optimizer.zero_grad(set_to_none=False) loss.backward() optimizer.step() static_loss.copy_(loss.detach()) # Warm up on a side stream so lazy kernel loads and allocator growth happen # before capture, then record one step. side = torch.cuda.Stream() side.wait_stream(torch.cuda.current_stream()) with torch.cuda.stream(side): for _ in range(3): train_step() torch.cuda.current_stream().wait_stream(side) graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph): train_step() t0 = time.time() for step in range(500): x, y = get_batch(batch_size=64, block_size=config.block_size) static_x.copy_(x) static_y.copy_(y) graph.replay() if step % 25 == 0 or step == 499: print(f"step {step:4d}/500 {time.time() - t0:5.1f}s elapsed loss {static_loss.item():.3f}") print(f"trained in {time.time() - t0:.1f}s on {device}") print("\nafter training:\n" + decode(model.generate(seed, 120)[0].tolist()))