diff --git a/README.md b/README.md index a2916ce7..6584eab1 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,46 @@ torchrun --nproc_per_node gpu -m sparsify meta-llama/Meta-Llama-3-8B --distribut The above command trains an SAE for every _even_ layer of Llama 3 8B, using all available GPUs. It accumulates gradients over 8 minibatches, and splits each minibatch into 2 microbatches before feeding them into the SAE encoder, thus saving a lot of memory. It also loads the model in 8-bit precision using `bitsandbytes`. This command requires no more than 48GB of memory per GPU on an 8 GPU node. +Sharding whole layers has two limits: the number of hookpoints must be divisible by +the number of GPUs, and every rank still materialises the world-sized activation for +every hookpoint, including the ones it does not own. `--distribute_latents` shards the +*latent* dimension of every sparse coder instead, so each rank holds `num_latents / +world_size` rows of every coder: + +```bash +torchrun --nproc_per_node gpu -m sparsify HuggingFaceTB/SmolLM2-135M --distribute_latents --batch_size 8 +``` + +Parameters, gradients and optimizer state shrink with the world size, every hookpoint +stays on every rank so any number of GPUs works, and the ranks agree on a single global +top-k, which makes the result mathematically identical to single-GPU training. +Checkpoints are gathered and written unsharded, so they load with the ordinary loader. + +Measured on SmolLM2-135M, 30 hookpoints, batch size 8 per rank, A100-40GB. Peak +allocated per rank, and median time per step (the same number of tokens per step in +every column): + +| GPUs | DDP | `--distribute_modules` | `--distribute_latents` | +|---|---|---|---| +| 2 | 9.04 GiB / 0.33 s | 3.86 GiB / 0.38 s | **3.65 GiB / 0.33 s** | +| 4 | 9.04 GiB | not runnable (30 not divisible by 4) | **2.24 GiB / 0.36 s** | +| 8 | 9.04 GiB / 0.30 s | not runnable | **1.69 GiB / 0.43 s** | + +**Which to use.** `--distribute_latents` is strictly better than `--distribute_modules`: +lighter, faster, and it runs at GPU counts where `--distribute_modules` refuses to +start. Against DDP it buys a large memory saving for a modest slowdown -- 5.4x less +memory for 1.4x the step time at 8 GPUs -- because the ranks must agree on the top-k, +which costs two collectives per coder per forward and none in the backward. + +The slowdown grows with the batch, since the collectives scale with the gathered batch +while the per-rank shard does not: at batch size 64 the same 8-GPU run is 1.9x lighter +than DDP and 2.1x slower. Prefer a smaller per-rank batch with more gradient +accumulation when using many ranks. + +`--distribute_latents` supports the AuxK loss, `multi_topk` and `skip_connection`, but +requires `activation="topk"`; `groupmax` partitions the latent dimension itself and +cannot be split this way. + ## TODO There are several features that we'd like to add in the near future: diff --git a/sparsify/config.py b/sparsify/config.py index bf9ba997..6613ec3a 100644 --- a/sparsify/config.py +++ b/sparsify/config.py @@ -101,6 +101,14 @@ class TrainConfig(Serializable): distribute_modules: bool = False """Store one copy of each sparse coder, instead of copying them across devices.""" + distribute_latents: bool = False + """Shard the latent dimension of every sparse coder across devices. + + Like `distribute_modules` this stores one copy of each parameter, but it keeps + every hookpoint on every rank, so the number of hookpoints need not be + divisible by the world size and the `[N, num_latents]` pre-activation shrinks + with the world size instead of growing with it.""" + save_every: int = 1000 """Save sparse coders every `save_every` steps.""" @@ -121,6 +129,17 @@ def __post_init__(self): if self.layers and self.layer_stride != 1: raise ValueError("Cannot specify both `layers` and `layer_stride`.") + if self.distribute_modules and self.distribute_latents: + raise ValueError( + "Cannot combine `distribute_modules` and `distribute_latents`." + ) + + if self.distribute_latents and self.loss_fn in ("ce", "kl"): + raise ValueError( + "Sharding latents across ranks is not compatible with the " + "cross-entropy or KL divergence losses." + ) + if self.distribute_modules and self.loss_fn in ("ce", "kl"): raise ValueError( "Distributing modules across ranks is not compatible with the " diff --git a/sparsify/fused_encoder.py b/sparsify/fused_encoder.py index 272d47db..854e28ce 100644 --- a/sparsify/fused_encoder.py +++ b/sparsify/fused_encoder.py @@ -3,6 +3,13 @@ import torch import torch.nn.functional as F +# Upper bound on the temporary built inside the weight-gradient accumulation. The +# unchunked form allocates N * k * D elements at once, which for a wide model and +# a large k is the single biggest tensor in the backward pass -- at N=8192, k=32, +# D=1024 it is 1 GiB. Consuming it in row-blocks bounds that without changing the +# result. +BACKWARD_CHUNK_BYTES = 64 * 1024 * 1024 + class EncoderOutput(NamedTuple): top_acts: torch.Tensor @@ -66,15 +73,27 @@ def backward(ctx, grad_values, grad_indices, grad_preacts): # --- Grad w.r.t. weight --- if ctx.needs_input_grad[1]: grad_weight = torch.zeros_like(weight) - # Compute contributions from each top-k element: - # computed as grad_values * input for each top-k location. - contributions = grad_values.unsqueeze(2) * input.unsqueeze(1) - _, _, D = contributions.shape - # Flatten contributions to shape (N*k, D) - contributions = contributions.reshape(-1, D) - - # Accumulate contributions into the correct rows of grad_weight. - grad_weight.index_add_(0, indices.flatten(), contributions.type_as(weight)) + # Each top-k location contributes `grad_values * input` to its row of + # grad_weight. Materialising all of them at once costs N * k * D + # elements, so walk the batch in blocks sized to BACKWARD_CHUNK_BYTES + # and fold each block in as it is built. The arithmetic is unchanged; + # only the accumulation order inside index_add_ differs, and that was + # already unspecified on CUDA. + N, k = grad_values.shape + D = input.shape[-1] + itemsize = torch.promote_types(grad_values.dtype, input.dtype).itemsize + rows = max(1, BACKWARD_CHUNK_BYTES // max(1, k * D * itemsize)) + + for start in range(0, N, rows): + stop = min(start + rows, N) + block = grad_values[start:stop].unsqueeze(2) * input[ + start:stop + ].unsqueeze(1) + grad_weight.index_add_( + 0, + indices[start:stop].flatten(), + block.reshape(-1, D).type_as(weight), + ) # --- Grad w.r.t. bias --- if bias is not None and ctx.needs_input_grad[2]: diff --git a/sparsify/latent_parallel.py b/sparsify/latent_parallel.py new file mode 100644 index 00000000..98055f0b --- /dev/null +++ b/sparsify/latent_parallel.py @@ -0,0 +1,242 @@ +"""Tensor parallelism over the latent dimension of a sparse coder. + +`--distribute_modules` shards whole hookpoints across ranks: each rank owns a few +SAEs outright. That removes the replicated parameter/optimizer state, but it pays +for it twice: every rank must all-gather the world-sized activation for *every* +hookpoint, including the ones it does not own, and the number of hookpoints has to +be divisible by the world size. + +Sharding the latent dimension instead keeps every hookpoint on every rank and +splits each sparse coder's rows. Rank `r` owns latents +`[r * M/W, (r+1) * M/W)` of every coder, so parameters, gradients and optimizer +state all shrink by `W` exactly as they do under `--distribute_modules`, while the +`[N, M]` pre-activation, the dominant transient, shrinks to `[N, M/W]` instead of +growing. + +Two collectives per coder per forward, and none in the backward: + +1. `global_topk`: each rank offers its local top-k as candidates and all ranks + agree on the same global winners. The global top-k of size k can draw at most k + entries from any one rank, so local top-k candidates are sufficient. +2. `all_reduce_sum`: each rank decodes only the winners it owns and the partial + reconstructions are summed. The backward of that sum is the identity, because + every rank computes the same loss from the same summed output. +""" + +from typing import NamedTuple + +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch import Tensor + +from .fused_encoder import BACKWARD_CHUNK_BYTES + + +def shard_size(num_latents: int, world_size: int) -> int: + """Number of latents owned by each rank.""" + if num_latents % world_size: + raise ValueError( + f"num_latents ({num_latents}) must be divisible by the world size " + f"({world_size}) to shard the latent dimension" + ) + return num_latents // world_size + + +class AllReduceSum(torch.autograd.Function): + """Sum a tensor across ranks; the backward is the identity. + + Valid only when every rank goes on to compute the same scalar loss from the + summed result, which is the case here: all ranks see the same input batch and + the same reconstruction, so `dL/d(partial_r)` is `dL/d(sum)` on every rank and + no gradient communication is needed. + """ + + @staticmethod + def forward(ctx, x: Tensor) -> Tensor: # type: ignore[override] + out = x.contiguous().clone() + dist.all_reduce(out, op=dist.ReduceOp.SUM) + return out + + @staticmethod + def backward(ctx, grad: Tensor): # type: ignore[override] + return grad + + +def all_reduce_sum(x: Tensor) -> Tensor: + return AllReduceSum.apply(x) # type: ignore[return-value] + + +def global_sum(x: Tensor) -> Tensor: + """Sum a tensor across ranks in place, outside the autograd graph.""" + total = x.detach().clone() + dist.all_reduce(total, op=dist.ReduceOp.SUM) + return total + + +def global_topk_mask(local_acts: Tensor, k: int) -> Tensor: + """Which of this rank's local top-k entries survive the *global* top-k. + + `local_acts` is `[N, k]`, the activations this rank offers as candidates. + Returns a `[N, k]` bool mask. Ranks agree because they all run the same + deterministic reduction over the same gathered tensor. + """ + world_size = dist.get_world_size() + rank = dist.get_rank() + n, k_local = local_acts.shape + + # all_gather_into_tensor concatenates along dim 0, so gather to [W, N, k] and + # transpose to put the candidate axis last: position p means rank p // k. + gathered = local_acts.new_empty(world_size * n, k_local) + dist.all_gather_into_tensor(gathered, local_acts.detach().contiguous()) + candidates = gathered.view(world_size, n, k_local).permute(1, 0, 2).reshape(n, -1) + + winners = candidates.topk(k, dim=-1, sorted=False).indices # [N, k] + mine = winners.div(k_local, rounding_mode="floor") == rank + local_pos = winners % k_local + + mask = torch.zeros(n * k_local, dtype=torch.bool, device=local_acts.device) + rows = torch.arange(n, device=local_acts.device).unsqueeze(1).expand_as(local_pos) + mask[(rows * k_local + local_pos)[mine]] = True + return mask.view(n, k_local) + + +# Parameters split along dim 0; everything else in a coder is replicated. +SHARDED_KEYS = ("encoder.weight", "encoder.bias", "W_dec") + + +def gather_full_state(sae) -> dict[str, Tensor]: + """Reassemble the unsharded state dict of a latent-sharded coder. + + Collective: every rank must call it. Only rank 0's result is meaningful to + write, but all ranks get the same tensors. Checkpoints written from this load + with the ordinary `SparseCoder.load_from_disk`, so a sharded run and a + single-GPU run produce interchangeable artifacts. + """ + world_size = dist.get_world_size() + full: dict[str, Tensor] = {} + for name, tensor in sae.state_dict().items(): + if name in SHARDED_KEYS: + buf = tensor.new_empty(tensor.shape[0] * world_size, *tensor.shape[1:]) + dist.all_gather_into_tensor(buf, tensor.contiguous()) + full[name] = buf + else: + full[name] = tensor + return full + + +def sync_replicated_grads(saes) -> None: + """Sum the gradients of the parameters that are *not* sharded. + + The sharded rows see every token on every rank, so their gradients are already + complete and need no communication. Only `b_dec` (and `W_skip`) are replicated. + """ + for sae in saes: + for param in (sae.b_dec, sae.W_skip): + if param is not None and param.grad is not None: + dist.all_reduce(param.grad, op=dist.ReduceOp.SUM) + + +# -------------------------------------------------------------------------------- +# Ragged winners. +# +# After the global reduction a rank owns only some of the k candidates it offered +# -- k / world_size of them on average. Keeping the [N, k] layout and zeroing the +# losers is correct but wasteful: the decode and the encoder weight-gradient are +# both O(N * k * d) regardless of how few entries survive, so per-rank work stays +# flat as the world grows even though each rank owns fewer latents. +# +# Compacting the survivors into a flat (token, latent, value) list makes both +# O(nnz * d) with nnz = N * k / world_size, which is the scaling the sharding was +# supposed to buy. Padding to the busiest token instead would only recover a +# factor of about k / max_per_token, roughly 2.5x at k=32 and 8 ranks, against +# world_size here. +# -------------------------------------------------------------------------------- + + +class Winners(NamedTuple): + values: Tensor + """Pre-activation of each surviving (token, latent) pair, `[nnz]`.""" + + rows: Tensor + """Token index of each pair, ascending, `[nnz]`.""" + + latents: Tensor + """Shard-local latent index of each pair, `[nnz]`.""" + + offsets: Tensor + """Start of each token's run in the flat list, `[N]`, for `embedding_bag`.""" + + +def compact_winners(mask: Tensor, cand_acts: Tensor, cand_idx: Tensor) -> Winners: + """Flatten the surviving candidates, dropping the losers entirely.""" + rows, cols = mask.nonzero(as_tuple=True) # row-major, so rows is ascending + counts = mask.sum(1) + offsets = torch.zeros_like(counts) + offsets[1:] = counts.cumsum(0)[:-1] + return Winners(cand_acts[rows, cols], rows, cand_idx[rows, cols], offsets) + + +def ragged_decode(winners: Winners, acts: Tensor, w_dec: Tensor) -> Tensor: + """Sum `acts * w_dec[latent]` per token, over the flat winner list.""" + return F.embedding_bag( + winners.latents, + w_dec, + offsets=winners.offsets, + mode="sum", + per_sample_weights=acts.to(w_dec.dtype), + ) + + +class SparseEncoderGrad(torch.autograd.Function): + """Carry the encoder's gradient for a ragged set of selected latents. + + The dense pre-activation is computed outside the graph, because it exists + only to run the top-k. This hands the selected values back to autograd with + the gradient the encoder would have produced, so the backward touches the + surviving pairs rather than every candidate the rank offered. The arithmetic + matches `FusedEncoder.backward`, restricted to those pairs -- which is exact, + since the losers were multiplied by zero and carry no gradient anyway. + """ + + @staticmethod + def forward(ctx, values, x, weight, bias, rows, latents): # type: ignore[override] + ctx.save_for_backward(x, weight, bias, rows, latents) + return values.clone() + + @staticmethod + def backward(ctx, grad_values): # type: ignore[override] + x, weight, bias, rows, latents = ctx.saved_tensors + _, needs_x, needs_w, needs_b, _, _ = ctx.needs_input_grad + + grad_x = torch.zeros_like(x) if needs_x else None + grad_w = torch.zeros_like(weight) if needs_w else None + + d = x.shape[-1] + itemsize = torch.promote_types(grad_values.dtype, x.dtype).itemsize + block = max(1, BACKWARD_CHUNK_BYTES // max(1, d * itemsize)) + + for start in range(0, latents.numel(), block): + stop = start + block + gv = grad_values[start:stop].unsqueeze(1) + row = rows[start:stop] + latent = latents[start:stop] + if grad_x is not None: + grad_x.index_add_(0, row, (weight[latent] * gv).type_as(x)) + if grad_w is not None: + grad_w.index_add_(0, latent, (x[row] * gv).type_as(weight)) + + grad_b = None + if bias is not None and needs_b: + grad_b = torch.zeros_like(bias) + grad_b.index_add_(0, latents, grad_values.type_as(bias)) + + return None, grad_x, grad_w, grad_b, None, None + + +def sparse_encoder_grad( + values: Tensor, x: Tensor, weight: Tensor, bias: Tensor, winners: Winners +) -> Tensor: + return SparseEncoderGrad.apply( # type: ignore[return-value] + values, x, weight, bias, winners.rows, winners.latents + ) diff --git a/sparsify/sparse_coder.py b/sparsify/sparse_coder.py index d0e155a1..253a4271 100644 --- a/sparsify/sparse_coder.py +++ b/sparsify/sparse_coder.py @@ -5,6 +5,7 @@ import einops import torch +import torch.nn.functional as F from huggingface_hub import snapshot_download from natsort import natsorted from safetensors import safe_open @@ -13,6 +14,16 @@ from .config import SparseCoderConfig from .fused_encoder import EncoderOutput, fused_encoder +from .latent_parallel import ( + Winners, + all_reduce_sum, + compact_winners, + global_sum, + global_topk_mask, + ragged_decode, + shard_size, + sparse_encoder_grad, +) from .utils import decoder_impl @@ -44,13 +55,48 @@ def __init__( dtype: torch.dtype | None = None, *, decoder: bool = True, + latent_shard: tuple[int, int] | None = None, ): super().__init__() self.cfg = cfg self.d_in = d_in - self.num_latents = cfg.num_latents or d_in * cfg.expansion_factor + self.global_num_latents = cfg.num_latents or d_in * cfg.expansion_factor + + # `latent_shard` is (rank, world_size): this coder holds only its slice of + # the latent dimension and cooperates with its peers in `forward`. + if latent_shard is None: + self.num_latents = self.global_num_latents + self.latent_offset = 0 + self.latent_world_size = 1 + else: + shard_rank, world_size = latent_shard + if cfg.activation != "topk": + raise ValueError( + "Latent sharding currently supports activation='topk' only; " + f"'{cfg.activation}' partitions the latent dimension itself." + ) + self.num_latents = shard_size(self.global_num_latents, world_size) + self.latent_offset = shard_rank * self.num_latents + self.latent_world_size = world_size + + if latent_shard is None: + self.encoder = nn.Linear(d_in, self.num_latents, device=device, dtype=dtype) + else: + # Draw the *global* initialization and keep only this rank's rows. + # Every rank runs the same seed, so drawing at the sharded shape + # directly would give all of them identical weights, collapsing the + # dictionary to num_latents // world_size distinct latents. Drawing at + # the full shape also makes a sharded run reproduce an unsharded run + # with the same seed, row for row. The full draw is transient. + full = nn.Linear(d_in, self.global_num_latents, device=device, dtype=dtype) + self.encoder = nn.Linear(d_in, self.num_latents, device=device, dtype=dtype) + self.encoder.weight.data.copy_( + full.weight.data[ + self.latent_offset : self.latent_offset + self.num_latents + ] + ) + del full - self.encoder = nn.Linear(d_in, self.num_latents, device=device, dtype=dtype) self.encoder.bias.data.zero_() if decoder: @@ -186,6 +232,10 @@ def device(self): def dtype(self): return self.encoder.weight.dtype + @property + def is_latent_sharded(self) -> bool: + return self.latent_world_size > 1 + def encode(self, x: Tensor) -> EncoderOutput: """Encode the input and select the top-k latents.""" if not self.cfg.transcode: @@ -195,6 +245,58 @@ def encode(self, x: Tensor) -> EncoderOutput: x, self.encoder.weight, self.encoder.bias, self.cfg.k, self.cfg.activation ) + def _count_dead(self, dead_mask: Tensor) -> int: + """Number of dead latents, counted over the whole dictionary.""" + if not self.is_latent_sharded: + return int(dead_mask.sum()) + + return int(global_sum(dead_mask.sum())) + + def _sharded_encode(self, x: Tensor) -> tuple[Tensor, Winners, Tensor]: + """Select globally, then rejoin the graph on the surviving pairs only. + + The dense pre-activation exists solely to run the top-k, so it is built + outside the graph; `sparse_encoder_grad` reattaches the encoder gradient + to the winners, which keeps the backward proportional to how many latents + this rank actually owns rather than to how many it offered. + """ + shifted = x if self.cfg.transcode else x - self.b_dec + + with torch.no_grad(): + pre_acts = F.relu(F.linear(shifted, self.encoder.weight, self.encoder.bias)) + cand_acts, cand_idx = pre_acts.topk(self.cfg.k, dim=-1, sorted=False) + winners = compact_winners( + global_topk_mask(cand_acts, self.cfg.k), cand_acts, cand_idx + ) + + acts = sparse_encoder_grad( + winners.values, shifted, self.encoder.weight, self.encoder.bias, winners + ) + return acts, winners, pre_acts + + def _sharded_decode(self, acts: Tensor, winners: Winners, x: Tensor) -> Tensor: + """Decode this rank's winners and sum the partial reconstructions. + + `b_dec` and `W_skip` are replicated rather than sharded, so each rank + contributes 1/W of them: the sum reproduces each exactly once, and each + rank ends up holding 1/W of their gradient, which + `latent_parallel.sync_replicated_grads` then adds back up. + """ + assert self.W_dec is not None, "Decoder weight was not initialized." + world_size = self.latent_world_size + + partial = ragged_decode(winners, acts, self.W_dec) + partial = partial + self.b_dec / world_size + if self.W_skip is not None: + partial = partial + (x.to(self.dtype) @ self.W_skip.mT) / world_size + return all_reduce_sum(partial) + + def _sharded_global_winners(self, scores: Tensor, k: int) -> Winners: + """Global top-k over a shard-local score matrix, as a flat winner list.""" + candidates = min(k, self.num_latents) + vals, idx = scores.topk(candidates, sorted=False) + return compact_winners(global_topk_mask(vals, k), vals, idx) + def decode(self, top_acts: Tensor, top_indices: Tensor) -> Tensor: assert self.W_dec is not None, "Decoder weight was not initialized." @@ -215,16 +317,23 @@ def forward( dead_mask: Tensor | None = None, total_variance: Tensor | None = None, ) -> ForwardOutput: - top_acts, top_indices, pre_acts = self.encode(x) + if self.is_latent_sharded: + top_acts, winners, pre_acts = self._sharded_encode(x) + top_indices = winners.latents + else: + top_acts, top_indices, pre_acts = self.encode(x) # If we aren't given a distinct target, we're autoencoding if y is None: y = x # Decode - sae_out = self.decode(top_acts, top_indices) - if self.W_skip is not None: - sae_out += x.to(self.dtype) @ self.W_skip.mT + if self.is_latent_sharded: + sae_out = self._sharded_decode(top_acts, winners, x) + else: + sae_out = self.decode(top_acts, top_indices) + if self.W_skip is not None: + sae_out = sae_out + x.to(self.dtype) @ self.W_skip.mT # Compute the residual e = y - sae_out @@ -234,7 +343,7 @@ def forward( total_variance = (y - y.mean(0)).pow(2).sum() # Second decoder pass for AuxK loss - if dead_mask is not None and (num_dead := int(dead_mask.sum())) > 0: + if dead_mask is not None and (num_dead := self._count_dead(dead_mask)) > 0: # Heuristic from Appendix B.1 in the paper k_aux = y.shape[-1] // 2 @@ -245,15 +354,20 @@ def forward( # Don't include living latents in this loss auxk_latents = torch.where(dead_mask[None], pre_acts, -torch.inf) - # Top-k dead latents - auxk_acts, auxk_indices = auxk_latents.topk(k_aux, sorted=False) - # Encourage the top ~50% of dead latents to predict the residual of the # top k living latents. We call decoder_impl directly rather than # self.decode because the residual target e already accounts for b_dec # (sae_out includes it), so adding b_dec again here would double-count it. assert self.W_dec is not None, "Decoder weight was not initialized." - e_hat = decoder_impl(auxk_indices, auxk_acts.to(self.dtype), self.W_dec.mT) + + if self.is_latent_sharded: + aux = self._sharded_global_winners(auxk_latents, k_aux) + e_hat = all_reduce_sum(ragged_decode(aux, aux.values, self.W_dec)) + else: + auxk_acts, auxk_indices = auxk_latents.topk(k_aux, sorted=False) + e_hat = decoder_impl( + auxk_indices, auxk_acts.to(self.dtype), self.W_dec.mT + ) auxk_loss = (e_hat - e.detach()).pow(2).sum() auxk_loss = scale * auxk_loss / total_variance else: @@ -263,8 +377,15 @@ def forward( fvu = l2_loss / total_variance if self.cfg.multi_topk: - top_acts, top_indices = pre_acts.topk(4 * self.cfg.k, sorted=False) - sae_out = self.decode(top_acts, top_indices) + wide_k = 4 * self.cfg.k + if self.is_latent_sharded: + wide = self._sharded_global_winners(pre_acts, wide_k) + sae_out = self._sharded_decode(wide.values, wide, x) + else: + top_acts, top_indices = pre_acts.topk(wide_k, sorted=False) + sae_out = self.decode(top_acts, top_indices) + if self.W_skip is not None: + sae_out = sae_out + x.to(self.dtype) @ self.W_skip.mT multi_topk_fvu = (sae_out - y).pow(2).sum() / total_variance else: diff --git a/sparsify/trainer.py b/sparsify/trainer.py index 88b73276..556595b9 100644 --- a/sparsify/trainer.py +++ b/sparsify/trainer.py @@ -1,15 +1,17 @@ +import json import os from collections import defaultdict from dataclasses import asdict from fnmatch import fnmatchcase from glob import glob +from pathlib import Path from typing import Sized import torch import torch.distributed as dist from datasets import Dataset as HfDataset from natsort import natsorted -from safetensors.torch import load_model +from safetensors.torch import load_model, save_file from schedulefree import ScheduleFreeWrapper from torch import Tensor, nn from torch.nn.parallel import DistributedDataParallel as DDP @@ -19,12 +21,25 @@ from .config import TrainConfig from .data import MemmapDataset +from .latent_parallel import gather_full_state, sync_replicated_grads from .muon import Muon from .sign_sgd import SignSGD -from .sparse_coder import SparseCoder +from .sparse_coder import ForwardOutput, SparseCoder from .utils import get_layer_list, resolve_widths, set_submodule +def save_full_coder(sae: SparseCoder, state: dict[str, Tensor], path: str) -> None: + """Write a reassembled (unsharded) coder so ordinary loaders can read it.""" + directory = Path(path) + directory.mkdir(parents=True, exist_ok=True) + save_file( + {k: v.contiguous() for k, v in state.items()}, + str(directory / "sae.safetensors"), + ) + with open(directory / "cfg.json", "w") as f: + json.dump({**sae.cfg.to_dict(), "d_in": sae.d_in}, f) + + def get_saes_by_layer_name(saes: dict, module_name: str): return { k: v @@ -97,7 +112,15 @@ def __init__( # Add suffix to the name to disambiguate multiple seeds name = f"{hook}/seed{seed}" if len(cfg.init_seeds) > 1 else hook self.saes[name] = SparseCoder( - input_widths[hook], cfg.sae, device, dtype=torch.float32 + input_widths[hook], + cfg.sae, + device, + dtype=torch.float32, + latent_shard=( + (dist.get_rank(), dist.get_world_size()) + if cfg.distribute_latents + else None + ), ) assert isinstance(dataset, Sized) @@ -120,7 +143,7 @@ def __init__( pgs = [ dict( params=sae.parameters(), - lr=cfg.lr or 2e-4 / (sae.num_latents / (2**14)) ** 0.5, + lr=cfg.lr or 2e-4 / (sae.global_num_latents / (2**14)) ** 0.5, ) for sae in self.saes.values() ] @@ -163,7 +186,7 @@ def __init__( pgs = [ dict( params=sae.parameters(), - lr=cfg.lr or 5e-3 / (sae.num_latents / (2**14)) ** 0.5, + lr=cfg.lr or 5e-3 / (sae.global_num_latents / (2**14)) ** 0.5, ) for sae in self.saes.values() ] @@ -187,7 +210,7 @@ def __init__( self.cfg.exclude_tokens, device=device, dtype=torch.long ) - num_latents = list(self.saes.values())[0].num_latents + num_latents = list(self.saes.values())[0].global_num_latents self.initial_k = min(num_latents, round(list(input_widths.values())[0] * 10)) self.final_k = self.cfg.sae.k @@ -256,7 +279,9 @@ def fit(self): self.model.requires_grad_(False) rank_zero = not dist.is_initialized() or dist.get_rank() == 0 - ddp = dist.is_initialized() and not self.cfg.distribute_modules + ddp = dist.is_initialized() and not ( + self.cfg.distribute_modules or self.cfg.distribute_latents + ) wandb = None if self.cfg.log_to_wandb and rank_zero: @@ -370,7 +395,7 @@ def hook(module: nn.Module, inputs, outputs): out_shape = outputs.shape # Scatter and gather the hidden states across ranks if necessary - if self.cfg.distribute_modules: + if self.cfg.distribute_modules or self.cfg.distribute_latents: world_outputs = outputs.new_empty( outputs.shape[0] * dist.get_world_size(), *outputs.shape[1:] ) @@ -391,7 +416,9 @@ def hook(module: nn.Module, inputs, outputs): dist.all_gather_into_tensor(world_mask, mask) mask = world_mask.bool() - if name not in self.module_plan[dist.get_rank()]: + if self.cfg.distribute_modules and ( + name not in self.module_plan[dist.get_rank()] + ): return # Flatten the batch and sequence dimensions @@ -436,7 +463,7 @@ def hook(module: nn.Module, inputs, outputs): out = wrapped(x=inputs, y=outputs, dead_mask=dead_mask) # Update the did_fire mask - did_fire[sae_name][out.latent_indices.flatten()] = True + did_fire[sae_name][self.fired_indices(out)] = True self.maybe_all_reduce(did_fire[sae_name], "max") # Replace the normal output with the SAE output @@ -460,7 +487,7 @@ def hook(module: nn.Module, inputs, outputs): ) # Update the did_fire mask - did_fire[sae_name][out.latent_indices.flatten()] = True + did_fire[sae_name][self.fired_indices(out)] = True self.maybe_all_reduce(did_fire[sae_name], "max") # max is "any" # Metrics that only make sense for local @@ -506,7 +533,10 @@ def hook(module: nn.Module, inputs, outputs): ) # Bookkeeping for dead feature detection - N = tokens_mask.sum().item() + n_tokens = tokens_mask.sum() + if self.cfg.distribute_latents: + dist.all_reduce(n_tokens, op=dist.ReduceOp.SUM) + N = int(n_tokens.item()) num_tokens_in_step += N # Compute clean logits if using KL loss @@ -552,6 +582,9 @@ def hook(module: nn.Module, inputs, outputs): for sae in self.saes.values(): sae.remove_gradient_parallel_to_decoder_directions() + if self.cfg.distribute_latents: + sync_replicated_grads(self.saes.values()) + for optimizer in self.optimizers: optimizer.step() optimizer.zero_grad() @@ -633,6 +666,16 @@ def hook(module: nn.Module, inputs, outputs): pbar.close() + def fired_indices(self, out: ForwardOutput) -> Tensor: + """Shard-local indices of the latents that actually fired. + + Under `distribute_latents` the coder returns a flat list of the winners + this rank owns, so the only entries to drop are the ones ReLU zeroed. + """ + if not self.cfg.distribute_latents: + return out.latent_indices.flatten() + return out.latent_indices[out.latent_acts > 0].flatten() + def local_hookpoints(self) -> list[str]: return ( self.module_plan[dist.get_rank()] @@ -649,8 +692,21 @@ def maybe_all_cat(self, x: Tensor) -> Tensor: dist.all_gather_into_tensor(buffer, x) return buffer + @property + def coders_are_sharded(self) -> bool: + """Whether each rank owns a distinct slice of the coders. + + True for both distributed modes, and false under DDP, where every rank + holds a replica. It decides two things: that per-latent quantities must + not be reduced across ranks, and that every rank holds checkpoint state. + """ + return self.cfg.distribute_modules or self.cfg.distribute_latents + def maybe_all_reduce(self, x: Tensor, op: str = "mean") -> Tensor: - if not dist.is_initialized() or self.cfg.distribute_modules: + # Under `distribute_latents` every rank already sees the same global batch, + # so the scalars are identical and need no reduction, while the per-latent + # masks are indexed in *shard-local* space and must not be reduced at all. + if not dist.is_initialized() or self.coders_are_sharded: return x if op == "sum": @@ -694,7 +750,13 @@ def _checkpoint(self, saes: dict[str, SparseCoder], path: str, rank_zero: bool): for name, sae in saes.items(): assert isinstance(sae, SparseCoder) - sae.save_to_disk(f"{path}/{name}") + if self.cfg.distribute_latents: + # Collective on every rank; only rank 0 writes the result. + full = gather_full_state(sae) + if rank_zero: + save_full_coder(sae, full, f"{path}/{name}") + else: + sae.save_to_disk(f"{path}/{name}") if rank_zero: for i, scheduler in enumerate(self.lr_schedulers): @@ -729,7 +791,7 @@ def save(self): rank_zero = not dist.is_initialized() or dist.get_rank() == 0 - if rank_zero or self.cfg.distribute_modules: + if rank_zero or self.coders_are_sharded: self._checkpoint(self.saes, path, rank_zero) # Barrier to ensure all ranks have saved before continuing @@ -746,7 +808,7 @@ def save_best(self, avg_loss: float | dict[str, float]): if avg_loss[name] < self.best_loss[name]: # type: ignore self.best_loss[name] = avg_loss[name] # type: ignore - if rank_zero or self.cfg.distribute_modules: + if rank_zero or self.coders_are_sharded: self._checkpoint( {name: self.saes[name]}, f"{base_path}/{name}", rank_zero ) @@ -754,7 +816,7 @@ def save_best(self, avg_loss: float | dict[str, float]): if avg_loss < self.best_loss: # type: ignore self.best_loss = avg_loss # type: ignore - if rank_zero or self.cfg.distribute_modules: + if rank_zero or self.coders_are_sharded: self._checkpoint(self.saes, base_path, rank_zero) # Barrier to ensure all ranks have saved before continuing diff --git a/tests/test_latent_parallel.py b/tests/test_latent_parallel.py new file mode 100644 index 00000000..2347d633 --- /dev/null +++ b/tests/test_latent_parallel.py @@ -0,0 +1,214 @@ +"""A latent-sharded coder must be indistinguishable from the unsharded one. + +Runs on CPU over gloo, so it needs no GPU and no launcher: the equivalence being +checked is algebraic, not hardware-dependent. +""" + +import os + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +import sparsify.sparse_coder as sparse_coder_module +from sparsify.config import SparseCoderConfig +from sparsify.latent_parallel import gather_full_state, sync_replicated_grads +from sparsify.sparse_coder import SparseCoder +from sparsify.utils import eager_decode + + +def _use_eager_decoder() -> None: + """These workers run on CPU over gloo whether or not the host has a GPU. + + `decoder_impl` is bound at import time and prefers the Triton kernel, which + rejects CPU tensors, so pin the eager path in the worker rather than relying + on the host having no CUDA. + """ + sparse_coder_module.decoder_impl = eager_decode + + +D_IN = 32 +WORLD_SIZE = 2 +SHARDED = ("encoder.weight", "encoder.bias", "W_dec") + + +def _build_pair(rank: int, cfg: SparseCoderConfig): + """A full coder and this rank's shard of it, holding identical weights.""" + torch.manual_seed(0) + full = SparseCoder(D_IN, cfg, dtype=torch.float32) + + shard = SparseCoder(D_IN, cfg, dtype=torch.float32, latent_shard=(rank, WORLD_SIZE)) + lo = rank * shard.num_latents + hi = lo + shard.num_latents + shard.encoder.weight.data.copy_(full.encoder.weight.data[lo:hi]) + shard.encoder.bias.data.copy_(full.encoder.bias.data[lo:hi]) + assert full.W_dec is not None and shard.W_dec is not None + shard.W_dec.data.copy_(full.W_dec.data[lo:hi]) + shard.b_dec.data.copy_(full.b_dec.data) + return full, shard, lo, hi + + +def _worker(rank: int, port: int, case: dict, errors): + try: + os.environ.update( + MASTER_ADDR="127.0.0.1", + MASTER_PORT=str(port), + RANK=str(rank), + WORLD_SIZE=str(WORLD_SIZE), + ) + dist.init_process_group("gloo", rank=rank, world_size=WORLD_SIZE) + _use_eager_decoder() + + case = dict(case) + wants_dead = case.pop("dead", False) + cfg = SparseCoderConfig(expansion_factor=8, **case) + full, shard, lo, hi = _build_pair(rank, cfg) + + # The same latents must be dead in both, so derive one global mask and + # hand each coder its slice. + dead_full = dead_shard = None + if wants_dead: + torch.manual_seed(99) + dead_full = torch.rand(full.num_latents) < 0.5 + dead_shard = dead_full[lo:hi] + + torch.manual_seed(1234) + x = torch.randn(64, D_IN) + + out_full = full(x, dead_mask=dead_full) + out_shard = shard(x, dead_mask=dead_shard) + + # 1. Same reconstruction, hence the same losses. + tol = dict(atol=2e-5, rtol=2e-5) + torch.testing.assert_close(out_shard.sae_out, out_full.sae_out, **tol) + torch.testing.assert_close(out_shard.fvu, out_full.fvu, **tol) + torch.testing.assert_close(out_shard.auxk_loss, out_full.auxk_loss, **tol) + torch.testing.assert_close( + out_shard.multi_topk_fvu, out_full.multi_topk_fvu, **tol + ) + if wants_dead: + assert out_full.auxk_loss > 0, "auxk case is vacuous" + if cfg.multi_topk: + assert out_full.multi_topk_fvu > 0, "multi_topk case is vacuous" + + # 2. Same gradients on the rows this rank owns. + loss = out_full.fvu + out_full.auxk_loss + out_full.multi_topk_fvu + loss.backward() + (out_shard.fvu + out_shard.auxk_loss + out_shard.multi_topk_fvu).backward() + sync_replicated_grads([shard]) + + for got, want in ( + (shard.encoder.weight, full.encoder.weight), + (shard.encoder.bias, full.encoder.bias), + (shard.W_dec, full.W_dec), + ): + assert got.grad is not None and want.grad is not None + torch.testing.assert_close(got.grad, want.grad[lo:hi], **tol) + + # Replicated parameters: correct only after the reduction. + for got, want in ((shard.b_dec, full.b_dec), (shard.W_skip, full.W_skip)): + if got is None: + continue + assert got.grad is not None and want.grad is not None + torch.testing.assert_close(got.grad, want.grad, **tol) + + # 3. The gathered checkpoint is the unsharded coder. + gathered = gather_full_state(shard) + reference = full.state_dict() + assert set(gathered) == set(reference) + for name in SHARDED: + torch.testing.assert_close(gathered[name], reference[name]) + except Exception: + import traceback + + errors.put((rank, traceback.format_exc())) + raise + finally: + if dist.is_initialized(): + dist.destroy_process_group() + + +def _spawn(target, port: int, *extra): + ctx = mp.get_context("spawn") + errors = ctx.Queue() + procs = [ + ctx.Process(target=target, args=(r, port, *extra, errors)) + for r in range(WORLD_SIZE) + ] + for proc in procs: + proc.start() + for proc in procs: + proc.join(timeout=180) + + failures = [] + while not errors.empty(): + failures.append(errors.get()) + assert not failures, "\n\n".join(f"rank {r}:\n{tb}" for r, tb in failures) + assert all(proc.exitcode == 0 for proc in procs), [p.exitcode for p in procs] + + +def _init_worker(rank: int, port: int, errors): + """A shard built from a seed must equal that slice of the unsharded draw.""" + try: + os.environ.update( + MASTER_ADDR="127.0.0.1", + MASTER_PORT=str(port), + RANK=str(rank), + WORLD_SIZE=str(WORLD_SIZE), + ) + dist.init_process_group("gloo", rank=rank, world_size=WORLD_SIZE) + _use_eager_decoder() + cfg = SparseCoderConfig(expansion_factor=8, k=4) + + torch.manual_seed(7) + full = SparseCoder(D_IN, cfg, dtype=torch.float32) + torch.manual_seed(7) + shard = SparseCoder( + D_IN, cfg, dtype=torch.float32, latent_shard=(rank, WORLD_SIZE) + ) + + lo = rank * shard.num_latents + hi = lo + shard.num_latents + torch.testing.assert_close(shard.encoder.weight, full.encoder.weight[lo:hi]) + assert full.W_dec is not None and shard.W_dec is not None + torch.testing.assert_close(shard.W_dec, full.W_dec[lo:hi]) + + # The real failure this guards against: every rank runs the same seed, so + # drawing at the sharded shape hands them all identical rows and the + # global dictionary silently collapses to num_latents // world_size. + peer = shard.encoder.weight.detach().clone() + dist.broadcast(peer, src=0) + if rank != 0: + assert not torch.allclose( + peer, shard.encoder.weight + ), "ranks hold identical latents; the global dictionary collapsed" + except Exception: + import traceback + + errors.put((rank, traceback.format_exc())) + raise + finally: + if dist.is_initialized(): + dist.destroy_process_group() + + +def test_shard_initialization_matches_global_draw(): + _spawn(_init_worker, 29600) + + +CASES = { + "k1": dict(k=1), + "k4": dict(k=4), + "k32": dict(k=32), + "multi_topk": dict(k=4, multi_topk=True), + "skip_connection": dict(k=4, skip_connection=True), + "auxk": dict(k=4, dead=True), + "auxk_and_skip": dict(k=4, dead=True, skip_connection=True), +} + + +@pytest.mark.parametrize("name", list(CASES)) +def test_latent_shard_matches_unsharded(name: str): + """Equivalence must hold across the loss terms, not just the plain top-k.""" + _spawn(_worker, 29500 + list(CASES).index(name), CASES[name])