From 35657c3c70fa1279c3e902a5109256543461da71 Mon Sep 17 00:00:00 2001 From: Alessandro Potenza Date: Sun, 23 Aug 2026 16:44:29 +0200 Subject: [PATCH 1/2] perf: bound the weight-gradient temporary in FusedEncoder.backward The backward pass builds `grad_values[:, :, None] * input[:, None, :]`, an `[N, k, D]` tensor holding every top-k contribution at once. It is the largest single allocation in SAE training: 288 MiB at batch 8, ctx 512, d_model 576, k 32, and 1 GiB in the shape `tests/test_encode.py` already exercises. It scales with k, so the `--k 192` configuration the README recommends pays six times that. Consume it in row-blocks sized to a fixed byte budget instead. The arithmetic is unchanged; only the accumulation order inside `index_add_` differs, and that was already unspecified on CUDA. Peak allocated per rank on SmolLM2-135M with 30 hookpoints at 2 GPUs: 9.19 -> 9.04 GiB under DDP, 4.24 -> 3.86 GiB under --distribute_modules. Step time is unchanged within run-to-run variance. --- sparsify/fused_encoder.py | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/sparsify/fused_encoder.py b/sparsify/fused_encoder.py index 272d47db..854e28ce 100644 --- a/sparsify/fused_encoder.py +++ b/sparsify/fused_encoder.py @@ -3,6 +3,13 @@ import torch import torch.nn.functional as F +# Upper bound on the temporary built inside the weight-gradient accumulation. The +# unchunked form allocates N * k * D elements at once, which for a wide model and +# a large k is the single biggest tensor in the backward pass -- at N=8192, k=32, +# D=1024 it is 1 GiB. Consuming it in row-blocks bounds that without changing the +# result. +BACKWARD_CHUNK_BYTES = 64 * 1024 * 1024 + class EncoderOutput(NamedTuple): top_acts: torch.Tensor @@ -66,15 +73,27 @@ def backward(ctx, grad_values, grad_indices, grad_preacts): # --- Grad w.r.t. weight --- if ctx.needs_input_grad[1]: grad_weight = torch.zeros_like(weight) - # Compute contributions from each top-k element: - # computed as grad_values * input for each top-k location. - contributions = grad_values.unsqueeze(2) * input.unsqueeze(1) - _, _, D = contributions.shape - # Flatten contributions to shape (N*k, D) - contributions = contributions.reshape(-1, D) - - # Accumulate contributions into the correct rows of grad_weight. - grad_weight.index_add_(0, indices.flatten(), contributions.type_as(weight)) + # Each top-k location contributes `grad_values * input` to its row of + # grad_weight. Materialising all of them at once costs N * k * D + # elements, so walk the batch in blocks sized to BACKWARD_CHUNK_BYTES + # and fold each block in as it is built. The arithmetic is unchanged; + # only the accumulation order inside index_add_ differs, and that was + # already unspecified on CUDA. + N, k = grad_values.shape + D = input.shape[-1] + itemsize = torch.promote_types(grad_values.dtype, input.dtype).itemsize + rows = max(1, BACKWARD_CHUNK_BYTES // max(1, k * D * itemsize)) + + for start in range(0, N, rows): + stop = min(start + rows, N) + block = grad_values[start:stop].unsqueeze(2) * input[ + start:stop + ].unsqueeze(1) + grad_weight.index_add_( + 0, + indices[start:stop].flatten(), + block.reshape(-1, D).type_as(weight), + ) # --- Grad w.r.t. bias --- if bias is not None and ctx.needs_input_grad[2]: From bbbe0da08647678c5975831022f0de9a423b94a5 Mon Sep 17 00:00:00 2001 From: Alessandro Potenza Date: Sun, 23 Aug 2026 16:44:44 +0200 Subject: [PATCH 2/2] feat: shard the latent dimension across ranks with --distribute_latents DDP replicates every sparse coder on every rank, and `--distribute_modules` removes that replication only when the number of hookpoints divides evenly by the world size. Sharding the latent dimension instead keeps every hookpoint on every rank and splits each coder's rows, so parameter, gradient and optimizer state shrink with the world size for any number of GPUs. Ranks agree on one global top-k through a candidate all-gather, decode only the latents they own, and sum the partial reconstructions. The backward needs no communication, because every rank computes the same loss from the same summed output. Only `b_dec` and `W_skip` are replicated, so only their gradients are reduced. Checkpoints are gathered on rank 0 and written unsharded, so they load with the ordinary loader and a sharded run is interchangeable with a single-GPU one. The survivors are kept as a ragged (token, latent, value) list rather than as the `[N, k]` candidate block with the losers zeroed. That matters: with the block layout both the decode and the encoder weight-gradient stay `O(N * k * d)` however few entries survive, so per-rank work would not fall as the world grows. Compacting makes them `O(nnz * d)` with `nnz = N * k / world_size`. Supports the AuxK loss, multi-TopK and skip connections. `groupmax` partitions the latent dimension itself and is rejected. Adds the first distributed tests in the suite: gloo on CPU, checking forward, gradient and checkpoint equivalence against an unsharded coder across every loss term, plus a regression test for the shard initialization. --- README.md | 40 ++++++ sparsify/config.py | 19 +++ sparsify/latent_parallel.py | 242 ++++++++++++++++++++++++++++++++++ sparsify/sparse_coder.py | 147 +++++++++++++++++++-- sparsify/trainer.py | 96 +++++++++++--- tests/test_latent_parallel.py | 214 ++++++++++++++++++++++++++++++ 6 files changed, 728 insertions(+), 30 deletions(-) create mode 100644 sparsify/latent_parallel.py create mode 100644 tests/test_latent_parallel.py diff --git a/README.md b/README.md index a2916ce7..6584eab1 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,46 @@ torchrun --nproc_per_node gpu -m sparsify meta-llama/Meta-Llama-3-8B --distribut The above command trains an SAE for every _even_ layer of Llama 3 8B, using all available GPUs. It accumulates gradients over 8 minibatches, and splits each minibatch into 2 microbatches before feeding them into the SAE encoder, thus saving a lot of memory. It also loads the model in 8-bit precision using `bitsandbytes`. This command requires no more than 48GB of memory per GPU on an 8 GPU node. +Sharding whole layers has two limits: the number of hookpoints must be divisible by +the number of GPUs, and every rank still materialises the world-sized activation for +every hookpoint, including the ones it does not own. `--distribute_latents` shards the +*latent* dimension of every sparse coder instead, so each rank holds `num_latents / +world_size` rows of every coder: + +```bash +torchrun --nproc_per_node gpu -m sparsify HuggingFaceTB/SmolLM2-135M --distribute_latents --batch_size 8 +``` + +Parameters, gradients and optimizer state shrink with the world size, every hookpoint +stays on every rank so any number of GPUs works, and the ranks agree on a single global +top-k, which makes the result mathematically identical to single-GPU training. +Checkpoints are gathered and written unsharded, so they load with the ordinary loader. + +Measured on SmolLM2-135M, 30 hookpoints, batch size 8 per rank, A100-40GB. Peak +allocated per rank, and median time per step (the same number of tokens per step in +every column): + +| GPUs | DDP | `--distribute_modules` | `--distribute_latents` | +|---|---|---|---| +| 2 | 9.04 GiB / 0.33 s | 3.86 GiB / 0.38 s | **3.65 GiB / 0.33 s** | +| 4 | 9.04 GiB | not runnable (30 not divisible by 4) | **2.24 GiB / 0.36 s** | +| 8 | 9.04 GiB / 0.30 s | not runnable | **1.69 GiB / 0.43 s** | + +**Which to use.** `--distribute_latents` is strictly better than `--distribute_modules`: +lighter, faster, and it runs at GPU counts where `--distribute_modules` refuses to +start. Against DDP it buys a large memory saving for a modest slowdown -- 5.4x less +memory for 1.4x the step time at 8 GPUs -- because the ranks must agree on the top-k, +which costs two collectives per coder per forward and none in the backward. + +The slowdown grows with the batch, since the collectives scale with the gathered batch +while the per-rank shard does not: at batch size 64 the same 8-GPU run is 1.9x lighter +than DDP and 2.1x slower. Prefer a smaller per-rank batch with more gradient +accumulation when using many ranks. + +`--distribute_latents` supports the AuxK loss, `multi_topk` and `skip_connection`, but +requires `activation="topk"`; `groupmax` partitions the latent dimension itself and +cannot be split this way. + ## TODO There are several features that we'd like to add in the near future: diff --git a/sparsify/config.py b/sparsify/config.py index bf9ba997..6613ec3a 100644 --- a/sparsify/config.py +++ b/sparsify/config.py @@ -101,6 +101,14 @@ class TrainConfig(Serializable): distribute_modules: bool = False """Store one copy of each sparse coder, instead of copying them across devices.""" + distribute_latents: bool = False + """Shard the latent dimension of every sparse coder across devices. + + Like `distribute_modules` this stores one copy of each parameter, but it keeps + every hookpoint on every rank, so the number of hookpoints need not be + divisible by the world size and the `[N, num_latents]` pre-activation shrinks + with the world size instead of growing with it.""" + save_every: int = 1000 """Save sparse coders every `save_every` steps.""" @@ -121,6 +129,17 @@ def __post_init__(self): if self.layers and self.layer_stride != 1: raise ValueError("Cannot specify both `layers` and `layer_stride`.") + if self.distribute_modules and self.distribute_latents: + raise ValueError( + "Cannot combine `distribute_modules` and `distribute_latents`." + ) + + if self.distribute_latents and self.loss_fn in ("ce", "kl"): + raise ValueError( + "Sharding latents across ranks is not compatible with the " + "cross-entropy or KL divergence losses." + ) + if self.distribute_modules and self.loss_fn in ("ce", "kl"): raise ValueError( "Distributing modules across ranks is not compatible with the " diff --git a/sparsify/latent_parallel.py b/sparsify/latent_parallel.py new file mode 100644 index 00000000..98055f0b --- /dev/null +++ b/sparsify/latent_parallel.py @@ -0,0 +1,242 @@ +"""Tensor parallelism over the latent dimension of a sparse coder. + +`--distribute_modules` shards whole hookpoints across ranks: each rank owns a few +SAEs outright. That removes the replicated parameter/optimizer state, but it pays +for it twice: every rank must all-gather the world-sized activation for *every* +hookpoint, including the ones it does not own, and the number of hookpoints has to +be divisible by the world size. + +Sharding the latent dimension instead keeps every hookpoint on every rank and +splits each sparse coder's rows. Rank `r` owns latents +`[r * M/W, (r+1) * M/W)` of every coder, so parameters, gradients and optimizer +state all shrink by `W` exactly as they do under `--distribute_modules`, while the +`[N, M]` pre-activation, the dominant transient, shrinks to `[N, M/W]` instead of +growing. + +Two collectives per coder per forward, and none in the backward: + +1. `global_topk`: each rank offers its local top-k as candidates and all ranks + agree on the same global winners. The global top-k of size k can draw at most k + entries from any one rank, so local top-k candidates are sufficient. +2. `all_reduce_sum`: each rank decodes only the winners it owns and the partial + reconstructions are summed. The backward of that sum is the identity, because + every rank computes the same loss from the same summed output. +""" + +from typing import NamedTuple + +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch import Tensor + +from .fused_encoder import BACKWARD_CHUNK_BYTES + + +def shard_size(num_latents: int, world_size: int) -> int: + """Number of latents owned by each rank.""" + if num_latents % world_size: + raise ValueError( + f"num_latents ({num_latents}) must be divisible by the world size " + f"({world_size}) to shard the latent dimension" + ) + return num_latents // world_size + + +class AllReduceSum(torch.autograd.Function): + """Sum a tensor across ranks; the backward is the identity. + + Valid only when every rank goes on to compute the same scalar loss from the + summed result, which is the case here: all ranks see the same input batch and + the same reconstruction, so `dL/d(partial_r)` is `dL/d(sum)` on every rank and + no gradient communication is needed. + """ + + @staticmethod + def forward(ctx, x: Tensor) -> Tensor: # type: ignore[override] + out = x.contiguous().clone() + dist.all_reduce(out, op=dist.ReduceOp.SUM) + return out + + @staticmethod + def backward(ctx, grad: Tensor): # type: ignore[override] + return grad + + +def all_reduce_sum(x: Tensor) -> Tensor: + return AllReduceSum.apply(x) # type: ignore[return-value] + + +def global_sum(x: Tensor) -> Tensor: + """Sum a tensor across ranks in place, outside the autograd graph.""" + total = x.detach().clone() + dist.all_reduce(total, op=dist.ReduceOp.SUM) + return total + + +def global_topk_mask(local_acts: Tensor, k: int) -> Tensor: + """Which of this rank's local top-k entries survive the *global* top-k. + + `local_acts` is `[N, k]`, the activations this rank offers as candidates. + Returns a `[N, k]` bool mask. Ranks agree because they all run the same + deterministic reduction over the same gathered tensor. + """ + world_size = dist.get_world_size() + rank = dist.get_rank() + n, k_local = local_acts.shape + + # all_gather_into_tensor concatenates along dim 0, so gather to [W, N, k] and + # transpose to put the candidate axis last: position p means rank p // k. + gathered = local_acts.new_empty(world_size * n, k_local) + dist.all_gather_into_tensor(gathered, local_acts.detach().contiguous()) + candidates = gathered.view(world_size, n, k_local).permute(1, 0, 2).reshape(n, -1) + + winners = candidates.topk(k, dim=-1, sorted=False).indices # [N, k] + mine = winners.div(k_local, rounding_mode="floor") == rank + local_pos = winners % k_local + + mask = torch.zeros(n * k_local, dtype=torch.bool, device=local_acts.device) + rows = torch.arange(n, device=local_acts.device).unsqueeze(1).expand_as(local_pos) + mask[(rows * k_local + local_pos)[mine]] = True + return mask.view(n, k_local) + + +# Parameters split along dim 0; everything else in a coder is replicated. +SHARDED_KEYS = ("encoder.weight", "encoder.bias", "W_dec") + + +def gather_full_state(sae) -> dict[str, Tensor]: + """Reassemble the unsharded state dict of a latent-sharded coder. + + Collective: every rank must call it. Only rank 0's result is meaningful to + write, but all ranks get the same tensors. Checkpoints written from this load + with the ordinary `SparseCoder.load_from_disk`, so a sharded run and a + single-GPU run produce interchangeable artifacts. + """ + world_size = dist.get_world_size() + full: dict[str, Tensor] = {} + for name, tensor in sae.state_dict().items(): + if name in SHARDED_KEYS: + buf = tensor.new_empty(tensor.shape[0] * world_size, *tensor.shape[1:]) + dist.all_gather_into_tensor(buf, tensor.contiguous()) + full[name] = buf + else: + full[name] = tensor + return full + + +def sync_replicated_grads(saes) -> None: + """Sum the gradients of the parameters that are *not* sharded. + + The sharded rows see every token on every rank, so their gradients are already + complete and need no communication. Only `b_dec` (and `W_skip`) are replicated. + """ + for sae in saes: + for param in (sae.b_dec, sae.W_skip): + if param is not None and param.grad is not None: + dist.all_reduce(param.grad, op=dist.ReduceOp.SUM) + + +# -------------------------------------------------------------------------------- +# Ragged winners. +# +# After the global reduction a rank owns only some of the k candidates it offered +# -- k / world_size of them on average. Keeping the [N, k] layout and zeroing the +# losers is correct but wasteful: the decode and the encoder weight-gradient are +# both O(N * k * d) regardless of how few entries survive, so per-rank work stays +# flat as the world grows even though each rank owns fewer latents. +# +# Compacting the survivors into a flat (token, latent, value) list makes both +# O(nnz * d) with nnz = N * k / world_size, which is the scaling the sharding was +# supposed to buy. Padding to the busiest token instead would only recover a +# factor of about k / max_per_token, roughly 2.5x at k=32 and 8 ranks, against +# world_size here. +# -------------------------------------------------------------------------------- + + +class Winners(NamedTuple): + values: Tensor + """Pre-activation of each surviving (token, latent) pair, `[nnz]`.""" + + rows: Tensor + """Token index of each pair, ascending, `[nnz]`.""" + + latents: Tensor + """Shard-local latent index of each pair, `[nnz]`.""" + + offsets: Tensor + """Start of each token's run in the flat list, `[N]`, for `embedding_bag`.""" + + +def compact_winners(mask: Tensor, cand_acts: Tensor, cand_idx: Tensor) -> Winners: + """Flatten the surviving candidates, dropping the losers entirely.""" + rows, cols = mask.nonzero(as_tuple=True) # row-major, so rows is ascending + counts = mask.sum(1) + offsets = torch.zeros_like(counts) + offsets[1:] = counts.cumsum(0)[:-1] + return Winners(cand_acts[rows, cols], rows, cand_idx[rows, cols], offsets) + + +def ragged_decode(winners: Winners, acts: Tensor, w_dec: Tensor) -> Tensor: + """Sum `acts * w_dec[latent]` per token, over the flat winner list.""" + return F.embedding_bag( + winners.latents, + w_dec, + offsets=winners.offsets, + mode="sum", + per_sample_weights=acts.to(w_dec.dtype), + ) + + +class SparseEncoderGrad(torch.autograd.Function): + """Carry the encoder's gradient for a ragged set of selected latents. + + The dense pre-activation is computed outside the graph, because it exists + only to run the top-k. This hands the selected values back to autograd with + the gradient the encoder would have produced, so the backward touches the + surviving pairs rather than every candidate the rank offered. The arithmetic + matches `FusedEncoder.backward`, restricted to those pairs -- which is exact, + since the losers were multiplied by zero and carry no gradient anyway. + """ + + @staticmethod + def forward(ctx, values, x, weight, bias, rows, latents): # type: ignore[override] + ctx.save_for_backward(x, weight, bias, rows, latents) + return values.clone() + + @staticmethod + def backward(ctx, grad_values): # type: ignore[override] + x, weight, bias, rows, latents = ctx.saved_tensors + _, needs_x, needs_w, needs_b, _, _ = ctx.needs_input_grad + + grad_x = torch.zeros_like(x) if needs_x else None + grad_w = torch.zeros_like(weight) if needs_w else None + + d = x.shape[-1] + itemsize = torch.promote_types(grad_values.dtype, x.dtype).itemsize + block = max(1, BACKWARD_CHUNK_BYTES // max(1, d * itemsize)) + + for start in range(0, latents.numel(), block): + stop = start + block + gv = grad_values[start:stop].unsqueeze(1) + row = rows[start:stop] + latent = latents[start:stop] + if grad_x is not None: + grad_x.index_add_(0, row, (weight[latent] * gv).type_as(x)) + if grad_w is not None: + grad_w.index_add_(0, latent, (x[row] * gv).type_as(weight)) + + grad_b = None + if bias is not None and needs_b: + grad_b = torch.zeros_like(bias) + grad_b.index_add_(0, latents, grad_values.type_as(bias)) + + return None, grad_x, grad_w, grad_b, None, None + + +def sparse_encoder_grad( + values: Tensor, x: Tensor, weight: Tensor, bias: Tensor, winners: Winners +) -> Tensor: + return SparseEncoderGrad.apply( # type: ignore[return-value] + values, x, weight, bias, winners.rows, winners.latents + ) diff --git a/sparsify/sparse_coder.py b/sparsify/sparse_coder.py index d0e155a1..253a4271 100644 --- a/sparsify/sparse_coder.py +++ b/sparsify/sparse_coder.py @@ -5,6 +5,7 @@ import einops import torch +import torch.nn.functional as F from huggingface_hub import snapshot_download from natsort import natsorted from safetensors import safe_open @@ -13,6 +14,16 @@ from .config import SparseCoderConfig from .fused_encoder import EncoderOutput, fused_encoder +from .latent_parallel import ( + Winners, + all_reduce_sum, + compact_winners, + global_sum, + global_topk_mask, + ragged_decode, + shard_size, + sparse_encoder_grad, +) from .utils import decoder_impl @@ -44,13 +55,48 @@ def __init__( dtype: torch.dtype | None = None, *, decoder: bool = True, + latent_shard: tuple[int, int] | None = None, ): super().__init__() self.cfg = cfg self.d_in = d_in - self.num_latents = cfg.num_latents or d_in * cfg.expansion_factor + self.global_num_latents = cfg.num_latents or d_in * cfg.expansion_factor + + # `latent_shard` is (rank, world_size): this coder holds only its slice of + # the latent dimension and cooperates with its peers in `forward`. + if latent_shard is None: + self.num_latents = self.global_num_latents + self.latent_offset = 0 + self.latent_world_size = 1 + else: + shard_rank, world_size = latent_shard + if cfg.activation != "topk": + raise ValueError( + "Latent sharding currently supports activation='topk' only; " + f"'{cfg.activation}' partitions the latent dimension itself." + ) + self.num_latents = shard_size(self.global_num_latents, world_size) + self.latent_offset = shard_rank * self.num_latents + self.latent_world_size = world_size + + if latent_shard is None: + self.encoder = nn.Linear(d_in, self.num_latents, device=device, dtype=dtype) + else: + # Draw the *global* initialization and keep only this rank's rows. + # Every rank runs the same seed, so drawing at the sharded shape + # directly would give all of them identical weights, collapsing the + # dictionary to num_latents // world_size distinct latents. Drawing at + # the full shape also makes a sharded run reproduce an unsharded run + # with the same seed, row for row. The full draw is transient. + full = nn.Linear(d_in, self.global_num_latents, device=device, dtype=dtype) + self.encoder = nn.Linear(d_in, self.num_latents, device=device, dtype=dtype) + self.encoder.weight.data.copy_( + full.weight.data[ + self.latent_offset : self.latent_offset + self.num_latents + ] + ) + del full - self.encoder = nn.Linear(d_in, self.num_latents, device=device, dtype=dtype) self.encoder.bias.data.zero_() if decoder: @@ -186,6 +232,10 @@ def device(self): def dtype(self): return self.encoder.weight.dtype + @property + def is_latent_sharded(self) -> bool: + return self.latent_world_size > 1 + def encode(self, x: Tensor) -> EncoderOutput: """Encode the input and select the top-k latents.""" if not self.cfg.transcode: @@ -195,6 +245,58 @@ def encode(self, x: Tensor) -> EncoderOutput: x, self.encoder.weight, self.encoder.bias, self.cfg.k, self.cfg.activation ) + def _count_dead(self, dead_mask: Tensor) -> int: + """Number of dead latents, counted over the whole dictionary.""" + if not self.is_latent_sharded: + return int(dead_mask.sum()) + + return int(global_sum(dead_mask.sum())) + + def _sharded_encode(self, x: Tensor) -> tuple[Tensor, Winners, Tensor]: + """Select globally, then rejoin the graph on the surviving pairs only. + + The dense pre-activation exists solely to run the top-k, so it is built + outside the graph; `sparse_encoder_grad` reattaches the encoder gradient + to the winners, which keeps the backward proportional to how many latents + this rank actually owns rather than to how many it offered. + """ + shifted = x if self.cfg.transcode else x - self.b_dec + + with torch.no_grad(): + pre_acts = F.relu(F.linear(shifted, self.encoder.weight, self.encoder.bias)) + cand_acts, cand_idx = pre_acts.topk(self.cfg.k, dim=-1, sorted=False) + winners = compact_winners( + global_topk_mask(cand_acts, self.cfg.k), cand_acts, cand_idx + ) + + acts = sparse_encoder_grad( + winners.values, shifted, self.encoder.weight, self.encoder.bias, winners + ) + return acts, winners, pre_acts + + def _sharded_decode(self, acts: Tensor, winners: Winners, x: Tensor) -> Tensor: + """Decode this rank's winners and sum the partial reconstructions. + + `b_dec` and `W_skip` are replicated rather than sharded, so each rank + contributes 1/W of them: the sum reproduces each exactly once, and each + rank ends up holding 1/W of their gradient, which + `latent_parallel.sync_replicated_grads` then adds back up. + """ + assert self.W_dec is not None, "Decoder weight was not initialized." + world_size = self.latent_world_size + + partial = ragged_decode(winners, acts, self.W_dec) + partial = partial + self.b_dec / world_size + if self.W_skip is not None: + partial = partial + (x.to(self.dtype) @ self.W_skip.mT) / world_size + return all_reduce_sum(partial) + + def _sharded_global_winners(self, scores: Tensor, k: int) -> Winners: + """Global top-k over a shard-local score matrix, as a flat winner list.""" + candidates = min(k, self.num_latents) + vals, idx = scores.topk(candidates, sorted=False) + return compact_winners(global_topk_mask(vals, k), vals, idx) + def decode(self, top_acts: Tensor, top_indices: Tensor) -> Tensor: assert self.W_dec is not None, "Decoder weight was not initialized." @@ -215,16 +317,23 @@ def forward( dead_mask: Tensor | None = None, total_variance: Tensor | None = None, ) -> ForwardOutput: - top_acts, top_indices, pre_acts = self.encode(x) + if self.is_latent_sharded: + top_acts, winners, pre_acts = self._sharded_encode(x) + top_indices = winners.latents + else: + top_acts, top_indices, pre_acts = self.encode(x) # If we aren't given a distinct target, we're autoencoding if y is None: y = x # Decode - sae_out = self.decode(top_acts, top_indices) - if self.W_skip is not None: - sae_out += x.to(self.dtype) @ self.W_skip.mT + if self.is_latent_sharded: + sae_out = self._sharded_decode(top_acts, winners, x) + else: + sae_out = self.decode(top_acts, top_indices) + if self.W_skip is not None: + sae_out = sae_out + x.to(self.dtype) @ self.W_skip.mT # Compute the residual e = y - sae_out @@ -234,7 +343,7 @@ def forward( total_variance = (y - y.mean(0)).pow(2).sum() # Second decoder pass for AuxK loss - if dead_mask is not None and (num_dead := int(dead_mask.sum())) > 0: + if dead_mask is not None and (num_dead := self._count_dead(dead_mask)) > 0: # Heuristic from Appendix B.1 in the paper k_aux = y.shape[-1] // 2 @@ -245,15 +354,20 @@ def forward( # Don't include living latents in this loss auxk_latents = torch.where(dead_mask[None], pre_acts, -torch.inf) - # Top-k dead latents - auxk_acts, auxk_indices = auxk_latents.topk(k_aux, sorted=False) - # Encourage the top ~50% of dead latents to predict the residual of the # top k living latents. We call decoder_impl directly rather than # self.decode because the residual target e already accounts for b_dec # (sae_out includes it), so adding b_dec again here would double-count it. assert self.W_dec is not None, "Decoder weight was not initialized." - e_hat = decoder_impl(auxk_indices, auxk_acts.to(self.dtype), self.W_dec.mT) + + if self.is_latent_sharded: + aux = self._sharded_global_winners(auxk_latents, k_aux) + e_hat = all_reduce_sum(ragged_decode(aux, aux.values, self.W_dec)) + else: + auxk_acts, auxk_indices = auxk_latents.topk(k_aux, sorted=False) + e_hat = decoder_impl( + auxk_indices, auxk_acts.to(self.dtype), self.W_dec.mT + ) auxk_loss = (e_hat - e.detach()).pow(2).sum() auxk_loss = scale * auxk_loss / total_variance else: @@ -263,8 +377,15 @@ def forward( fvu = l2_loss / total_variance if self.cfg.multi_topk: - top_acts, top_indices = pre_acts.topk(4 * self.cfg.k, sorted=False) - sae_out = self.decode(top_acts, top_indices) + wide_k = 4 * self.cfg.k + if self.is_latent_sharded: + wide = self._sharded_global_winners(pre_acts, wide_k) + sae_out = self._sharded_decode(wide.values, wide, x) + else: + top_acts, top_indices = pre_acts.topk(wide_k, sorted=False) + sae_out = self.decode(top_acts, top_indices) + if self.W_skip is not None: + sae_out = sae_out + x.to(self.dtype) @ self.W_skip.mT multi_topk_fvu = (sae_out - y).pow(2).sum() / total_variance else: diff --git a/sparsify/trainer.py b/sparsify/trainer.py index 88b73276..556595b9 100644 --- a/sparsify/trainer.py +++ b/sparsify/trainer.py @@ -1,15 +1,17 @@ +import json import os from collections import defaultdict from dataclasses import asdict from fnmatch import fnmatchcase from glob import glob +from pathlib import Path from typing import Sized import torch import torch.distributed as dist from datasets import Dataset as HfDataset from natsort import natsorted -from safetensors.torch import load_model +from safetensors.torch import load_model, save_file from schedulefree import ScheduleFreeWrapper from torch import Tensor, nn from torch.nn.parallel import DistributedDataParallel as DDP @@ -19,12 +21,25 @@ from .config import TrainConfig from .data import MemmapDataset +from .latent_parallel import gather_full_state, sync_replicated_grads from .muon import Muon from .sign_sgd import SignSGD -from .sparse_coder import SparseCoder +from .sparse_coder import ForwardOutput, SparseCoder from .utils import get_layer_list, resolve_widths, set_submodule +def save_full_coder(sae: SparseCoder, state: dict[str, Tensor], path: str) -> None: + """Write a reassembled (unsharded) coder so ordinary loaders can read it.""" + directory = Path(path) + directory.mkdir(parents=True, exist_ok=True) + save_file( + {k: v.contiguous() for k, v in state.items()}, + str(directory / "sae.safetensors"), + ) + with open(directory / "cfg.json", "w") as f: + json.dump({**sae.cfg.to_dict(), "d_in": sae.d_in}, f) + + def get_saes_by_layer_name(saes: dict, module_name: str): return { k: v @@ -97,7 +112,15 @@ def __init__( # Add suffix to the name to disambiguate multiple seeds name = f"{hook}/seed{seed}" if len(cfg.init_seeds) > 1 else hook self.saes[name] = SparseCoder( - input_widths[hook], cfg.sae, device, dtype=torch.float32 + input_widths[hook], + cfg.sae, + device, + dtype=torch.float32, + latent_shard=( + (dist.get_rank(), dist.get_world_size()) + if cfg.distribute_latents + else None + ), ) assert isinstance(dataset, Sized) @@ -120,7 +143,7 @@ def __init__( pgs = [ dict( params=sae.parameters(), - lr=cfg.lr or 2e-4 / (sae.num_latents / (2**14)) ** 0.5, + lr=cfg.lr or 2e-4 / (sae.global_num_latents / (2**14)) ** 0.5, ) for sae in self.saes.values() ] @@ -163,7 +186,7 @@ def __init__( pgs = [ dict( params=sae.parameters(), - lr=cfg.lr or 5e-3 / (sae.num_latents / (2**14)) ** 0.5, + lr=cfg.lr or 5e-3 / (sae.global_num_latents / (2**14)) ** 0.5, ) for sae in self.saes.values() ] @@ -187,7 +210,7 @@ def __init__( self.cfg.exclude_tokens, device=device, dtype=torch.long ) - num_latents = list(self.saes.values())[0].num_latents + num_latents = list(self.saes.values())[0].global_num_latents self.initial_k = min(num_latents, round(list(input_widths.values())[0] * 10)) self.final_k = self.cfg.sae.k @@ -256,7 +279,9 @@ def fit(self): self.model.requires_grad_(False) rank_zero = not dist.is_initialized() or dist.get_rank() == 0 - ddp = dist.is_initialized() and not self.cfg.distribute_modules + ddp = dist.is_initialized() and not ( + self.cfg.distribute_modules or self.cfg.distribute_latents + ) wandb = None if self.cfg.log_to_wandb and rank_zero: @@ -370,7 +395,7 @@ def hook(module: nn.Module, inputs, outputs): out_shape = outputs.shape # Scatter and gather the hidden states across ranks if necessary - if self.cfg.distribute_modules: + if self.cfg.distribute_modules or self.cfg.distribute_latents: world_outputs = outputs.new_empty( outputs.shape[0] * dist.get_world_size(), *outputs.shape[1:] ) @@ -391,7 +416,9 @@ def hook(module: nn.Module, inputs, outputs): dist.all_gather_into_tensor(world_mask, mask) mask = world_mask.bool() - if name not in self.module_plan[dist.get_rank()]: + if self.cfg.distribute_modules and ( + name not in self.module_plan[dist.get_rank()] + ): return # Flatten the batch and sequence dimensions @@ -436,7 +463,7 @@ def hook(module: nn.Module, inputs, outputs): out = wrapped(x=inputs, y=outputs, dead_mask=dead_mask) # Update the did_fire mask - did_fire[sae_name][out.latent_indices.flatten()] = True + did_fire[sae_name][self.fired_indices(out)] = True self.maybe_all_reduce(did_fire[sae_name], "max") # Replace the normal output with the SAE output @@ -460,7 +487,7 @@ def hook(module: nn.Module, inputs, outputs): ) # Update the did_fire mask - did_fire[sae_name][out.latent_indices.flatten()] = True + did_fire[sae_name][self.fired_indices(out)] = True self.maybe_all_reduce(did_fire[sae_name], "max") # max is "any" # Metrics that only make sense for local @@ -506,7 +533,10 @@ def hook(module: nn.Module, inputs, outputs): ) # Bookkeeping for dead feature detection - N = tokens_mask.sum().item() + n_tokens = tokens_mask.sum() + if self.cfg.distribute_latents: + dist.all_reduce(n_tokens, op=dist.ReduceOp.SUM) + N = int(n_tokens.item()) num_tokens_in_step += N # Compute clean logits if using KL loss @@ -552,6 +582,9 @@ def hook(module: nn.Module, inputs, outputs): for sae in self.saes.values(): sae.remove_gradient_parallel_to_decoder_directions() + if self.cfg.distribute_latents: + sync_replicated_grads(self.saes.values()) + for optimizer in self.optimizers: optimizer.step() optimizer.zero_grad() @@ -633,6 +666,16 @@ def hook(module: nn.Module, inputs, outputs): pbar.close() + def fired_indices(self, out: ForwardOutput) -> Tensor: + """Shard-local indices of the latents that actually fired. + + Under `distribute_latents` the coder returns a flat list of the winners + this rank owns, so the only entries to drop are the ones ReLU zeroed. + """ + if not self.cfg.distribute_latents: + return out.latent_indices.flatten() + return out.latent_indices[out.latent_acts > 0].flatten() + def local_hookpoints(self) -> list[str]: return ( self.module_plan[dist.get_rank()] @@ -649,8 +692,21 @@ def maybe_all_cat(self, x: Tensor) -> Tensor: dist.all_gather_into_tensor(buffer, x) return buffer + @property + def coders_are_sharded(self) -> bool: + """Whether each rank owns a distinct slice of the coders. + + True for both distributed modes, and false under DDP, where every rank + holds a replica. It decides two things: that per-latent quantities must + not be reduced across ranks, and that every rank holds checkpoint state. + """ + return self.cfg.distribute_modules or self.cfg.distribute_latents + def maybe_all_reduce(self, x: Tensor, op: str = "mean") -> Tensor: - if not dist.is_initialized() or self.cfg.distribute_modules: + # Under `distribute_latents` every rank already sees the same global batch, + # so the scalars are identical and need no reduction, while the per-latent + # masks are indexed in *shard-local* space and must not be reduced at all. + if not dist.is_initialized() or self.coders_are_sharded: return x if op == "sum": @@ -694,7 +750,13 @@ def _checkpoint(self, saes: dict[str, SparseCoder], path: str, rank_zero: bool): for name, sae in saes.items(): assert isinstance(sae, SparseCoder) - sae.save_to_disk(f"{path}/{name}") + if self.cfg.distribute_latents: + # Collective on every rank; only rank 0 writes the result. + full = gather_full_state(sae) + if rank_zero: + save_full_coder(sae, full, f"{path}/{name}") + else: + sae.save_to_disk(f"{path}/{name}") if rank_zero: for i, scheduler in enumerate(self.lr_schedulers): @@ -729,7 +791,7 @@ def save(self): rank_zero = not dist.is_initialized() or dist.get_rank() == 0 - if rank_zero or self.cfg.distribute_modules: + if rank_zero or self.coders_are_sharded: self._checkpoint(self.saes, path, rank_zero) # Barrier to ensure all ranks have saved before continuing @@ -746,7 +808,7 @@ def save_best(self, avg_loss: float | dict[str, float]): if avg_loss[name] < self.best_loss[name]: # type: ignore self.best_loss[name] = avg_loss[name] # type: ignore - if rank_zero or self.cfg.distribute_modules: + if rank_zero or self.coders_are_sharded: self._checkpoint( {name: self.saes[name]}, f"{base_path}/{name}", rank_zero ) @@ -754,7 +816,7 @@ def save_best(self, avg_loss: float | dict[str, float]): if avg_loss < self.best_loss: # type: ignore self.best_loss = avg_loss # type: ignore - if rank_zero or self.cfg.distribute_modules: + if rank_zero or self.coders_are_sharded: self._checkpoint(self.saes, base_path, rank_zero) # Barrier to ensure all ranks have saved before continuing diff --git a/tests/test_latent_parallel.py b/tests/test_latent_parallel.py new file mode 100644 index 00000000..2347d633 --- /dev/null +++ b/tests/test_latent_parallel.py @@ -0,0 +1,214 @@ +"""A latent-sharded coder must be indistinguishable from the unsharded one. + +Runs on CPU over gloo, so it needs no GPU and no launcher: the equivalence being +checked is algebraic, not hardware-dependent. +""" + +import os + +import pytest +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +import sparsify.sparse_coder as sparse_coder_module +from sparsify.config import SparseCoderConfig +from sparsify.latent_parallel import gather_full_state, sync_replicated_grads +from sparsify.sparse_coder import SparseCoder +from sparsify.utils import eager_decode + + +def _use_eager_decoder() -> None: + """These workers run on CPU over gloo whether or not the host has a GPU. + + `decoder_impl` is bound at import time and prefers the Triton kernel, which + rejects CPU tensors, so pin the eager path in the worker rather than relying + on the host having no CUDA. + """ + sparse_coder_module.decoder_impl = eager_decode + + +D_IN = 32 +WORLD_SIZE = 2 +SHARDED = ("encoder.weight", "encoder.bias", "W_dec") + + +def _build_pair(rank: int, cfg: SparseCoderConfig): + """A full coder and this rank's shard of it, holding identical weights.""" + torch.manual_seed(0) + full = SparseCoder(D_IN, cfg, dtype=torch.float32) + + shard = SparseCoder(D_IN, cfg, dtype=torch.float32, latent_shard=(rank, WORLD_SIZE)) + lo = rank * shard.num_latents + hi = lo + shard.num_latents + shard.encoder.weight.data.copy_(full.encoder.weight.data[lo:hi]) + shard.encoder.bias.data.copy_(full.encoder.bias.data[lo:hi]) + assert full.W_dec is not None and shard.W_dec is not None + shard.W_dec.data.copy_(full.W_dec.data[lo:hi]) + shard.b_dec.data.copy_(full.b_dec.data) + return full, shard, lo, hi + + +def _worker(rank: int, port: int, case: dict, errors): + try: + os.environ.update( + MASTER_ADDR="127.0.0.1", + MASTER_PORT=str(port), + RANK=str(rank), + WORLD_SIZE=str(WORLD_SIZE), + ) + dist.init_process_group("gloo", rank=rank, world_size=WORLD_SIZE) + _use_eager_decoder() + + case = dict(case) + wants_dead = case.pop("dead", False) + cfg = SparseCoderConfig(expansion_factor=8, **case) + full, shard, lo, hi = _build_pair(rank, cfg) + + # The same latents must be dead in both, so derive one global mask and + # hand each coder its slice. + dead_full = dead_shard = None + if wants_dead: + torch.manual_seed(99) + dead_full = torch.rand(full.num_latents) < 0.5 + dead_shard = dead_full[lo:hi] + + torch.manual_seed(1234) + x = torch.randn(64, D_IN) + + out_full = full(x, dead_mask=dead_full) + out_shard = shard(x, dead_mask=dead_shard) + + # 1. Same reconstruction, hence the same losses. + tol = dict(atol=2e-5, rtol=2e-5) + torch.testing.assert_close(out_shard.sae_out, out_full.sae_out, **tol) + torch.testing.assert_close(out_shard.fvu, out_full.fvu, **tol) + torch.testing.assert_close(out_shard.auxk_loss, out_full.auxk_loss, **tol) + torch.testing.assert_close( + out_shard.multi_topk_fvu, out_full.multi_topk_fvu, **tol + ) + if wants_dead: + assert out_full.auxk_loss > 0, "auxk case is vacuous" + if cfg.multi_topk: + assert out_full.multi_topk_fvu > 0, "multi_topk case is vacuous" + + # 2. Same gradients on the rows this rank owns. + loss = out_full.fvu + out_full.auxk_loss + out_full.multi_topk_fvu + loss.backward() + (out_shard.fvu + out_shard.auxk_loss + out_shard.multi_topk_fvu).backward() + sync_replicated_grads([shard]) + + for got, want in ( + (shard.encoder.weight, full.encoder.weight), + (shard.encoder.bias, full.encoder.bias), + (shard.W_dec, full.W_dec), + ): + assert got.grad is not None and want.grad is not None + torch.testing.assert_close(got.grad, want.grad[lo:hi], **tol) + + # Replicated parameters: correct only after the reduction. + for got, want in ((shard.b_dec, full.b_dec), (shard.W_skip, full.W_skip)): + if got is None: + continue + assert got.grad is not None and want.grad is not None + torch.testing.assert_close(got.grad, want.grad, **tol) + + # 3. The gathered checkpoint is the unsharded coder. + gathered = gather_full_state(shard) + reference = full.state_dict() + assert set(gathered) == set(reference) + for name in SHARDED: + torch.testing.assert_close(gathered[name], reference[name]) + except Exception: + import traceback + + errors.put((rank, traceback.format_exc())) + raise + finally: + if dist.is_initialized(): + dist.destroy_process_group() + + +def _spawn(target, port: int, *extra): + ctx = mp.get_context("spawn") + errors = ctx.Queue() + procs = [ + ctx.Process(target=target, args=(r, port, *extra, errors)) + for r in range(WORLD_SIZE) + ] + for proc in procs: + proc.start() + for proc in procs: + proc.join(timeout=180) + + failures = [] + while not errors.empty(): + failures.append(errors.get()) + assert not failures, "\n\n".join(f"rank {r}:\n{tb}" for r, tb in failures) + assert all(proc.exitcode == 0 for proc in procs), [p.exitcode for p in procs] + + +def _init_worker(rank: int, port: int, errors): + """A shard built from a seed must equal that slice of the unsharded draw.""" + try: + os.environ.update( + MASTER_ADDR="127.0.0.1", + MASTER_PORT=str(port), + RANK=str(rank), + WORLD_SIZE=str(WORLD_SIZE), + ) + dist.init_process_group("gloo", rank=rank, world_size=WORLD_SIZE) + _use_eager_decoder() + cfg = SparseCoderConfig(expansion_factor=8, k=4) + + torch.manual_seed(7) + full = SparseCoder(D_IN, cfg, dtype=torch.float32) + torch.manual_seed(7) + shard = SparseCoder( + D_IN, cfg, dtype=torch.float32, latent_shard=(rank, WORLD_SIZE) + ) + + lo = rank * shard.num_latents + hi = lo + shard.num_latents + torch.testing.assert_close(shard.encoder.weight, full.encoder.weight[lo:hi]) + assert full.W_dec is not None and shard.W_dec is not None + torch.testing.assert_close(shard.W_dec, full.W_dec[lo:hi]) + + # The real failure this guards against: every rank runs the same seed, so + # drawing at the sharded shape hands them all identical rows and the + # global dictionary silently collapses to num_latents // world_size. + peer = shard.encoder.weight.detach().clone() + dist.broadcast(peer, src=0) + if rank != 0: + assert not torch.allclose( + peer, shard.encoder.weight + ), "ranks hold identical latents; the global dictionary collapsed" + except Exception: + import traceback + + errors.put((rank, traceback.format_exc())) + raise + finally: + if dist.is_initialized(): + dist.destroy_process_group() + + +def test_shard_initialization_matches_global_draw(): + _spawn(_init_worker, 29600) + + +CASES = { + "k1": dict(k=1), + "k4": dict(k=4), + "k32": dict(k=32), + "multi_topk": dict(k=4, multi_topk=True), + "skip_connection": dict(k=4, skip_connection=True), + "auxk": dict(k=4, dead=True), + "auxk_and_skip": dict(k=4, dead=True, skip_connection=True), +} + + +@pytest.mark.parametrize("name", list(CASES)) +def test_latent_shard_matches_unsharded(name: str): + """Equivalence must hold across the loss terms, not just the plain top-k.""" + _spawn(_worker, 29500 + list(CASES).index(name), CASES[name])