Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ This is simple, but very memory inefficient. If you want to train SAEs for many
torchrun --nproc_per_node gpu -m sparsify meta-llama/Meta-Llama-3-8B --distribute_modules --batch_size 1 --layer_stride 2 --grad_acc_steps 8 --ctx_len 2048 --k 192 --load_in_8bit --micro_acc_steps 2
```

The above command trains an SAE for every _even_ layer of Llama 3 8B, using all available GPUs. It accumulates gradients over 8 minibatches, and splits each minibatch into 2 microbatches before feeding them into the SAE encoder, thus saving a lot of memory. It also loads the model in 8-bit precision using `bitsandbytes`. This command requires no more than 48GB of memory per GPU on an 8 GPU node.
The above command trains an SAE for every _even_ layer of Llama 3 8B, using all available GPUs. It accumulates gradients over 8 minibatches, and splits each minibatch into 2 microbatches before feeding them into the SAE encoder, thus saving a lot of memory. (`--micro_acc_steps` only works with the default `fvu` loss; with `ce` or `kl`, use `--grad_acc_steps` instead.) It also loads the model in 8-bit precision using `bitsandbytes`. This command requires no more than 48GB of memory per GPU on an 8 GPU node.

## TODO

Expand Down
8 changes: 7 additions & 1 deletion sparsify/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,13 @@ class TrainConfig(Serializable):
"""Number of steps over which to accumulate gradients."""

micro_acc_steps: int = 1
"""Chunk the activations into this number of microbatches for training."""
"""Chunk the activations into this number of microbatches for training.

Each chunk gets its own forward and backward pass through the sparse coder, so
the large latent activations are only held in memory one chunk at a time. The
result is equivalent to an unchunked step. Only supported with `loss_fn="fvu"`;
use `grad_acc_steps` to reduce memory with the `ce` and `kl` losses.
"""

loss_fn: Literal["ce", "fvu", "kl"] = "fvu"
"""Loss function to use for training the sparse coders.
Expand Down
7 changes: 6 additions & 1 deletion sparsify/trainer.py
Original file line number Diff line number Diff line change
Expand Up @@ -313,7 +313,12 @@ def fit(self):

tokens_mask: torch.Tensor

acc_steps = self.cfg.grad_acc_steps * self.cfg.micro_acc_steps
# Every micro-batch chunk is normalized by the variance of the full batch (see
# `total_variance` in the hook below), so the chunk losses already sum to one
# batch's loss. Only `grad_acc_steps` should divide the loss, not
# `micro_acc_steps`; dividing by both would scale gradients and logged
# metrics down by `micro_acc_steps`.
acc_steps = self.cfg.grad_acc_steps
denom = acc_steps * self.cfg.wandb_log_frequency
num_tokens_in_step = 0

Expand Down
129 changes: 118 additions & 11 deletions tests/test_micro_acc_steps.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,16 @@
import sys
import types

import pytest
import torch
import torch.nn.functional as F
from datasets import Dataset
from transformers import GPT2Config, GPT2LMHeadModel

from sparsify import SaeConfig, TrainConfig, Trainer

VOCAB_SIZE = 64
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"


def make_model(n_layer: int = 1, n_positions: int = 128) -> GPT2LMHeadModel:
Expand All @@ -17,7 +22,7 @@ def make_model(n_layer: int = 1, n_positions: int = 128) -> GPT2LMHeadModel:
vocab_size=VOCAB_SIZE,
n_positions=n_positions,
)
return GPT2LMHeadModel(config).to("cuda")
return GPT2LMHeadModel(config).to(DEVICE)


def make_dataset(num_seqs: int = 64, seq_len: int = 128) -> Dataset:
Expand All @@ -26,11 +31,11 @@ def make_dataset(num_seqs: int = 64, seq_len: int = 128) -> Dataset:
return Dataset.from_dict({"input_ids": tokens.tolist()}).with_format("torch")


def make_config(tmp_path, **kwargs) -> TrainConfig:
def make_config(tmp_path, multi_topk: bool = False, **kwargs) -> TrainConfig:
kwargs.setdefault("log_to_wandb", False)
return TrainConfig(
SaeConfig(expansion_factor=2, k=4),
SaeConfig(expansion_factor=2, k=4, multi_topk=multi_topk),
batch_size=16,
log_to_wandb=False,
save_dir=str(tmp_path),
layers=[0],
lr=1e-3,
Expand All @@ -55,14 +60,75 @@ def train_deltas(tmp_path, micro_acc_steps: int) -> dict[str, torch.Tensor]:
}


def first_step_grads(tmp_path, *, grad_acc_steps: int = 1, **kwargs):
"""Train up to the first optimizer step and return every SAE parameter's grad."""
# One batch per accumulation step, so there is exactly one optimizer step.
trainer = Trainer(
make_config(tmp_path, grad_acc_steps=grad_acc_steps, **kwargs),
make_dataset(num_seqs=16 * grad_acc_steps),
make_model(),
)

if kwargs.get("auxk_alpha", 0.0) > 0:
# Nothing is dead on step 0, so mark half the latents dead by hand to make
# sure the AuxK branch actually runs.
for counts in trainer.num_tokens_since_fired.values():
counts[: counts.numel() // 2] = trainer.cfg.dead_feature_threshold + 1

grads: dict[str, torch.Tensor] = {}

def capture(optimizer, args, kw):
if grads: # Only the first step
return

for name, sae in trainer.saes.items():
for pname, p in sae.named_parameters():
if p.grad is not None:
grads[f"{name}.{pname}"] = p.grad.detach().clone().flatten()

for optimizer in trainer.optimizers:
optimizer.register_step_pre_hook(capture)

trainer.fit()
return grads


class FakeWandb(types.ModuleType):
"""Minimal stand-in for `wandb` that records what the trainer logs."""

def __init__(self):
super().__init__("wandb")
self.logs: list[dict] = []

def init(self, *args, **kwargs):
pass

def log(self, info, step=None):
self.logs.append(dict(info))


def logged_fvu(tmp_path, monkeypatch, micro_acc_steps: int) -> dict[str, float]:
fake = FakeWandb()
monkeypatch.setitem(sys.modules, "wandb", fake)

trainer = Trainer(
make_config(tmp_path, micro_acc_steps=micro_acc_steps, log_to_wandb=True),
make_dataset(num_seqs=16),
make_model(),
)
trainer.fit()

assert len(fake.logs) == 1
return {k: v for k, v in fake.logs[0].items() if k.startswith("fvu/")}


def test_micro_acc_steps_matches_unchunked_update(tmp_path):
# Chunking must not change the optimizer step. `acc_steps` scales the loss and
# `total_variance` is a sum (not a mean), so the two updates coincide.
# Chunking must not change the optimizer step. Every chunk is normalized by the
# variance of the full batch, so the chunked update matches the unchunked one up
# to float noise.
#
# This is exact only in the limit of large chunks: each chunk normalizes FVU by
# its own y.mean(0), so tiny chunks diverge (~22% at 32 tokens/chunk, ~9% at
# 256). At the 8x128=1024 tokens/chunk used here it is already float-noise
# exact, and realistic training chunks are far larger still.
# NOTE: Adam's first step is roughly lr * sign(grad), so this test cannot see a
# wrong gradient *scale*. test_micro_acc_steps_gradients_match_unchunked does.
unchunked = train_deltas(tmp_path / "a", micro_acc_steps=1)
chunked = train_deltas(tmp_path / "b", micro_acc_steps=2)

Expand All @@ -73,11 +139,52 @@ def test_micro_acc_steps_matches_unchunked_update(tmp_path):

# Compare the updates relative to the size of the update itself.
rel_gap = (d2 - d1).norm() / d1.norm()
cos = torch.nn.functional.cosine_similarity(d1, d2, dim=0)
cos = F.cosine_similarity(d1, d2, dim=0)
assert rel_gap < 0.01, f"{name} update differs by {rel_gap:.3f} of its norm"
assert cos > 0.999, f"{name} update direction diverged (cos={cos:.4f})"


@pytest.mark.parametrize(
"kwargs",
[
dict(micro_acc_steps=2),
dict(micro_acc_steps=3), # Uneven chunks
dict(micro_acc_steps=8),
dict(micro_acc_steps=2, grad_acc_steps=2),
dict(micro_acc_steps=3, multi_topk=True),
dict(micro_acc_steps=3, auxk_alpha=1 / 32),
],
ids=lambda kw: ",".join(f"{k}={v}" for k, v in kw.items()),
)
def test_micro_acc_steps_gradients_match_unchunked(tmp_path, kwargs):
# The gradients of a chunked step must equal those of an unchunked step. Compared
# before the optimizer runs, because Adam, Muon and Signum all hide a constant
# gradient scale (e.g. dividing the loss by `micro_acc_steps` twice).
unchunked_kwargs = {k: v for k, v in kwargs.items() if k != "micro_acc_steps"}
unchunked = first_step_grads(tmp_path / "a", **unchunked_kwargs)
chunked = first_step_grads(tmp_path / "b", **kwargs)

assert unchunked and set(unchunked) == set(chunked)
for name, g1 in unchunked.items():
g2 = chunked[name]
assert g1.norm() > 1e-8, f"{name} has no gradient; test would be vacuous"

ratio = g2.norm() / g1.norm()
cos = F.cosine_similarity(g1, g2, dim=0)
assert abs(ratio - 1) < 0.05, f"{name} gradient scaled by {ratio:.3f}"
assert cos > 0.99, f"{name} gradient direction diverged (cos={cos:.4f})"


def test_micro_acc_steps_logged_fvu_matches_unchunked(tmp_path, monkeypatch):
# The FVU sent to wandb should not depend on how the batch was chunked.
unchunked = logged_fvu(tmp_path / "a", monkeypatch, micro_acc_steps=1)
chunked = logged_fvu(tmp_path / "b", monkeypatch, micro_acc_steps=4)

assert unchunked and set(unchunked) == set(chunked)
for name, fvu in unchunked.items():
assert chunked[name] == pytest.approx(fvu, rel=0.05), name


def test_micro_acc_steps_actually_trains(tmp_path):
# Guards the regression this file exists for: micro_acc_steps was silently
# ignored (it only scaled the loss denominator), so chunking was a no-op.
Expand Down
Loading