Skip to content

Commit e93b327

Browse files
authored
fix(update): keep snapshot timestamps strictly increasing (#1152)
The v4 spec requires a snapshot's timestamp-ms to be strictly greater than its parent's. SnapshotUpdate stamped new snapshots with the current time, so two commits in the same millisecond, or a commit on top of a parent from a clock running ahead, broke that rule. New snapshots now get max(now, parent.timestamp_ms + 1), where the parent is the head of the target branch. This matches Java's SnapshotProducer.snapshotTimestampMillis (apache/iceberg#16293) and, like Java, applies to every format version. Tests cover a first snapshot, back-to-back commits, a parent in the future, and a branch whose parent differs from main's head.
1 parent 7640fe5 commit e93b327

2 files changed

Lines changed: 124 additions & 4 deletions

File tree

‎src/iceberg/test/fast_append_test.cc‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,15 @@
1919

2020
#include "iceberg/update/fast_append.h"
2121

22+
#include <chrono>
2223
#include <format>
2324
#include <functional>
2425
#include <limits>
2526
#include <mutex>
2627
#include <optional>
2728
#include <stdexcept>
2829
#include <string>
30+
#include <string_view>
2931
#include <thread>
3032
#include <unordered_set>
3133
#include <variant>
@@ -59,6 +61,7 @@
5961
#include "iceberg/update/overwrite_files.h"
6062
#include "iceberg/update/rewrite_files.h"
6163
#include "iceberg/update/update_properties.h"
64+
#include "iceberg/util/timepoint.h"
6265
#include "iceberg/util/uuid.h"
6366

6467
namespace iceberg {
@@ -594,6 +597,116 @@ TEST_F(SnapshotUpdateTest, ConcurrentManifestPaths) {
594597
}
595598
}
596599

600+
class SnapshotTimestampTest : public UpdateTestBase {
601+
protected:
602+
static constexpr int64_t kOldSnapshotId = 3051729675574597004;
603+
static constexpr int64_t kCurrentSnapshotId = 3055729675574597004;
604+
static constexpr std::string_view kAuditBranch = "audit";
605+
606+
// Register TableMetadataV2Valid with main's head moved one hour into the
607+
// future, which acts as a clock that does not advance past it, and an
608+
// "audit" branch pointing at the older snapshot from the past.
609+
void RegisterWithFutureMainHead() {
610+
ICEBERG_UNWRAP_OR_FAIL(auto metadata,
611+
ReadTableMetadataFromResource("TableMetadataV2Valid.json"));
612+
metadata->location = table_location_;
613+
future_ms_ = CurrentTimePointMs() + std::chrono::hours(1);
614+
for (auto& snapshot : metadata->snapshots) {
615+
if (snapshot->snapshot_id == kCurrentSnapshotId) {
616+
snapshot->timestamp_ms = future_ms_;
617+
}
618+
}
619+
for (auto& entry : metadata->snapshot_log) {
620+
if (entry.snapshot_id == kCurrentSnapshotId) {
621+
entry.timestamp_ms = future_ms_;
622+
}
623+
}
624+
ICEBERG_UNWRAP_OR_FAIL(auto audit, SnapshotRef::MakeBranch(kOldSnapshotId));
625+
metadata->refs[std::string(kAuditBranch)] = std::move(audit);
626+
627+
std::ignore = catalog_->DropTable(table_ident_, /*purge=*/false);
628+
auto metadata_location = std::format("{}/metadata/00001-{}.metadata.json",
629+
table_location_, Uuid::GenerateV7().ToString());
630+
ASSERT_THAT(TableMetadataUtil::Write(*file_io_, metadata_location, *metadata),
631+
IsOk());
632+
ICEBERG_UNWRAP_OR_FAIL(table_,
633+
catalog_->RegisterTable(table_ident_, metadata_location));
634+
}
635+
636+
// Commit an empty snapshot to the branch and return the branch's new head.
637+
Result<std::shared_ptr<Snapshot>> CommitTo(
638+
std::string_view branch = SnapshotRef::kMainBranch) {
639+
ICEBERG_ASSIGN_OR_RAISE(auto ctx,
640+
TransactionContext::Make(table_, TransactionKind::kUpdate));
641+
auto update = std::make_shared<TestSnapshotUpdate>(ctx);
642+
update->ToBranch(std::string(branch));
643+
ICEBERG_RETURN_UNEXPECTED(update->Commit());
644+
ICEBERG_RETURN_UNEXPECTED(table_->Refresh());
645+
const auto& metadata = table_->metadata();
646+
auto it = metadata->refs.find(std::string(branch));
647+
ICEBERG_CHECK(it != metadata->refs.end(), "Missing branch {}", branch);
648+
return metadata->SnapshotById(it->second->snapshot_id);
649+
}
650+
651+
TimePointMs future_ms_;
652+
};
653+
654+
TEST_F(SnapshotTimestampTest, FirstSnapshotUsesCurrentTime) {
655+
RegisterTableFromResource("TableMetadataV2ValidMinimal.json");
656+
ASSERT_EQ(table_->metadata()->snapshots.size(), 0);
657+
658+
auto before = CurrentTimePointMs();
659+
ICEBERG_UNWRAP_OR_FAIL(auto snapshot, CommitTo());
660+
auto after = CurrentTimePointMs();
661+
662+
EXPECT_FALSE(snapshot->parent_snapshot_id.has_value());
663+
EXPECT_GE(snapshot->timestamp_ms, before);
664+
EXPECT_LE(snapshot->timestamp_ms, after);
665+
}
666+
667+
TEST_F(SnapshotTimestampTest, SuccessiveCommitsStrictlyIncrease) {
668+
RegisterTableFromResource("TableMetadataV2ValidMinimal.json");
669+
670+
// Commits run faster than the clock ticks, so some share a millisecond.
671+
ICEBERG_UNWRAP_OR_FAIL(auto previous, CommitTo());
672+
for (int i = 0; i < 20; ++i) {
673+
ICEBERG_UNWRAP_OR_FAIL(auto snapshot, CommitTo());
674+
ASSERT_EQ(snapshot->parent_snapshot_id, previous->snapshot_id);
675+
EXPECT_GT(snapshot->timestamp_ms, previous->timestamp_ms);
676+
previous = std::move(snapshot);
677+
}
678+
}
679+
680+
TEST_F(SnapshotTimestampTest, FutureParentGetsParentPlusOne) {
681+
RegisterWithFutureMainHead();
682+
683+
ICEBERG_UNWRAP_OR_FAIL(auto first, CommitTo());
684+
EXPECT_EQ(first->parent_snapshot_id, kCurrentSnapshotId);
685+
EXPECT_EQ(first->timestamp_ms, future_ms_ + std::chrono::milliseconds(1));
686+
687+
// The clock is still behind the parent, so the next commit bumps again.
688+
ICEBERG_UNWRAP_OR_FAIL(auto second, CommitTo());
689+
EXPECT_EQ(second->parent_snapshot_id, first->snapshot_id);
690+
EXPECT_EQ(second->timestamp_ms, future_ms_ + std::chrono::milliseconds(2));
691+
}
692+
693+
TEST_F(SnapshotTimestampTest, UsesParentOnTargetBranch) {
694+
RegisterWithFutureMainHead();
695+
696+
// The audit branch's parent is in the past, so its commit uses the current
697+
// time even though main's head is in the future.
698+
auto before = CurrentTimePointMs();
699+
ICEBERG_UNWRAP_OR_FAIL(auto audit, CommitTo(kAuditBranch));
700+
auto after = CurrentTimePointMs();
701+
EXPECT_EQ(audit->parent_snapshot_id, kOldSnapshotId);
702+
EXPECT_GE(audit->timestamp_ms, before);
703+
EXPECT_LE(audit->timestamp_ms, after);
704+
705+
ICEBERG_UNWRAP_OR_FAIL(auto main, CommitTo());
706+
EXPECT_EQ(main->parent_snapshot_id, kCurrentSnapshotId);
707+
EXPECT_EQ(main->timestamp_ms, future_ms_ + std::chrono::milliseconds(1));
708+
}
709+
597710
namespace {
598711

599712
class CapturingReporter final : public MetricsReporter {

‎src/iceberg/update/snapshot_update.cc‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include "iceberg/update/snapshot_update.h"
2121

2222
#include <algorithm>
23+
#include <chrono>
2324
#include <format>
2425
#include <ranges>
2526
#include <span>
@@ -46,6 +47,7 @@
4647
#include "iceberg/util/snapshot_util.h"
4748
#include "iceberg/util/string_util.h"
4849
#include "iceberg/util/task_group.h"
50+
#include "iceberg/util/timepoint.h"
4951
#include "iceberg/util/uuid.h"
5052

5153
namespace iceberg {
@@ -386,12 +388,17 @@ Result<SnapshotUpdate::ApplyResult> SnapshotUpdate::Apply() {
386388
}
387389

388390
ICEBERG_ASSIGN_OR_RAISE(auto summary, ComputeSummary(base()));
391+
// The v4 spec requires timestamps to increase strictly along a branch.
392+
// Like Java, apply it to every format version.
393+
TimePointMs timestamp_ms = CurrentTimePointMs();
394+
if (parent_snapshot != nullptr && timestamp_ms <= parent_snapshot->timestamp_ms) {
395+
timestamp_ms = parent_snapshot->timestamp_ms + std::chrono::milliseconds(1);
396+
}
389397
ICEBERG_ASSIGN_OR_RAISE(
390398
staged_snapshot_,
391-
Snapshot::Make(sequence_number, SnapshotId(), parent_snapshot_id,
392-
CurrentTimePointMs(), std::move(op), std::move(summary),
393-
base().current_schema_id, std::move(manifest_list_path), next_row_id,
394-
assigned_rows));
399+
Snapshot::Make(sequence_number, SnapshotId(), parent_snapshot_id, timestamp_ms,
400+
std::move(op), std::move(summary), base().current_schema_id,
401+
std::move(manifest_list_path), next_row_id, assigned_rows));
395402

396403
return ApplyResult{.snapshot = staged_snapshot_,
397404
.target_branch = target_branch_,

0 commit comments

Comments
 (0)