|
19 | 19 |
|
20 | 20 | #include "iceberg/update/fast_append.h" |
21 | 21 |
|
| 22 | +#include <chrono> |
22 | 23 | #include <format> |
23 | 24 | #include <functional> |
24 | 25 | #include <limits> |
25 | 26 | #include <mutex> |
26 | 27 | #include <optional> |
27 | 28 | #include <stdexcept> |
28 | 29 | #include <string> |
| 30 | +#include <string_view> |
29 | 31 | #include <thread> |
30 | 32 | #include <unordered_set> |
31 | 33 | #include <variant> |
|
59 | 61 | #include "iceberg/update/overwrite_files.h" |
60 | 62 | #include "iceberg/update/rewrite_files.h" |
61 | 63 | #include "iceberg/update/update_properties.h" |
| 64 | +#include "iceberg/util/timepoint.h" |
62 | 65 | #include "iceberg/util/uuid.h" |
63 | 66 |
|
64 | 67 | namespace iceberg { |
@@ -594,6 +597,116 @@ TEST_F(SnapshotUpdateTest, ConcurrentManifestPaths) { |
594 | 597 | } |
595 | 598 | } |
596 | 599 |
|
| 600 | +class SnapshotTimestampTest : public UpdateTestBase { |
| 601 | + protected: |
| 602 | + static constexpr int64_t kOldSnapshotId = 3051729675574597004; |
| 603 | + static constexpr int64_t kCurrentSnapshotId = 3055729675574597004; |
| 604 | + static constexpr std::string_view kAuditBranch = "audit"; |
| 605 | + |
| 606 | + // Register TableMetadataV2Valid with main's head moved one hour into the |
| 607 | + // future, which acts as a clock that does not advance past it, and an |
| 608 | + // "audit" branch pointing at the older snapshot from the past. |
| 609 | + void RegisterWithFutureMainHead() { |
| 610 | + ICEBERG_UNWRAP_OR_FAIL(auto metadata, |
| 611 | + ReadTableMetadataFromResource("TableMetadataV2Valid.json")); |
| 612 | + metadata->location = table_location_; |
| 613 | + future_ms_ = CurrentTimePointMs() + std::chrono::hours(1); |
| 614 | + for (auto& snapshot : metadata->snapshots) { |
| 615 | + if (snapshot->snapshot_id == kCurrentSnapshotId) { |
| 616 | + snapshot->timestamp_ms = future_ms_; |
| 617 | + } |
| 618 | + } |
| 619 | + for (auto& entry : metadata->snapshot_log) { |
| 620 | + if (entry.snapshot_id == kCurrentSnapshotId) { |
| 621 | + entry.timestamp_ms = future_ms_; |
| 622 | + } |
| 623 | + } |
| 624 | + ICEBERG_UNWRAP_OR_FAIL(auto audit, SnapshotRef::MakeBranch(kOldSnapshotId)); |
| 625 | + metadata->refs[std::string(kAuditBranch)] = std::move(audit); |
| 626 | + |
| 627 | + std::ignore = catalog_->DropTable(table_ident_, /*purge=*/false); |
| 628 | + auto metadata_location = std::format("{}/metadata/00001-{}.metadata.json", |
| 629 | + table_location_, Uuid::GenerateV7().ToString()); |
| 630 | + ASSERT_THAT(TableMetadataUtil::Write(*file_io_, metadata_location, *metadata), |
| 631 | + IsOk()); |
| 632 | + ICEBERG_UNWRAP_OR_FAIL(table_, |
| 633 | + catalog_->RegisterTable(table_ident_, metadata_location)); |
| 634 | + } |
| 635 | + |
| 636 | + // Commit an empty snapshot to the branch and return the branch's new head. |
| 637 | + Result<std::shared_ptr<Snapshot>> CommitTo( |
| 638 | + std::string_view branch = SnapshotRef::kMainBranch) { |
| 639 | + ICEBERG_ASSIGN_OR_RAISE(auto ctx, |
| 640 | + TransactionContext::Make(table_, TransactionKind::kUpdate)); |
| 641 | + auto update = std::make_shared<TestSnapshotUpdate>(ctx); |
| 642 | + update->ToBranch(std::string(branch)); |
| 643 | + ICEBERG_RETURN_UNEXPECTED(update->Commit()); |
| 644 | + ICEBERG_RETURN_UNEXPECTED(table_->Refresh()); |
| 645 | + const auto& metadata = table_->metadata(); |
| 646 | + auto it = metadata->refs.find(std::string(branch)); |
| 647 | + ICEBERG_CHECK(it != metadata->refs.end(), "Missing branch {}", branch); |
| 648 | + return metadata->SnapshotById(it->second->snapshot_id); |
| 649 | + } |
| 650 | + |
| 651 | + TimePointMs future_ms_; |
| 652 | +}; |
| 653 | + |
| 654 | +TEST_F(SnapshotTimestampTest, FirstSnapshotUsesCurrentTime) { |
| 655 | + RegisterTableFromResource("TableMetadataV2ValidMinimal.json"); |
| 656 | + ASSERT_EQ(table_->metadata()->snapshots.size(), 0); |
| 657 | + |
| 658 | + auto before = CurrentTimePointMs(); |
| 659 | + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, CommitTo()); |
| 660 | + auto after = CurrentTimePointMs(); |
| 661 | + |
| 662 | + EXPECT_FALSE(snapshot->parent_snapshot_id.has_value()); |
| 663 | + EXPECT_GE(snapshot->timestamp_ms, before); |
| 664 | + EXPECT_LE(snapshot->timestamp_ms, after); |
| 665 | +} |
| 666 | + |
| 667 | +TEST_F(SnapshotTimestampTest, SuccessiveCommitsStrictlyIncrease) { |
| 668 | + RegisterTableFromResource("TableMetadataV2ValidMinimal.json"); |
| 669 | + |
| 670 | + // Commits run faster than the clock ticks, so some share a millisecond. |
| 671 | + ICEBERG_UNWRAP_OR_FAIL(auto previous, CommitTo()); |
| 672 | + for (int i = 0; i < 20; ++i) { |
| 673 | + ICEBERG_UNWRAP_OR_FAIL(auto snapshot, CommitTo()); |
| 674 | + ASSERT_EQ(snapshot->parent_snapshot_id, previous->snapshot_id); |
| 675 | + EXPECT_GT(snapshot->timestamp_ms, previous->timestamp_ms); |
| 676 | + previous = std::move(snapshot); |
| 677 | + } |
| 678 | +} |
| 679 | + |
| 680 | +TEST_F(SnapshotTimestampTest, FutureParentGetsParentPlusOne) { |
| 681 | + RegisterWithFutureMainHead(); |
| 682 | + |
| 683 | + ICEBERG_UNWRAP_OR_FAIL(auto first, CommitTo()); |
| 684 | + EXPECT_EQ(first->parent_snapshot_id, kCurrentSnapshotId); |
| 685 | + EXPECT_EQ(first->timestamp_ms, future_ms_ + std::chrono::milliseconds(1)); |
| 686 | + |
| 687 | + // The clock is still behind the parent, so the next commit bumps again. |
| 688 | + ICEBERG_UNWRAP_OR_FAIL(auto second, CommitTo()); |
| 689 | + EXPECT_EQ(second->parent_snapshot_id, first->snapshot_id); |
| 690 | + EXPECT_EQ(second->timestamp_ms, future_ms_ + std::chrono::milliseconds(2)); |
| 691 | +} |
| 692 | + |
| 693 | +TEST_F(SnapshotTimestampTest, UsesParentOnTargetBranch) { |
| 694 | + RegisterWithFutureMainHead(); |
| 695 | + |
| 696 | + // The audit branch's parent is in the past, so its commit uses the current |
| 697 | + // time even though main's head is in the future. |
| 698 | + auto before = CurrentTimePointMs(); |
| 699 | + ICEBERG_UNWRAP_OR_FAIL(auto audit, CommitTo(kAuditBranch)); |
| 700 | + auto after = CurrentTimePointMs(); |
| 701 | + EXPECT_EQ(audit->parent_snapshot_id, kOldSnapshotId); |
| 702 | + EXPECT_GE(audit->timestamp_ms, before); |
| 703 | + EXPECT_LE(audit->timestamp_ms, after); |
| 704 | + |
| 705 | + ICEBERG_UNWRAP_OR_FAIL(auto main, CommitTo()); |
| 706 | + EXPECT_EQ(main->parent_snapshot_id, kCurrentSnapshotId); |
| 707 | + EXPECT_EQ(main->timestamp_ms, future_ms_ + std::chrono::milliseconds(1)); |
| 708 | +} |
| 709 | + |
597 | 710 | namespace { |
598 | 711 |
|
599 | 712 | class CapturingReporter final : public MetricsReporter { |
|
0 commit comments