Examples with IcebergTable org.projectnessie.model.IcebergTable used on opensource projects

Example 6 with IcebergTable

use of org.projectnessie.model.IcebergTable in project iceberg by apache.

the class TestNessieTable method testCreate.

@Test
public void testCreate() throws IOException {
    // Table should be created in iceberg
    // Table should be renamed in iceberg
    String tableName = TABLE_IDENTIFIER.name();
    Table icebergTable = catalog.loadTable(TABLE_IDENTIFIER);
    // add a column
    icebergTable.updateSchema().addColumn("mother", Types.LongType.get()).commit();
    IcebergTable table = getTable(KEY);
    // check parameters are in expected state
    String expected = (temp.toUri() + DB_NAME + "/" + tableName).replace("///", "/");
    Assertions.assertThat(getTableLocation(tableName)).isEqualTo(expected);
    // Only 1 snapshotFile Should exist and no manifests should exist
    Assertions.assertThat(metadataVersionFiles(tableName)).isNotNull().hasSize(2);
    Assertions.assertThat(manifestFiles(tableName)).isNotNull().isEmpty();
    verifyCommitMetadata();
}

Example 7 with IcebergTable

use of org.projectnessie.model.IcebergTable in project iceberg by apache.

the class TestNessieTable method verifyGlobalStateMovesForDML.

/**
 * Verify that Nessie always returns the globally-current global-content w/ only DMLs.
 */
@Test
public void verifyGlobalStateMovesForDML() throws Exception {
    // 1. initialize table
    Table icebergTable = catalog.loadTable(TABLE_IDENTIFIER);
    icebergTable.updateSchema().addColumn("initial_column", Types.LongType.get()).commit();
    // 2. create 2nd branch
    String testCaseBranch = "verify-global-moving";
    api.createReference().sourceRefName(BRANCH).reference(Branch.of(testCaseBranch, catalog.currentHash())).create();
    NessieCatalog branchCatalog = initCatalog(testCaseBranch);
    IcebergTable contentInitialMain = getTable(BRANCH, KEY);
    IcebergTable contentInitialBranch = getTable(testCaseBranch, KEY);
    Table tableInitialMain = catalog.loadTable(TABLE_IDENTIFIER);
    // verify table-metadata-location + snapshot-id
    Assertions.assertThat(contentInitialMain).as("global-contents + snapshot-id equal on both branches in Nessie").isEqualTo(contentInitialBranch);
    Assertions.assertThat(tableInitialMain.currentSnapshot()).isNull();
    // 3. modify table in "main" branch (add some data)
    DataFile file1 = makeDataFile(icebergTable, addRecordsToFile(icebergTable, "file1"));
    icebergTable.newAppend().appendFile(file1).commit();
    IcebergTable contentsAfter1Main = getTable(KEY);
    IcebergTable contentsAfter1Branch = getTable(testCaseBranch, KEY);
    Table tableAfter1Main = catalog.loadTable(TABLE_IDENTIFIER);
    // --> assert getValue() against both branches returns the updated metadata-location
    // verify table-metadata-location
    Assertions.assertThat(contentInitialMain.getMetadataLocation()).describedAs("metadata-location must change on %s", BRANCH).isNotEqualTo(contentsAfter1Main.getMetadataLocation());
    Assertions.assertThat(contentInitialBranch.getMetadataLocation()).describedAs("metadata-location must change on %s", testCaseBranch).isNotEqualTo(contentsAfter1Branch.getMetadataLocation());
    Assertions.assertThat(contentsAfter1Main).extracting(IcebergTable::getSchemaId).describedAs("on-reference-state must not be equal on both branches").isEqualTo(contentsAfter1Branch.getSchemaId());
    // verify manifests
    Assertions.assertThat(tableAfter1Main.currentSnapshot().allManifests()).describedAs("verify number of manifests on 'main'").hasSize(1);
    // 4. modify table in "main" branch (add some data) again
    DataFile file2 = makeDataFile(icebergTable, addRecordsToFile(icebergTable, "file2"));
    icebergTable.newAppend().appendFile(file2).commit();
    IcebergTable contentsAfter2Main = getTable(KEY);
    IcebergTable contentsAfter2Branch = getTable(testCaseBranch, KEY);
    Table tableAfter2Main = catalog.loadTable(TABLE_IDENTIFIER);
    // --> assert getValue() against both branches returns the updated metadata-location
    // verify table-metadata-location
    Assertions.assertThat(contentsAfter2Main.getMetadataLocation()).describedAs("metadata-location must change on %s", BRANCH).isNotEqualTo(contentsAfter1Main.getMetadataLocation());
    Assertions.assertThat(contentsAfter2Branch.getMetadataLocation()).describedAs("on-reference-state must change on %s", testCaseBranch).isNotEqualTo(contentsAfter1Branch.getMetadataLocation());
    // verify manifests
    Assertions.assertThat(tableAfter2Main.currentSnapshot().allManifests()).describedAs("verify number of manifests on 'main'").hasSize(2);
}

Example 8 with IcebergTable

use of org.projectnessie.model.IcebergTable in project nessie by projectnessie.

the class ITUpgradePath method commit.

@Test
@Order(103)
void commit() throws Exception {
    ContentKey key = ContentKey.of("my", "tables", "table_name");
    IcebergTable content = IcebergTable.of("metadata-location", 42L, 43, 44, 45, "content-id-" + version);
    String commitMessage = "hello world " + version;
    Put operation = Put.of(key, content);
    Branch branchNew = commitMaybeRetry(api.commitMultipleOperations().commitMeta(CommitMeta.fromMessage(commitMessage)).operation(operation).branch(versionBranch));
    assertThat(branchNew).isNotEqualTo(versionBranch).extracting(Branch::getName).isEqualTo(versionBranch.getName());
    expectedRefLogEntry("COMMIT");
}

Example 9 with IcebergTable

use of org.projectnessie.model.IcebergTable in project nessie by projectnessie.

the class TableCommitMetaStoreWorker method toStoreOnReferenceState.

@Override
public ByteString toStoreOnReferenceState(Content content) {
    ObjectTypes.Content.Builder builder = ObjectTypes.Content.newBuilder().setId(content.getId());
    if (content instanceof IcebergTable) {
        IcebergTable state = (IcebergTable) content;
        ObjectTypes.IcebergRefState.Builder stateBuilder = ObjectTypes.IcebergRefState.newBuilder().setSnapshotId(state.getSnapshotId()).setSchemaId(state.getSchemaId()).setSpecId(state.getSpecId()).setSortOrderId(state.getSortOrderId());
        builder.setIcebergRefState(stateBuilder);
    } else if (content instanceof IcebergView) {
        IcebergView view = (IcebergView) content;
        builder.setIcebergViewState(ObjectTypes.IcebergViewState.newBuilder().setVersionId(view.getVersionId()).setSchemaId(view.getSchemaId()).setDialect(view.getDialect()).setSqlText(view.getSqlText()));
    } else if (content instanceof DeltaLakeTable) {
        ObjectTypes.DeltaLakeTable.Builder table = ObjectTypes.DeltaLakeTable.newBuilder().addAllMetadataLocationHistory(((DeltaLakeTable) content).getMetadataLocationHistory()).addAllCheckpointLocationHistory(((DeltaLakeTable) content).getCheckpointLocationHistory());
        String lastCheckpoint = ((DeltaLakeTable) content).getLastCheckpoint();
        if (lastCheckpoint != null) {
            table.setLastCheckpoint(lastCheckpoint);
        }
        builder.setDeltaLakeTable(table);
    } else if (content instanceof Namespace) {
        builder.setNamespace(ObjectTypes.Namespace.newBuilder().setName(((Namespace) content).name()));
    } else {
        throw new IllegalArgumentException("Unknown type " + content);
    }
    return builder.build().toByteString();
}

Example 10 with IcebergTable

use of org.projectnessie.model.IcebergTable in project nessie by projectnessie.

the class TestStoreWorker method testSerdeIceberg.

@Test
void testSerdeIceberg() {
    String path = "foo/bar";
    IcebergTable table = IcebergTable.of(path, 42, 43, 44, 45, ID);
    ObjectTypes.Content protoTableGlobal = ObjectTypes.Content.newBuilder().setId(ID).setIcebergMetadataPointer(IcebergMetadataPointer.newBuilder().setMetadataLocation(path)).build();
    ObjectTypes.Content protoOnRef = ObjectTypes.Content.newBuilder().setId(ID).setIcebergRefState(IcebergRefState.newBuilder().setSnapshotId(42).setSchemaId(43).setSpecId(44).setSortOrderId(45)).build();
    ByteString tableGlobalBytes = worker.toStoreGlobalState(table);
    ByteString snapshotBytes = worker.toStoreOnReferenceState(table);
    Assertions.assertEquals(protoTableGlobal.toByteString(), tableGlobalBytes);
    Assertions.assertEquals(protoOnRef.toByteString(), snapshotBytes);
    Content deserialized = worker.valueFromStore(snapshotBytes, Optional.of(tableGlobalBytes));
    Assertions.assertEquals(table, deserialized);
}