Examples with DeltaLakeTable org.projectnessie.model.DeltaLakeTable used on opensource projects

Example 1 with DeltaLakeTable

use of org.projectnessie.model.DeltaLakeTable in project nessie by projectnessie.

the class TableCommitMetaStoreWorker method toStoreOnReferenceState.

@Override
public ByteString toStoreOnReferenceState(Content content) {
    ObjectTypes.Content.Builder builder = ObjectTypes.Content.newBuilder().setId(content.getId());
    if (content instanceof IcebergTable) {
        IcebergTable state = (IcebergTable) content;
        ObjectTypes.IcebergRefState.Builder stateBuilder = ObjectTypes.IcebergRefState.newBuilder().setSnapshotId(state.getSnapshotId()).setSchemaId(state.getSchemaId()).setSpecId(state.getSpecId()).setSortOrderId(state.getSortOrderId());
        builder.setIcebergRefState(stateBuilder);
    } else if (content instanceof IcebergView) {
        IcebergView view = (IcebergView) content;
        builder.setIcebergViewState(ObjectTypes.IcebergViewState.newBuilder().setVersionId(view.getVersionId()).setSchemaId(view.getSchemaId()).setDialect(view.getDialect()).setSqlText(view.getSqlText()));
    } else if (content instanceof DeltaLakeTable) {
        ObjectTypes.DeltaLakeTable.Builder table = ObjectTypes.DeltaLakeTable.newBuilder().addAllMetadataLocationHistory(((DeltaLakeTable) content).getMetadataLocationHistory()).addAllCheckpointLocationHistory(((DeltaLakeTable) content).getCheckpointLocationHistory());
        String lastCheckpoint = ((DeltaLakeTable) content).getLastCheckpoint();
        if (lastCheckpoint != null) {
            table.setLastCheckpoint(lastCheckpoint);
        }
        builder.setDeltaLakeTable(table);
    } else if (content instanceof Namespace) {
        builder.setNamespace(ObjectTypes.Namespace.newBuilder().setName(((Namespace) content).name()));
    } else {
        throw new IllegalArgumentException("Unknown type " + content);
    }
    return builder.build().toByteString();
}

Example 2 with DeltaLakeTable

use of org.projectnessie.model.DeltaLakeTable in project nessie by projectnessie.

the class ITDeltaLogBranches method testCheckpoint.

@Test
void testCheckpoint() throws NessieNotFoundException {
    Dataset<Row> targetTable = createKVDataSet(Arrays.asList(tuple2(1, 10), tuple2(2, 20), tuple2(3, 30), tuple2(4, 40)), "key", "value");
    // write some data to table
    targetTable.write().format("delta").save(tempPath.getAbsolutePath());
    // write enough to trigger a checkpoint generation
    for (int i = 0; i < 15; i++) {
        targetTable.write().format("delta").mode("append").save(tempPath.getAbsolutePath());
    }
    DeltaTable target = DeltaTable.forPath(spark, tempPath.getAbsolutePath());
    int expectedSize = target.toDF().collectAsList().size();
    Assertions.assertEquals(64, expectedSize);
    String tableName = tempPath.getAbsolutePath() + "/_delta_log";
    ContentKey key = ContentKey.of(tableName.split("/"));
    Content content = api.getContent().key(key).refName("main").get().get(key);
    Optional<DeltaLakeTable> table = content.unwrap(DeltaLakeTable.class);
    Assertions.assertTrue(table.isPresent());
    Assertions.assertEquals(1, table.get().getCheckpointLocationHistory().size());
    Assertions.assertEquals(5, table.get().getMetadataLocationHistory().size());
    Assertions.assertNotNull(table.get().getLastCheckpoint());
}