Examples with Snapshot org.apache.iceberg.Snapshot used on opensource projects

Example 1 with Snapshot

use of org.apache.iceberg.Snapshot in project presto by prestodb.

the class HistoryTable method cursor.

@Override
public RecordCursor cursor(ConnectorTransactionHandle transactionHandle, ConnectorSession session, TupleDomain<Integer> constraint) {
    InMemoryRecordSet.Builder table = InMemoryRecordSet.builder(COLUMNS);
    Set<Long> ancestorIds = ImmutableSet.copyOf(SnapshotUtil.currentAncestors(icebergTable));
    for (HistoryEntry historyEntry : icebergTable.history()) {
        long snapshotId = historyEntry.snapshotId();
        Snapshot snapshot = icebergTable.snapshot(snapshotId);
        table.addRow(packDateTimeWithZone(historyEntry.timestampMillis(), session.getSqlFunctionProperties().getTimeZoneKey()), snapshotId, snapshot != null ? snapshot.parentId() : null, ancestorIds.contains(snapshotId));
    }
    return table.build().cursor();
}

Example 2 with Snapshot

use of org.apache.iceberg.Snapshot in project presto by prestodb.

the class ManifestsTable method buildPages.

private static List<Page> buildPages(ConnectorTableMetadata tableMetadata, Table icebergTable, long snapshotId) {
    PageListBuilder pagesBuilder = PageListBuilder.forTable(tableMetadata);
    Snapshot snapshot = icebergTable.snapshot(snapshotId);
    if (snapshot == null) {
        throw new PrestoException(ICEBERG_INVALID_METADATA, format("Snapshot ID [%s] does not exist for table: %s", snapshotId, icebergTable));
    }
    Map<Integer, PartitionSpec> partitionSpecsById = icebergTable.specs();
    snapshot.allManifests().forEach(file -> {
        pagesBuilder.beginRow();
        pagesBuilder.appendVarchar(file.path());
        pagesBuilder.appendBigint(file.length());
        pagesBuilder.appendInteger(file.partitionSpecId());
        pagesBuilder.appendBigint(file.snapshotId());
        pagesBuilder.appendInteger(file.addedFilesCount());
        pagesBuilder.appendInteger(file.existingFilesCount());
        pagesBuilder.appendInteger(file.deletedFilesCount());
        writePartitionSummaries(pagesBuilder.nextColumn(), file.partitions(), partitionSpecsById.get(file.partitionSpecId()));
        pagesBuilder.endRow();
    });
    return pagesBuilder.build();
}

Example 3 with Snapshot

use of org.apache.iceberg.Snapshot in project drill by apache.

the class IcebergQueriesTest method testSelectSnapshotsMetadata.

@Test
public void testSelectSnapshotsMetadata() throws Exception {
    String query = "select * from dfs.tmp.`testAllTypes#snapshots`";
    List<Snapshot> snapshots = new ArrayList<>();
    table.snapshots().forEach(snapshots::add);
    JsonStringHashMap<Object, Object> summaryMap = new JsonStringHashMap<>();
    snapshots.get(0).summary().forEach((k, v) -> summaryMap.put(new Text(k.getBytes(StandardCharsets.UTF_8)), new Text(v.getBytes(StandardCharsets.UTF_8))));
    JsonStringHashMap<Object, Object> secondSummaryMap = new JsonStringHashMap<>();
    snapshots.get(1).summary().forEach((k, v) -> secondSummaryMap.put(new Text(k.getBytes(StandardCharsets.UTF_8)), new Text(v.getBytes(StandardCharsets.UTF_8))));
    testBuilder().sqlQuery(query).unOrdered().baselineColumns("committed_at", "snapshot_id", "parent_id", "operation", "manifest_list", "summary").baselineValues(LocalDateTime.ofInstant(Instant.ofEpochMilli(snapshots.get(0).timestampMillis()), ZoneId.of("UTC")), snapshots.get(0).snapshotId(), snapshots.get(0).parentId(), snapshots.get(0).operation(), snapshots.get(0).manifestListLocation(), summaryMap).baselineValues(LocalDateTime.ofInstant(Instant.ofEpochMilli(snapshots.get(1).timestampMillis()), ZoneId.of("UTC")), snapshots.get(1).snapshotId(), snapshots.get(1).parentId(), snapshots.get(1).operation(), snapshots.get(1).manifestListLocation(), secondSummaryMap).go();
}

Example 4 with Snapshot

use of org.apache.iceberg.Snapshot in project hive by apache.

the class HiveCreateReplaceTableTest method testCreateTableTxnAndAppend.

@Test
public void testCreateTableTxnAndAppend() {
    Assert.assertFalse("Table should not exist", catalog.tableExists(TABLE_IDENTIFIER));
    Transaction txn = catalog.newCreateTableTransaction(TABLE_IDENTIFIER, SCHEMA, SPEC, tableLocation, Maps.newHashMap());
    AppendFiles append = txn.newAppend();
    DataFile dataFile = DataFiles.builder(SPEC).withPath("/path/to/data-a.parquet").withFileSizeInBytes(0).withRecordCount(1).build();
    append.appendFile(dataFile);
    append.commit();
    txn.commitTransaction();
    Table table = catalog.loadTable(TABLE_IDENTIFIER);
    Snapshot snapshot = table.currentSnapshot();
    Assert.assertTrue("Table should have one manifest file", snapshot.allManifests().size() == 1);
}