Examples with StructLikeWrapper org.apache.iceberg.util.StructLikeWrapper used on opensource projects

Example 1 with StructLikeWrapper

use of org.apache.iceberg.util.StructLikeWrapper in project presto by prestodb.

the class PartitionTable method getPartitions.

private Map<StructLikeWrapper, Partition> getPartitions(TableScan tableScan) {
    try (CloseableIterable<FileScanTask> fileScanTasks = tableScan.planFiles()) {
        Map<StructLikeWrapper, Partition> partitions = new HashMap<>();
        for (FileScanTask fileScanTask : fileScanTasks) {
            DataFile dataFile = fileScanTask.file();
            Types.StructType structType = fileScanTask.spec().partitionType();
            StructLike partitionStruct = dataFile.partition();
            StructLikeWrapper partitionWrapper = StructLikeWrapper.forType(structType).set(partitionStruct);
            if (!partitions.containsKey(partitionWrapper)) {
                Partition partition = new Partition(idToTypeMapping, nonPartitionPrimitiveColumns, partitionStruct, dataFile.recordCount(), dataFile.fileSizeInBytes(), toMap(dataFile.lowerBounds()), toMap(dataFile.upperBounds()), dataFile.nullValueCounts(), dataFile.columnSizes());
                partitions.put(partitionWrapper, partition);
                continue;
            }
            Partition partition = partitions.get(partitionWrapper);
            partition.incrementFileCount();
            partition.incrementRecordCount(dataFile.recordCount());
            partition.incrementSize(dataFile.fileSizeInBytes());
            partition.updateMin(toMap(dataFile.lowerBounds()), dataFile.nullValueCounts(), dataFile.recordCount());
            partition.updateMax(toMap(dataFile.upperBounds()), dataFile.nullValueCounts(), dataFile.recordCount());
            partition.updateNullCount(dataFile.nullValueCounts());
        }
        return partitions;
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}

Example 2 with StructLikeWrapper

use of org.apache.iceberg.util.StructLikeWrapper in project presto by prestodb.

the class PartitionTable method buildRecordCursor.

private RecordCursor buildRecordCursor(Map<StructLikeWrapper, Partition> partitions, List<PartitionField> partitionFields) {
    List<Type> partitionTypes = partitionTypes(partitionFields);
    List<? extends Class<?>> partitionColumnClass = partitionTypes.stream().map(type -> type.typeId().javaClass()).collect(toImmutableList());
    int columnCounts = partitionColumnTypes.size() + 3 + columnMetricTypes.size();
    ImmutableList.Builder<List<Object>> records = ImmutableList.builder();
    for (Partition partition : partitions.values()) {
        List<Object> row = new ArrayList<>(columnCounts);
        // add data for partition columns
        for (int i = 0; i < partitionColumnTypes.size(); i++) {
            row.add(convert(partition.getValues().get(i, partitionColumnClass.get(i)), partitionTypes.get(i)));
        }
        // add the top level metrics.
        row.add(partition.getRecordCount());
        row.add(partition.getFileCount());
        row.add(partition.getSize());
        // add column level metrics
        for (int i = 0; i < columnMetricTypes.size(); i++) {
            if (!partition.hasValidColumnMetrics()) {
                row.add(null);
                continue;
            }
            Integer fieldId = nonPartitionPrimitiveColumns.get(i).fieldId();
            Type.PrimitiveType type = idToTypeMapping.get(fieldId);
            Object min = convert(partition.getMinValues().get(fieldId), type);
            Object max = convert(partition.getMaxValues().get(fieldId), type);
            Long nullCount = partition.getNullCounts().get(fieldId);
            row.add(getColumnMetricBlock(columnMetricTypes.get(i), min, max, nullCount));
        }
        records.add(row);
    }
    return new InMemoryRecordSet(resultTypes, records.build()).cursor();
}