Examples with PARTITION com.facebook.presto.delta.DeltaColumnHandle.ColumnType.PARTITION used on opensource projects

Example 1 with PARTITION

use of com.facebook.presto.delta.DeltaColumnHandle.ColumnType.PARTITION in project presto by prestodb.

the class DeltaExpressionUtils method iterateWithPartitionPruning.

/**
 * Utility method that takes an iterator of {@link AddFile}s and a predicate and returns an iterator of {@link AddFile}s
 * that satisfy the predicate (predicate evaluates to a deterministic NO)
 */
public static CloseableIterator<AddFile> iterateWithPartitionPruning(CloseableIterator<AddFile> inputIterator, TupleDomain<DeltaColumnHandle> predicate, TypeManager typeManager) {
    TupleDomain<String> partitionPredicate = extractPartitionColumnsPredicate(predicate);
    if (partitionPredicate.isAll()) {
        // there is no partition filter, return the input iterator as is.
        return inputIterator;
    }
    if (partitionPredicate.isNone()) {
        // nothing passes the partition predicate, return empty iterator
        return new CloseableIterator<AddFile>() {

            @Override
            public boolean hasNext() {
                return false;
            }

            @Override
            public AddFile next() {
                throw new NoSuchElementException();
            }

            @Override
            public void close() throws IOException {
                inputIterator.close();
            }
        };
    }
    List<DeltaColumnHandle> partitionColumns = predicate.getColumnDomains().get().stream().filter(entry -> entry.getColumn().getColumnType() == PARTITION).map(entry -> entry.getColumn()).collect(Collectors.toList());
    return new CloseableIterator<AddFile>() {

        private AddFile nextItem;

        @Override
        public boolean hasNext() {
            if (nextItem != null) {
                return true;
            }
            while (inputIterator.hasNext()) {
                AddFile nextFile = inputIterator.next();
                if (evaluatePartitionPredicate(partitionPredicate, partitionColumns, typeManager, nextFile)) {
                    nextItem = nextFile;
                    break;
                }
            }
            return nextItem != null;
        }

        @Override
        public AddFile next() {
            if (!hasNext()) {
                throw new NoSuchElementException("there are no more files");
            }
            AddFile toReturn = nextItem;
            nextItem = null;
            return toReturn;
        }

        @Override
        public void close() throws IOException {
            inputIterator.close();
        }
    };
}

Example 2 with PARTITION

use of com.facebook.presto.delta.DeltaColumnHandle.ColumnType.PARTITION in project presto by prestodb.

the class DeltaPageSourceProvider method createPageSource.

@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorSplit split, ConnectorTableLayoutHandle layout, List<ColumnHandle> columns, SplitContext splitContext) {
    DeltaSplit deltaSplit = (DeltaSplit) split;
    DeltaTableLayoutHandle deltaTableLayoutHandle = (DeltaTableLayoutHandle) layout;
    DeltaTableHandle deltaTableHandle = deltaTableLayoutHandle.getTable();
    HdfsContext hdfsContext = new HdfsContext(session, deltaSplit.getSchema(), deltaSplit.getTable(), deltaSplit.getFilePath(), false);
    Path filePath = new Path(deltaSplit.getFilePath());
    List<DeltaColumnHandle> deltaColumnHandles = columns.stream().map(DeltaColumnHandle.class::cast).collect(Collectors.toList());
    List<DeltaColumnHandle> regularColumnHandles = deltaColumnHandles.stream().filter(columnHandle -> columnHandle.getColumnType() != PARTITION).collect(Collectors.toList());
    ConnectorPageSource dataPageSource = createParquetPageSource(hdfsEnvironment, session.getUser(), hdfsEnvironment.getConfiguration(hdfsContext, filePath), filePath, deltaSplit.getStart(), deltaSplit.getLength(), deltaSplit.getFileSize(), regularColumnHandles, deltaTableHandle.toSchemaTableName(), getParquetMaxReadBlockSize(session), isParquetBatchReadsEnabled(session), isParquetBatchReaderVerificationEnabled(session), typeManager, deltaTableLayoutHandle.getPredicate(), fileFormatDataSourceStats, false);
    return new DeltaPageSource(deltaColumnHandles, convertPartitionValues(deltaColumnHandles, deltaSplit.getPartitionValues()), dataPageSource);
}