Search in sources :

Example 1 with AddFile

use of io.delta.standalone.actions.AddFile in project presto by prestodb.

the class DeltaExpressionUtils method iterateWithPartitionPruning.

/**
 * Utility method that takes an iterator of {@link AddFile}s and a predicate and returns an iterator of {@link AddFile}s
 * that satisfy the predicate (predicate evaluates to a deterministic NO)
 */
public static CloseableIterator<AddFile> iterateWithPartitionPruning(CloseableIterator<AddFile> inputIterator, TupleDomain<DeltaColumnHandle> predicate, TypeManager typeManager) {
    TupleDomain<String> partitionPredicate = extractPartitionColumnsPredicate(predicate);
    if (partitionPredicate.isAll()) {
        // there is no partition filter, return the input iterator as is.
        return inputIterator;
    }
    if (partitionPredicate.isNone()) {
        // nothing passes the partition predicate, return empty iterator
        return new CloseableIterator<AddFile>() {

            @Override
            public boolean hasNext() {
                return false;
            }

            @Override
            public AddFile next() {
                throw new NoSuchElementException();
            }

            @Override
            public void close() throws IOException {
                inputIterator.close();
            }
        };
    }
    List<DeltaColumnHandle> partitionColumns = predicate.getColumnDomains().get().stream().filter(entry -> entry.getColumn().getColumnType() == PARTITION).map(entry -> entry.getColumn()).collect(Collectors.toList());
    return new CloseableIterator<AddFile>() {

        private AddFile nextItem;

        @Override
        public boolean hasNext() {
            if (nextItem != null) {
                return true;
            }
            while (inputIterator.hasNext()) {
                AddFile nextFile = inputIterator.next();
                if (evaluatePartitionPredicate(partitionPredicate, partitionColumns, typeManager, nextFile)) {
                    nextItem = nextFile;
                    break;
                }
            }
            return nextItem != null;
        }

        @Override
        public AddFile next() {
            if (!hasNext()) {
                throw new NoSuchElementException("there are no more files");
            }
            AddFile toReturn = nextItem;
            nextItem = null;
            return toReturn;
        }

        @Override
        public void close() throws IOException {
            inputIterator.close();
        }
    };
}
Also used : PARTITION(com.facebook.presto.delta.DeltaColumnHandle.ColumnType.PARTITION) StandardTypes(com.facebook.presto.common.type.StandardTypes) Slice(io.airlift.slice.Slice) PrestoException(com.facebook.presto.spi.PrestoException) DELTA_UNSUPPORTED_COLUMN_TYPE(com.facebook.presto.delta.DeltaErrorCode.DELTA_UNSUPPORTED_COLUMN_TYPE) Float.floatToRawIntBits(java.lang.Float.floatToRawIntBits) Float.parseFloat(java.lang.Float.parseFloat) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) ImmutableList(com.google.common.collect.ImmutableList) CloseableIterator(io.delta.standalone.data.CloseableIterator) TypeManager(com.facebook.presto.common.type.TypeManager) Map(java.util.Map) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) NoSuchElementException(java.util.NoSuchElementException) Type(com.facebook.presto.common.type.Type) Double.parseDouble(java.lang.Double.parseDouble) ImmutableMap(com.google.common.collect.ImmutableMap) Timestamp(java.sql.Timestamp) DELTA_INVALID_PARTITION_VALUE(com.facebook.presto.delta.DeltaErrorCode.DELTA_INVALID_PARTITION_VALUE) IOException(java.io.IOException) AddFile(io.delta.standalone.actions.AddFile) Collectors(java.util.stream.Collectors) Domain(com.facebook.presto.common.predicate.Domain) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) String.format(java.lang.String.format) Date(java.sql.Date) Double.doubleToRawLongBits(java.lang.Double.doubleToRawLongBits) List(java.util.List) ColumnHandle(com.facebook.presto.spi.ColumnHandle) Optional(java.util.Optional) Long.parseLong(java.lang.Long.parseLong) ValueSet(com.facebook.presto.common.predicate.ValueSet) AddFile(io.delta.standalone.actions.AddFile) CloseableIterator(io.delta.standalone.data.CloseableIterator) NoSuchElementException(java.util.NoSuchElementException)

Aggregations

Domain (com.facebook.presto.common.predicate.Domain)1 TupleDomain (com.facebook.presto.common.predicate.TupleDomain)1 ValueSet (com.facebook.presto.common.predicate.ValueSet)1 StandardTypes (com.facebook.presto.common.type.StandardTypes)1 Type (com.facebook.presto.common.type.Type)1 TypeManager (com.facebook.presto.common.type.TypeManager)1 PARTITION (com.facebook.presto.delta.DeltaColumnHandle.ColumnType.PARTITION)1 DELTA_INVALID_PARTITION_VALUE (com.facebook.presto.delta.DeltaErrorCode.DELTA_INVALID_PARTITION_VALUE)1 DELTA_UNSUPPORTED_COLUMN_TYPE (com.facebook.presto.delta.DeltaErrorCode.DELTA_UNSUPPORTED_COLUMN_TYPE)1 ColumnHandle (com.facebook.presto.spi.ColumnHandle)1 PrestoException (com.facebook.presto.spi.PrestoException)1 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)1 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 Slice (io.airlift.slice.Slice)1 Slices.utf8Slice (io.airlift.slice.Slices.utf8Slice)1 AddFile (io.delta.standalone.actions.AddFile)1 CloseableIterator (io.delta.standalone.data.CloseableIterator)1 IOException (java.io.IOException)1 Double.doubleToRawLongBits (java.lang.Double.doubleToRawLongBits)1