Examples with TupleDomain com.facebook.presto.common.predicate.TupleDomain used on opensource projects

Example 1 with TupleDomain

use of com.facebook.presto.common.predicate.TupleDomain in project presto by prestodb.

the class TestShardMetadataRecordCursor method testSimple.

@Test
public void testSimple() {
    ShardManager shardManager = createShardManager(dbi);
    // Add shards to the table
    long tableId = 1;
    OptionalInt bucketNumber = OptionalInt.empty();
    UUID uuid1 = UUID.randomUUID();
    UUID uuid2 = UUID.randomUUID();
    UUID uuid3 = UUID.randomUUID();
    ShardInfo shardInfo1 = new ShardInfo(uuid1, bucketNumber, ImmutableSet.of("node1"), ImmutableList.of(), 1, 10, 100, 0x1234);
    ShardInfo shardInfo2 = new ShardInfo(uuid2, bucketNumber, ImmutableSet.of("node2"), ImmutableList.of(), 2, 20, 200, 0xCAFEBABEDEADBEEFL);
    ShardInfo shardInfo3 = new ShardInfo(uuid3, bucketNumber, ImmutableSet.of("node3"), ImmutableList.of(), 3, 30, 300, 0xFEDCBA0987654321L);
    List<ShardInfo> shards = ImmutableList.of(shardInfo1, shardInfo2, shardInfo3);
    long transactionId = shardManager.beginTransaction();
    shardManager.commitShards(transactionId, tableId, ImmutableList.of(new ColumnInfo(1, BIGINT), new ColumnInfo(2, DATE)), shards, Optional.empty(), 0);
    Slice schema = utf8Slice(DEFAULT_TEST_ORDERS.getSchemaName());
    Slice table = utf8Slice(DEFAULT_TEST_ORDERS.getTableName());
    DateTime date1 = DateTime.parse("2015-01-01T00:00");
    DateTime date2 = DateTime.parse("2015-01-02T00:00");
    TupleDomain<Integer> tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.<Integer, Domain>builder().put(0, Domain.singleValue(createVarcharType(10), schema)).put(1, Domain.create(ValueSet.ofRanges(lessThanOrEqual(createVarcharType(10), table)), true)).put(8, Domain.create(ValueSet.ofRanges(lessThanOrEqual(BIGINT, date1.getMillis()), greaterThan(BIGINT, date2.getMillis())), true)).put(9, Domain.create(ValueSet.ofRanges(lessThanOrEqual(BIGINT, date1.getMillis()), greaterThan(BIGINT, date2.getMillis())), true)).build());
    List<MaterializedRow> actual;
    try (RecordCursor cursor = new ShardMetadataSystemTable(dbi).cursor(null, SESSION, tupleDomain)) {
        actual = getMaterializedResults(cursor, SHARD_METADATA.getColumns());
    }
    assertEquals(actual.size(), 3);
    List<MaterializedRow> expected = ImmutableList.of(new MaterializedRow(DEFAULT_PRECISION, schema, table, utf8Slice(uuid1.toString()), null, 100L, 10L, 1L, utf8Slice("0000000000001234"), null, null, null, null), new MaterializedRow(DEFAULT_PRECISION, schema, table, utf8Slice(uuid2.toString()), null, 200L, 20L, 2L, utf8Slice("cafebabedeadbeef"), null, null, null, null), new MaterializedRow(DEFAULT_PRECISION, schema, table, utf8Slice(uuid3.toString()), null, 300L, 30L, 3L, utf8Slice("fedcba0987654321"), null, null, null, null));
    assertEquals(actual, expected);
}

Example 2 with TupleDomain

use of com.facebook.presto.common.predicate.TupleDomain in project presto by prestodb.

the class TestPrometheusSplit method testPredicatePushDownLowerBoundDirect.

@Test
public void testPredicatePushDownLowerBoundDirect() {
    Range lowRange = Range.greaterThanOrEqual(TIMESTAMP_WITH_TIME_ZONE, packDateTimeWithZone(1570460709643L, UTC_KEY));
    ValueSet valueSet = ValueSet.ofRanges(lowRange);
    Domain testDomain = Domain.create(valueSet, false);
    TupleDomain<ColumnHandle> testTupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of(new PrometheusColumnHandle("timestamp", TIMESTAMP_WITH_TIME_ZONE, 2), testDomain));
    PrometheusPredicateTimeInfo predicateTimes = PrometheusSplitManager.determinePredicateTimes(testTupleDomain).orElseThrow(() -> new AssertionError("predicate pushdown error on Prometheus Column"));
    Instant expected = ofEpochMilli(1570460709643L);
    assertEquals(predicateTimes.getPredicateLowerTimeBound().orElseThrow(() -> new AssertionError("predicate pushdown error on Prometheus Column")), expected);
}

Example 3 with TupleDomain

use of com.facebook.presto.common.predicate.TupleDomain in project presto by prestodb.

the class RaptorPageSourceProvider method createPageSource.

@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorSplit split, List<ColumnHandle> columns, SplitContext splitContext) {
    RaptorSplit raptorSplit = (RaptorSplit) split;
    OptionalInt bucketNumber = raptorSplit.getBucketNumber();
    TupleDomain<RaptorColumnHandle> predicate = raptorSplit.getEffectivePredicate();
    ReaderAttributes attributes = ReaderAttributes.from(session);
    OptionalLong transactionId = raptorSplit.getTransactionId();
    Optional<Map<String, Type>> columnTypes = raptorSplit.getColumnTypes();
    boolean tableSupportsDeltaDelete = raptorSplit.isTableSupportsDeltaDelete();
    HdfsContext context = new HdfsContext(session);
    Map<UUID, UUID> shardDeltaMap = raptorSplit.getShardDeltaMap();
    if (raptorSplit.getShardUuids().size() == 1) {
        UUID shardUuid = raptorSplit.getShardUuids().iterator().next();
        return createPageSource(context, DEFAULT_HIVE_FILE_CONTEXT, shardUuid, Optional.ofNullable(shardDeltaMap.get(shardUuid)), tableSupportsDeltaDelete, bucketNumber, columns, predicate, attributes, transactionId, columnTypes);
    }
    Iterator<ConnectorPageSource> iterator = raptorSplit.getShardUuids().stream().map(shardUuid -> createPageSource(context, DEFAULT_HIVE_FILE_CONTEXT, shardUuid, Optional.ofNullable(shardDeltaMap.get(shardUuid)), tableSupportsDeltaDelete, bucketNumber, columns, predicate, attributes, transactionId, columnTypes)).iterator();
    return new ConcatPageSource(iterator);
}

Example 4 with TupleDomain

use of com.facebook.presto.common.predicate.TupleDomain in project presto by prestodb.

the class TestJdbcQueryBuilder method testNormalBuildSql.

@Test
public void testNormalBuildSql() throws SQLException {
    TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.<ColumnHandle, Domain>builder().put(columns.get(0), Domain.create(SortedRangeSet.copyOf(BIGINT, ImmutableList.of(Range.equal(BIGINT, 128L), Range.equal(BIGINT, 180L), Range.equal(BIGINT, 233L), Range.lessThan(BIGINT, 25L), Range.range(BIGINT, 66L, true, 96L, true), Range.greaterThan(BIGINT, 192L))), false)).put(columns.get(1), Domain.create(SortedRangeSet.copyOf(DOUBLE, ImmutableList.of(Range.equal(DOUBLE, 200011.0), Range.equal(DOUBLE, 200014.0), Range.equal(DOUBLE, 200017.0), Range.equal(DOUBLE, 200116.5), Range.range(DOUBLE, 200030.0, true, 200036.0, true), Range.range(DOUBLE, 200048.0, true, 200099.0, true))), false)).put(columns.get(7), Domain.create(SortedRangeSet.copyOf(TINYINT, ImmutableList.of(Range.range(TINYINT, 60L, true, 70L, false), Range.range(TINYINT, 52L, true, 55L, false))), false)).put(columns.get(8), Domain.create(SortedRangeSet.copyOf(SMALLINT, ImmutableList.of(Range.range(SMALLINT, -75L, true, -68L, true), Range.range(SMALLINT, -200L, true, -100L, false))), false)).put(columns.get(9), Domain.create(SortedRangeSet.copyOf(INTEGER, ImmutableList.of(Range.equal(INTEGER, 80L), Range.equal(INTEGER, 96L), Range.lessThan(INTEGER, 0L))), false)).put(columns.get(2), Domain.create(SortedRangeSet.copyOf(BOOLEAN, ImmutableList.of(Range.equal(BOOLEAN, true))), false)).build());
    Connection connection = database.getConnection();
    try (PreparedStatement preparedStatement = new QueryBuilder("\"").buildSql(jdbcClient, session, connection, "", "", "test_table", columns, tupleDomain, Optional.empty());
        ResultSet resultSet = preparedStatement.executeQuery()) {
        ImmutableSet.Builder<Long> builder = ImmutableSet.builder();
        while (resultSet.next()) {
            builder.add((Long) resultSet.getObject("col_0"));
        }
        assertEquals(builder.build(), ImmutableSet.of(68L, 180L, 196L));
    }
}

Example 5 with TupleDomain

use of com.facebook.presto.common.predicate.TupleDomain in project presto by prestodb.

the class DeltaPageSourceProvider method createParquetPageSource.

private static ConnectorPageSource createParquetPageSource(HdfsEnvironment hdfsEnvironment, String user, Configuration configuration, Path path, long start, long length, long fileSize, List<DeltaColumnHandle> columns, SchemaTableName tableName, DataSize maxReadBlockSize, boolean batchReaderEnabled, boolean verificationEnabled, TypeManager typeManager, TupleDomain<DeltaColumnHandle> effectivePredicate, FileFormatDataSourceStats stats, boolean columnIndexFilterEnabled) {
    AggregatedMemoryContext systemMemoryContext = newSimpleAggregatedMemoryContext();
    ParquetDataSource dataSource = null;
    try {
        FSDataInputStream inputStream = hdfsEnvironment.getFileSystem(user, path, configuration).open(path);
        dataSource = buildHdfsParquetDataSource(inputStream, path, stats);
        ParquetMetadata parquetMetadata = MetadataReader.readFooter(dataSource, fileSize).getParquetMetadata();
        FileMetaData fileMetaData = parquetMetadata.getFileMetaData();
        MessageType fileSchema = fileMetaData.getSchema();
        Optional<MessageType> message = columns.stream().filter(column -> column.getColumnType() == REGULAR || isPushedDownSubfield(column)).map(column -> getColumnType(typeManager.getType(column.getDataType()), fileSchema, column, tableName, path)).filter(Optional::isPresent).map(Optional::get).map(type -> new MessageType(fileSchema.getName(), type)).reduce(MessageType::union);
        MessageType requestedSchema = message.orElse(new MessageType(fileSchema.getName(), ImmutableList.of()));
        ImmutableList.Builder<BlockMetaData> footerBlocks = ImmutableList.builder();
        for (BlockMetaData block : parquetMetadata.getBlocks()) {
            long firstDataPage = block.getColumns().get(0).getFirstDataPageOffset();
            if (firstDataPage >= start && firstDataPage < start + length) {
                footerBlocks.add(block);
            }
        }
        Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, requestedSchema);
        TupleDomain<ColumnDescriptor> parquetTupleDomain = getParquetTupleDomain(descriptorsByPath, effectivePredicate);
        Predicate parquetPredicate = buildPredicate(requestedSchema, parquetTupleDomain, descriptorsByPath);
        final ParquetDataSource finalDataSource = dataSource;
        ImmutableList.Builder<BlockMetaData> blocks = ImmutableList.builder();
        List<ColumnIndexStore> blockIndexStores = new ArrayList<>();
        for (BlockMetaData block : footerBlocks.build()) {
            Optional<ColumnIndexStore> columnIndexStore = ColumnIndexFilterUtils.getColumnIndexStore(parquetPredicate, finalDataSource, block, descriptorsByPath, columnIndexFilterEnabled);
            if (predicateMatches(parquetPredicate, block, finalDataSource, descriptorsByPath, parquetTupleDomain, columnIndexStore, columnIndexFilterEnabled)) {
                blocks.add(block);
                blockIndexStores.add(columnIndexStore.orElse(null));
            }
        }
        MessageColumnIO messageColumnIO = getColumnIO(fileSchema, requestedSchema);
        ParquetReader parquetReader = new ParquetReader(messageColumnIO, blocks.build(), dataSource, systemMemoryContext, maxReadBlockSize, batchReaderEnabled, verificationEnabled, parquetPredicate, blockIndexStores, columnIndexFilterEnabled);
        ImmutableList.Builder<String> namesBuilder = ImmutableList.builder();
        ImmutableList.Builder<Type> typesBuilder = ImmutableList.builder();
        ImmutableList.Builder<Optional<Field>> fieldsBuilder = ImmutableList.builder();
        for (DeltaColumnHandle column : columns) {
            checkArgument(column.getColumnType() == REGULAR || column.getColumnType() == SUBFIELD, "column type must be regular or subfield column");
            String name = column.getName();
            Type type = typeManager.getType(column.getDataType());
            namesBuilder.add(name);
            typesBuilder.add(type);
            if (isPushedDownSubfield(column)) {
                Subfield pushedDownSubfield = getPushedDownSubfield(column);
                List<String> nestedColumnPath = nestedColumnPath(pushedDownSubfield);
                Optional<ColumnIO> columnIO = findNestedColumnIO(lookupColumnByName(messageColumnIO, pushedDownSubfield.getRootName()), nestedColumnPath);
                if (columnIO.isPresent()) {
                    fieldsBuilder.add(constructField(type, columnIO.get()));
                } else {
                    fieldsBuilder.add(Optional.empty());
                }
            } else if (getParquetType(type, fileSchema, column, tableName, path).isPresent()) {
                fieldsBuilder.add(constructField(type, lookupColumnByName(messageColumnIO, name)));
            } else {
                fieldsBuilder.add(Optional.empty());
            }
        }
        return new ParquetPageSource(parquetReader, typesBuilder.build(), fieldsBuilder.build(), namesBuilder.build(), new RuntimeStats());
    } catch (Exception exception) {
        try {
            if (dataSource != null) {
                dataSource.close();
            }
        } catch (IOException ignored) {
        }
        if (exception instanceof PrestoException) {
            throw (PrestoException) exception;
        }
        if (exception instanceof ParquetCorruptionException) {
            throw new PrestoException(DELTA_BAD_DATA, exception);
        }
        if (exception instanceof AccessControlException) {
            throw new PrestoException(PERMISSION_DENIED, exception.getMessage(), exception);
        }
        if (nullToEmpty(exception.getMessage()).trim().equals("Filesystem closed") || exception instanceof FileNotFoundException) {
            throw new PrestoException(DELTA_CANNOT_OPEN_SPLIT, exception);
        }
        String message = format("Error opening Hive split %s (offset=%s, length=%s): %s", path, start, length, exception.getMessage());
        if (exception.getClass().getSimpleName().equals("BlockMissingException")) {
            throw new PrestoException(DELTA_MISSING_DATA, message, exception);
        }
        throw new PrestoException(DELTA_CANNOT_OPEN_SPLIT, message, exception);
    }
}