Examples with NullableValue com.facebook.presto.common.predicate.NullableValue used on opensource projects

Example 21 with NullableValue

use of com.facebook.presto.common.predicate.NullableValue in project presto by prestodb.

the class HivePartitionManager method parseValuesAndFilterPartition.

private Optional<HivePartition> parseValuesAndFilterPartition(SchemaTableName tableName, String partitionId, List<HiveColumnHandle> partitionColumns, List<Type> partitionColumnTypes, Constraint<ColumnHandle> constraint) {
    HivePartition partition = parsePartition(tableName, partitionId, partitionColumns, partitionColumnTypes, timeZone);
    Map<ColumnHandle, Domain> domains = constraint.getSummary().getDomains().get();
    for (HiveColumnHandle column : partitionColumns) {
        NullableValue value = partition.getKeys().get(column);
        Domain allowedDomain = domains.get(column);
        if (allowedDomain != null && !allowedDomain.includesNullableValue(value.getValue())) {
            return Optional.empty();
        }
    }
    if (constraint.predicate().isPresent() && !constraint.predicate().get().test(partition.getKeys())) {
        return Optional.empty();
    }
    return Optional.of(partition);
}

Example 22 with NullableValue

use of com.facebook.presto.common.predicate.NullableValue in project presto by prestodb.

the class TestMetastoreHiveStatisticsProvider method testNullablePartitionValue.

@Test
public void testNullablePartitionValue() {
    HivePartition partitionWithNull = partition("p1=__HIVE_DEFAULT_PARTITION__/p2=1");
    assertTrue(partitionWithNull.getKeys().containsValue(new NullableValue(VARCHAR, null)));
    HivePartition partitionNameWithSlashN = partition("p1=\\N/p2=2");
    assertTrue(partitionNameWithSlashN.getKeys().containsValue(new NullableValue(VARCHAR, Slices.utf8Slice("\\N"))));
    assertFalse(partitionNameWithSlashN.getKeys().containsValue(new NullableValue(VARCHAR, null)));
}

Example 23 with NullableValue

use of com.facebook.presto.common.predicate.NullableValue in project presto by prestodb.

the class TestInformationSchemaMetadata method testInformationSchemaPredicatePushdownWithConstraintPredicate.

@Test
public void testInformationSchemaPredicatePushdownWithConstraintPredicate() {
    TransactionId transactionId = transactionManager.beginTransaction(false);
    Constraint<ColumnHandle> constraint = new Constraint<>(TupleDomain.all(), // test_schema has a table named "another_table" and we filter that out in this predicate
    bindings -> {
        NullableValue catalog = bindings.get(new InformationSchemaColumnHandle("table_catalog"));
        NullableValue schema = bindings.get(new InformationSchemaColumnHandle("table_schema"));
        NullableValue table = bindings.get(new InformationSchemaColumnHandle("table_name"));
        boolean isValid = true;
        if (catalog != null) {
            isValid = ((Slice) catalog.getValue()).toStringUtf8().equals("test_catalog");
        }
        if (schema != null) {
            isValid &= ((Slice) schema.getValue()).toStringUtf8().equals("test_schema");
        }
        if (table != null) {
            isValid &= ((Slice) table.getValue()).toStringUtf8().equals("test_view");
        }
        return isValid;
    });
    InformationSchemaMetadata informationSchemaMetadata = new InformationSchemaMetadata("test_catalog", metadata);
    List<ConnectorTableLayoutResult> layoutResults = informationSchemaMetadata.getTableLayouts(createNewSession(transactionId), new InformationSchemaTableHandle("test_catalog", "information_schema", "views"), constraint, Optional.empty());
    assertEquals(layoutResults.size(), 1);
    ConnectorTableLayoutHandle handle = layoutResults.get(0).getTableLayout().getHandle();
    assertTrue(handle instanceof InformationSchemaTableLayoutHandle);
    InformationSchemaTableLayoutHandle tableHandle = (InformationSchemaTableLayoutHandle) handle;
    assertEquals(tableHandle.getPrefixes(), ImmutableSet.of(new QualifiedTablePrefix("test_catalog", "test_schema", "test_view")));
}

Example 24 with NullableValue

use of com.facebook.presto.common.predicate.NullableValue in project presto by prestodb.

the class AbstractTestHiveClient method setupHive.

protected void setupHive(String connectorId, String databaseName, String timeZoneId) {
    clientId = connectorId;
    database = databaseName;
    tablePartitionFormat = new SchemaTableName(database, "presto_test_partition_format");
    tableUnpartitioned = new SchemaTableName(database, "presto_test_unpartitioned");
    tableOffline = new SchemaTableName(database, "presto_test_offline");
    tableOfflinePartition = new SchemaTableName(database, "presto_test_offline_partition");
    tableNotReadable = new SchemaTableName(database, "presto_test_not_readable");
    view = new SchemaTableName(database, "presto_test_view");
    invalidTable = new SchemaTableName(database, INVALID_TABLE);
    tableBucketedStringInt = new SchemaTableName(database, "presto_test_bucketed_by_string_int");
    tableBucketedBigintBoolean = new SchemaTableName(database, "presto_test_bucketed_by_bigint_boolean");
    tableBucketedDoubleFloat = new SchemaTableName(database, "presto_test_bucketed_by_double_float");
    tablePartitionSchemaChange = new SchemaTableName(database, "presto_test_partition_schema_change");
    tablePartitionSchemaChangeNonCanonical = new SchemaTableName(database, "presto_test_partition_schema_change_non_canonical");
    tableBucketEvolution = new SchemaTableName(database, "presto_test_bucket_evolution");
    invalidClientId = "hive";
    invalidTableHandle = new HiveTableHandle(database, INVALID_TABLE);
    invalidTableLayoutHandle = new HiveTableLayoutHandle(invalidTable, "path", ImmutableList.of(), ImmutableList.of(), ImmutableMap.of(), ImmutableList.of(new HivePartition(invalidTable, "unknown", ImmutableMap.of())), TupleDomain.all(), TRUE_CONSTANT, ImmutableMap.of(), TupleDomain.all(), Optional.empty(), Optional.empty(), false, "layout", Optional.empty(), false);
    int partitionColumnIndex = MAX_PARTITION_KEY_COLUMN_INDEX;
    dsColumn = new HiveColumnHandle("ds", HIVE_STRING, parseTypeSignature(StandardTypes.VARCHAR), partitionColumnIndex--, PARTITION_KEY, Optional.empty(), Optional.empty());
    fileFormatColumn = new HiveColumnHandle("file_format", HIVE_STRING, parseTypeSignature(StandardTypes.VARCHAR), partitionColumnIndex--, PARTITION_KEY, Optional.empty(), Optional.empty());
    dummyColumn = new HiveColumnHandle("dummy", HIVE_INT, parseTypeSignature(StandardTypes.INTEGER), partitionColumnIndex--, PARTITION_KEY, Optional.empty(), Optional.empty());
    intColumn = new HiveColumnHandle("t_int", HIVE_INT, parseTypeSignature(StandardTypes.INTEGER), partitionColumnIndex--, PARTITION_KEY, Optional.empty(), Optional.empty());
    invalidColumnHandle = new HiveColumnHandle(INVALID_COLUMN, HIVE_STRING, parseTypeSignature(StandardTypes.VARCHAR), 0, REGULAR, Optional.empty(), Optional.empty());
    List<HiveColumnHandle> partitionColumns = ImmutableList.of(dsColumn, fileFormatColumn, dummyColumn);
    List<HivePartition> partitions = ImmutableList.<HivePartition>builder().add(new HivePartition(tablePartitionFormat, "ds=2012-12-29/file_format=textfile/dummy=1", ImmutableMap.<ColumnHandle, NullableValue>builder().put(dsColumn, NullableValue.of(createUnboundedVarcharType(), utf8Slice("2012-12-29"))).put(fileFormatColumn, NullableValue.of(createUnboundedVarcharType(), utf8Slice("textfile"))).put(dummyColumn, NullableValue.of(INTEGER, 1L)).build())).add(new HivePartition(tablePartitionFormat, "ds=2012-12-29/file_format=sequencefile/dummy=2", ImmutableMap.<ColumnHandle, NullableValue>builder().put(dsColumn, NullableValue.of(createUnboundedVarcharType(), utf8Slice("2012-12-29"))).put(fileFormatColumn, NullableValue.of(createUnboundedVarcharType(), utf8Slice("sequencefile"))).put(dummyColumn, NullableValue.of(INTEGER, 2L)).build())).add(new HivePartition(tablePartitionFormat, "ds=2012-12-29/file_format=rctext/dummy=3", ImmutableMap.<ColumnHandle, NullableValue>builder().put(dsColumn, NullableValue.of(createUnboundedVarcharType(), utf8Slice("2012-12-29"))).put(fileFormatColumn, NullableValue.of(createUnboundedVarcharType(), utf8Slice("rctext"))).put(dummyColumn, NullableValue.of(INTEGER, 3L)).build())).add(new HivePartition(tablePartitionFormat, "ds=2012-12-29/file_format=rcbinary/dummy=4", ImmutableMap.<ColumnHandle, NullableValue>builder().put(dsColumn, NullableValue.of(createUnboundedVarcharType(), utf8Slice("2012-12-29"))).put(fileFormatColumn, NullableValue.of(createUnboundedVarcharType(), utf8Slice("rcbinary"))).put(dummyColumn, NullableValue.of(INTEGER, 4L)).build())).build();
    partitionCount = partitions.size();
    tupleDomain = TupleDomain.fromFixedValues(ImmutableMap.of(dsColumn, NullableValue.of(createUnboundedVarcharType(), utf8Slice("2012-12-29"))));
    TupleDomain<Subfield> domainPredicate = tupleDomain.transform(HiveColumnHandle.class::cast).transform(column -> new Subfield(column.getName(), ImmutableList.of()));
    tableLayout = new ConnectorTableLayout(new HiveTableLayoutHandle(tablePartitionFormat, "path", partitionColumns, ImmutableList.of(new Column("t_string", HIVE_STRING, Optional.empty(), Optional.empty()), new Column("t_tinyint", HIVE_BYTE, Optional.empty(), Optional.empty()), new Column("t_smallint", HIVE_SHORT, Optional.empty(), Optional.empty()), new Column("t_int", HIVE_INT, Optional.empty(), Optional.empty()), new Column("t_bigint", HIVE_LONG, Optional.empty(), Optional.empty()), new Column("t_float", HIVE_FLOAT, Optional.empty(), Optional.empty()), new Column("t_double", HIVE_DOUBLE, Optional.empty(), Optional.empty()), new Column("t_boolean", HIVE_BOOLEAN, Optional.empty(), Optional.empty())), ImmutableMap.of(), partitions, domainPredicate, TRUE_CONSTANT, ImmutableMap.of(dsColumn.getName(), dsColumn), tupleDomain, Optional.empty(), Optional.empty(), false, "layout", Optional.empty(), false), Optional.empty(), withColumnDomains(ImmutableMap.of(dsColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("2012-12-29"))), false), fileFormatColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("textfile")), Range.equal(createUnboundedVarcharType(), utf8Slice("sequencefile")), Range.equal(createUnboundedVarcharType(), utf8Slice("rctext")), Range.equal(createUnboundedVarcharType(), utf8Slice("rcbinary"))), false), dummyColumn, Domain.create(ValueSet.ofRanges(Range.equal(INTEGER, 1L), Range.equal(INTEGER, 2L), Range.equal(INTEGER, 3L), Range.equal(INTEGER, 4L)), false))), Optional.empty(), Optional.empty(), Optional.of(new DiscretePredicates(ImmutableList.copyOf(partitionColumns), ImmutableList.of(withColumnDomains(ImmutableMap.of(dsColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("2012-12-29"))), false), fileFormatColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("textfile"))), false), dummyColumn, Domain.create(ValueSet.ofRanges(Range.equal(INTEGER, 1L)), false))), withColumnDomains(ImmutableMap.of(dsColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("2012-12-29"))), false), fileFormatColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("sequencefile"))), false), dummyColumn, Domain.create(ValueSet.ofRanges(Range.equal(INTEGER, 2L)), false))), withColumnDomains(ImmutableMap.of(dsColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("2012-12-29"))), false), fileFormatColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("rctext"))), false), dummyColumn, Domain.create(ValueSet.ofRanges(Range.equal(INTEGER, 3L)), false))), withColumnDomains(ImmutableMap.of(dsColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("2012-12-29"))), false), fileFormatColumn, Domain.create(ValueSet.ofRanges(Range.equal(createUnboundedVarcharType(), utf8Slice("rcbinary"))), false), dummyColumn, Domain.create(ValueSet.ofRanges(Range.equal(INTEGER, 4L)), false)))))), ImmutableList.of());
    List<HivePartition> unpartitionedPartitions = ImmutableList.of(new HivePartition(tableUnpartitioned));
    unpartitionedTableLayout = new ConnectorTableLayout(new HiveTableLayoutHandle(tableUnpartitioned, "path", ImmutableList.of(), ImmutableList.of(new Column("t_string", HIVE_STRING, Optional.empty(), Optional.empty()), new Column("t_tinyint", HIVE_BYTE, Optional.empty(), Optional.empty())), ImmutableMap.of(), unpartitionedPartitions, TupleDomain.all(), TRUE_CONSTANT, ImmutableMap.of(), TupleDomain.all(), Optional.empty(), Optional.empty(), false, "layout", Optional.empty(), false));
    timeZone = DateTimeZone.forTimeZone(TimeZone.getTimeZone(ZoneId.of(timeZoneId)));
}

Example 25 with NullableValue

use of com.facebook.presto.common.predicate.NullableValue in project presto by prestodb.

the class AbstractTestHiveClient method doTestBucketedTableEvolution.

private void doTestBucketedTableEvolution(HiveStorageFormat storageFormat, SchemaTableName tableName) throws Exception {
    int rowCount = 100;
    // 
    // Produce a table with 8 buckets.
    // The table has 3 partitions of 3 different bucket count (4, 8, 16).
    createEmptyTable(tableName, storageFormat, ImmutableList.of(new Column("id", HIVE_LONG, Optional.empty(), Optional.empty()), new Column("name", HIVE_STRING, Optional.empty(), Optional.empty())), ImmutableList.of(new Column("pk", HIVE_STRING, Optional.empty(), Optional.empty())), Optional.of(new HiveBucketProperty(ImmutableList.of("id"), 4, ImmutableList.of(), HIVE_COMPATIBLE, Optional.empty())));
    // write a 4-bucket partition
    MaterializedResult.Builder bucket4Builder = MaterializedResult.resultBuilder(SESSION, BIGINT, VARCHAR, VARCHAR);
    IntStream.range(0, rowCount).forEach(i -> bucket4Builder.row((long) i, String.valueOf(i), "four"));
    insertData(tableName, bucket4Builder.build());
    // write a 16-bucket partition
    alterBucketProperty(tableName, Optional.of(new HiveBucketProperty(ImmutableList.of("id"), 16, ImmutableList.of(), HIVE_COMPATIBLE, Optional.empty())));
    MaterializedResult.Builder bucket16Builder = MaterializedResult.resultBuilder(SESSION, BIGINT, VARCHAR, VARCHAR);
    IntStream.range(0, rowCount).forEach(i -> bucket16Builder.row((long) i, String.valueOf(i), "sixteen"));
    insertData(tableName, bucket16Builder.build());
    // write an 8-bucket partition
    alterBucketProperty(tableName, Optional.of(new HiveBucketProperty(ImmutableList.of("id"), 8, ImmutableList.of(), HIVE_COMPATIBLE, Optional.empty())));
    MaterializedResult.Builder bucket8Builder = MaterializedResult.resultBuilder(SESSION, BIGINT, VARCHAR, VARCHAR);
    IntStream.range(0, rowCount).forEach(i -> bucket8Builder.row((long) i, String.valueOf(i), "eight"));
    insertData(tableName, bucket8Builder.build());
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
        // read entire table
        List<ColumnHandle> columnHandles = ImmutableList.<ColumnHandle>builder().addAll(metadata.getColumnHandles(session, tableHandle).values()).build();
        MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.empty());
        assertBucketTableEvolutionResult(result, columnHandles, ImmutableSet.of(0, 1, 2, 3, 4, 5, 6, 7), rowCount);
        // read single bucket (table/logical bucket)
        NullableValue singleBucket = NullableValue.of(INTEGER, 6L);
        ConnectorTableLayoutHandle layoutHandle;
        if (HiveSessionProperties.isPushdownFilterEnabled(session)) {
            TupleDomain<VariableReferenceExpression> bucketDomain = TupleDomain.fromFixedValues(ImmutableMap.of(new VariableReferenceExpression(Optional.empty(), BUCKET_COLUMN_NAME, BIGINT), singleBucket));
            RowExpression predicate = ROW_EXPRESSION_SERVICE.getDomainTranslator().toPredicate(bucketDomain);
            layoutHandle = pushdownFilter(session, metadata, transaction.getMetastore(), ROW_EXPRESSION_SERVICE, FUNCTION_RESOLUTION, hivePartitionManager, METADATA.getFunctionAndTypeManager(), tableHandle, predicate, Optional.empty()).getLayout().getHandle();
        } else {
            layoutHandle = getOnlyElement(metadata.getTableLayouts(session, tableHandle, new Constraint<>(TupleDomain.fromFixedValues(ImmutableMap.of(bucketColumnHandle(), singleBucket))), Optional.empty())).getTableLayout().getHandle();
        }
        result = readTable(transaction, tableHandle, layoutHandle, columnHandles, session, OptionalInt.empty(), Optional.empty());
        assertBucketTableEvolutionResult(result, columnHandles, ImmutableSet.of(6), rowCount);
        // read single bucket, without selecting the bucketing column (i.e. id column)
        columnHandles = ImmutableList.<ColumnHandle>builder().addAll(metadata.getColumnHandles(session, tableHandle).values().stream().filter(columnHandle -> !"id".equals(((HiveColumnHandle) columnHandle).getName())).collect(toImmutableList())).build();
        result = readTable(transaction, tableHandle, layoutHandle, columnHandles, session, OptionalInt.empty(), Optional.empty());
        assertBucketTableEvolutionResult(result, columnHandles, ImmutableSet.of(6), rowCount);
    }
}