Examples with MaterializedDataPredicates com.facebook.presto.spi.MaterializedViewStatus.MaterializedDataPredicates used on opensource projects

Example 6 with MaterializedDataPredicates

use of com.facebook.presto.spi.MaterializedViewStatus.MaterializedDataPredicates in project presto by prestodb.

the class TestHiveMaterializedViewUtils method testMaterializedDataPredicatesWithEmptyPartitions.

@Test
public void testMaterializedDataPredicatesWithEmptyPartitions() {
    TestingTypeManager typeManager = new TestingTypeManager();
    TestingSemiTransactionalHiveMetastore testMetastore = TestingSemiTransactionalHiveMetastore.create();
    List<String> keys = ImmutableList.of("ds", "category");
    Column dsColumn = new Column("ds", HIVE_STRING, Optional.empty(), Optional.empty());
    Column categoryColumn = new Column("category", HIVE_STRING, Optional.empty(), Optional.empty());
    List<Column> partitionColumns = ImmutableList.of(dsColumn, categoryColumn);
    List<String> partitions = ImmutableList.of();
    testMetastore.setPartitionNames(partitions);
    ImmutableList.Builder<List<TestingPartitionResult>> partitionResults = ImmutableList.builder();
    MaterializedDataPredicates materializedDataPredicates = getMaterializedDataPredicates(testMetastore, metastoreContext, typeManager, getTable(partitionColumns), DateTimeZone.UTC);
    comparePredicates(materializedDataPredicates, keys, partitionResults.build());
}

Example 7 with MaterializedDataPredicates

use of com.facebook.presto.spi.MaterializedViewStatus.MaterializedDataPredicates in project presto by prestodb.

the class TestHiveMaterializedViewUtils method testDifferenceDataPredicates.

@Test
public void testDifferenceDataPredicates() {
    TestingTypeManager typeManager = new TestingTypeManager();
    TestingSemiTransactionalHiveMetastore testMetastore = TestingSemiTransactionalHiveMetastore.create();
    List<String> keys = ImmutableList.of("ds");
    Column dsColumn = new Column("ds", HIVE_STRING, Optional.empty(), Optional.empty());
    List<Column> partitionColumns = ImmutableList.of(dsColumn);
    List<String> partitions = ImmutableList.of("ds=2020-01-01", "ds=2020-01-02", "ds=2020-01-03", "ds=2020-01-04", "ds=2020-01-05", "ds=2020-01-06");
    testMetastore.setPartitionNames(partitions);
    MaterializedDataPredicates baseDataPredicates = getMaterializedDataPredicates(testMetastore, metastoreContext, typeManager, getTable(partitionColumns), DateTimeZone.UTC);
    List<String> viewPartitions = ImmutableList.of("ds=2020-01-02", "ds=2020-01-03", "ds=2020-01-05");
    testMetastore.setPartitionNames(viewPartitions);
    MaterializedDataPredicates materializedDataPredicates = getMaterializedDataPredicates(testMetastore, metastoreContext, typeManager, getTable(partitionColumns), DateTimeZone.UTC);
    Map<String, String> materializedViewToBaseColumnMap = ImmutableMap.of("ds", "ds");
    ImmutableList.Builder<List<TestingPartitionResult>> partitionResults = ImmutableList.builder();
    partitionResults.add(ImmutableList.of(new TestingPartitionResult("ds", VARCHAR, "CAST('2020-01-01' AS varchar)")));
    partitionResults.add(ImmutableList.of(new TestingPartitionResult("ds", VARCHAR, "CAST('2020-01-04' AS varchar)")));
    partitionResults.add(ImmutableList.of(new TestingPartitionResult("ds", VARCHAR, "CAST('2020-01-06' AS varchar)")));
    MaterializedDataPredicates diffDataPredicates = differenceDataPredicates(baseDataPredicates, materializedDataPredicates, materializedViewToBaseColumnMap);
    comparePredicates(diffDataPredicates, keys, partitionResults.build());
}

Example 8 with MaterializedDataPredicates

use of com.facebook.presto.spi.MaterializedViewStatus.MaterializedDataPredicates in project presto by prestodb.

the class HiveMaterializedViewUtils method getMaterializedDataPredicates.

public static MaterializedDataPredicates getMaterializedDataPredicates(SemiTransactionalHiveMetastore metastore, MetastoreContext metastoreContext, TypeManager typeManager, Table table, DateTimeZone timeZone) {
    List<Column> partitionColumns = table.getPartitionColumns();
    for (Column partitionColumn : partitionColumns) {
        HiveType hiveType = partitionColumn.getType();
        if (!hiveType.isSupportedType()) {
            throw new PrestoException(NOT_SUPPORTED, String.format("Unsupported Hive type %s found in partition keys of table %s.%s", hiveType, table.getDatabaseName(), table.getTableName()));
        }
    }
    List<HiveColumnHandle> partitionKeyColumnHandles = getPartitionKeyColumnHandles(table);
    Map<String, Type> partitionTypes = partitionKeyColumnHandles.stream().collect(toImmutableMap(HiveColumnHandle::getName, column -> typeManager.getType(column.getTypeSignature())));
    List<String> partitionNames = metastore.getPartitionNames(metastoreContext, table.getDatabaseName(), table.getTableName()).orElseThrow(() -> new TableNotFoundException(new SchemaTableName(table.getDatabaseName(), table.getTableName())));
    ImmutableList.Builder<TupleDomain<String>> partitionNamesAndValues = ImmutableList.builder();
    for (String partitionName : partitionNames) {
        ImmutableMap.Builder<String, NullableValue> partitionNameAndValuesMap = ImmutableMap.builder();
        Map<String, String> partitions = toPartitionNamesAndValues(partitionName);
        if (partitionColumns.size() != partitions.size()) {
            throw new PrestoException(HIVE_INVALID_METADATA, String.format("Expected %d partition key values, but got %d", partitionColumns.size(), partitions.size()));
        }
        partitionTypes.forEach((name, type) -> {
            String value = partitions.get(name);
            if (value == null) {
                throw new PrestoException(HIVE_INVALID_PARTITION_VALUE, String.format("partition key value cannot be null for field: %s", name));
            }
            partitionNameAndValuesMap.put(name, parsePartitionValue(name, value, type, timeZone));
        });
        TupleDomain<String> tupleDomain = TupleDomain.fromFixedValues(partitionNameAndValuesMap.build());
        partitionNamesAndValues.add(tupleDomain);
    }
    return new MaterializedDataPredicates(partitionNamesAndValues.build(), partitionColumns.stream().map(Column::getName).collect(toImmutableList()));
}

Example 9 with MaterializedDataPredicates

use of com.facebook.presto.spi.MaterializedViewStatus.MaterializedDataPredicates in project urban-eureka by errir503.

the class TestHiveMaterializedViewUtils method testDifferenceDataPredicatesWithDifferentExtraPartitions.

@Test
public void testDifferenceDataPredicatesWithDifferentExtraPartitions() {
    TestingTypeManager typeManager = new TestingTypeManager();
    TestingSemiTransactionalHiveMetastore testMetastore = TestingSemiTransactionalHiveMetastore.create();
    List<String> keys = ImmutableList.of("ds", "shipmode");
    Column dsColumn = new Column("ds", HIVE_STRING, Optional.empty(), Optional.empty());
    Column shipmodeColumn = new Column("shipmode", HIVE_STRING, Optional.empty(), Optional.empty());
    List<Column> partitionColumns = ImmutableList.of(dsColumn, shipmodeColumn);
    List<String> partitions = ImmutableList.of("ds=2020-01-01/shipmode=air", "ds=2020-01-01/shipmode=road", "ds=2020-01-02/shipmode=air", "ds=2020-01-02/shipmode=road");
    testMetastore.setPartitionNames(partitions);
    MaterializedDataPredicates baseDataPredicates = getMaterializedDataPredicates(testMetastore, metastoreContext, typeManager, getTable(partitionColumns), DateTimeZone.UTC);
    Column viewShipModeColumn = new Column("view_shipmode", HIVE_STRING, Optional.empty(), Optional.empty());
    List<Column> viewPartitionColumns = ImmutableList.of(dsColumn, viewShipModeColumn);
    List<String> viewPartitions = ImmutableList.of("ds=2020-01-01/view_shipmode=air", "ds=2020-01-01/view_shipmode=road");
    testMetastore.setPartitionNames(viewPartitions);
    MaterializedDataPredicates materializedDataPredicates = getMaterializedDataPredicates(testMetastore, metastoreContext, typeManager, getTable(viewPartitionColumns), DateTimeZone.UTC);
    Map<String, String> materializedViewToBaseColumnMap = ImmutableMap.of("ds", "ds");
    ImmutableList.Builder<List<TestingPartitionResult>> partitionResults = ImmutableList.builder();
    partitionResults.add(ImmutableList.of(new TestingPartitionResult("ds", VARCHAR, "CAST('2020-01-02' AS varchar)"), new TestingPartitionResult("shipmode", VARCHAR, "CAST('air' AS varchar)")));
    partitionResults.add(ImmutableList.of(new TestingPartitionResult("ds", VARCHAR, "CAST('2020-01-02' AS varchar)"), new TestingPartitionResult("shipmode", VARCHAR, "CAST('road' AS varchar)")));
    MaterializedDataPredicates diffDataPredicates = differenceDataPredicates(baseDataPredicates, materializedDataPredicates, materializedViewToBaseColumnMap);
    comparePredicates(diffDataPredicates, keys, partitionResults.build());
}

Example 10 with MaterializedDataPredicates

use of com.facebook.presto.spi.MaterializedViewStatus.MaterializedDataPredicates in project urban-eureka by errir503.

the class TestHiveMaterializedViewUtils method testDifferenceDataPredicatesNotMaterialized.

@Test
public void testDifferenceDataPredicatesNotMaterialized() {
    TestingTypeManager typeManager = new TestingTypeManager();
    TestingSemiTransactionalHiveMetastore testMetastore = TestingSemiTransactionalHiveMetastore.create();
    List<String> keys = ImmutableList.of("ds", "shipmode");
    Column dsColumn = new Column("ds", HIVE_STRING, Optional.empty(), Optional.empty());
    Column shipmodeColumn = new Column("shipmode", HIVE_STRING, Optional.empty(), Optional.empty());
    List<Column> partitionColumns = ImmutableList.of(dsColumn, shipmodeColumn);
    List<String> partitions = ImmutableList.of("ds=2020-01-01/shipmode=air", "ds=2020-01-01/shipmode=road", "ds=2020-01-02/shipmode=air", "ds=2020-01-02/shipmode=road");
    testMetastore.setPartitionNames(partitions);
    MaterializedDataPredicates baseDataPredicates = getMaterializedDataPredicates(testMetastore, metastoreContext, typeManager, getTable(partitionColumns), DateTimeZone.UTC);
    Column viewShipModeColumn = new Column("view_shipmode", HIVE_STRING, Optional.empty(), Optional.empty());
    List<Column> viewPartitionColumns = ImmutableList.of(dsColumn, viewShipModeColumn);
    List<String> viewPartitions = ImmutableList.of();
    testMetastore.setPartitionNames(viewPartitions);
    MaterializedDataPredicates materializedDataPredicates = getMaterializedDataPredicates(testMetastore, metastoreContext, typeManager, getTable(viewPartitionColumns), DateTimeZone.UTC);
    Map<String, String> materializedViewToBaseColumnMap = ImmutableMap.of("ds", "ds", "view_shipmode", "shipmode");
    ImmutableList.Builder<List<TestingPartitionResult>> partitionResults = ImmutableList.builder();
    partitionResults.add(ImmutableList.of(new TestingPartitionResult("ds", VARCHAR, "CAST('2020-01-01' AS varchar)"), new TestingPartitionResult("shipmode", VARCHAR, "CAST('air' AS varchar)")));
    partitionResults.add(ImmutableList.of(new TestingPartitionResult("ds", VARCHAR, "CAST('2020-01-01' AS varchar)"), new TestingPartitionResult("shipmode", VARCHAR, "CAST('road' AS varchar)")));
    partitionResults.add(ImmutableList.of(new TestingPartitionResult("ds", VARCHAR, "CAST('2020-01-02' AS varchar)"), new TestingPartitionResult("shipmode", VARCHAR, "CAST('air' AS varchar)")));
    partitionResults.add(ImmutableList.of(new TestingPartitionResult("ds", VARCHAR, "CAST('2020-01-02' AS varchar)"), new TestingPartitionResult("shipmode", VARCHAR, "CAST('road' AS varchar)")));
    MaterializedDataPredicates diffDataPredicates = differenceDataPredicates(baseDataPredicates, materializedDataPredicates, materializedViewToBaseColumnMap);
    comparePredicates(diffDataPredicates, keys, partitionResults.build());
}