Examples with TestingTypeManager com.facebook.presto.common.type.TestingTypeManager used on opensource projects

Example 11 with TestingTypeManager

use of com.facebook.presto.common.type.TestingTypeManager in project presto by prestodb.

the class TestHiveMaterializedViewUtils method testMaterializedDataPredicatesWithEmptyPartitions.

@Test
public void testMaterializedDataPredicatesWithEmptyPartitions() {
    TestingTypeManager typeManager = new TestingTypeManager();
    TestingSemiTransactionalHiveMetastore testMetastore = TestingSemiTransactionalHiveMetastore.create();
    List<String> keys = ImmutableList.of("ds", "category");
    Column dsColumn = new Column("ds", HIVE_STRING, Optional.empty(), Optional.empty());
    Column categoryColumn = new Column("category", HIVE_STRING, Optional.empty(), Optional.empty());
    List<Column> partitionColumns = ImmutableList.of(dsColumn, categoryColumn);
    List<String> partitions = ImmutableList.of();
    testMetastore.setPartitionNames(partitions);
    ImmutableList.Builder<List<TestingPartitionResult>> partitionResults = ImmutableList.builder();
    MaterializedDataPredicates materializedDataPredicates = getMaterializedDataPredicates(testMetastore, metastoreContext, typeManager, getTable(partitionColumns), DateTimeZone.UTC);
    comparePredicates(materializedDataPredicates, keys, partitionResults.build());
}

Example 12 with TestingTypeManager

use of com.facebook.presto.common.type.TestingTypeManager in project presto by prestodb.

the class TestHiveMaterializedViewUtils method testDifferenceDataPredicates.

@Test
public void testDifferenceDataPredicates() {
    TestingTypeManager typeManager = new TestingTypeManager();
    TestingSemiTransactionalHiveMetastore testMetastore = TestingSemiTransactionalHiveMetastore.create();
    List<String> keys = ImmutableList.of("ds");
    Column dsColumn = new Column("ds", HIVE_STRING, Optional.empty(), Optional.empty());
    List<Column> partitionColumns = ImmutableList.of(dsColumn);
    List<String> partitions = ImmutableList.of("ds=2020-01-01", "ds=2020-01-02", "ds=2020-01-03", "ds=2020-01-04", "ds=2020-01-05", "ds=2020-01-06");
    testMetastore.setPartitionNames(partitions);
    MaterializedDataPredicates baseDataPredicates = getMaterializedDataPredicates(testMetastore, metastoreContext, typeManager, getTable(partitionColumns), DateTimeZone.UTC);
    List<String> viewPartitions = ImmutableList.of("ds=2020-01-02", "ds=2020-01-03", "ds=2020-01-05");
    testMetastore.setPartitionNames(viewPartitions);
    MaterializedDataPredicates materializedDataPredicates = getMaterializedDataPredicates(testMetastore, metastoreContext, typeManager, getTable(partitionColumns), DateTimeZone.UTC);
    Map<String, String> materializedViewToBaseColumnMap = ImmutableMap.of("ds", "ds");
    ImmutableList.Builder<List<TestingPartitionResult>> partitionResults = ImmutableList.builder();
    partitionResults.add(ImmutableList.of(new TestingPartitionResult("ds", VARCHAR, "CAST('2020-01-01' AS varchar)")));
    partitionResults.add(ImmutableList.of(new TestingPartitionResult("ds", VARCHAR, "CAST('2020-01-04' AS varchar)")));
    partitionResults.add(ImmutableList.of(new TestingPartitionResult("ds", VARCHAR, "CAST('2020-01-06' AS varchar)")));
    MaterializedDataPredicates diffDataPredicates = differenceDataPredicates(baseDataPredicates, materializedDataPredicates, materializedViewToBaseColumnMap);
    comparePredicates(diffDataPredicates, keys, partitionResults.build());
}

Example 13 with TestingTypeManager

use of com.facebook.presto.common.type.TestingTypeManager in project presto by prestodb.

the class TestHivePageSourceProvider method testUseRecordReaderWithInputFormatAnnotationAndCustomSplit.

@Test
public void testUseRecordReaderWithInputFormatAnnotationAndCustomSplit() {
    StorageFormat storageFormat = StorageFormat.create(ParquetHiveSerDe.class.getName(), HoodieParquetInputFormat.class.getName(), "");
    Storage storage = new Storage(storageFormat, "test", Optional.empty(), true, ImmutableMap.of(), ImmutableMap.of());
    Map<String, String> customSplitInfo = ImmutableMap.of(CUSTOM_FILE_SPLIT_CLASS_KEY, HoodieRealtimeFileSplit.class.getName(), HUDI_BASEPATH_KEY, "/test/file.parquet", HUDI_DELTA_FILEPATHS_KEY, "/test/.file_100.log", HUDI_MAX_COMMIT_TIME_KEY, "100");
    HiveRecordCursorProvider recordCursorProvider = new MockHiveRecordCursorProvider();
    HiveBatchPageSourceFactory hiveBatchPageSourceFactory = new MockHiveBatchPageSourceFactory();
    Optional<ConnectorPageSource> pageSource = HivePageSourceProvider.createHivePageSource(ImmutableSet.of(recordCursorProvider), ImmutableSet.of(hiveBatchPageSourceFactory), new Configuration(), new TestingConnectorSession(new HiveSessionProperties(new HiveClientConfig().setUseRecordPageSourceForCustomSplit(true), new OrcFileWriterConfig(), new ParquetFileWriterConfig(), new CacheConfig()).getSessionProperties()), new Path("/test/"), OptionalInt.empty(), 0, 100, 200, Instant.now().toEpochMilli(), storage, TupleDomain.none(), ImmutableList.of(), ImmutableMap.of(), ImmutableList.of(), DateTimeZone.UTC, new TestingTypeManager(), new SchemaTableName("test", "test"), ImmutableList.of(), ImmutableList.of(), ImmutableMap.of(), 0, TableToPartitionMapping.empty(), Optional.empty(), false, null, null, false, null, Optional.empty(), customSplitInfo);
    assertTrue(pageSource.isPresent());
    assertTrue(pageSource.get() instanceof RecordPageSource);
}

Example 14 with TestingTypeManager

use of com.facebook.presto.common.type.TestingTypeManager in project presto by prestodb.

the class TestHiveMaterializedViewUtils method testDifferenceDataPredicatesNotMaterialized.

@Test
public void testDifferenceDataPredicatesNotMaterialized() {
    TestingTypeManager typeManager = new TestingTypeManager();
    TestingSemiTransactionalHiveMetastore testMetastore = TestingSemiTransactionalHiveMetastore.create();
    List<String> keys = ImmutableList.of("ds", "shipmode");
    Column dsColumn = new Column("ds", HIVE_STRING, Optional.empty(), Optional.empty());
    Column shipmodeColumn = new Column("shipmode", HIVE_STRING, Optional.empty(), Optional.empty());
    List<Column> partitionColumns = ImmutableList.of(dsColumn, shipmodeColumn);
    List<String> partitions = ImmutableList.of("ds=2020-01-01/shipmode=air", "ds=2020-01-01/shipmode=road", "ds=2020-01-02/shipmode=air", "ds=2020-01-02/shipmode=road");
    testMetastore.setPartitionNames(partitions);
    MaterializedDataPredicates baseDataPredicates = getMaterializedDataPredicates(testMetastore, metastoreContext, typeManager, getTable(partitionColumns), DateTimeZone.UTC);
    Column viewShipModeColumn = new Column("view_shipmode", HIVE_STRING, Optional.empty(), Optional.empty());
    List<Column> viewPartitionColumns = ImmutableList.of(dsColumn, viewShipModeColumn);
    List<String> viewPartitions = ImmutableList.of();
    testMetastore.setPartitionNames(viewPartitions);
    MaterializedDataPredicates materializedDataPredicates = getMaterializedDataPredicates(testMetastore, metastoreContext, typeManager, getTable(viewPartitionColumns), DateTimeZone.UTC);
    Map<String, String> materializedViewToBaseColumnMap = ImmutableMap.of("ds", "ds", "view_shipmode", "shipmode");
    ImmutableList.Builder<List<TestingPartitionResult>> partitionResults = ImmutableList.builder();
    partitionResults.add(ImmutableList.of(new TestingPartitionResult("ds", VARCHAR, "CAST('2020-01-01' AS varchar)"), new TestingPartitionResult("shipmode", VARCHAR, "CAST('air' AS varchar)")));
    partitionResults.add(ImmutableList.of(new TestingPartitionResult("ds", VARCHAR, "CAST('2020-01-01' AS varchar)"), new TestingPartitionResult("shipmode", VARCHAR, "CAST('road' AS varchar)")));
    partitionResults.add(ImmutableList.of(new TestingPartitionResult("ds", VARCHAR, "CAST('2020-01-02' AS varchar)"), new TestingPartitionResult("shipmode", VARCHAR, "CAST('air' AS varchar)")));
    partitionResults.add(ImmutableList.of(new TestingPartitionResult("ds", VARCHAR, "CAST('2020-01-02' AS varchar)"), new TestingPartitionResult("shipmode", VARCHAR, "CAST('road' AS varchar)")));
    MaterializedDataPredicates diffDataPredicates = differenceDataPredicates(baseDataPredicates, materializedDataPredicates, materializedViewToBaseColumnMap);
    comparePredicates(diffDataPredicates, keys, partitionResults.build());
}

Example 15 with TestingTypeManager

use of com.facebook.presto.common.type.TestingTypeManager in project presto by prestodb.

the class TestHiveMaterializedViewUtils method testDifferenceDataPredicatesWithAlias.

@Test
public void testDifferenceDataPredicatesWithAlias() {
    TestingTypeManager typeManager = new TestingTypeManager();
    TestingSemiTransactionalHiveMetastore testMetastore = TestingSemiTransactionalHiveMetastore.create();
    List<String> keys = ImmutableList.of("ds", "shipmode");
    Column dsColumn = new Column("ds", HIVE_STRING, Optional.empty(), Optional.empty());
    Column shipmodeColumn = new Column("shipmode", HIVE_STRING, Optional.empty(), Optional.empty());
    List<Column> partitionColumns = ImmutableList.of(dsColumn, shipmodeColumn);
    List<String> partitions = ImmutableList.of("ds=2020-01-01/shipmode=air", "ds=2020-01-01/shipmode=road", "ds=2020-01-02/shipmode=air", "ds=2020-01-02/shipmode=road");
    testMetastore.setPartitionNames(partitions);
    MaterializedDataPredicates baseDataPredicates = getMaterializedDataPredicates(testMetastore, metastoreContext, typeManager, getTable(partitionColumns), DateTimeZone.UTC);
    Column viewShipModeColumn = new Column("view_shipmode", HIVE_STRING, Optional.empty(), Optional.empty());
    List<Column> viewPartitionColumns = ImmutableList.of(dsColumn, viewShipModeColumn);
    List<String> viewPartitions = ImmutableList.of("ds=2020-01-01/view_shipmode=air", "ds=2020-01-01/view_shipmode=road");
    testMetastore.setPartitionNames(viewPartitions);
    MaterializedDataPredicates materializedDataPredicates = getMaterializedDataPredicates(testMetastore, metastoreContext, typeManager, getTable(viewPartitionColumns), DateTimeZone.UTC);
    Map<String, String> materializedViewToBaseColumnMap = ImmutableMap.of("ds", "ds", "view_shipmode", "shipmode");
    ImmutableList.Builder<List<TestingPartitionResult>> partitionResults = ImmutableList.builder();
    partitionResults.add(ImmutableList.of(new TestingPartitionResult("ds", VARCHAR, "CAST('2020-01-02' AS varchar)"), new TestingPartitionResult("shipmode", VARCHAR, "CAST('air' AS varchar)")));
    partitionResults.add(ImmutableList.of(new TestingPartitionResult("ds", VARCHAR, "CAST('2020-01-02' AS varchar)"), new TestingPartitionResult("shipmode", VARCHAR, "CAST('road' AS varchar)")));
    MaterializedDataPredicates diffDataPredicates = differenceDataPredicates(baseDataPredicates, materializedDataPredicates, materializedViewToBaseColumnMap);
    comparePredicates(diffDataPredicates, keys, partitionResults.build());
}