Examples with TestingConnectorSession io.prestosql.testing.TestingConnectorSession used on opensource projects

Example 1 with TestingConnectorSession

use of io.prestosql.testing.TestingConnectorSession in project hetu-core by openlookeng.

the class TestBackgroundHiveSplitLoader method testPartitionedTableWithDynamicFilter.

@Test
public void testPartitionedTableWithDynamicFilter() throws Exception {
    TypeManager typeManager = new TestingTypeManager();
    List<HivePartitionMetadata> hivePartitionMetadatas = ImmutableList.of(new HivePartitionMetadata(new HivePartition(new SchemaTableName("testSchema", "table_name")), Optional.of(new Partition("testSchema", "table_name", ImmutableList.of("1"), TABLE_STORAGE, ImmutableList.of(TABLE_COLUMN), ImmutableMap.of("param", "value"))), ImmutableMap.of()));
    ConnectorSession connectorSession = new TestingConnectorSession(new HiveSessionProperties(new HiveConfig().setMaxSplitSize(new DataSize(1.0, GIGABYTE)).setDynamicFilterPartitionFilteringEnabled(true), new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties());
    BackgroundHiveSplitLoader backgroundHiveSplitLoader = new BackgroundHiveSplitLoader(PARTITIONED_TABLE, hivePartitionMetadatas, TupleDomain.all(), BackgroundHiveSplitLoader.BucketSplitInfo.createBucketSplitInfo(Optional.empty(), Optional.empty()), connectorSession, new TestingHdfsEnvironment(TEST_FILES), new NamenodeStats(), new CachingDirectoryLister(new HiveConfig()), directExecutor(), 2, false, Optional.empty(), createTestDynamicFilterSupplier("partitionColumn", ImmutableList.of(0L, 2L, 3L)), Optional.empty(), ImmutableMap.of(), typeManager);
    HiveSplitSource hiveSplitSource = hiveSplitSource(backgroundHiveSplitLoader);
    backgroundHiveSplitLoader.start(hiveSplitSource);
    List<HiveSplit> splits = drainSplits(hiveSplitSource);
    assertEquals(splits.size(), 0, "Splits should be filtered");
}

Example 2 with TestingConnectorSession

use of io.prestosql.testing.TestingConnectorSession in project hetu-core by openlookeng.

the class TestColumnTypeCacheable method testBigintCacheable.

@Test
public void testBigintCacheable() {
    ConnectorSession session = new TestingConnectorSession(new HiveSessionProperties(new HiveConfig().setDynamicFilterPartitionFilteringEnabled(false), new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties());
    ColumnMetadata ptdMetadata = new ColumnMetadata("pt_d", BIGINT);
    Set<TupleDomain<ColumnMetadata>> cachePredicates = ImmutableSet.of(TupleDomain.withColumnDomains(ImmutableMap.of(ptdMetadata, Domain.singleValue(BIGINT, 20200522L))), TupleDomain.withColumnDomains(ImmutableMap.of(ptdMetadata, Domain.singleValue(BIGINT, 20200521L))));
    HiveSplitSource hiveSplitSource = HiveSplitSource.allAtOnce(session, "database", "table", 10, 10000, new DataSize(10, MEGABYTE), Integer.MAX_VALUE, new TestingHiveSplitLoader(), Executors.newFixedThreadPool(5), new CounterStat(), null, cachePredicates, null, new HiveConfig(), HiveStorageFormat.ORC);
    int[] idPrefix = new int[] { 1 };
    ImmutableMap.of("__HIVE_DEFAULT_PARTITION__", 1, "20200520", 2, "20200521", 3, "20200522", 2).forEach((ptdValue, splitCount) -> {
        for (int i = 1; i <= splitCount; i++) {
            hiveSplitSource.addToQueue(new TestPartitionSplit(idPrefix[0] * 10 + i, ImmutableList.of(new HivePartitionKey("pt_d", ptdValue)), "pt_d=" + ptdValue));
        }
        idPrefix[0] = idPrefix[0] + 1;
    });
    List<ConnectorSplit> splits = getSplits(hiveSplitSource, 10);
    assertEquals(splits.size(), 8);
    assertEquals(splits.stream().filter(ConnectorSplit::isCacheable).count(), 5);
    assertEquals(splits.stream().filter(ConnectorSplit::isCacheable).map(HiveSplitWrapper::getOnlyHiveSplit).filter(hiveSplit -> hiveSplit.getPartitionKeys().contains(new HivePartitionKey("pt_d", "20200521"))).count(), 3);
    assertEquals(splits.stream().filter(ConnectorSplit::isCacheable).map(HiveSplitWrapper::getOnlyHiveSplit).filter(hiveSplit -> hiveSplit.getPartitionKeys().contains(new HivePartitionKey("pt_d", "20200522"))).count(), 2);
    assertEquals(splits.stream().filter(ConnectorSplit::isCacheable).map(HiveSplitWrapper::getOnlyHiveSplit).filter(hiveSplit -> hiveSplit.getPartitionKeys().contains(new HivePartitionKey("pt_d", "20200520"))).count(), 0);
}

Example 3 with TestingConnectorSession

use of io.prestosql.testing.TestingConnectorSession in project hetu-core by openlookeng.

the class TestColumnTypeCacheable method testDoubleTypeCacheable.

@Test
public void testDoubleTypeCacheable() {
    ConnectorSession session = new TestingConnectorSession(new HiveSessionProperties(new HiveConfig().setDynamicFilterPartitionFilteringEnabled(false), new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties());
    ColumnMetadata ptdMetadata = new ColumnMetadata("pt_d", DOUBLE);
    Set<TupleDomain<ColumnMetadata>> cachePredicates = ImmutableSet.of(TupleDomain.withColumnDomains(ImmutableMap.of(ptdMetadata, Domain.singleValue(DOUBLE, 1.0d))), TupleDomain.withColumnDomains(ImmutableMap.of(ptdMetadata, Domain.singleValue(DOUBLE, 1000.10d))));
    HiveSplitSource hiveSplitSource = HiveSplitSource.allAtOnce(session, "database", "table", 10, 10000, new DataSize(10, MEGABYTE), Integer.MAX_VALUE, new TestingHiveSplitLoader(), Executors.newFixedThreadPool(5), new CounterStat(), null, cachePredicates, null, new HiveConfig(), HiveStorageFormat.ORC);
    int[] idPrefix = new int[] { 1 };
    ImmutableMap.of("__HIVE_DEFAULT_PARTITION__", 1, "1.0", 2, "2", 3, "1000.10", 4).forEach((ptdValue, splitCount) -> {
        for (int i = 1; i <= splitCount; i++) {
            hiveSplitSource.addToQueue(new TestPartitionSplit(idPrefix[0] * 10 + i, ImmutableList.of(new HivePartitionKey("pt_d", ptdValue)), "pt_d=" + ptdValue));
        }
        idPrefix[0] = idPrefix[0] + 1;
    });
    List<ConnectorSplit> splits = getSplits(hiveSplitSource, 10);
    assertEquals(splits.size(), 10);
    assertEquals(splits.stream().filter(ConnectorSplit::isCacheable).count(), 6);
    assertEquals(splits.stream().filter(ConnectorSplit::isCacheable).map(HiveSplitWrapper::getOnlyHiveSplit).filter(hiveSplit -> hiveSplit.getPartitionKeys().contains(new HivePartitionKey("pt_d", "1.0"))).count(), 2);
    assertEquals(splits.stream().filter(ConnectorSplit::isCacheable).map(HiveSplitWrapper::getOnlyHiveSplit).filter(hiveSplit -> hiveSplit.getPartitionKeys().contains(new HivePartitionKey("pt_d", "2"))).count(), 0);
    assertEquals(splits.stream().filter(ConnectorSplit::isCacheable).map(HiveSplitWrapper::getOnlyHiveSplit).filter(hiveSplit -> hiveSplit.getPartitionKeys().contains(new HivePartitionKey("pt_d", "1000.10"))).count(), 4);
}

Example 4 with TestingConnectorSession

use of io.prestosql.testing.TestingConnectorSession in project hetu-core by openlookeng.

the class TestColumnTypeCacheable method testBooleanCacheable.

@Test
public void testBooleanCacheable() {
    ConnectorSession session = new TestingConnectorSession(new HiveSessionProperties(new HiveConfig().setDynamicFilterPartitionFilteringEnabled(false), new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties());
    ColumnMetadata ptdMetadata = new ColumnMetadata("pt_d", BOOLEAN);
    Set<TupleDomain<ColumnMetadata>> cachePredicates = ImmutableSet.of(TupleDomain.withColumnDomains(ImmutableMap.of(ptdMetadata, Domain.singleValue(BOOLEAN, true))), TupleDomain.withColumnDomains(ImmutableMap.of(ptdMetadata, Domain.singleValue(BOOLEAN, false))));
    HiveSplitSource hiveSplitSource = HiveSplitSource.allAtOnce(session, "database", "table", 10, 10000, new DataSize(10, MEGABYTE), Integer.MAX_VALUE, new TestingHiveSplitLoader(), Executors.newFixedThreadPool(5), new CounterStat(), null, cachePredicates, null, new HiveConfig(), HiveStorageFormat.ORC);
    int[] idPrefix = new int[] { 1 };
    ImmutableMap.of("__HIVE_DEFAULT_PARTITION__", 1, "false", 2, "true", 3).forEach((ptdValue, splitCount) -> {
        for (int i = 1; i <= splitCount; i++) {
            hiveSplitSource.addToQueue(new TestPartitionSplit(idPrefix[0] * 10 + i, ImmutableList.of(new HivePartitionKey("pt_d", ptdValue)), "pt_d=" + ptdValue));
        }
        idPrefix[0] = idPrefix[0] + 1;
    });
    List<ConnectorSplit> splits = getSplits(hiveSplitSource, 10);
    assertEquals(splits.size(), 6);
    assertEquals(splits.stream().filter(ConnectorSplit::isCacheable).count(), 5);
    assertEquals(splits.stream().filter(ConnectorSplit::isCacheable).map(HiveSplitWrapper::getOnlyHiveSplit).filter(hiveSplit -> hiveSplit.getPartitionKeys().contains(new HivePartitionKey("pt_d", "true"))).count(), 3);
    assertEquals(splits.stream().filter(ConnectorSplit::isCacheable).map(HiveSplitWrapper::getOnlyHiveSplit).filter(hiveSplit -> hiveSplit.getPartitionKeys().contains(new HivePartitionKey("pt_d", "false"))).count(), 2);
}

Example 5 with TestingConnectorSession

use of io.prestosql.testing.TestingConnectorSession in project hetu-core by openlookeng.

the class TestColumnTypeCacheable method testCharCacheable.

@Test
public void testCharCacheable() {
    ConnectorSession session = new TestingConnectorSession(new HiveSessionProperties(new HiveConfig().setDynamicFilterPartitionFilteringEnabled(false), new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties());
    ColumnMetadata ptdMetadata = new ColumnMetadata("pt_d", createCharType(3));
    Set<TupleDomain<ColumnMetadata>> cachePredicates = ImmutableSet.of(TupleDomain.withColumnDomains(ImmutableMap.of(ptdMetadata, Domain.singleValue(createCharType(3), Slices.utf8Slice("abc")))), TupleDomain.withColumnDomains(ImmutableMap.of(ptdMetadata, Domain.singleValue(createCharType(3), Slices.utf8Slice("xyz")))));
    HiveSplitSource hiveSplitSource = HiveSplitSource.allAtOnce(session, "database", "table", 10, 10000, new DataSize(10, MEGABYTE), Integer.MAX_VALUE, new TestingHiveSplitLoader(), Executors.newFixedThreadPool(5), new CounterStat(), null, cachePredicates, null, new HiveConfig(), HiveStorageFormat.ORC);
    int[] idPrefix = new int[] { 1 };
    ImmutableMap.of("__HIVE_DEFAULT_PARTITION__", 1, "abc", 2, "def", 3, "xyz", 4).forEach((ptdValue, splitCount) -> {
        for (int i = 1; i <= splitCount; i++) {
            hiveSplitSource.addToQueue(new TestPartitionSplit(idPrefix[0] * 10 + i, ImmutableList.of(new HivePartitionKey("pt_d", ptdValue)), "pt_d=" + ptdValue));
        }
        idPrefix[0] = idPrefix[0] + 1;
    });
    List<ConnectorSplit> splits = getSplits(hiveSplitSource, 10);
    assertEquals(splits.size(), 10);
    assertEquals(splits.stream().filter(ConnectorSplit::isCacheable).count(), 6);
    assertEquals(splits.stream().filter(ConnectorSplit::isCacheable).map(HiveSplitWrapper::getOnlyHiveSplit).filter(hiveSplit -> hiveSplit.getPartitionKeys().contains(new HivePartitionKey("pt_d", "abc"))).count(), 2);
    assertEquals(splits.stream().filter(ConnectorSplit::isCacheable).map(HiveSplitWrapper::getOnlyHiveSplit).filter(hiveSplit -> hiveSplit.getPartitionKeys().contains(new HivePartitionKey("pt_d", "def"))).count(), 0);
    assertEquals(splits.stream().filter(ConnectorSplit::isCacheable).map(HiveSplitWrapper::getOnlyHiveSplit).filter(hiveSplit -> hiveSplit.getPartitionKeys().contains(new HivePartitionKey("pt_d", "xyz"))).count(), 4);
}