Search in sources :

Example 1 with DynamicFilterCacheManager

use of io.prestosql.dynamicfilter.DynamicFilterCacheManager in project hetu-core by openlookeng.

the class TestDynamicFiltersCollector method TestCollectingGlobalDynamicFilters.

@Test
public void TestCollectingGlobalDynamicFilters() throws InterruptedException {
    final QueryId queryId = new QueryId("test_query");
    final String filterId = "1";
    final String columnName = "column";
    final TestingColumnHandle columnHandle = new TestingColumnHandle(columnName);
    final Set<String> valueSet = ImmutableSet.of("1", "2", "3");
    TaskContext taskContext = mock(TaskContext.class);
    Session session = testSessionBuilder().setQueryId(queryId).setSystemProperty(ENABLE_DYNAMIC_FILTERING, "true").setSystemProperty(DYNAMIC_FILTERING_DATA_TYPE, "HASHSET").build();
    when(taskContext.getSession()).thenReturn(session);
    // set up state store and merged dynamic filters map
    Map mockMap = new HashMap<>();
    StateStoreProvider stateStoreProvider = mock(StateStoreProvider.class);
    StateStore stateStore = mock(StateStore.class);
    StateMap stateMap = new MockStateMap<>("test-map", mockMap);
    when(stateStoreProvider.getStateStore()).thenReturn(stateStore);
    when(stateStore.getStateCollection(any())).thenReturn(stateMap);
    when(stateStore.createStateMap(any())).thenReturn(stateMap);
    when(stateStore.getOrCreateStateCollection(any(), any())).thenReturn(stateMap);
    // set up state store listener and dynamic filter cache
    StateStoreListenerManager stateStoreListenerManager = new StateStoreListenerManager(stateStoreProvider);
    DynamicFilterCacheManager dynamicFilterCacheManager = new DynamicFilterCacheManager();
    stateStoreListenerManager.addStateStoreListener(new DynamicFilterListener(dynamicFilterCacheManager), MERGED_DYNAMIC_FILTERS);
    LocalDynamicFiltersCollector collector = new LocalDynamicFiltersCollector(taskContext, Optional.empty(), dynamicFilterCacheManager);
    TableScanNode tableScan = mock(TableScanNode.class);
    when(tableScan.getAssignments()).thenReturn(ImmutableMap.of(new Symbol(columnName), columnHandle));
    List<DynamicFilters.Descriptor> dynamicFilterDescriptors = ImmutableList.of(new DynamicFilters.Descriptor(filterId, new VariableReferenceExpression(columnName, BIGINT)));
    collector.initContext(ImmutableList.of(dynamicFilterDescriptors), SymbolUtils.toLayOut(tableScan.getOutputSymbols()));
    assertTrue(collector.getDynamicFilters(tableScan).isEmpty(), "there should be no dynamic filter available");
    // put some values in state store as a new dynamic filter
    // and wait for the listener to process the event
    stateMap.put(createKey(DynamicFilterUtils.FILTERPREFIX, filterId, queryId.getId()), valueSet);
    TimeUnit.MILLISECONDS.sleep(100);
    // get available dynamic filter and verify it
    List<Map<ColumnHandle, DynamicFilter>> dynamicFilters = collector.getDynamicFilters(tableScan);
    assertEquals(dynamicFilters.size(), 1, "there should be a new dynamic filter");
    assertEquals(dynamicFilters.size(), 1);
    DynamicFilter dynamicFilter = dynamicFilters.get(0).get(columnHandle);
    assertTrue(dynamicFilter instanceof HashSetDynamicFilter, "new dynamic filter should be hashset");
    assertEquals(dynamicFilter.getSize(), valueSet.size(), "new dynamic filter should have correct size");
    for (String value : valueSet) {
        assertTrue(dynamicFilter.contains(value), "new dynamic filter should contain correct values");
    }
    // clean up when task finishes
    collector.removeDynamicFilter(true);
    DynamicFilter cachedFilter = dynamicFilterCacheManager.getDynamicFilter(DynamicFilterCacheManager.createCacheKey(filterId, queryId.getId()));
    assertNull(cachedFilter, "cached dynamic filter should have been removed");
}
Also used : HashMap(java.util.HashMap) Symbol(io.prestosql.spi.plan.Symbol) MockStateMap(io.prestosql.statestore.MockStateMap) StateMap(io.prestosql.spi.statestore.StateMap) StateStoreListenerManager(io.prestosql.statestore.listener.StateStoreListenerManager) StateStoreProvider(io.prestosql.statestore.StateStoreProvider) MockStateMap(io.prestosql.statestore.MockStateMap) TestingColumnHandle(io.prestosql.spi.connector.TestingColumnHandle) DynamicFilters(io.prestosql.sql.DynamicFilters) DynamicFilterListener(io.prestosql.dynamicfilter.DynamicFilterListener) DynamicFilterCacheManager(io.prestosql.dynamicfilter.DynamicFilterCacheManager) TaskContext(io.prestosql.operator.TaskContext) DynamicFilter(io.prestosql.spi.dynamicfilter.DynamicFilter) HashSetDynamicFilter(io.prestosql.spi.dynamicfilter.HashSetDynamicFilter) QueryId(io.prestosql.spi.QueryId) StateStore(io.prestosql.spi.statestore.StateStore) HashSetDynamicFilter(io.prestosql.spi.dynamicfilter.HashSetDynamicFilter) TableScanNode(io.prestosql.spi.plan.TableScanNode) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) MockStateMap(io.prestosql.statestore.MockStateMap) HashMap(java.util.HashMap) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) StateMap(io.prestosql.spi.statestore.StateMap) Session(io.prestosql.Session) Test(org.testng.annotations.Test)

Example 2 with DynamicFilterCacheManager

use of io.prestosql.dynamicfilter.DynamicFilterCacheManager in project hetu-core by openlookeng.

the class TaskTestUtils method createTestingPlanner.

public static LocalExecutionPlanner createTestingPlanner() {
    Metadata metadata = createTestMetadataManager();
    PageSourceManager pageSourceManager = new PageSourceManager();
    HetuMetaStoreManager hetuMetaStoreManager = new HetuMetaStoreManager();
    FeaturesConfig featuresConfig = new FeaturesConfig();
    CubeManager cubeManager = new CubeManager(featuresConfig, hetuMetaStoreManager);
    pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
    // we don't start the finalizer so nothing will be collected, which is ok for a test
    FinalizerService finalizerService = new FinalizerService();
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService));
    NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler);
    PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(metadata, 0);
    NodeInfo nodeInfo = new NodeInfo("test");
    FileSystemClientManager fileSystemClientManager = new FileSystemClientManager();
    SeedStoreManager seedStoreManager = new SeedStoreManager(fileSystemClientManager);
    StateStoreProvider stateStoreProvider = new LocalStateStoreProvider(seedStoreManager);
    HeuristicIndexerManager heuristicIndexerManager = new HeuristicIndexerManager(new FileSystemClientManager(), new HetuMetaStoreManager());
    return new LocalExecutionPlanner(metadata, new TypeAnalyzer(new SqlParser(), metadata), Optional.empty(), pageSourceManager, new IndexManager(), nodePartitioningManager, new PageSinkManager(), new MockExchangeClientSupplier(), new ExpressionCompiler(metadata, pageFunctionCompiler), pageFunctionCompiler, new JoinFilterFunctionCompiler(metadata), new IndexJoinLookupStats(), new TaskManagerConfig(), new GenericSpillerFactory((types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }), (types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, (types, partitionFunction, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, new PagesIndex.TestingFactory(false), new JoinCompiler(metadata), new LookupJoinOperators(), new OrderingCompiler(), nodeInfo, stateStoreProvider, new StateStoreListenerManager(stateStoreProvider), new DynamicFilterCacheManager(), heuristicIndexerManager, cubeManager);
}
Also used : NodeInfo(io.airlift.node.NodeInfo) SqlParser(io.prestosql.sql.parser.SqlParser) PlanFragmentId(io.prestosql.sql.planner.plan.PlanFragmentId) FileSystemClientManager(io.prestosql.filesystem.FileSystemClientManager) TEST_TABLE_HANDLE(io.prestosql.testing.TestingHandles.TEST_TABLE_HANDLE) PartitioningScheme(io.prestosql.sql.planner.PartitioningScheme) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) ObjectMapperProvider(io.airlift.json.ObjectMapperProvider) StateStoreListenerManager(io.prestosql.statestore.listener.StateStoreListenerManager) PageFunctionCompiler(io.prestosql.sql.gen.PageFunctionCompiler) Partitioning(io.prestosql.sql.planner.Partitioning) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) MockExchangeClientSupplier(io.prestosql.execution.TestSqlTaskManager.MockExchangeClientSupplier) EventListenerManager(io.prestosql.eventlistener.EventListenerManager) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) OrderingCompiler(io.prestosql.sql.gen.OrderingCompiler) ImmutableMap(com.google.common.collect.ImmutableMap) MetadataManager.createTestMetadataManager(io.prestosql.metadata.MetadataManager.createTestMetadataManager) CatalogName(io.prestosql.spi.connector.CatalogName) TableScanNode(io.prestosql.spi.plan.TableScanNode) UUID(java.util.UUID) NodeSchedulerConfig(io.prestosql.execution.scheduler.NodeSchedulerConfig) Metadata(io.prestosql.metadata.Metadata) ReuseExchangeOperator(io.prestosql.spi.operator.ReuseExchangeOperator) List(java.util.List) SOURCE_DISTRIBUTION(io.prestosql.sql.planner.SystemPartitioningHandle.SOURCE_DISTRIBUTION) PageSinkManager(io.prestosql.split.PageSinkManager) Optional(java.util.Optional) TEST_SESSION(io.prestosql.SessionTestUtils.TEST_SESSION) NodePartitioningManager(io.prestosql.sql.planner.NodePartitioningManager) LocalExecutionPlanner(io.prestosql.sql.planner.LocalExecutionPlanner) LegacyNetworkTopology(io.prestosql.execution.scheduler.LegacyNetworkTopology) SINGLE_DISTRIBUTION(io.prestosql.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION) ExpressionCompiler(io.prestosql.sql.gen.ExpressionCompiler) StatsAndCosts(io.prestosql.cost.StatsAndCosts) NodeScheduler(io.prestosql.execution.scheduler.NodeScheduler) Split(io.prestosql.metadata.Split) OutputBuffers(io.prestosql.execution.buffer.OutputBuffers) IndexJoinLookupStats(io.prestosql.operator.index.IndexJoinLookupStats) OptionalInt(java.util.OptionalInt) DynamicFilterCacheManager(io.prestosql.dynamicfilter.DynamicFilterCacheManager) GenericSpillerFactory(io.prestosql.spiller.GenericSpillerFactory) LookupJoinOperators(io.prestosql.operator.LookupJoinOperators) SeedStoreManager(io.prestosql.seedstore.SeedStoreManager) VARCHAR(io.prestosql.spi.type.VarcharType.VARCHAR) ImmutableList(com.google.common.collect.ImmutableList) TestingSplit(io.prestosql.testing.TestingSplit) LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) InMemoryNodeManager(io.prestosql.metadata.InMemoryNodeManager) PageSourceManager(io.prestosql.split.PageSourceManager) TestingColumnHandle(io.prestosql.testing.TestingMetadata.TestingColumnHandle) PagesIndex(io.prestosql.operator.PagesIndex) Symbol(io.prestosql.spi.plan.Symbol) PlanFragment(io.prestosql.sql.planner.PlanFragment) StageExecutionDescriptor.ungroupedExecution(io.prestosql.operator.StageExecutionDescriptor.ungroupedExecution) IndexManager(io.prestosql.index.IndexManager) SplitMonitor(io.prestosql.event.SplitMonitor) FinalizerService(io.prestosql.util.FinalizerService) CubeManager(io.prestosql.cube.CubeManager) JoinFilterFunctionCompiler(io.prestosql.sql.gen.JoinFilterFunctionCompiler) HetuMetaStoreManager(io.prestosql.metastore.HetuMetaStoreManager) StateStoreProvider(io.prestosql.statestore.StateStoreProvider) FeaturesConfig(io.prestosql.sql.analyzer.FeaturesConfig) JoinCompiler(io.prestosql.sql.gen.JoinCompiler) PageFunctionCompiler(io.prestosql.sql.gen.PageFunctionCompiler) FeaturesConfig(io.prestosql.sql.analyzer.FeaturesConfig) Metadata(io.prestosql.metadata.Metadata) NodeSchedulerConfig(io.prestosql.execution.scheduler.NodeSchedulerConfig) StateStoreListenerManager(io.prestosql.statestore.listener.StateStoreListenerManager) PagesIndex(io.prestosql.operator.PagesIndex) LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) StateStoreProvider(io.prestosql.statestore.StateStoreProvider) PageSourceManager(io.prestosql.split.PageSourceManager) NodePartitioningManager(io.prestosql.sql.planner.NodePartitioningManager) LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) SeedStoreManager(io.prestosql.seedstore.SeedStoreManager) OrderingCompiler(io.prestosql.sql.gen.OrderingCompiler) NodeScheduler(io.prestosql.execution.scheduler.NodeScheduler) HetuMetaStoreManager(io.prestosql.metastore.HetuMetaStoreManager) GenericSpillerFactory(io.prestosql.spiller.GenericSpillerFactory) PageSinkManager(io.prestosql.split.PageSinkManager) LookupJoinOperators(io.prestosql.operator.LookupJoinOperators) JoinCompiler(io.prestosql.sql.gen.JoinCompiler) DynamicFilterCacheManager(io.prestosql.dynamicfilter.DynamicFilterCacheManager) LocalExecutionPlanner(io.prestosql.sql.planner.LocalExecutionPlanner) MockExchangeClientSupplier(io.prestosql.execution.TestSqlTaskManager.MockExchangeClientSupplier) IndexJoinLookupStats(io.prestosql.operator.index.IndexJoinLookupStats) JoinFilterFunctionCompiler(io.prestosql.sql.gen.JoinFilterFunctionCompiler) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) SqlParser(io.prestosql.sql.parser.SqlParser) CubeManager(io.prestosql.cube.CubeManager) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) InMemoryNodeManager(io.prestosql.metadata.InMemoryNodeManager) FileSystemClientManager(io.prestosql.filesystem.FileSystemClientManager) IndexManager(io.prestosql.index.IndexManager) FinalizerService(io.prestosql.util.FinalizerService) NodeInfo(io.airlift.node.NodeInfo) LegacyNetworkTopology(io.prestosql.execution.scheduler.LegacyNetworkTopology) ExpressionCompiler(io.prestosql.sql.gen.ExpressionCompiler)

Example 3 with DynamicFilterCacheManager

use of io.prestosql.dynamicfilter.DynamicFilterCacheManager in project hetu-core by openlookeng.

the class TestCrossRegionDynamicFilterOperator method testBloomFilterWithoutValue.

@Test
public void testBloomFilterWithoutValue() {
    List<Type> types = ImmutableList.of(VarcharType.VARCHAR, VarcharType.VARCHAR);
    String queryId = "query-123456";
    DynamicFilterCacheManager dynamicFilterCacheManager = new DynamicFilterCacheManager();
    CrossRegionDynamicFilterOperator operator = createBloomFilterOperator(queryId, dynamicFilterCacheManager);
    assertFalse(operator.isFinished());
    List<Page> pages = rowPagesBuilder(types).row("10001", "0001").row("10002", "0002").pageBreak().row("10003", "0003").build();
    assertNull(operator.getOutput());
    assertFalse(operator.isFinished());
    operator.addInput(pages.get(0));
    assertFalse(operator.needsInput());
    Page page1 = operator.getOutput();
    assertEquals(page1, pages.get(0));
    assertTrue(operator.needsInput());
    operator.addInput(pages.get(1));
    Page page2 = operator.getOutput();
    assertEquals(page2, pages.get(1));
    operator.close();
}
Also used : DynamicFilterCacheManager(io.prestosql.dynamicfilter.DynamicFilterCacheManager) Type(io.prestosql.spi.type.Type) VarcharType(io.prestosql.spi.type.VarcharType) Page(io.prestosql.spi.Page) Test(org.testng.annotations.Test)

Example 4 with DynamicFilterCacheManager

use of io.prestosql.dynamicfilter.DynamicFilterCacheManager in project hetu-core by openlookeng.

the class TestCrossRegionDynamicFilterOperator method testBloomFilter.

@Test
public void testBloomFilter() {
    List<Type> types = ImmutableList.of(VarcharType.VARCHAR, VarcharType.VARCHAR);
    String queryId = "query-123456-2";
    DynamicFilterCacheManager dynamicFilterCacheManager = new DynamicFilterCacheManager();
    CrossRegionDynamicFilterOperator operator = createBloomFilterOperator(queryId, dynamicFilterCacheManager);
    addBloomFilter("orderId", ImmutableList.of("10001"), dynamicFilterCacheManager, queryId);
    List<Page> pages = rowPagesBuilder(types).row("10001", "0001").row("10002", "0002").row("10003", "0003").build();
    operator.addInput(pages.get(0));
    Page page = operator.getOutput();
    assertEquals(page.getPositionCount(), 1);
    Block block = page.getBlock(0).getLoadedBlock();
    String nativeValue = TypeUtils.readNativeValueForDynamicFilter(types.get(0), block, 0);
    assertEquals(nativeValue, "10001");
    operator.close();
}
Also used : DynamicFilterCacheManager(io.prestosql.dynamicfilter.DynamicFilterCacheManager) Type(io.prestosql.spi.type.Type) VarcharType(io.prestosql.spi.type.VarcharType) Block(io.prestosql.spi.block.Block) Page(io.prestosql.spi.Page) Test(org.testng.annotations.Test)

Example 5 with DynamicFilterCacheManager

use of io.prestosql.dynamicfilter.DynamicFilterCacheManager in project hetu-core by openlookeng.

the class TestCrossRegionDynamicFilterOperator method testMultipleFilters.

@Test
public void testMultipleFilters() {
    List<Type> types = ImmutableList.of(VarcharType.VARCHAR, VarcharType.VARCHAR);
    String queryId = "query-123456-3";
    DynamicFilterCacheManager dynamicFilterCacheManager = new DynamicFilterCacheManager();
    CrossRegionDynamicFilterOperator operator = createBloomFilterOperator(queryId, dynamicFilterCacheManager);
    addBloomFilter("orderId", ImmutableList.of("10003", "10004"), dynamicFilterCacheManager, queryId);
    addBloomFilter("custkey", ImmutableList.of("0001", "0002"), dynamicFilterCacheManager, queryId);
    List<Page> pages = rowPagesBuilder(types).row("10001", "0001").row("10002", "0002").row("10003", "0003").build();
    operator.addInput(pages.get(0));
    Page page = operator.getOutput();
    assertEquals(page.getPositionCount(), 0);
    operator.close();
}
Also used : DynamicFilterCacheManager(io.prestosql.dynamicfilter.DynamicFilterCacheManager) Type(io.prestosql.spi.type.Type) VarcharType(io.prestosql.spi.type.VarcharType) Page(io.prestosql.spi.Page) Test(org.testng.annotations.Test)

Aggregations

DynamicFilterCacheManager (io.prestosql.dynamicfilter.DynamicFilterCacheManager)6 Test (org.testng.annotations.Test)4 Page (io.prestosql.spi.Page)3 TableScanNode (io.prestosql.spi.plan.TableScanNode)3 Type (io.prestosql.spi.type.Type)3 VarcharType (io.prestosql.spi.type.VarcharType)3 StateStoreProvider (io.prestosql.statestore.StateStoreProvider)3 StateStoreListenerManager (io.prestosql.statestore.listener.StateStoreListenerManager)3 ImmutableMap (com.google.common.collect.ImmutableMap)2 NodeInfo (io.airlift.node.NodeInfo)2 FileSystemClientManager (io.prestosql.filesystem.FileSystemClientManager)2 Split (io.prestosql.metadata.Split)2 LookupJoinOperators (io.prestosql.operator.LookupJoinOperators)2 PagesIndex (io.prestosql.operator.PagesIndex)2 IndexJoinLookupStats (io.prestosql.operator.index.IndexJoinLookupStats)2 SeedStoreManager (io.prestosql.seedstore.SeedStoreManager)2 Symbol (io.prestosql.spi.plan.Symbol)2 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableSet (com.google.common.collect.ImmutableSet)1 ObjectMapperProvider (io.airlift.json.ObjectMapperProvider)1