use of io.prestosql.dynamicfilter.DynamicFilterCacheManager in project hetu-core by openlookeng.
the class TestDynamicFiltersCollector method TestCollectingGlobalDynamicFilters.
@Test
public void TestCollectingGlobalDynamicFilters() throws InterruptedException {
final QueryId queryId = new QueryId("test_query");
final String filterId = "1";
final String columnName = "column";
final TestingColumnHandle columnHandle = new TestingColumnHandle(columnName);
final Set<String> valueSet = ImmutableSet.of("1", "2", "3");
TaskContext taskContext = mock(TaskContext.class);
Session session = testSessionBuilder().setQueryId(queryId).setSystemProperty(ENABLE_DYNAMIC_FILTERING, "true").setSystemProperty(DYNAMIC_FILTERING_DATA_TYPE, "HASHSET").build();
when(taskContext.getSession()).thenReturn(session);
// set up state store and merged dynamic filters map
Map mockMap = new HashMap<>();
StateStoreProvider stateStoreProvider = mock(StateStoreProvider.class);
StateStore stateStore = mock(StateStore.class);
StateMap stateMap = new MockStateMap<>("test-map", mockMap);
when(stateStoreProvider.getStateStore()).thenReturn(stateStore);
when(stateStore.getStateCollection(any())).thenReturn(stateMap);
when(stateStore.createStateMap(any())).thenReturn(stateMap);
when(stateStore.getOrCreateStateCollection(any(), any())).thenReturn(stateMap);
// set up state store listener and dynamic filter cache
StateStoreListenerManager stateStoreListenerManager = new StateStoreListenerManager(stateStoreProvider);
DynamicFilterCacheManager dynamicFilterCacheManager = new DynamicFilterCacheManager();
stateStoreListenerManager.addStateStoreListener(new DynamicFilterListener(dynamicFilterCacheManager), MERGED_DYNAMIC_FILTERS);
LocalDynamicFiltersCollector collector = new LocalDynamicFiltersCollector(taskContext, Optional.empty(), dynamicFilterCacheManager);
TableScanNode tableScan = mock(TableScanNode.class);
when(tableScan.getAssignments()).thenReturn(ImmutableMap.of(new Symbol(columnName), columnHandle));
List<DynamicFilters.Descriptor> dynamicFilterDescriptors = ImmutableList.of(new DynamicFilters.Descriptor(filterId, new VariableReferenceExpression(columnName, BIGINT)));
collector.initContext(ImmutableList.of(dynamicFilterDescriptors), SymbolUtils.toLayOut(tableScan.getOutputSymbols()));
assertTrue(collector.getDynamicFilters(tableScan).isEmpty(), "there should be no dynamic filter available");
// put some values in state store as a new dynamic filter
// and wait for the listener to process the event
stateMap.put(createKey(DynamicFilterUtils.FILTERPREFIX, filterId, queryId.getId()), valueSet);
TimeUnit.MILLISECONDS.sleep(100);
// get available dynamic filter and verify it
List<Map<ColumnHandle, DynamicFilter>> dynamicFilters = collector.getDynamicFilters(tableScan);
assertEquals(dynamicFilters.size(), 1, "there should be a new dynamic filter");
assertEquals(dynamicFilters.size(), 1);
DynamicFilter dynamicFilter = dynamicFilters.get(0).get(columnHandle);
assertTrue(dynamicFilter instanceof HashSetDynamicFilter, "new dynamic filter should be hashset");
assertEquals(dynamicFilter.getSize(), valueSet.size(), "new dynamic filter should have correct size");
for (String value : valueSet) {
assertTrue(dynamicFilter.contains(value), "new dynamic filter should contain correct values");
}
// clean up when task finishes
collector.removeDynamicFilter(true);
DynamicFilter cachedFilter = dynamicFilterCacheManager.getDynamicFilter(DynamicFilterCacheManager.createCacheKey(filterId, queryId.getId()));
assertNull(cachedFilter, "cached dynamic filter should have been removed");
}
use of io.prestosql.dynamicfilter.DynamicFilterCacheManager in project hetu-core by openlookeng.
the class TaskTestUtils method createTestingPlanner.
public static LocalExecutionPlanner createTestingPlanner() {
Metadata metadata = createTestMetadataManager();
PageSourceManager pageSourceManager = new PageSourceManager();
HetuMetaStoreManager hetuMetaStoreManager = new HetuMetaStoreManager();
FeaturesConfig featuresConfig = new FeaturesConfig();
CubeManager cubeManager = new CubeManager(featuresConfig, hetuMetaStoreManager);
pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
// we don't start the finalizer so nothing will be collected, which is ok for a test
FinalizerService finalizerService = new FinalizerService();
NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService));
NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler);
PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(metadata, 0);
NodeInfo nodeInfo = new NodeInfo("test");
FileSystemClientManager fileSystemClientManager = new FileSystemClientManager();
SeedStoreManager seedStoreManager = new SeedStoreManager(fileSystemClientManager);
StateStoreProvider stateStoreProvider = new LocalStateStoreProvider(seedStoreManager);
HeuristicIndexerManager heuristicIndexerManager = new HeuristicIndexerManager(new FileSystemClientManager(), new HetuMetaStoreManager());
return new LocalExecutionPlanner(metadata, new TypeAnalyzer(new SqlParser(), metadata), Optional.empty(), pageSourceManager, new IndexManager(), nodePartitioningManager, new PageSinkManager(), new MockExchangeClientSupplier(), new ExpressionCompiler(metadata, pageFunctionCompiler), pageFunctionCompiler, new JoinFilterFunctionCompiler(metadata), new IndexJoinLookupStats(), new TaskManagerConfig(), new GenericSpillerFactory((types, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}), (types, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}, (types, partitionFunction, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}, new PagesIndex.TestingFactory(false), new JoinCompiler(metadata), new LookupJoinOperators(), new OrderingCompiler(), nodeInfo, stateStoreProvider, new StateStoreListenerManager(stateStoreProvider), new DynamicFilterCacheManager(), heuristicIndexerManager, cubeManager);
}
use of io.prestosql.dynamicfilter.DynamicFilterCacheManager in project hetu-core by openlookeng.
the class TestCrossRegionDynamicFilterOperator method testBloomFilterWithoutValue.
@Test
public void testBloomFilterWithoutValue() {
List<Type> types = ImmutableList.of(VarcharType.VARCHAR, VarcharType.VARCHAR);
String queryId = "query-123456";
DynamicFilterCacheManager dynamicFilterCacheManager = new DynamicFilterCacheManager();
CrossRegionDynamicFilterOperator operator = createBloomFilterOperator(queryId, dynamicFilterCacheManager);
assertFalse(operator.isFinished());
List<Page> pages = rowPagesBuilder(types).row("10001", "0001").row("10002", "0002").pageBreak().row("10003", "0003").build();
assertNull(operator.getOutput());
assertFalse(operator.isFinished());
operator.addInput(pages.get(0));
assertFalse(operator.needsInput());
Page page1 = operator.getOutput();
assertEquals(page1, pages.get(0));
assertTrue(operator.needsInput());
operator.addInput(pages.get(1));
Page page2 = operator.getOutput();
assertEquals(page2, pages.get(1));
operator.close();
}
use of io.prestosql.dynamicfilter.DynamicFilterCacheManager in project hetu-core by openlookeng.
the class TestCrossRegionDynamicFilterOperator method testBloomFilter.
@Test
public void testBloomFilter() {
List<Type> types = ImmutableList.of(VarcharType.VARCHAR, VarcharType.VARCHAR);
String queryId = "query-123456-2";
DynamicFilterCacheManager dynamicFilterCacheManager = new DynamicFilterCacheManager();
CrossRegionDynamicFilterOperator operator = createBloomFilterOperator(queryId, dynamicFilterCacheManager);
addBloomFilter("orderId", ImmutableList.of("10001"), dynamicFilterCacheManager, queryId);
List<Page> pages = rowPagesBuilder(types).row("10001", "0001").row("10002", "0002").row("10003", "0003").build();
operator.addInput(pages.get(0));
Page page = operator.getOutput();
assertEquals(page.getPositionCount(), 1);
Block block = page.getBlock(0).getLoadedBlock();
String nativeValue = TypeUtils.readNativeValueForDynamicFilter(types.get(0), block, 0);
assertEquals(nativeValue, "10001");
operator.close();
}
use of io.prestosql.dynamicfilter.DynamicFilterCacheManager in project hetu-core by openlookeng.
the class TestCrossRegionDynamicFilterOperator method testMultipleFilters.
@Test
public void testMultipleFilters() {
List<Type> types = ImmutableList.of(VarcharType.VARCHAR, VarcharType.VARCHAR);
String queryId = "query-123456-3";
DynamicFilterCacheManager dynamicFilterCacheManager = new DynamicFilterCacheManager();
CrossRegionDynamicFilterOperator operator = createBloomFilterOperator(queryId, dynamicFilterCacheManager);
addBloomFilter("orderId", ImmutableList.of("10003", "10004"), dynamicFilterCacheManager, queryId);
addBloomFilter("custkey", ImmutableList.of("0001", "0002"), dynamicFilterCacheManager, queryId);
List<Page> pages = rowPagesBuilder(types).row("10001", "0001").row("10002", "0002").row("10003", "0003").build();
operator.addInput(pages.get(0));
Page page = operator.getOutput();
assertEquals(page.getPositionCount(), 0);
operator.close();
}
Aggregations