Search in sources :

Example 1 with HashSetDynamicFilter

use of io.prestosql.spi.dynamicfilter.HashSetDynamicFilter in project hetu-core by openlookeng.

the class TestDynamicFiltersCollector method TestCollectingGlobalDynamicFilters.

@Test
public void TestCollectingGlobalDynamicFilters() throws InterruptedException {
    final QueryId queryId = new QueryId("test_query");
    final String filterId = "1";
    final String columnName = "column";
    final TestingColumnHandle columnHandle = new TestingColumnHandle(columnName);
    final Set<String> valueSet = ImmutableSet.of("1", "2", "3");
    TaskContext taskContext = mock(TaskContext.class);
    Session session = testSessionBuilder().setQueryId(queryId).setSystemProperty(ENABLE_DYNAMIC_FILTERING, "true").setSystemProperty(DYNAMIC_FILTERING_DATA_TYPE, "HASHSET").build();
    when(taskContext.getSession()).thenReturn(session);
    // set up state store and merged dynamic filters map
    Map mockMap = new HashMap<>();
    StateStoreProvider stateStoreProvider = mock(StateStoreProvider.class);
    StateStore stateStore = mock(StateStore.class);
    StateMap stateMap = new MockStateMap<>("test-map", mockMap);
    when(stateStoreProvider.getStateStore()).thenReturn(stateStore);
    when(stateStore.getStateCollection(any())).thenReturn(stateMap);
    when(stateStore.createStateMap(any())).thenReturn(stateMap);
    when(stateStore.getOrCreateStateCollection(any(), any())).thenReturn(stateMap);
    // set up state store listener and dynamic filter cache
    StateStoreListenerManager stateStoreListenerManager = new StateStoreListenerManager(stateStoreProvider);
    DynamicFilterCacheManager dynamicFilterCacheManager = new DynamicFilterCacheManager();
    stateStoreListenerManager.addStateStoreListener(new DynamicFilterListener(dynamicFilterCacheManager), MERGED_DYNAMIC_FILTERS);
    LocalDynamicFiltersCollector collector = new LocalDynamicFiltersCollector(taskContext, Optional.empty(), dynamicFilterCacheManager);
    TableScanNode tableScan = mock(TableScanNode.class);
    when(tableScan.getAssignments()).thenReturn(ImmutableMap.of(new Symbol(columnName), columnHandle));
    List<DynamicFilters.Descriptor> dynamicFilterDescriptors = ImmutableList.of(new DynamicFilters.Descriptor(filterId, new VariableReferenceExpression(columnName, BIGINT)));
    collector.initContext(ImmutableList.of(dynamicFilterDescriptors), SymbolUtils.toLayOut(tableScan.getOutputSymbols()));
    assertTrue(collector.getDynamicFilters(tableScan).isEmpty(), "there should be no dynamic filter available");
    // put some values in state store as a new dynamic filter
    // and wait for the listener to process the event
    stateMap.put(createKey(DynamicFilterUtils.FILTERPREFIX, filterId, queryId.getId()), valueSet);
    TimeUnit.MILLISECONDS.sleep(100);
    // get available dynamic filter and verify it
    List<Map<ColumnHandle, DynamicFilter>> dynamicFilters = collector.getDynamicFilters(tableScan);
    assertEquals(dynamicFilters.size(), 1, "there should be a new dynamic filter");
    assertEquals(dynamicFilters.size(), 1);
    DynamicFilter dynamicFilter = dynamicFilters.get(0).get(columnHandle);
    assertTrue(dynamicFilter instanceof HashSetDynamicFilter, "new dynamic filter should be hashset");
    assertEquals(dynamicFilter.getSize(), valueSet.size(), "new dynamic filter should have correct size");
    for (String value : valueSet) {
        assertTrue(dynamicFilter.contains(value), "new dynamic filter should contain correct values");
    }
    // clean up when task finishes
    collector.removeDynamicFilter(true);
    DynamicFilter cachedFilter = dynamicFilterCacheManager.getDynamicFilter(DynamicFilterCacheManager.createCacheKey(filterId, queryId.getId()));
    assertNull(cachedFilter, "cached dynamic filter should have been removed");
}
Also used : HashMap(java.util.HashMap) Symbol(io.prestosql.spi.plan.Symbol) MockStateMap(io.prestosql.statestore.MockStateMap) StateMap(io.prestosql.spi.statestore.StateMap) StateStoreListenerManager(io.prestosql.statestore.listener.StateStoreListenerManager) StateStoreProvider(io.prestosql.statestore.StateStoreProvider) MockStateMap(io.prestosql.statestore.MockStateMap) TestingColumnHandle(io.prestosql.spi.connector.TestingColumnHandle) DynamicFilters(io.prestosql.sql.DynamicFilters) DynamicFilterListener(io.prestosql.dynamicfilter.DynamicFilterListener) DynamicFilterCacheManager(io.prestosql.dynamicfilter.DynamicFilterCacheManager) TaskContext(io.prestosql.operator.TaskContext) DynamicFilter(io.prestosql.spi.dynamicfilter.DynamicFilter) HashSetDynamicFilter(io.prestosql.spi.dynamicfilter.HashSetDynamicFilter) QueryId(io.prestosql.spi.QueryId) StateStore(io.prestosql.spi.statestore.StateStore) HashSetDynamicFilter(io.prestosql.spi.dynamicfilter.HashSetDynamicFilter) TableScanNode(io.prestosql.spi.plan.TableScanNode) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) MockStateMap(io.prestosql.statestore.MockStateMap) HashMap(java.util.HashMap) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) StateMap(io.prestosql.spi.statestore.StateMap) Session(io.prestosql.Session) Test(org.testng.annotations.Test)

Example 2 with HashSetDynamicFilter

use of io.prestosql.spi.dynamicfilter.HashSetDynamicFilter in project hetu-core by openlookeng.

the class TestDynamicFilterServiceWithHashSet method testRegisterAndMergeDynamicFiltersHashSet.

@Test
public void testRegisterAndMergeDynamicFiltersHashSet() throws InterruptedException {
    setUpHashSet();
    filterId = "df2";
    registerDf(filterId, session, PARTITIONED, dynamicFilterService);
    // Test getDynamicFilterSupplier
    VariableReferenceExpression mockExpression = mock(VariableReferenceExpression.class);
    when(mockExpression.getName()).thenReturn("name");
    ColumnHandle mockColumnHandle = mock(ColumnHandle.class);
    Supplier<List<Set<DynamicFilter>>> dynamicFilterSupplier = DynamicFilterService.getDynamicFilterSupplier(session.getQueryId(), ImmutableList.of(ImmutableList.of(new DynamicFilters.Descriptor(filterId, mockExpression))), ImmutableMap.of(new Symbol("name"), mockColumnHandle));
    assertTrue(dynamicFilterSupplier.get().isEmpty(), "should return empty dynamic filter set when dynamic filters are not available");
    mockLocalDynamicFilterHashSet("task1.0", filterId, session.getQueryId().toString(), Arrays.asList("11", "12", "13", "14"));
    mockLocalDynamicFilterHashSet("task1.1", filterId, session.getQueryId().toString(), Arrays.asList("15", "16", "17", "18"));
    Thread.sleep(2000);
    Set hs = fetchDynamicFilterHashSet(filterId, session.getQueryId().toString());
    for (int i = 11; i < 19; i++) {
        Assert.assertEquals(true, hs.contains(i + ""));
    }
    Assert.assertEquals(false, hs.contains("10"));
    // Test getDynamicFilterSupplier
    dynamicFilterSupplier = DynamicFilterService.getDynamicFilterSupplier(session.getQueryId(), ImmutableList.of(ImmutableList.of(new DynamicFilters.Descriptor(filterId, mockExpression))), ImmutableMap.of(new Symbol("name"), mockColumnHandle));
    List<Set<DynamicFilter>> dynamicFilters = dynamicFilterSupplier.get();
    assertFalse(dynamicFilters == null, "dynamic filters should be ready");
    assertEquals(dynamicFilters.size(), 1, "there should be 1 dynamic filter in supplier");
    HashSetDynamicFilter hsDF = ((HashSetDynamicFilter) dynamicFilters.get(0).toArray()[0]);
    assertEquals(hs, hsDF.getSetValues(), "dynamic filter in supplier should be the same as the one merged");
    dynamicFilterSupplier = DynamicFilterService.getDynamicFilterSupplier(new QueryId("invalid"), ImmutableList.of(ImmutableList.of(new DynamicFilters.Descriptor(filterId, mockExpression))), ImmutableMap.of(new Symbol("name"), mockColumnHandle));
    assertTrue(dynamicFilterSupplier.get().isEmpty(), "should return empty dynamic filter set for invalid or non-existing queryId");
    String queryId = session.getQueryId().getId();
    assertEquals(stateStoreProvider.getStateStore().getStateCollection(createKey(DynamicFilterUtils.PARTIALPREFIX, filterId, queryId)).size(), 2);
    assertEquals(stateStoreProvider.getStateStore().getStateCollection(createKey(DynamicFilterUtils.TASKSPREFIX, filterId, queryId)).size(), 2);
    dynamicFilterService.clearDynamicFiltersForQuery(queryId);
    Thread.sleep(1000);
    assertEquals(stateStoreProvider.getStateStore().getStateCollection(createKey(DynamicFilterUtils.PARTIALPREFIX, filterId, queryId)).size(), 0);
    assertEquals(stateStoreProvider.getStateStore().getStateCollection(createKey(DynamicFilterUtils.TASKSPREFIX, filterId, queryId)).size(), 0);
}
Also used : ColumnHandle(io.prestosql.spi.connector.ColumnHandle) HashSet(java.util.HashSet) Set(java.util.Set) StateSet(io.prestosql.spi.statestore.StateSet) DynamicFilter(io.prestosql.spi.dynamicfilter.DynamicFilter) HashSetDynamicFilter(io.prestosql.spi.dynamicfilter.HashSetDynamicFilter) Symbol(io.prestosql.spi.plan.Symbol) QueryId(io.prestosql.spi.QueryId) HashSetDynamicFilter(io.prestosql.spi.dynamicfilter.HashSetDynamicFilter) DynamicFilters(io.prestosql.sql.DynamicFilters) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) ImmutableList(com.google.common.collect.ImmutableList) List(java.util.List) Test(org.testng.annotations.Test) BeforeTest(org.testng.annotations.BeforeTest)

Example 3 with HashSetDynamicFilter

use of io.prestosql.spi.dynamicfilter.HashSetDynamicFilter in project hetu-core by openlookeng.

the class TestHiveUtil method testIsPartitionFilteredWithNonPartitionFilter.

@Test
public void testIsPartitionFilteredWithNonPartitionFilter() {
    TypeManager typeManager = new TestingTypeManager();
    Set<DynamicFilter> dynamicFilters = new HashSet<>();
    List<HivePartitionKey> partitions = new ArrayList<>();
    partitions.add(new HivePartitionKey("pt_d", "0"));
    partitions.add(new HivePartitionKey("app_id", "10000"));
    ColumnHandle nameColumn = new HiveColumnHandle("name", HIVE_STRING, parseTypeSignature(VARCHAR), 0, REGULAR, Optional.empty());
    Set nameFilter = new HashSet();
    nameFilter.add("Alice");
    dynamicFilters.add(new HashSetDynamicFilter("1", nameColumn, nameFilter, DynamicFilter.Type.GLOBAL));
    assertFalse(isPartitionFiltered(partitions, ImmutableList.of(dynamicFilters), typeManager), "Should not filter partition if dynamicFilter is on non-partition column");
}
Also used : ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Set(java.util.Set) HashSet(java.util.HashSet) DynamicFilter(io.prestosql.spi.dynamicfilter.DynamicFilter) HashSetDynamicFilter(io.prestosql.spi.dynamicfilter.HashSetDynamicFilter) BloomFilterDynamicFilter(io.prestosql.spi.dynamicfilter.BloomFilterDynamicFilter) ArrayList(java.util.ArrayList) TestingTypeManager(io.prestosql.spi.type.TestingTypeManager) TypeManager(io.prestosql.spi.type.TypeManager) TestingTypeManager(io.prestosql.spi.type.TestingTypeManager) HashSet(java.util.HashSet) HashSetDynamicFilter(io.prestosql.spi.dynamicfilter.HashSetDynamicFilter) Test(org.testng.annotations.Test)

Example 4 with HashSetDynamicFilter

use of io.prestosql.spi.dynamicfilter.HashSetDynamicFilter in project hetu-core by openlookeng.

the class DataCenterPageSource method applyDynamicFilters.

private void applyDynamicFilters(Map<ColumnHandle, DynamicFilter> dynamicFilters) {
    ImmutableMap.Builder<String, byte[]> builder = new ImmutableMap.Builder();
    for (Map.Entry<ColumnHandle, DynamicFilter> entry : dynamicFilters.entrySet()) {
        if (!appliedDynamicFilters.contains(entry.getKey().getColumnName())) {
            DynamicFilter df = entry.getValue();
            String columnName = entry.getKey().getColumnName();
            if (df instanceof HashSetDynamicFilter) {
                // FIXME: Read fpp from config
                BloomFilterDynamicFilter bloomFilterDynamicFilter = BloomFilterDynamicFilter.fromHashSetDynamicFilter((HashSetDynamicFilter) df);
                builder.put(columnName, bloomFilterDynamicFilter.createSerializedBloomFilter());
            } else if (df instanceof CombinedDynamicFilter) {
                BloomFilterDynamicFilter bloomFilterDynamicFilter = BloomFilterDynamicFilter.fromCombinedDynamicFilter((CombinedDynamicFilter) df);
                if (bloomFilterDynamicFilter != null) {
                    builder.put(columnName, bloomFilterDynamicFilter.createSerializedBloomFilter());
                }
            } else if (df instanceof BloomFilterDynamicFilter) {
                builder.put(columnName, ((BloomFilterDynamicFilter) df).getBloomFilterSerialized());
            } else {
                LOGGER.info("Dynamic Filter (type: " + df.getClass().getSimpleName() + ") skipped for DC connector");
            }
        }
    }
    Map<String, byte[]> newDynamicFilters = builder.build();
    if (!newDynamicFilters.isEmpty()) {
        if (client.applyDynamicFilters(newDynamicFilters)) {
            appliedDynamicFilters.addAll(newDynamicFilters.keySet());
        }
    }
}
Also used : ColumnHandle(io.prestosql.spi.connector.ColumnHandle) DynamicFilter(io.prestosql.spi.dynamicfilter.DynamicFilter) CombinedDynamicFilter(io.prestosql.spi.dynamicfilter.CombinedDynamicFilter) HashSetDynamicFilter(io.prestosql.spi.dynamicfilter.HashSetDynamicFilter) BloomFilterDynamicFilter(io.prestosql.spi.dynamicfilter.BloomFilterDynamicFilter) CombinedDynamicFilter(io.prestosql.spi.dynamicfilter.CombinedDynamicFilter) BloomFilterDynamicFilter(io.prestosql.spi.dynamicfilter.BloomFilterDynamicFilter) ImmutableMap(com.google.common.collect.ImmutableMap) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) HashSetDynamicFilter(io.prestosql.spi.dynamicfilter.HashSetDynamicFilter)

Example 5 with HashSetDynamicFilter

use of io.prestosql.spi.dynamicfilter.HashSetDynamicFilter in project hetu-core by openlookeng.

the class TestDynamicFilterSourceOperator method testGlobalDynamicFilterSourceOperatorHashSet.

@Test
public void testGlobalDynamicFilterSourceOperatorHashSet() {
    String filterId = "22";
    DynamicFilterSourceOperatorFactory operatorFactory = createOperatorFactory(GLOBAL, HASHSET, 1, channel(0, BIGINT, filterId));
    // will finish before noMoreOperators()
    DynamicFilterSourceOperator op1 = createOperator(operatorFactory);
    verifyPassthrough(op1, ImmutableList.of(BIGINT), new Page(createLongsBlock(1, 2)), new Page(createLongsBlock(12, 21)), new Page(createLongsBlock(13, 22)), new Page(createLongsBlock(3, 5)));
    String key = DynamicFilterUtils.createKey(PARTIALPREFIX, filterId, TEST_SESSION.getQueryId().toString());
    StateSet states = ((StateSet) stateStoreProvider.getStateStore().getStateCollection(key));
    for (Object bfSerialized : states.getAll()) {
        HashSetDynamicFilter bfdf = new HashSetDynamicFilter(filterId, null, (Set) bfSerialized, GLOBAL);
        assertTrue(bfdf.contains(22L));
        assertEquals(bfdf.getSize(), 8);
    }
    assertEquals(stateStoreProvider.getStateStore().getStateCollection(DynamicFilterUtils.createKey(TASKSPREFIX, filterId, TEST_SESSION.getQueryId().toString())).size(), 1);
}
Also used : DynamicFilterSourceOperatorFactory(io.prestosql.operator.DynamicFilterSourceOperator.DynamicFilterSourceOperatorFactory) SequencePageBuilder.createSequencePage(io.prestosql.SequencePageBuilder.createSequencePage) Page(io.prestosql.spi.Page) StateSet(io.prestosql.spi.statestore.StateSet) HashSetDynamicFilter(io.prestosql.spi.dynamicfilter.HashSetDynamicFilter) Test(org.testng.annotations.Test) AfterTest(org.testng.annotations.AfterTest) BeforeTest(org.testng.annotations.BeforeTest)

Aggregations

HashSetDynamicFilter (io.prestosql.spi.dynamicfilter.HashSetDynamicFilter)5 DynamicFilter (io.prestosql.spi.dynamicfilter.DynamicFilter)4 Test (org.testng.annotations.Test)4 ColumnHandle (io.prestosql.spi.connector.ColumnHandle)3 ImmutableMap (com.google.common.collect.ImmutableMap)2 QueryId (io.prestosql.spi.QueryId)2 BloomFilterDynamicFilter (io.prestosql.spi.dynamicfilter.BloomFilterDynamicFilter)2 Symbol (io.prestosql.spi.plan.Symbol)2 VariableReferenceExpression (io.prestosql.spi.relation.VariableReferenceExpression)2 StateSet (io.prestosql.spi.statestore.StateSet)2 DynamicFilters (io.prestosql.sql.DynamicFilters)2 HashSet (java.util.HashSet)2 Map (java.util.Map)2 Set (java.util.Set)2 BeforeTest (org.testng.annotations.BeforeTest)2 ImmutableList (com.google.common.collect.ImmutableList)1 SequencePageBuilder.createSequencePage (io.prestosql.SequencePageBuilder.createSequencePage)1 Session (io.prestosql.Session)1 DynamicFilterCacheManager (io.prestosql.dynamicfilter.DynamicFilterCacheManager)1 DynamicFilterListener (io.prestosql.dynamicfilter.DynamicFilterListener)1