Examples with DynamicFilterSourceOperatorFactory io.prestosql.operator.DynamicFilterSourceOperator.DynamicFilterSourceOperatorFactory used on opensource projects

Example 1 with DynamicFilterSourceOperatorFactory

use of io.prestosql.operator.DynamicFilterSourceOperator.DynamicFilterSourceOperatorFactory in project hetu-core by openlookeng.

the class TestDynamicFilterSourceOperator method testCollectMultipleColumns.

@Test
public void testCollectMultipleColumns() {
    String filterId1 = "multiple_columns_1";
    String filterId2 = "multiple_columns_2";
    OperatorFactory operatorFactory = createOperatorFactory(LOCAL, HASHSET, 1, channel(0, BOOLEAN, filterId1), channel(1, DOUBLE, filterId2));
    verifyPassthrough(createOperator((DynamicFilterSourceOperatorFactory) operatorFactory), ImmutableList.of(BOOLEAN, DOUBLE), new Page(createBooleansBlock(true, 2), createDoublesBlock(1.5, 3.0)), new Page(createBooleansBlock(false, 1), createDoublesBlock(4.5)));
    operatorFactory.noMoreOperators();
    String key1 = DynamicFilterUtils.createKey(PARTIALPREFIX, filterId1, TEST_SESSION.getQueryId().toString());
    Set<Boolean> set1 = new HashSet<>(Arrays.asList(true, false));
    StateSet states1 = ((StateSet) stateStoreProvider.getStateStore().getStateCollection(key1));
    for (Object bfSerialized : states1.getAll()) {
        assertEquals((Set) bfSerialized, set1);
    }
    String key2 = DynamicFilterUtils.createKey(PARTIALPREFIX, filterId2, TEST_SESSION.getQueryId().toString());
    Set<Double> set2 = new HashSet<>(Arrays.asList(1.5, 3.0, 4.5));
    StateSet states2 = ((StateSet) stateStoreProvider.getStateStore().getStateCollection(key2));
    for (Object bfSerialized : states2.getAll()) {
        assertEquals((Set) bfSerialized, set2);
    }
}

Example 2 with DynamicFilterSourceOperatorFactory

use of io.prestosql.operator.DynamicFilterSourceOperator.DynamicFilterSourceOperatorFactory in project hetu-core by openlookeng.

the class TestDynamicFilterSourceOperator method testCollectTooMuchRows.

@Test
public void testCollectTooMuchRows() {
    String filterId = "too_much_rows";
    final int maxRowCount = getDynamicFilteringMaxPerDriverValueCount(pipelineContext.getSession());
    Page largePage = createSequencePage(ImmutableList.of(BIGINT), maxRowCount + 1);
    OperatorFactory operatorFactory = createOperatorFactory(LOCAL, HASHSET, 1, channel(0, BIGINT, filterId));
    verifyPassthrough(createOperator((DynamicFilterSourceOperatorFactory) operatorFactory), ImmutableList.of(BIGINT), largePage);
    operatorFactory.noMoreOperators();
    String key = DynamicFilterUtils.createKey(PARTIALPREFIX, filterId, TEST_SESSION.getQueryId().toString());
    assertNull(stateStoreProvider.getStateStore().getStateCollection(key));
}

Example 3 with DynamicFilterSourceOperatorFactory

use of io.prestosql.operator.DynamicFilterSourceOperator.DynamicFilterSourceOperatorFactory in project hetu-core by openlookeng.

the class TestDynamicFilterSourceOperator method createOperatorFactory.

private DynamicFilterSourceOperatorFactory createOperatorFactory(DynamicFilter.Type dfType, FeaturesConfig.DynamicFilterDataType dataType, int partitionCount, Channel... buildChannels) {
    NodeInfo nodeInfo = new NodeInfo("test");
    Multimap<String, Symbol> probeSymbols = MultimapBuilder.treeKeys().arrayListValues().build();
    Map<String, Integer> buildChannelMap = new HashMap<>();
    Arrays.stream(buildChannels).map(channel -> buildChannelMap.put(channel.getFilterId(), channel.getIndex()));
    Arrays.stream(buildChannels).map(channel -> probeSymbols.put(channel.getFilterId(), new Symbol(String.valueOf(channel.getIndex()))));
    TaskId taskId = new TaskId("test0.0");
    LocalDynamicFilter localDynamicFilter = new LocalDynamicFilter(probeSymbols, buildChannelMap, partitionCount, dfType, dataType, 0.1D, taskId, stateStoreProvider);
    return new DynamicFilterSourceOperatorFactory(0, new PlanNodeId("PLAN_NODE_ID"), localDynamicFilter.getValueConsumer(), Arrays.stream(buildChannels).collect(toList()), getDynamicFilteringMaxPerDriverValueCount(TEST_SESSION), getDynamicFilteringMaxPerDriverSize(TEST_SESSION));
}

Example 4 with DynamicFilterSourceOperatorFactory

use of io.prestosql.operator.DynamicFilterSourceOperator.DynamicFilterSourceOperatorFactory in project hetu-core by openlookeng.

the class TestDynamicFilterSourceOperator method testCollectDeduplication.

@Test
public void testCollectDeduplication() {
    String filterId = "deduplication";
    final int maxRowCount = getDynamicFilteringMaxPerDriverValueCount(pipelineContext.getSession());
    // lots of zeros
    Page largePage = new Page(createLongRepeatBlock(7, maxRowCount * 10));
    // lots of nulls
    Page nullsPage = new Page(createLongsBlock(Arrays.asList(new Long[maxRowCount * 10])));
    OperatorFactory operatorFactory = createOperatorFactory(LOCAL, BLOOM_FILTER, 1, channel(0, BIGINT, filterId));
    verifyPassthrough(createOperator((DynamicFilterSourceOperatorFactory) operatorFactory), ImmutableList.of(BIGINT), largePage, nullsPage);
    operatorFactory.noMoreOperators();
    String key = DynamicFilterUtils.createKey(PARTIALPREFIX, filterId, TEST_SESSION.getQueryId().toString());
    Set<Long> set = new HashSet<>();
    set.add(7L);
    StateSet states = ((StateSet) stateStoreProvider.getStateStore().getStateCollection(key));
    for (Object bfSerialized : states.getAll()) {
        assertEquals((Set) bfSerialized, set);
    }
}

Example 5 with DynamicFilterSourceOperatorFactory

use of io.prestosql.operator.DynamicFilterSourceOperator.DynamicFilterSourceOperatorFactory in project hetu-core by openlookeng.

the class TestDynamicFilterSourceOperator method testCollectMultipleOperators.

@Test
public void testCollectMultipleOperators() {
    String filterId = "0";
    DynamicFilterSourceOperatorFactory operatorFactory = createOperatorFactory(LOCAL, HASHSET, 2, channel(0, BIGINT, filterId));
    // will finish before noMoreOperators()
    DynamicFilterSourceOperator op1 = createOperator(operatorFactory);
    verifyPassthrough(op1, ImmutableList.of(BIGINT), new Page(createLongsBlock(1, 2)), new Page(createLongsBlock(3, 5)));
    // will finish after noMoreOperators()
    Operator op2 = createOperator(operatorFactory);
    operatorFactory.noMoreOperators();
    verifyPassthrough(op2, ImmutableList.of(BIGINT), new Page(createLongsBlock(2, 3)), new Page(createLongsBlock(1, 4)));
    String key = DynamicFilterUtils.createKey(PARTIALPREFIX, filterId, TEST_SESSION.getQueryId().toString());
    Set<Long> set = new HashSet<>(Arrays.asList(1L, 2L, 3L, 4L, 5L));
    StateSet states = ((StateSet) stateStoreProvider.getStateStore().getStateCollection(key));
    for (Object bfSerialized : states.getAll()) {
        assertEquals(set, (Set) bfSerialized);
    }
}