Examples with InternalAggregateProcessAllWindowFunction org.apache.flink.streaming.runtime.operators.windowing.functions.InternalAggregateProcessAllWindowFunction used on opensource projects

Example 1 with InternalAggregateProcessAllWindowFunction

use of org.apache.flink.streaming.runtime.operators.windowing.functions.InternalAggregateProcessAllWindowFunction in project flink by apache.

the class InternalWindowFunctionTest method testInternalAggregateProcessAllWindowFunction.

@SuppressWarnings("unchecked")
@Test
public void testInternalAggregateProcessAllWindowFunction() throws Exception {
    AggregateProcessAllWindowFunctionMock mock = mock(AggregateProcessAllWindowFunctionMock.class);
    InternalAggregateProcessAllWindowFunction<Long, Set<Long>, Map<Long, Long>, String, TimeWindow> windowFunction = new InternalAggregateProcessAllWindowFunction<>(new AggregateFunction<Long, Set<Long>, Map<Long, Long>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Set<Long> createAccumulator() {
            return new HashSet<>();
        }

        @Override
        public Set<Long> add(Long value, Set<Long> accumulator) {
            accumulator.add(value);
            return accumulator;
        }

        @Override
        public Map<Long, Long> getResult(Set<Long> accumulator) {
            Map<Long, Long> result = new HashMap<>();
            for (Long in : accumulator) {
                result.put(in, in);
            }
            return result;
        }

        @Override
        public Set<Long> merge(Set<Long> a, Set<Long> b) {
            a.addAll(b);
            return a;
        }
    }, mock);
    // check setOutputType
    TypeInformation<String> stringType = BasicTypeInfo.STRING_TYPE_INFO;
    ExecutionConfig execConf = new ExecutionConfig();
    execConf.setParallelism(42);
    StreamingFunctionUtils.setOutputType(windowFunction, stringType, execConf);
    verify(mock).setOutputType(stringType, execConf);
    // check open
    Configuration config = new Configuration();
    windowFunction.open(config);
    verify(mock).open(config);
    // check setRuntimeContext
    RuntimeContext rCtx = mock(RuntimeContext.class);
    windowFunction.setRuntimeContext(rCtx);
    verify(mock).setRuntimeContext(rCtx);
    // check apply
    TimeWindow w = mock(TimeWindow.class);
    Collector<String> c = (Collector<String>) mock(Collector.class);
    List<Long> args = new LinkedList<>();
    args.add(23L);
    args.add(24L);
    InternalWindowFunction.InternalWindowContext ctx = mock(InternalWindowFunction.InternalWindowContext.class);
    windowFunction.process(((byte) 0), w, ctx, args, c);
    verify(mock).process((AggregateProcessAllWindowFunctionMock.Context) anyObject(), (Iterable) argThat(containsInAnyOrder(allOf(hasEntry(is(23L), is(23L)), hasEntry(is(24L), is(24L))))), eq(c));
    // check close
    windowFunction.close();
    verify(mock).close();
}

Example 2 with InternalAggregateProcessAllWindowFunction

use of org.apache.flink.streaming.runtime.operators.windowing.functions.InternalAggregateProcessAllWindowFunction in project flink by apache.

the class AllWindowedStream method aggregate.

/**
 * Applies the given window function to each window. The window function is called for each
 * evaluation of the window for each key individually. The output of the window function is
 * interpreted as a regular non-windowed stream.
 *
 * <p>Arriving data is incrementally aggregated using the given aggregate function. This means
 * that the window function typically has only a single value to process when called.
 *
 * @param aggregateFunction The aggregation function that is used for incremental aggregation.
 * @param windowFunction The process window function.
 * @param accumulatorType Type information for the internal accumulator type of the aggregation
 *     function
 * @param resultType Type information for the result type of the window function
 * @return The data stream that is the result of applying the window function to the window.
 * @param <ACC> The type of the AggregateFunction's accumulator
 * @param <V> The type of AggregateFunction's result, and the WindowFunction's input
 * @param <R> The type of the elements in the resulting stream, equal to the WindowFunction's
 *     result type
 */
@PublicEvolving
public <ACC, V, R> SingleOutputStreamOperator<R> aggregate(AggregateFunction<T, ACC, V> aggregateFunction, ProcessAllWindowFunction<V, R, W> windowFunction, TypeInformation<ACC> accumulatorType, TypeInformation<V> aggregateResultType, TypeInformation<R> resultType) {
    checkNotNull(aggregateFunction, "aggregateFunction");
    checkNotNull(windowFunction, "windowFunction");
    checkNotNull(accumulatorType, "accumulatorType");
    checkNotNull(aggregateResultType, "aggregateResultType");
    checkNotNull(resultType, "resultType");
    if (aggregateFunction instanceof RichFunction) {
        throw new UnsupportedOperationException("This aggregate function cannot be a RichFunction.");
    }
    // clean the closures
    windowFunction = input.getExecutionEnvironment().clean(windowFunction);
    aggregateFunction = input.getExecutionEnvironment().clean(aggregateFunction);
    final String callLocation = Utils.getCallLocationName();
    final String udfName = "AllWindowedStream." + callLocation;
    final String opName = windowAssigner.getClass().getSimpleName();
    final String opDescription;
    final KeySelector<T, Byte> keySel = input.getKeySelector();
    OneInputStreamOperator<T, R> operator;
    if (evictor != null) {
        @SuppressWarnings({ "unchecked", "rawtypes" }) TypeSerializer<StreamRecord<T>> streamRecordSerializer = (TypeSerializer<StreamRecord<T>>) new StreamElementSerializer(input.getType().createSerializer(getExecutionEnvironment().getConfig()));
        ListStateDescriptor<StreamRecord<T>> stateDesc = new ListStateDescriptor<>("window-contents", streamRecordSerializer);
        opDescription = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + evictor + ", " + udfName + ")";
        operator = new EvictingWindowOperator<>(windowAssigner, windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), keySel, input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), stateDesc, new InternalAggregateProcessAllWindowFunction<>(aggregateFunction, windowFunction), trigger, evictor, allowedLateness, lateDataOutputTag);
    } else {
        AggregatingStateDescriptor<T, ACC, V> stateDesc = new AggregatingStateDescriptor<>("window-contents", aggregateFunction, accumulatorType.createSerializer(getExecutionEnvironment().getConfig()));
        opDescription = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + udfName + ")";
        operator = new WindowOperator<>(windowAssigner, windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), keySel, input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), stateDesc, new InternalSingleValueProcessAllWindowFunction<>(windowFunction), trigger, allowedLateness, lateDataOutputTag);
    }
    return input.transform(opName, resultType, operator).setDescription(opDescription).forceNonParallel();
}