Examples with ListStateDescriptor org.apache.flink.api.common.state.ListStateDescriptor used on opensource projects

Example 41 with ListStateDescriptor

use of org.apache.flink.api.common.state.ListStateDescriptor in project flink by apache.

the class WindowTranslationTest method testProcessProcessingTime.

@Test
@SuppressWarnings("rawtypes")
public void testProcessProcessingTime() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));
    DataStream<Tuple2<String, Integer>> window1 = source.keyBy(new TupleKeySelector()).window(TumblingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS))).process(new ProcessWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, String, TimeWindow>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void process(String key, Context ctx, Iterable<Tuple2<String, Integer>> values, Collector<Tuple2<String, Integer>> out) throws Exception {
            for (Tuple2<String, Integer> in : values) {
                out.collect(in);
            }
        }
    });
    OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
    OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
    Assert.assertTrue(operator instanceof WindowOperator);
    WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
    Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger);
    Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingProcessingTimeWindows);
    Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);
    processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}

Example 42 with ListStateDescriptor

use of org.apache.flink.api.common.state.ListStateDescriptor in project beam by apache.

the class UnboundedSourceWrapper method initializeState.

@Override
public void initializeState(FunctionInitializationContext context) throws Exception {
    if (checkpointCoder == null) {
        // no checkpoint coder available in this source
        return;
    }
    OperatorStateStore stateStore = context.getOperatorStateStore();
    CoderTypeInformation<KV<? extends UnboundedSource<OutputT, CheckpointMarkT>, CheckpointMarkT>> typeInformation = (CoderTypeInformation) new CoderTypeInformation<>(checkpointCoder);
    stateForCheckpoint = stateStore.getOperatorState(new ListStateDescriptor<>(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME, typeInformation.createSerializer(new ExecutionConfig())));
    if (context.isRestored()) {
        isRestored = true;
        LOG.info("Having restore state in the UnbounedSourceWrapper.");
    } else {
        LOG.info("No restore state for UnbounedSourceWrapper.");
    }
}

Example 43 with ListStateDescriptor

use of org.apache.flink.api.common.state.ListStateDescriptor in project flink by apache.

the class RocksDBListStateTest method testAddAndGet.

// ------------------------------------------------------------------------
@Test
public void testAddAndGet() throws Exception {
    final ListStateDescriptor<Long> stateDescr = new ListStateDescriptor<>("my-state", Long.class);
    stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());
    final RocksDBStateBackend backend = new RocksDBStateBackend(tmp.newFolder().toURI());
    backend.setDbStoragePath(tmp.newFolder().getAbsolutePath());
    final RocksDBKeyedStateBackend<String> keyedBackend = createKeyedBackend(backend);
    try {
        InternalListState<VoidNamespace, Long> state = keyedBackend.createListState(VoidNamespaceSerializer.INSTANCE, stateDescr);
        state.setCurrentNamespace(VoidNamespace.INSTANCE);
        keyedBackend.setCurrentKey("abc");
        assertNull(state.get());
        keyedBackend.setCurrentKey("def");
        assertNull(state.get());
        state.add(17L);
        state.add(11L);
        assertEquals(asList(17L, 11L), state.get());
        keyedBackend.setCurrentKey("abc");
        assertNull(state.get());
        keyedBackend.setCurrentKey("g");
        assertNull(state.get());
        state.add(1L);
        state.add(2L);
        keyedBackend.setCurrentKey("def");
        assertEquals(asList(17L, 11L), state.get());
        state.clear();
        assertNull(state.get());
        keyedBackend.setCurrentKey("g");
        state.add(3L);
        state.add(2L);
        state.add(1L);
        keyedBackend.setCurrentKey("def");
        assertNull(state.get());
        keyedBackend.setCurrentKey("g");
        assertEquals(asList(1L, 2L, 3L, 2L, 1L), state.get());
    } finally {
        keyedBackend.close();
        keyedBackend.dispose();
    }
}

Example 44 with ListStateDescriptor

use of org.apache.flink.api.common.state.ListStateDescriptor in project flink by apache.

the class HeapKeyedStateBackendSnapshotMigrationTest method testRestore1_2ToMaster.

/**
	 * [FLINK-5979]
	 *
	 * This test takes a snapshot that was created with Flink 1.2 and tries to restore it in master to check
	 * the backwards compatibility of the serialization format of {@link StateTable}s.
	 */
@Test
public void testRestore1_2ToMaster() throws Exception {
    ClassLoader cl = getClass().getClassLoader();
    URL resource = cl.getResource("heap_keyed_statebackend_1_2.snapshot");
    Preconditions.checkNotNull(resource, "Binary snapshot resource not found!");
    final Integer namespace1 = 1;
    final Integer namespace2 = 2;
    final Integer namespace3 = 3;
    try (final HeapKeyedStateBackend<String> keyedBackend = createKeyedBackend()) {
        final KeyGroupsStateHandle stateHandle;
        try (BufferedInputStream bis = new BufferedInputStream((new FileInputStream(resource.getFile())))) {
            stateHandle = InstantiationUtil.deserializeObject(bis, Thread.currentThread().getContextClassLoader());
        }
        keyedBackend.restore(Collections.singleton(stateHandle));
        final ListStateDescriptor<Long> stateDescr = new ListStateDescriptor<>("my-state", Long.class);
        stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());
        InternalListState<Integer, Long> state = keyedBackend.createListState(IntSerializer.INSTANCE, stateDescr);
        assertEquals(7, keyedBackend.numStateEntries());
        keyedBackend.setCurrentKey("abc");
        state.setCurrentNamespace(namespace1);
        assertEquals(asList(33L, 55L), state.get());
        state.setCurrentNamespace(namespace2);
        assertEquals(asList(22L, 11L), state.get());
        state.setCurrentNamespace(namespace3);
        assertEquals(Collections.singletonList(44L), state.get());
        keyedBackend.setCurrentKey("def");
        state.setCurrentNamespace(namespace1);
        assertEquals(asList(11L, 44L), state.get());
        state.setCurrentNamespace(namespace3);
        assertEquals(asList(22L, 55L, 33L), state.get());
        keyedBackend.setCurrentKey("jkl");
        state.setCurrentNamespace(namespace1);
        assertEquals(asList(11L, 22L, 33L, 44L, 55L), state.get());
        keyedBackend.setCurrentKey("mno");
        state.setCurrentNamespace(namespace3);
        assertEquals(asList(11L, 22L, 33L, 44L, 55L), state.get());
    }
}

Example 45 with ListStateDescriptor

use of org.apache.flink.api.common.state.ListStateDescriptor in project flink by apache.

the class AllWindowedStream method aggregate.

/**
	 * Applies the given window function to each window. The window function is called for each
	 * evaluation of the window for each key individually. The output of the window function is
	 * interpreted as a regular non-windowed stream.
	 *
	 * <p>Arriving data is incrementally aggregated using the given aggregate function. This means
	 * that the window function typically has only a single value to process when called.
	 *
	 * @param aggregateFunction The aggregation function that is used for incremental aggregation.
	 * @param windowFunction The process window function.
	 * @param accumulatorType Type information for the internal accumulator type of the aggregation function
	 * @param resultType Type information for the result type of the window function
	 *
	 * @return The data stream that is the result of applying the window function to the window.
	 *
	 * @param <ACC> The type of the AggregateFunction's accumulator
	 * @param <V> The type of AggregateFunction's result, and the WindowFunction's input
	 * @param <R> The type of the elements in the resulting stream, equal to the
	 *            WindowFunction's result type
	 */
@PublicEvolving
public <ACC, V, R> SingleOutputStreamOperator<R> aggregate(AggregateFunction<T, ACC, V> aggregateFunction, ProcessAllWindowFunction<V, R, W> windowFunction, TypeInformation<ACC> accumulatorType, TypeInformation<V> aggregateResultType, TypeInformation<R> resultType) {
    checkNotNull(aggregateFunction, "aggregateFunction");
    checkNotNull(windowFunction, "windowFunction");
    checkNotNull(accumulatorType, "accumulatorType");
    checkNotNull(aggregateResultType, "aggregateResultType");
    checkNotNull(resultType, "resultType");
    if (aggregateFunction instanceof RichFunction) {
        throw new UnsupportedOperationException("This aggregate function cannot be a RichFunction.");
    }
    //clean the closures
    windowFunction = input.getExecutionEnvironment().clean(windowFunction);
    aggregateFunction = input.getExecutionEnvironment().clean(aggregateFunction);
    final String callLocation = Utils.getCallLocationName();
    final String udfName = "AllWindowedStream." + callLocation;
    final String opName;
    final KeySelector<T, Byte> keySel = input.getKeySelector();
    OneInputStreamOperator<T, R> operator;
    if (evictor != null) {
        @SuppressWarnings({ "unchecked", "rawtypes" }) TypeSerializer<StreamRecord<T>> streamRecordSerializer = (TypeSerializer<StreamRecord<T>>) new StreamElementSerializer(input.getType().createSerializer(getExecutionEnvironment().getConfig()));
        ListStateDescriptor<StreamRecord<T>> stateDesc = new ListStateDescriptor<>("window-contents", streamRecordSerializer);
        opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + evictor + ", " + udfName + ")";
        operator = new EvictingWindowOperator<>(windowAssigner, windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), keySel, input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), stateDesc, new InternalAggregateProcessAllWindowFunction<>(aggregateFunction, windowFunction), trigger, evictor, allowedLateness, lateDataOutputTag);
    } else {
        AggregatingStateDescriptor<T, ACC, V> stateDesc = new AggregatingStateDescriptor<>("window-contents", aggregateFunction, accumulatorType.createSerializer(getExecutionEnvironment().getConfig()));
        opName = "TriggerWindow(" + windowAssigner + ", " + stateDesc + ", " + trigger + ", " + udfName + ")";
        operator = new WindowOperator<>(windowAssigner, windowAssigner.getWindowSerializer(getExecutionEnvironment().getConfig()), keySel, input.getKeyType().createSerializer(getExecutionEnvironment().getConfig()), stateDesc, new InternalSingleValueProcessAllWindowFunction<>(windowFunction), trigger, allowedLateness, lateDataOutputTag);
    }
    return input.transform(opName, resultType, operator).forceNonParallel();
}