Examples with InternalValueState org.apache.flink.runtime.state.internal.InternalValueState used on opensource projects

Example 1 with InternalValueState

use of org.apache.flink.runtime.state.internal.InternalValueState in project flink by apache.

the class WindowOperator method open.

@Override
public void open() throws Exception {
    super.open();
    collector = new TimestampedCollector<>(output);
    collector.eraseTimestamp();
    internalTimerService = getInternalTimerService("window-timers", windowSerializer, this);
    triggerContext = new TriggerContext();
    triggerContext.open();
    StateDescriptor<ValueState<RowData>, RowData> windowStateDescriptor = new ValueStateDescriptor<>("window-aggs", new RowDataSerializer(accumulatorTypes));
    this.windowState = (InternalValueState<K, W, RowData>) getOrCreateKeyedState(windowSerializer, windowStateDescriptor);
    if (produceUpdates) {
        LogicalType[] valueTypes = ArrayUtils.addAll(aggResultTypes, windowPropertyTypes);
        StateDescriptor<ValueState<RowData>, RowData> previousStateDescriptor = new ValueStateDescriptor<>("previous-aggs", new RowDataSerializer(valueTypes));
        this.previousState = (InternalValueState<K, W, RowData>) getOrCreateKeyedState(windowSerializer, previousStateDescriptor);
    }
    compileGeneratedCode();
    WindowContext windowContext = new WindowContext();
    windowAggregator.open(new PerWindowStateDataViewStore(getKeyedStateBackend(), windowSerializer, getRuntimeContext()));
    if (windowAssigner instanceof MergingWindowAssigner) {
        this.windowFunction = new MergingWindowProcessFunction<>((MergingWindowAssigner<W>) windowAssigner, windowAggregator, windowSerializer, allowedLateness);
    } else if (windowAssigner instanceof PanedWindowAssigner) {
        this.windowFunction = new PanedWindowProcessFunction<>((PanedWindowAssigner<W>) windowAssigner, windowAggregator, allowedLateness);
    } else {
        this.windowFunction = new GeneralWindowProcessFunction<>(windowAssigner, windowAggregator, allowedLateness);
    }
    windowFunction.open(windowContext);
    // metrics
    this.numLateRecordsDropped = metrics.counter(LATE_ELEMENTS_DROPPED_METRIC_NAME);
    this.lateRecordsDroppedRate = metrics.meter(LATE_ELEMENTS_DROPPED_RATE_METRIC_NAME, new MeterView(numLateRecordsDropped));
    this.watermarkLatency = metrics.gauge(WATERMARK_LATENCY_METRIC_NAME, () -> {
        long watermark = internalTimerService.currentWatermark();
        if (watermark < 0) {
            return 0L;
        } else {
            return internalTimerService.currentProcessingTime() - watermark;
        }
    });
}

Example 2 with InternalValueState

use of org.apache.flink.runtime.state.internal.InternalValueState in project flink by apache.

the class StateBackendTestBase method testCheckConcurrencyProblemWhenPerformingCheckpointAsync.

@Test
public void testCheckConcurrencyProblemWhenPerformingCheckpointAsync() throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    ExecutorService executorService = Executors.newScheduledThreadPool(1);
    CheckpointableKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env);
    try {
        long checkpointID = 0;
        List<Future> futureList = new ArrayList();
        for (int i = 0; i < 10; ++i) {
            ValueStateDescriptor<Integer> kvId = new ValueStateDescriptor<>("id" + i, IntSerializer.INSTANCE);
            ValueState<Integer> state = backend.getOrCreateKeyedState(VoidNamespaceSerializer.INSTANCE, kvId);
            ((InternalValueState) state).setCurrentNamespace(VoidNamespace.INSTANCE);
            backend.setCurrentKey(i);
            state.update(i);
            futureList.add(runSnapshotAsync(executorService, backend.snapshot(checkpointID++, System.currentTimeMillis(), streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation())));
        }
        for (Future future : futureList) {
            future.get(20, TimeUnit.SECONDS);
        }
    } catch (Exception e) {
        fail();
    } finally {
        IOUtils.closeQuietly(backend);
        backend.dispose();
        executorService.shutdown();
    }
}

Example 3 with InternalValueState

use of org.apache.flink.runtime.state.internal.InternalValueState in project flink by apache.

the class StateBackendTestBase method testBackendUsesRegisteredKryoDefaultSerializerUsingGetOrCreate.

@Test
@SuppressWarnings("unchecked")
public void testBackendUsesRegisteredKryoDefaultSerializerUsingGetOrCreate() throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    CheckpointableKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env);
    try {
        // cast because our test serializer is not typed to TestPojo
        env.getExecutionConfig().addDefaultKryoSerializer(TestPojo.class, (Class) ExceptionThrowingTestSerializer.class);
        TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
        // make sure that we are in fact using the KryoSerializer
        assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
        pojoType.createSerializer(env.getExecutionConfig());
        ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id", pojoType);
        ValueState<TestPojo> state = backend.getOrCreateKeyedState(VoidNamespaceSerializer.INSTANCE, kvId);
        assertTrue(state instanceof InternalValueState);
        ((InternalValueState) state).setCurrentNamespace(VoidNamespace.INSTANCE);
        // we will be expecting ExpectedKryoTestException to be thrown,
        // because the ExceptionThrowingTestSerializer should be used
        int numExceptions = 0;
        backend.setCurrentKey(1);
        try {
            // backends that eagerly serializes (such as RocksDB) will fail here
            state.update(new TestPojo("u1", 1));
        } catch (ExpectedKryoTestException e) {
            numExceptions++;
        } catch (Exception e) {
            if (e.getCause() instanceof ExpectedKryoTestException) {
                numExceptions++;
            } else {
                throw e;
            }
        }
        try {
            // backends that lazily serializes (such as memory state backend) will fail here
            runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        } catch (ExpectedKryoTestException e) {
            numExceptions++;
        } catch (Exception e) {
            if (e.getCause() instanceof ExpectedKryoTestException) {
                numExceptions++;
            } else {
                throw e;
            }
        }
        assertTrue("Didn't see the expected Kryo exception.", numExceptions > 0);
    } finally {
        IOUtils.closeQuietly(backend);
        backend.dispose();
    }
}

Example 4 with InternalValueState

use of org.apache.flink.runtime.state.internal.InternalValueState in project flink by apache.

the class AbstractWindowAggProcessor method open.

@Override
public void open(Context<Long> context) throws Exception {
    this.ctx = context;
    final LongSerializer namespaceSerializer = LongSerializer.INSTANCE;
    ValueState<RowData> state = ctx.getKeyedStateBackend().getOrCreateKeyedState(namespaceSerializer, new ValueStateDescriptor<>("window-aggs", accSerializer));
    this.windowState = new WindowValueState<>((InternalValueState<RowData, Long, RowData>) state);
    this.clockService = ClockService.of(ctx.getTimerService());
    this.windowTimerService = new WindowTimerServiceImpl(ctx.getTimerService(), shiftTimeZone);
    this.aggregator = genAggsHandler.newInstance(ctx.getRuntimeContext().getUserCodeClassLoader());
    this.aggregator.open(new PerWindowStateDataViewStore(ctx.getKeyedStateBackend(), namespaceSerializer, ctx.getRuntimeContext()));
    this.windowBuffer = windowBufferFactory.create(ctx.getOperatorOwner(), ctx.getMemoryManager(), ctx.getMemorySize(), ctx.getRuntimeContext(), windowTimerService, ctx.getKeyedStateBackend(), windowState, isEventTime, shiftTimeZone);
    this.reuseOutput = new JoinedRowData();
    this.currentProgress = Long.MIN_VALUE;
    this.nextTriggerProgress = Long.MIN_VALUE;
}

Example 5 with InternalValueState

use of org.apache.flink.runtime.state.internal.InternalValueState in project flink by apache.

the class StateBackendTestBase method testBackendUsesRegisteredKryoSerializerUsingGetOrCreate.

@Test
@SuppressWarnings("unchecked")
public void testBackendUsesRegisteredKryoSerializerUsingGetOrCreate() throws Exception {
    CheckpointStreamFactory streamFactory = createStreamFactory();
    SharedStateRegistry sharedStateRegistry = new SharedStateRegistryImpl();
    env.getExecutionConfig().registerTypeWithKryoSerializer(TestPojo.class, ExceptionThrowingTestSerializer.class);
    TypeInformation<TestPojo> pojoType = new GenericTypeInfo<>(TestPojo.class);
    // make sure that we are in fact using the KryoSerializer
    assertTrue(pojoType.createSerializer(env.getExecutionConfig()) instanceof KryoSerializer);
    ValueStateDescriptor<TestPojo> kvId = new ValueStateDescriptor<>("id", pojoType);
    CheckpointableKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env);
    try {
        ValueState<TestPojo> state = backend.getOrCreateKeyedState(VoidNamespaceSerializer.INSTANCE, kvId);
        assertTrue(state instanceof InternalValueState);
        ((InternalValueState) state).setCurrentNamespace(VoidNamespace.INSTANCE);
        // we will be expecting ExpectedKryoTestException to be thrown,
        // because the ExceptionThrowingTestSerializer should be used
        int numExceptions = 0;
        backend.setCurrentKey(1);
        try {
            // backends that eagerly serializes (such as RocksDB) will fail here
            state.update(new TestPojo("u1", 1));
        } catch (ExpectedKryoTestException e) {
            numExceptions++;
        } catch (Exception e) {
            if (e.getCause() instanceof ExpectedKryoTestException) {
                numExceptions++;
            } else {
                throw e;
            }
        }
        try {
            // backends that lazily serializes (such as memory state backend) will fail here
            runSnapshot(backend.snapshot(682375462378L, 2, streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()), sharedStateRegistry);
        } catch (ExpectedKryoTestException e) {
            numExceptions++;
        } catch (Exception e) {
            if (e.getCause() instanceof ExpectedKryoTestException) {
                numExceptions++;
            } else {
                throw e;
            }
        }
        assertTrue("Didn't see the expected Kryo exception.", numExceptions > 0);
    } finally {
        IOUtils.closeQuietly(backend);
        backend.dispose();
    }
}