Examples with StreamStateHandle org.apache.flink.runtime.state.StreamStateHandle used on opensource projects

Example 11 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class FsCheckpointStateOutputStreamTest method testEmptyState.

@Test
public void testEmptyState() throws Exception {
    FsCheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(TEMP_DIR_PATH, FileSystem.getLocalFileSystem(), 1024, 512);
    StreamStateHandle handle = stream.closeAndGetHandle();
    assertTrue(handle == null);
}

Example 12 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class GenericWriteAheadSink method saveHandleInState.

/**
	 * Called when a checkpoint barrier arrives. It closes any open streams to the backend
	 * and marks them as pending for committing to the external, third-party storage system.
	 *
	 * @param checkpointId the id of the latest received checkpoint.
	 * @throws IOException in case something went wrong when handling the stream to the backend.
	 */
private void saveHandleInState(final long checkpointId, final long timestamp) throws Exception {
    //only add handle if a new OperatorState was created since the last snapshot
    if (out != null) {
        int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask();
        StreamStateHandle handle = out.closeAndGetHandle();
        PendingCheckpoint pendingCheckpoint = new PendingCheckpoint(checkpointId, subtaskIdx, timestamp, handle);
        if (pendingCheckpoints.contains(pendingCheckpoint)) {
            //we already have a checkpoint stored for that ID that may have been partially written,
            //so we discard this "alternate version" and use the stored checkpoint
            handle.discardState();
        } else {
            pendingCheckpoints.add(pendingCheckpoint);
        }
        out = null;
    }
}

Example 13 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class AccumulatingAlignedProcessingTimeWindowOperatorTest method checkpointRestoreWithPendingWindowTumblingWithProcessFunction.

@Test
public void checkpointRestoreWithPendingWindowTumblingWithProcessFunction() {
    try {
        final int windowSize = 200;
        // tumbling window that triggers every 200 milliseconds
        AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op = new AccumulatingProcessingTimeWindowOperator<>(validatingIdentityProcessFunction, identitySelector, IntSerializer.INSTANCE, IntSerializer.INSTANCE, windowSize, windowSize);
        OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(op);
        testHarness.setup();
        testHarness.open();
        testHarness.setProcessingTime(0);
        // inject some elements
        final int numElementsFirst = 700;
        final int numElements = 1000;
        for (int i = 0; i < numElementsFirst; i++) {
            testHarness.processElement(new StreamRecord<>(i));
        }
        // draw a snapshot and dispose the window
        int beforeSnapShot = testHarness.getOutput().size();
        StreamStateHandle state = testHarness.snapshotLegacy(1L, System.currentTimeMillis());
        List<Integer> resultAtSnapshot = extractFromStreamRecords(testHarness.getOutput());
        int afterSnapShot = testHarness.getOutput().size();
        assertEquals("operator performed computation during snapshot", beforeSnapShot, afterSnapShot);
        assertTrue(afterSnapShot <= numElementsFirst);
        // inject some random elements, which should not show up in the state
        for (int i = 0; i < 300; i++) {
            testHarness.processElement(new StreamRecord<>(i + numElementsFirst));
        }
        testHarness.close();
        op.dispose();
        // re-create the operator and restore the state
        op = new AccumulatingProcessingTimeWindowOperator<>(validatingIdentityProcessFunction, identitySelector, IntSerializer.INSTANCE, IntSerializer.INSTANCE, windowSize, windowSize);
        testHarness = new OneInputStreamOperatorTestHarness<>(op);
        testHarness.setup();
        testHarness.restore(state);
        testHarness.open();
        // inject some more elements
        for (int i = numElementsFirst; i < numElements; i++) {
            testHarness.processElement(new StreamRecord<>(i));
        }
        testHarness.setProcessingTime(400);
        // get and verify the result
        List<Integer> finalResult = new ArrayList<>();
        finalResult.addAll(resultAtSnapshot);
        List<Integer> finalPartialResult = extractFromStreamRecords(testHarness.getOutput());
        finalResult.addAll(finalPartialResult);
        assertEquals(numElements, finalResult.size());
        Collections.sort(finalResult);
        for (int i = 0; i < numElements; i++) {
            assertEquals(i, finalResult.get(i).intValue());
        }
        testHarness.close();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 14 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class AccumulatingAlignedProcessingTimeWindowOperatorTest method checkpointRestoreWithPendingWindowSliding.

@Test
public void checkpointRestoreWithPendingWindowSliding() {
    try {
        final int factor = 4;
        final int windowSlide = 50;
        final int windowSize = factor * windowSlide;
        // sliding window (200 msecs) every 50 msecs
        AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op = new AccumulatingProcessingTimeWindowOperator<>(validatingIdentityFunction, identitySelector, IntSerializer.INSTANCE, IntSerializer.INSTANCE, windowSize, windowSlide);
        OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(op);
        testHarness.setProcessingTime(0);
        testHarness.setup();
        testHarness.open();
        // inject some elements
        final int numElements = 1000;
        final int numElementsFirst = 700;
        for (int i = 0; i < numElementsFirst; i++) {
            testHarness.processElement(new StreamRecord<>(i));
        }
        // draw a snapshot
        List<Integer> resultAtSnapshot = extractFromStreamRecords(testHarness.getOutput());
        int beforeSnapShot = testHarness.getOutput().size();
        StreamStateHandle state = testHarness.snapshotLegacy(1L, System.currentTimeMillis());
        int afterSnapShot = testHarness.getOutput().size();
        assertEquals("operator performed computation during snapshot", beforeSnapShot, afterSnapShot);
        assertTrue(resultAtSnapshot.size() <= factor * numElementsFirst);
        // inject the remaining elements - these should not influence the snapshot
        for (int i = numElementsFirst; i < numElements; i++) {
            testHarness.processElement(new StreamRecord<>(i));
        }
        testHarness.close();
        // re-create the operator and restore the state
        op = new AccumulatingProcessingTimeWindowOperator<>(validatingIdentityFunction, identitySelector, IntSerializer.INSTANCE, IntSerializer.INSTANCE, windowSize, windowSlide);
        testHarness = new OneInputStreamOperatorTestHarness<>(op);
        testHarness.setup();
        testHarness.restore(state);
        testHarness.open();
        // inject again the remaining elements
        for (int i = numElementsFirst; i < numElements; i++) {
            testHarness.processElement(new StreamRecord<>(i));
        }
        testHarness.setProcessingTime(50);
        testHarness.setProcessingTime(100);
        testHarness.setProcessingTime(150);
        testHarness.setProcessingTime(200);
        testHarness.setProcessingTime(250);
        testHarness.setProcessingTime(300);
        testHarness.setProcessingTime(350);
        // get and verify the result
        List<Integer> finalResult = new ArrayList<>(resultAtSnapshot);
        List<Integer> finalPartialResult = extractFromStreamRecords(testHarness.getOutput());
        finalResult.addAll(finalPartialResult);
        assertEquals(factor * numElements, finalResult.size());
        Collections.sort(finalResult);
        for (int i = 0; i < factor * numElements; i++) {
            assertEquals(i / factor, finalResult.get(i).intValue());
        }
        testHarness.close();
        op.dispose();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 15 with StreamStateHandle

use of org.apache.flink.runtime.state.StreamStateHandle in project flink by apache.

the class AccumulatingAlignedProcessingTimeWindowOperatorTest method checkpointRestoreWithPendingWindowTumbling.

@Test
public void checkpointRestoreWithPendingWindowTumbling() {
    try {
        final int windowSize = 200;
        // tumbling window that triggers every 200 milliseconds
        AccumulatingProcessingTimeWindowOperator<Integer, Integer, Integer> op = new AccumulatingProcessingTimeWindowOperator<>(validatingIdentityFunction, identitySelector, IntSerializer.INSTANCE, IntSerializer.INSTANCE, windowSize, windowSize);
        OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(op);
        testHarness.setup();
        testHarness.open();
        testHarness.setProcessingTime(0);
        // inject some elements
        final int numElementsFirst = 700;
        final int numElements = 1000;
        for (int i = 0; i < numElementsFirst; i++) {
            testHarness.processElement(new StreamRecord<>(i));
        }
        // draw a snapshot and dispose the window
        int beforeSnapShot = testHarness.getOutput().size();
        StreamStateHandle state = testHarness.snapshotLegacy(1L, System.currentTimeMillis());
        List<Integer> resultAtSnapshot = extractFromStreamRecords(testHarness.getOutput());
        int afterSnapShot = testHarness.getOutput().size();
        assertEquals("operator performed computation during snapshot", beforeSnapShot, afterSnapShot);
        assertTrue(afterSnapShot <= numElementsFirst);
        // inject some random elements, which should not show up in the state
        for (int i = 0; i < 300; i++) {
            testHarness.processElement(new StreamRecord<>(i + numElementsFirst));
        }
        testHarness.close();
        op.dispose();
        // re-create the operator and restore the state
        op = new AccumulatingProcessingTimeWindowOperator<>(validatingIdentityFunction, identitySelector, IntSerializer.INSTANCE, IntSerializer.INSTANCE, windowSize, windowSize);
        testHarness = new OneInputStreamOperatorTestHarness<>(op);
        testHarness.setup();
        testHarness.restore(state);
        testHarness.open();
        // inject some more elements
        for (int i = numElementsFirst; i < numElements; i++) {
            testHarness.processElement(new StreamRecord<>(i));
        }
        testHarness.setProcessingTime(400);
        // get and verify the result
        List<Integer> finalResult = new ArrayList<>();
        finalResult.addAll(resultAtSnapshot);
        List<Integer> finalPartialResult = extractFromStreamRecords(testHarness.getOutput());
        finalResult.addAll(finalPartialResult);
        assertEquals(numElements, finalResult.size());
        Collections.sort(finalResult);
        for (int i = 0; i < numElements; i++) {
            assertEquals(i, finalResult.get(i).intValue());
        }
        testHarness.close();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}