Examples with OperatorSubtaskState org.apache.flink.runtime.checkpoint.OperatorSubtaskState used on opensource projects

Example 76 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class CheckpointMessagesTest method testConfirmTaskCheckpointed.

@Test
public void testConfirmTaskCheckpointed() {
    final Random rnd = new Random();
    try {
        AcknowledgeCheckpoint noState = new AcknowledgeCheckpoint(new JobID(), new ExecutionAttemptID(), 569345L);
        KeyGroupRange keyGroupRange = KeyGroupRange.of(42, 42);
        TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot();
        OperatorSubtaskState subtaskState = OperatorSubtaskState.builder().setManagedOperatorState(generatePartitionableStateHandle(new JobVertexID(), 0, 2, 8, false)).setManagedKeyedState(generateKeyGroupState(keyGroupRange, Collections.singletonList(new MyHandle()))).setInputChannelState(singleton(createNewInputChannelStateHandle(10, rnd))).setResultSubpartitionState(singleton(createNewResultSubpartitionStateHandle(10, rnd))).build();
        checkpointStateHandles.putSubtaskStateByOperatorID(new OperatorID(), subtaskState);
        AcknowledgeCheckpoint withState = new AcknowledgeCheckpoint(new JobID(), new ExecutionAttemptID(), 87658976143L, new CheckpointMetrics(), checkpointStateHandles);
        testSerializabilityEqualsHashCode(noState);
        testSerializabilityEqualsHashCode(withState);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 77 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class HeapTimersSnapshottingTest method testSerializingTimersInRawStateForCheckpoints.

@Test
public void testSerializingTimersInRawStateForCheckpoints() throws Exception {
    try (KeyedOneInputStreamOperatorTestHarness<Integer, Integer, Integer> testHarness = getTestHarness()) {
        RocksDBStateBackend backend = new RocksDBStateBackend(temporaryFolder.newFolder().toURI());
        backend.setPriorityQueueStateType(PriorityQueueStateType.HEAP);
        testHarness.setStateBackend(backend);
        testHarness.open();
        testHarness.processElement(0, 0L);
        OperatorSubtaskState state = testHarness.snapshotWithLocalState(0L, 1L, CheckpointType.CHECKPOINT).getJobManagerOwnedState();
        assertThat(state.getRawKeyedState().isEmpty(), equalTo(false));
    }
}

Example 78 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class RocksDBAsyncSnapshotTest method testFullyAsyncSnapshot.

/**
 * This ensures that asynchronous state handles are actually materialized asynchronously.
 *
 * <p>We use latches to block at various stages and see if the code still continues through the
 * parts that are not asynchronous. If the checkpoint is not done asynchronously the test will
 * simply lock forever.
 */
@Test
public void testFullyAsyncSnapshot() throws Exception {
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    testHarness.configureForKeyedStream(new KeySelector<String, String>() {

        @Override
        public String getKey(String value) throws Exception {
            return value;
        }
    }, BasicTypeInfo.STRING_TYPE_INFO);
    StreamConfig streamConfig = testHarness.getStreamConfig();
    File dbDir = temporaryFolder.newFolder();
    RocksDBStateBackend backend = new RocksDBStateBackend(new MemoryStateBackend());
    backend.setDbStoragePath(dbDir.getAbsolutePath());
    streamConfig.setStateBackend(backend);
    streamConfig.setStreamOperator(new AsyncCheckpointOperator());
    streamConfig.setOperatorID(new OperatorID());
    final OneShotLatch delayCheckpointLatch = new OneShotLatch();
    final OneShotLatch ensureCheckpointLatch = new OneShotLatch();
    CheckpointResponder checkpointResponderMock = new CheckpointResponder() {

        @Override
        public void acknowledgeCheckpoint(JobID jobID, ExecutionAttemptID executionAttemptID, long checkpointId, CheckpointMetrics checkpointMetrics, TaskStateSnapshot subtaskState) {
            // even though the async checkpoint would not finish
            try {
                delayCheckpointLatch.await();
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
            boolean hasManagedKeyedState = false;
            for (Map.Entry<OperatorID, OperatorSubtaskState> entry : subtaskState.getSubtaskStateMappings()) {
                OperatorSubtaskState state = entry.getValue();
                if (state != null) {
                    hasManagedKeyedState |= state.getManagedKeyedState() != null;
                }
            }
            // should be one k/v state
            assertTrue(hasManagedKeyedState);
            // we now know that the checkpoint went through
            ensureCheckpointLatch.trigger();
        }

        @Override
        public void reportCheckpointMetrics(JobID jobID, ExecutionAttemptID executionAttemptID, long checkpointId, CheckpointMetrics checkpointMetrics) {
        }

        @Override
        public void declineCheckpoint(JobID jobID, ExecutionAttemptID executionAttemptID, long checkpointId, CheckpointException checkpointException) {
        }
    };
    JobID jobID = new JobID();
    ExecutionAttemptID executionAttemptID = new ExecutionAttemptID();
    TestTaskStateManager taskStateManagerTestMock = new TestTaskStateManager(jobID, executionAttemptID, checkpointResponderMock, TestLocalRecoveryConfig.disabled(), new InMemoryStateChangelogStorage(), new HashMap<>(), -1L, new OneShotLatch());
    StreamMockEnvironment mockEnv = new StreamMockEnvironment(testHarness.jobConfig, testHarness.taskConfig, testHarness.memorySize, new MockInputSplitProvider(), testHarness.bufferSize, taskStateManagerTestMock);
    AtomicReference<Throwable> errorRef = new AtomicReference<>();
    mockEnv.setExternalExceptionHandler(errorRef::set);
    testHarness.invoke(mockEnv);
    testHarness.waitForTaskRunning();
    final OneInputStreamTask<String, String> task = testHarness.getTask();
    task.triggerCheckpointAsync(new CheckpointMetaData(42, 17), CheckpointOptions.forCheckpointWithDefaultLocation()).get();
    testHarness.processElement(new StreamRecord<>("Wohoo", 0));
    // now we allow the checkpoint
    delayCheckpointLatch.trigger();
    // wait for the checkpoint to go through
    ensureCheckpointLatch.await();
    testHarness.endInput();
    ExecutorService threadPool = task.getAsyncOperationsThreadPool();
    threadPool.shutdown();
    Assert.assertTrue(threadPool.awaitTermination(60_000, TimeUnit.MILLISECONDS));
    testHarness.waitForTaskCompletion();
    if (errorRef.get() != null) {
        fail("Unexpected exception during execution.");
    }
}

Example 79 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class TwoPhaseCommitSinkFunctionTest method testFailBeforeNotify.

@Test
public void testFailBeforeNotify() throws Exception {
    harness.open();
    harness.processElement("42", 0);
    harness.snapshot(0, 1);
    harness.processElement("43", 2);
    OperatorSubtaskState snapshot = harness.snapshot(1, 3);
    tmpDirectory.setWritable(false);
    try {
        harness.processElement("44", 4);
        harness.snapshot(2, 5);
        fail("something should fail");
    } catch (Exception ex) {
        if (!(ex.getCause() instanceof ContentDump.NotWritableException)) {
            throw ex;
        }
    // ignore
    }
    closeTestHarness();
    tmpDirectory.setWritable(true);
    setUpTestHarness();
    harness.initializeState(snapshot);
    assertExactlyOnce(Arrays.asList("42", "43"));
    closeTestHarness();
    assertEquals(0, tmpDirectory.listFiles().size());
}

Example 80 with OperatorSubtaskState

use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.

the class TwoPhaseCommitSinkFunctionTest method testIgnoreCommitExceptionDuringRecovery.

@Test
public void testIgnoreCommitExceptionDuringRecovery() throws Exception {
    clock.setEpochMilli(0);
    harness.open();
    harness.processElement("42", 0);
    final OperatorSubtaskState snapshot = harness.snapshot(0, 1);
    harness.notifyOfCompletedCheckpoint(1);
    throwException.set(true);
    closeTestHarness();
    setUpTestHarness();
    final long transactionTimeout = 1000;
    sinkFunction.setTransactionTimeout(transactionTimeout);
    sinkFunction.ignoreFailuresAfterTransactionTimeout();
    try {
        harness.initializeState(snapshot);
        fail("Expected exception not thrown");
    } catch (RuntimeException e) {
        assertEquals("Expected exception", e.getMessage());
    }
    clock.setEpochMilli(transactionTimeout + 1);
    harness.initializeState(snapshot);
    assertExactlyOnce(Collections.singletonList("42"));
}