Examples with CheckpointMetricsBuilder org.apache.flink.runtime.checkpoint.CheckpointMetricsBuilder used on opensource projects

Example 6 with CheckpointMetricsBuilder

use of org.apache.flink.runtime.checkpoint.CheckpointMetricsBuilder in project flink by apache.

the class SubtaskCheckpointCoordinatorTest method testSavepointNotResultingInPriorityEvents.

@Test
public void testSavepointNotResultingInPriorityEvents() throws Exception {
    MockEnvironment mockEnvironment = MockEnvironment.builder().build();
    try (SubtaskCheckpointCoordinator coordinator = new MockSubtaskCheckpointCoordinatorBuilder().setUnalignedCheckpointEnabled(true).setEnvironment(mockEnvironment).build()) {
        AtomicReference<Boolean> broadcastedPriorityEvent = new AtomicReference<>(null);
        final OperatorChain<?, ?> operatorChain = new RegularOperatorChain(new MockStreamTaskBuilder(mockEnvironment).build(), new NonRecordWriter<>()) {

            @Override
            public void broadcastEvent(AbstractEvent event, boolean isPriorityEvent) throws IOException {
                super.broadcastEvent(event, isPriorityEvent);
                broadcastedPriorityEvent.set(isPriorityEvent);
            }
        };
        coordinator.checkpointState(new CheckpointMetaData(0, 0), new CheckpointOptions(SavepointType.savepoint(SavepointFormatType.CANONICAL), CheckpointStorageLocationReference.getDefault()), new CheckpointMetricsBuilder(), operatorChain, false, () -> true);
        assertEquals(false, broadcastedPriorityEvent.get());
    }
}

Example 7 with CheckpointMetricsBuilder

use of org.apache.flink.runtime.checkpoint.CheckpointMetricsBuilder in project flink by apache.

the class SubtaskCheckpointCoordinatorTest method testForceAlignedCheckpointResultingInPriorityEvents.

@Test
public void testForceAlignedCheckpointResultingInPriorityEvents() throws Exception {
    final long checkpointId = 42L;
    MockEnvironment mockEnvironment = MockEnvironment.builder().build();
    try (SubtaskCheckpointCoordinator coordinator = new MockSubtaskCheckpointCoordinatorBuilder().setUnalignedCheckpointEnabled(true).setEnvironment(mockEnvironment).build()) {
        AtomicReference<Boolean> broadcastedPriorityEvent = new AtomicReference<>(null);
        final OperatorChain<?, ?> operatorChain = new RegularOperatorChain(new MockStreamTaskBuilder(mockEnvironment).build(), new NonRecordWriter<>()) {

            @Override
            public void broadcastEvent(AbstractEvent event, boolean isPriorityEvent) throws IOException {
                super.broadcastEvent(event, isPriorityEvent);
                broadcastedPriorityEvent.set(isPriorityEvent);
                // test if we can write output data
                coordinator.getChannelStateWriter().addOutputData(checkpointId, new ResultSubpartitionInfo(0, 0), 0, BufferBuilderTestUtils.buildSomeBuffer(500));
            }
        };
        CheckpointOptions forcedAlignedOptions = CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, CheckpointStorageLocationReference.getDefault()).withUnalignedUnsupported();
        coordinator.checkpointState(new CheckpointMetaData(checkpointId, 0), forcedAlignedOptions, new CheckpointMetricsBuilder(), operatorChain, false, () -> true);
        assertEquals(true, broadcastedPriorityEvent.get());
    }
}

Example 8 with CheckpointMetricsBuilder

use of org.apache.flink.runtime.checkpoint.CheckpointMetricsBuilder in project flink by apache.

the class StreamTask method triggerCheckpointAsyncInMailbox.

private boolean triggerCheckpointAsyncInMailbox(CheckpointMetaData checkpointMetaData, CheckpointOptions checkpointOptions) throws Exception {
    FlinkSecurityManager.monitorUserSystemExitForCurrentThread();
    try {
        latestAsyncCheckpointStartDelayNanos = 1_000_000 * Math.max(0, System.currentTimeMillis() - checkpointMetaData.getTimestamp());
        // No alignment if we inject a checkpoint
        CheckpointMetricsBuilder checkpointMetrics = new CheckpointMetricsBuilder().setAlignmentDurationNanos(0L).setBytesProcessedDuringAlignment(0L).setCheckpointStartDelayNanos(latestAsyncCheckpointStartDelayNanos);
        subtaskCheckpointCoordinator.initInputsCheckpoint(checkpointMetaData.getCheckpointId(), checkpointOptions);
        boolean success = performCheckpoint(checkpointMetaData, checkpointOptions, checkpointMetrics);
        if (!success) {
            declineCheckpoint(checkpointMetaData.getCheckpointId());
        }
        return success;
    } catch (Exception e) {
        // propagate exceptions only if the task is still in "running" state
        if (isRunning) {
            throw new Exception("Could not perform checkpoint " + checkpointMetaData.getCheckpointId() + " for operator " + getName() + '.', e);
        } else {
            LOG.debug("Could not perform checkpoint {} for operator {} while the " + "invokable was not in state running.", checkpointMetaData.getCheckpointId(), getName(), e);
            return false;
        }
    } finally {
        FlinkSecurityManager.unmonitorUserSystemExitForCurrentThread();
    }
}

Example 9 with CheckpointMetricsBuilder

use of org.apache.flink.runtime.checkpoint.CheckpointMetricsBuilder in project flink by apache.

the class StreamTaskTest method testSyncSavepointWithEndInput.

/**
 * Test for SyncSavepoint and EndInput interactions. Targets following scenarios scenarios:
 *
 * <ol>
 *   <li>Thread1: notify sync savepoint
 *   <li>Thread2: endInput
 *   <li>Thread1: confirm/abort/abortAsync
 *   <li>assert inputEnded: confirmed - no, abort/abortAsync - yes
 * </ol>
 */
private void testSyncSavepointWithEndInput(BiConsumerWithException<StreamTask<?, ?>, Long, IOException> savepointResult, SnapshotType checkpointType, boolean expectEndInput) throws Exception {
    StreamTaskMailboxTestHarness<String> harness = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, STRING_TYPE_INFO).addInput(STRING_TYPE_INFO).setupOutputForSingletonOperatorChain(new TestBoundedOneInputStreamOperator()).build();
    final long checkpointId = 1L;
    CountDownLatch savepointTriggeredLatch = new CountDownLatch(1);
    CountDownLatch inputEndedLatch = new CountDownLatch(1);
    MailboxExecutor executor = harness.streamTask.getMailboxExecutorFactory().createExecutor(MAX_PRIORITY);
    executor.execute(() -> {
        try {
            harness.streamTask.triggerCheckpointOnBarrier(new CheckpointMetaData(checkpointId, checkpointId), new CheckpointOptions(checkpointType, getDefault()), new CheckpointMetricsBuilder());
        } catch (IOException e) {
            fail(e.getMessage());
        }
    }, "triggerCheckpointOnBarrier");
    new Thread(() -> {
        try {
            savepointTriggeredLatch.await();
            harness.endInput(expectEndInput);
            inputEndedLatch.countDown();
        } catch (InterruptedException e) {
            fail(e.getMessage());
        }
    }).start();
    // this mails should be executed from the one above (from triggerCheckpointOnBarrier)
    executor.execute(savepointTriggeredLatch::countDown, "savepointTriggeredLatch");
    executor.execute(() -> {
        inputEndedLatch.await();
        savepointResult.accept(harness.streamTask, checkpointId);
    }, "savepointResult");
    harness.processAll();
    Assert.assertEquals(expectEndInput, TestBoundedOneInputStreamOperator.isInputEnded());
}

Example 10 with CheckpointMetricsBuilder

use of org.apache.flink.runtime.checkpoint.CheckpointMetricsBuilder in project flink by apache.

the class StreamTaskTest method testCheckpointFailueOnClosedOperator.

/**
 * Tests exeptions is thrown by triggering checkpoint if operators are closed. This was
 * initially implemented for FLINK-16383. However after FLINK-2491 operators lifecycle has
 * changed and now we: (1) redefined close() to dispose(). After closing operators, there should
 * be no opportunity to invoke anything on the task. close() mentioned in FLINK-16383 is now
 * more like finish(). (2) We support triggering and performing checkpoints if operators are
 * finished.
 */
@Test
public void testCheckpointFailueOnClosedOperator() throws Throwable {
    ClosingOperator<Integer> operator = new ClosingOperator<>();
    StreamTaskMailboxTestHarnessBuilder<Integer> builder = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.INT_TYPE_INFO);
    try (StreamTaskMailboxTestHarness<Integer> harness = builder.setupOutputForSingletonOperatorChain(operator).build()) {
        // keeps the mailbox from suspending
        harness.setAutoProcess(false);
        harness.processElement(new StreamRecord<>(1));
        harness.streamTask.operatorChain.finishOperators(harness.streamTask.getActionExecutor(), StopMode.DRAIN);
        harness.streamTask.operatorChain.closeAllOperators();
        assertTrue(ClosingOperator.closed.get());
        harness.streamTask.triggerCheckpointOnBarrier(new CheckpointMetaData(1, 0), CheckpointOptions.forCheckpointWithDefaultLocation(), new CheckpointMetricsBuilder());
    } catch (Exception ex) {
        ExceptionUtils.assertThrowableWithMessage(ex, "OperatorChain and Task should never be closed at this point");
    }
}