Examples with AbstractStateBackend org.apache.flink.runtime.state.AbstractStateBackend used on opensource projects

Example 11 with AbstractStateBackend

use of org.apache.flink.runtime.state.AbstractStateBackend in project flink by apache.

the class StreamTaskTest method testAsyncCheckpointingConcurrentCloseAfterAcknowledge.

/**
	 * FLINK-5667
	 *
	 * Tests that a concurrent cancel operation does not discard the state handles of an
	 * acknowledged checkpoint. The situation can only happen if the cancel call is executed
	 * after Environment.acknowledgeCheckpoint() and before the
	 * CloseableRegistry.unregisterClosable() call.
	 */
@Test
public void testAsyncCheckpointingConcurrentCloseAfterAcknowledge() throws Exception {
    final long checkpointId = 42L;
    final long timestamp = 1L;
    final OneShotLatch acknowledgeCheckpointLatch = new OneShotLatch();
    final OneShotLatch completeAcknowledge = new OneShotLatch();
    TaskInfo mockTaskInfo = mock(TaskInfo.class);
    when(mockTaskInfo.getTaskNameWithSubtasks()).thenReturn("foobar");
    when(mockTaskInfo.getIndexOfThisSubtask()).thenReturn(0);
    Environment mockEnvironment = mock(Environment.class);
    when(mockEnvironment.getTaskInfo()).thenReturn(mockTaskInfo);
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            acknowledgeCheckpointLatch.trigger();
            // block here so that we can issue the concurrent cancel call
            completeAcknowledge.await();
            return null;
        }
    }).when(mockEnvironment).acknowledgeCheckpoint(anyLong(), any(CheckpointMetrics.class), any(SubtaskState.class));
    StreamTask<?, AbstractStreamOperator<?>> streamTask = mock(StreamTask.class, Mockito.CALLS_REAL_METHODS);
    CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp);
    streamTask.setEnvironment(mockEnvironment);
    StreamOperator<?> streamOperator = mock(StreamOperator.class, withSettings().extraInterfaces(StreamCheckpointedOperator.class));
    KeyGroupsStateHandle managedKeyedStateHandle = mock(KeyGroupsStateHandle.class);
    KeyGroupsStateHandle rawKeyedStateHandle = mock(KeyGroupsStateHandle.class);
    OperatorStateHandle managedOperatorStateHandle = mock(OperatorStateHandle.class);
    OperatorStateHandle rawOperatorStateHandle = mock(OperatorStateHandle.class);
    OperatorSnapshotResult operatorSnapshotResult = new OperatorSnapshotResult(new DoneFuture<>(managedKeyedStateHandle), new DoneFuture<>(rawKeyedStateHandle), new DoneFuture<>(managedOperatorStateHandle), new DoneFuture<>(rawOperatorStateHandle));
    when(streamOperator.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(operatorSnapshotResult);
    StreamOperator<?>[] streamOperators = { streamOperator };
    OperatorChain<Void, AbstractStreamOperator<Void>> operatorChain = mock(OperatorChain.class);
    when(operatorChain.getAllOperators()).thenReturn(streamOperators);
    StreamStateHandle streamStateHandle = mock(StreamStateHandle.class);
    CheckpointStreamFactory.CheckpointStateOutputStream outStream = mock(CheckpointStreamFactory.CheckpointStateOutputStream.class);
    when(outStream.closeAndGetHandle()).thenReturn(streamStateHandle);
    CheckpointStreamFactory mockStreamFactory = mock(CheckpointStreamFactory.class);
    when(mockStreamFactory.createCheckpointStateOutputStream(anyLong(), anyLong())).thenReturn(outStream);
    AbstractStateBackend mockStateBackend = mock(AbstractStateBackend.class);
    when(mockStateBackend.createStreamFactory(any(JobID.class), anyString())).thenReturn(mockStreamFactory);
    Whitebox.setInternalState(streamTask, "isRunning", true);
    Whitebox.setInternalState(streamTask, "lock", new Object());
    Whitebox.setInternalState(streamTask, "operatorChain", operatorChain);
    Whitebox.setInternalState(streamTask, "cancelables", new CloseableRegistry());
    Whitebox.setInternalState(streamTask, "asyncOperationsThreadPool", Executors.newFixedThreadPool(1));
    Whitebox.setInternalState(streamTask, "configuration", new StreamConfig(new Configuration()));
    Whitebox.setInternalState(streamTask, "stateBackend", mockStateBackend);
    streamTask.triggerCheckpoint(checkpointMetaData, CheckpointOptions.forFullCheckpoint());
    acknowledgeCheckpointLatch.await();
    ArgumentCaptor<SubtaskState> subtaskStateCaptor = ArgumentCaptor.forClass(SubtaskState.class);
    // check that the checkpoint has been completed
    verify(mockEnvironment).acknowledgeCheckpoint(eq(checkpointId), any(CheckpointMetrics.class), subtaskStateCaptor.capture());
    SubtaskState subtaskState = subtaskStateCaptor.getValue();
    // check that the subtask state contains the expected state handles
    assertEquals(managedKeyedStateHandle, subtaskState.getManagedKeyedState());
    assertEquals(rawKeyedStateHandle, subtaskState.getRawKeyedState());
    assertEquals(new ChainedStateHandle<>(Collections.singletonList(managedOperatorStateHandle)), subtaskState.getManagedOperatorState());
    assertEquals(new ChainedStateHandle<>(Collections.singletonList(rawOperatorStateHandle)), subtaskState.getRawOperatorState());
    // check that the state handles have not been discarded
    verify(managedKeyedStateHandle, never()).discardState();
    verify(rawKeyedStateHandle, never()).discardState();
    verify(managedOperatorStateHandle, never()).discardState();
    verify(rawOperatorStateHandle, never()).discardState();
    streamTask.cancel();
    completeAcknowledge.trigger();
    // canceling the stream task after it has acknowledged the checkpoint should not discard
    // the state handles
    verify(managedKeyedStateHandle, never()).discardState();
    verify(rawKeyedStateHandle, never()).discardState();
    verify(managedOperatorStateHandle, never()).discardState();
    verify(rawOperatorStateHandle, never()).discardState();
}

Example 12 with AbstractStateBackend

use of org.apache.flink.runtime.state.AbstractStateBackend in project beam by apache.

the class FlinkPipelineExecutionEnvironment method createStreamExecutionEnvironment.

/**
   * If the submitted job is a stream processing job, this method creates the adequate
   * Flink {@link org.apache.flink.streaming.api.environment.StreamExecutionEnvironment} depending
   * on the user-specified options.
   */
private StreamExecutionEnvironment createStreamExecutionEnvironment() {
    LOG.info("Creating the required Streaming Environment.");
    String masterUrl = options.getFlinkMaster();
    StreamExecutionEnvironment flinkStreamEnv = null;
    // depending on the master, create the right environment.
    if (masterUrl.equals("[local]")) {
        flinkStreamEnv = StreamExecutionEnvironment.createLocalEnvironment();
    } else if (masterUrl.equals("[auto]")) {
        flinkStreamEnv = StreamExecutionEnvironment.getExecutionEnvironment();
    } else if (masterUrl.matches(".*:\\d*")) {
        String[] parts = masterUrl.split(":");
        List<String> stagingFiles = options.getFilesToStage();
        flinkStreamEnv = StreamExecutionEnvironment.createRemoteEnvironment(parts[0], Integer.parseInt(parts[1]), stagingFiles.toArray(new String[stagingFiles.size()]));
    } else {
        LOG.warn("Unrecognized Flink Master URL {}. Defaulting to [auto].", masterUrl);
        flinkStreamEnv = StreamExecutionEnvironment.getExecutionEnvironment();
    }
    // set the correct parallelism.
    if (options.getParallelism() != -1) {
        flinkStreamEnv.setParallelism(options.getParallelism());
    }
    // set parallelism in the options (required by some execution code)
    options.setParallelism(flinkStreamEnv.getParallelism());
    if (options.getObjectReuse()) {
        flinkStreamEnv.getConfig().enableObjectReuse();
    } else {
        flinkStreamEnv.getConfig().disableObjectReuse();
    }
    // default to event time
    flinkStreamEnv.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    // for the following 2 parameters, a value of -1 means that Flink will use
    // the default values as specified in the configuration.
    int numRetries = options.getNumberOfExecutionRetries();
    if (numRetries != -1) {
        flinkStreamEnv.setNumberOfExecutionRetries(numRetries);
    }
    long retryDelay = options.getExecutionRetryDelay();
    if (retryDelay != -1) {
        flinkStreamEnv.getConfig().setExecutionRetryDelay(retryDelay);
    }
    // A value of -1 corresponds to disabled checkpointing (see CheckpointConfig in Flink).
    // If the value is not -1, then the validity checks are applied.
    // By default, checkpointing is disabled.
    long checkpointInterval = options.getCheckpointingInterval();
    if (checkpointInterval != -1) {
        if (checkpointInterval < 1) {
            throw new IllegalArgumentException("The checkpoint interval must be positive");
        }
        flinkStreamEnv.enableCheckpointing(checkpointInterval, options.getCheckpointingMode());
        flinkStreamEnv.getCheckpointConfig().setCheckpointTimeout(options.getCheckpointTimeoutMillis());
        boolean externalizedCheckpoint = options.isExternalizedCheckpointsEnabled();
        boolean retainOnCancellation = options.getRetainExternalizedCheckpointsOnCancellation();
        if (externalizedCheckpoint) {
            flinkStreamEnv.getCheckpointConfig().enableExternalizedCheckpoints(retainOnCancellation ? ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION : ExternalizedCheckpointCleanup.DELETE_ON_CANCELLATION);
        }
    }
    // State backend
    final AbstractStateBackend stateBackend = options.getStateBackend();
    if (stateBackend != null) {
        flinkStreamEnv.setStateBackend(stateBackend);
    }
    return flinkStreamEnv;
}