Examples with NetworkEnvironment org.apache.flink.runtime.io.network.NetworkEnvironment used on opensource projects

Example 11 with NetworkEnvironment

use of org.apache.flink.runtime.io.network.NetworkEnvironment in project flink by apache.

the class SingleInputGateTest method testRequestBackoffConfiguration.

/**
	 * Tests request back off configuration is correctly forwarded to the channels.
	 */
@Test
public void testRequestBackoffConfiguration() throws Exception {
    ResultPartitionID[] partitionIds = new ResultPartitionID[] { new ResultPartitionID(), new ResultPartitionID(), new ResultPartitionID() };
    InputChannelDeploymentDescriptor[] channelDescs = new InputChannelDeploymentDescriptor[] { // Local
    new InputChannelDeploymentDescriptor(partitionIds[0], ResultPartitionLocation.createLocal()), // Remote
    new InputChannelDeploymentDescriptor(partitionIds[1], ResultPartitionLocation.createRemote(new ConnectionID(new InetSocketAddress("localhost", 5000), 0))), // Unknown
    new InputChannelDeploymentDescriptor(partitionIds[2], ResultPartitionLocation.createUnknown()) };
    InputGateDeploymentDescriptor gateDesc = new InputGateDeploymentDescriptor(new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, channelDescs);
    int initialBackoff = 137;
    int maxBackoff = 1001;
    NetworkEnvironment netEnv = mock(NetworkEnvironment.class);
    when(netEnv.getResultPartitionManager()).thenReturn(new ResultPartitionManager());
    when(netEnv.getTaskEventDispatcher()).thenReturn(new TaskEventDispatcher());
    when(netEnv.getPartitionRequestInitialBackoff()).thenReturn(initialBackoff);
    when(netEnv.getPartitionRequestMaxBackoff()).thenReturn(maxBackoff);
    when(netEnv.getConnectionManager()).thenReturn(new LocalConnectionManager());
    SingleInputGate gate = SingleInputGate.create("TestTask", new JobID(), new ExecutionAttemptID(), gateDesc, netEnv, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    assertEquals(gateDesc.getConsumedPartitionType(), gate.getConsumedPartitionType());
    Map<IntermediateResultPartitionID, InputChannel> channelMap = gate.getInputChannels();
    assertEquals(3, channelMap.size());
    InputChannel localChannel = channelMap.get(partitionIds[0].getPartitionId());
    assertEquals(LocalInputChannel.class, localChannel.getClass());
    InputChannel remoteChannel = channelMap.get(partitionIds[1].getPartitionId());
    assertEquals(RemoteInputChannel.class, remoteChannel.getClass());
    InputChannel unknownChannel = channelMap.get(partitionIds[2].getPartitionId());
    assertEquals(UnknownInputChannel.class, unknownChannel.getClass());
    InputChannel[] channels = new InputChannel[] { localChannel, remoteChannel, unknownChannel };
    for (InputChannel ch : channels) {
        assertEquals(0, ch.getCurrentBackoff());
        assertTrue(ch.increaseBackoff());
        assertEquals(initialBackoff, ch.getCurrentBackoff());
        assertTrue(ch.increaseBackoff());
        assertEquals(initialBackoff * 2, ch.getCurrentBackoff());
        assertTrue(ch.increaseBackoff());
        assertEquals(initialBackoff * 2 * 2, ch.getCurrentBackoff());
        assertTrue(ch.increaseBackoff());
        assertEquals(maxBackoff, ch.getCurrentBackoff());
        assertFalse(ch.increaseBackoff());
    }
}

Example 12 with NetworkEnvironment

use of org.apache.flink.runtime.io.network.NetworkEnvironment in project flink by apache.

the class BlockingCheckpointsTest method createTask.

// ------------------------------------------------------------------------
//  Utilities
// ------------------------------------------------------------------------
private static Task createTask(Configuration taskConfig) throws IOException {
    JobInformation jobInformation = new JobInformation(new JobID(), "test job name", new SerializedValue<>(new ExecutionConfig()), new Configuration(), Collections.<BlobKey>emptyList(), Collections.<URL>emptyList());
    TaskInformation taskInformation = new TaskInformation(new JobVertexID(), "test task name", 1, 11, TestStreamTask.class.getName(), taskConfig);
    TaskKvStateRegistry mockKvRegistry = mock(TaskKvStateRegistry.class);
    NetworkEnvironment network = mock(NetworkEnvironment.class);
    when(network.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))).thenReturn(mockKvRegistry);
    return new Task(jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0, 0, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), 0, null, mock(MemoryManager.class), mock(IOManager.class), network, mock(BroadcastVariableManager.class), mock(TaskManagerActions.class), mock(InputSplitProvider.class), mock(CheckpointResponder.class), new FallbackLibraryCacheManager(), new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() }), new TestingTaskManagerRuntimeInfo(), new UnregisteredTaskMetricsGroup(), mock(ResultPartitionConsumableNotifier.class), mock(PartitionProducerStateChecker.class), Executors.directExecutor());
}

Example 13 with NetworkEnvironment

use of org.apache.flink.runtime.io.network.NetworkEnvironment in project flink by apache.

the class InterruptSensitiveRestoreTest method createTask.

// ------------------------------------------------------------------------
//  Utilities
// ------------------------------------------------------------------------
private static Task createTask(Configuration taskConfig, StreamStateHandle state, int mode) throws IOException {
    NetworkEnvironment networkEnvironment = mock(NetworkEnvironment.class);
    when(networkEnvironment.createKvStateTaskRegistry(any(JobID.class), any(JobVertexID.class))).thenReturn(mock(TaskKvStateRegistry.class));
    ChainedStateHandle<StreamStateHandle> operatorState = null;
    List<KeyGroupsStateHandle> keyGroupStateFromBackend = Collections.emptyList();
    List<KeyGroupsStateHandle> keyGroupStateFromStream = Collections.emptyList();
    List<Collection<OperatorStateHandle>> operatorStateBackend = Collections.emptyList();
    List<Collection<OperatorStateHandle>> operatorStateStream = Collections.emptyList();
    Map<String, OperatorStateHandle.StateMetaInfo> operatorStateMetadata = new HashMap<>(1);
    OperatorStateHandle.StateMetaInfo metaInfo = new OperatorStateHandle.StateMetaInfo(new long[] { 0 }, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE);
    operatorStateMetadata.put(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME, metaInfo);
    KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(new KeyGroupRange(0, 0));
    Collection<OperatorStateHandle> operatorStateHandles = Collections.singletonList(new OperatorStateHandle(operatorStateMetadata, state));
    List<KeyGroupsStateHandle> keyGroupsStateHandles = Collections.singletonList(new KeyGroupsStateHandle(keyGroupRangeOffsets, state));
    switch(mode) {
        case OPERATOR_MANAGED:
            operatorStateBackend = Collections.singletonList(operatorStateHandles);
            break;
        case OPERATOR_RAW:
            operatorStateStream = Collections.singletonList(operatorStateHandles);
            break;
        case KEYED_MANAGED:
            keyGroupStateFromBackend = keyGroupsStateHandles;
            break;
        case KEYED_RAW:
            keyGroupStateFromStream = keyGroupsStateHandles;
            break;
        case LEGACY:
            operatorState = new ChainedStateHandle<>(Collections.singletonList(state));
            break;
        default:
            throw new IllegalArgumentException();
    }
    TaskStateHandles taskStateHandles = new TaskStateHandles(operatorState, operatorStateBackend, operatorStateStream, keyGroupStateFromBackend, keyGroupStateFromStream);
    JobInformation jobInformation = new JobInformation(new JobID(), "test job name", new SerializedValue<>(new ExecutionConfig()), new Configuration(), Collections.<BlobKey>emptyList(), Collections.<URL>emptyList());
    TaskInformation taskInformation = new TaskInformation(new JobVertexID(), "test task name", 1, 1, SourceStreamTask.class.getName(), taskConfig);
    return new Task(jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0, 0, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), 0, taskStateHandles, mock(MemoryManager.class), mock(IOManager.class), networkEnvironment, mock(BroadcastVariableManager.class), mock(TaskManagerActions.class), mock(InputSplitProvider.class), mock(CheckpointResponder.class), new FallbackLibraryCacheManager(), new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() }), new TestingTaskManagerRuntimeInfo(), new UnregisteredTaskMetricsGroup(), mock(ResultPartitionConsumableNotifier.class), mock(PartitionProducerStateChecker.class), mock(Executor.class));
}