use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.
the class ResultPartitionTest method createResultPartition.
private BufferWritingResultPartition createResultPartition(ResultPartitionType partitionType) throws IOException {
NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNumNetworkBuffers(10).setBufferSize(bufferSize).build();
ResultPartition resultPartition = createPartition(network, fileChannelManager, partitionType, 2);
resultPartition.setup();
return (BufferWritingResultPartition) resultPartition;
}
use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.
the class ResultPartitionTest method testReleaseMemoryOnPipelinedPartition.
/**
* Tests {@link ResultPartition#close()} and {@link ResultPartition#release()} on a working
* pipelined partition.
*/
@Test
public void testReleaseMemoryOnPipelinedPartition() throws Exception {
final int numAllBuffers = 10;
final NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNumNetworkBuffers(numAllBuffers).setBufferSize(bufferSize).build();
final ResultPartition resultPartition = createPartition(network, ResultPartitionType.PIPELINED, 1);
try {
resultPartition.setup();
// take all buffers (more than the minimum required)
for (int i = 0; i < numAllBuffers; ++i) {
resultPartition.emitRecord(ByteBuffer.allocate(bufferSize - 1), 0);
}
assertEquals(0, resultPartition.getBufferPool().getNumberOfAvailableMemorySegments());
resultPartition.close();
assertTrue(resultPartition.getBufferPool().isDestroyed());
assertEquals(numAllBuffers, network.getNetworkBufferPool().getNumberOfUsedMemorySegments());
resultPartition.release();
assertEquals(0, network.getNetworkBufferPool().getNumberOfUsedMemorySegments());
} finally {
network.close();
}
}
use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.
the class InterruptSensitiveRestoreTest method createTask.
// ------------------------------------------------------------------------
// Utilities
// ------------------------------------------------------------------------
private static Task createTask(StreamConfig streamConfig, Configuration taskConfig, StreamStateHandle state, int mode) throws IOException {
ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
Collection<KeyedStateHandle> keyedStateFromBackend = Collections.emptyList();
Collection<KeyedStateHandle> keyedStateFromStream = Collections.emptyList();
Collection<OperatorStateHandle> operatorStateBackend = Collections.emptyList();
Collection<OperatorStateHandle> operatorStateStream = Collections.emptyList();
Map<String, OperatorStateHandle.StateMetaInfo> operatorStateMetadata = new HashMap<>(1);
OperatorStateHandle.StateMetaInfo metaInfo = new OperatorStateHandle.StateMetaInfo(new long[] { 0 }, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE);
operatorStateMetadata.put(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME, metaInfo);
KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(new KeyGroupRange(0, 0));
Collection<OperatorStateHandle> operatorStateHandles = Collections.singletonList(new OperatorStreamStateHandle(operatorStateMetadata, state));
List<KeyedStateHandle> keyedStateHandles = Collections.singletonList(new KeyGroupsStateHandle(keyGroupRangeOffsets, state));
switch(mode) {
case OPERATOR_MANAGED:
operatorStateBackend = operatorStateHandles;
break;
case OPERATOR_RAW:
operatorStateStream = operatorStateHandles;
break;
case KEYED_MANAGED:
keyedStateFromBackend = keyedStateHandles;
break;
case KEYED_RAW:
keyedStateFromStream = keyedStateHandles;
break;
default:
throw new IllegalArgumentException();
}
OperatorSubtaskState operatorSubtaskState = OperatorSubtaskState.builder().setManagedOperatorState(new StateObjectCollection<>(operatorStateBackend)).setRawOperatorState(new StateObjectCollection<>(operatorStateStream)).setManagedKeyedState(new StateObjectCollection<>(keyedStateFromBackend)).setRawKeyedState(new StateObjectCollection<>(keyedStateFromStream)).build();
JobVertexID jobVertexID = new JobVertexID();
OperatorID operatorID = OperatorID.fromJobVertexID(jobVertexID);
streamConfig.setOperatorID(operatorID);
TaskStateSnapshot stateSnapshot = new TaskStateSnapshot();
stateSnapshot.putSubtaskStateByOperatorID(operatorID, operatorSubtaskState);
JobManagerTaskRestore taskRestore = new JobManagerTaskRestore(1L, stateSnapshot);
JobInformation jobInformation = new JobInformation(new JobID(), "test job name", new SerializedValue<>(new ExecutionConfig()), new Configuration(), Collections.emptyList(), Collections.emptyList());
TaskInformation taskInformation = new TaskInformation(jobVertexID, "test task name", 1, 1, SourceStreamTask.class.getName(), taskConfig);
TestTaskStateManager taskStateManager = TestTaskStateManager.builder().setReportedCheckpointId(taskRestore.getRestoreCheckpointId()).setJobManagerTaskStateSnapshotsByCheckpointId(Collections.singletonMap(taskRestore.getRestoreCheckpointId(), taskRestore.getTaskStateSnapshot())).build();
return new Task(jobInformation, taskInformation, new ExecutionAttemptID(), new AllocationID(), 0, 0, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList(), mock(MemoryManager.class), mock(IOManager.class), shuffleEnvironment, new KvStateService(new KvStateRegistry(), null, null), mock(BroadcastVariableManager.class), new TaskEventDispatcher(), ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES, taskStateManager, mock(TaskManagerActions.class), mock(InputSplitProvider.class), mock(CheckpointResponder.class), new NoOpTaskOperatorEventGateway(), new TestGlobalAggregateManager(), TestingClassLoaderLease.newBuilder().build(), new FileCache(new String[] { EnvironmentInformation.getTemporaryFileDirectory() }, VoidPermanentBlobService.INSTANCE), new TestingTaskManagerRuntimeInfo(), UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(), new NoOpResultPartitionConsumableNotifier(), mock(PartitionProducerStateChecker.class), mock(Executor.class));
}
use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.
the class TaskExecutorTest method setup.
@Before
public void setup() throws IOException {
rpc = new TestingRpcService();
configuration = new Configuration();
TaskExecutorResourceUtils.adjustForLocalExecution(configuration);
unresolvedTaskManagerLocation = new LocalUnresolvedTaskManagerLocation();
jobId = new JobID();
jobId2 = new JobID();
testingFatalErrorHandler = new TestingFatalErrorHandler();
haServices = new TestingHighAvailabilityServices();
resourceManagerLeaderRetriever = new SettableLeaderRetrievalService();
jobManagerLeaderRetriever = new SettableLeaderRetrievalService();
jobManagerLeaderRetriever2 = new SettableLeaderRetrievalService();
haServices.setResourceManagerLeaderRetriever(resourceManagerLeaderRetriever);
haServices.setJobMasterLeaderRetriever(jobId, jobManagerLeaderRetriever);
haServices.setJobMasterLeaderRetriever(jobId2, jobManagerLeaderRetriever2);
nettyShuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
}
use of org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder in project flink by apache.
the class TaskAsyncCallTest method createQueuesAndActors.
@Before
public void createQueuesAndActors() {
numCalls = 1000;
awaitLatch = new OneShotLatch();
triggerLatch = new OneShotLatch();
shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
}
Aggregations