Examples with TaskExecutorPartitionTrackerImpl org.apache.flink.runtime.io.network.partition.TaskExecutorPartitionTrackerImpl used on opensource projects

Example 1 with TaskExecutorPartitionTrackerImpl

use of org.apache.flink.runtime.io.network.partition.TaskExecutorPartitionTrackerImpl in project flink by apache.

the class TaskExecutorTest method createPartitionTrackerWithFixedPartitionReport.

private static TaskExecutorPartitionTracker createPartitionTrackerWithFixedPartitionReport(ShuffleEnvironment<?, ?> shuffleEnvironment) {
    final ClusterPartitionReport.ClusterPartitionReportEntry clusterPartitionReportEntry = new ClusterPartitionReport.ClusterPartitionReportEntry(new IntermediateDataSetID(), Collections.singleton(new ResultPartitionID()), 4);
    final ClusterPartitionReport clusterPartitionReport = new ClusterPartitionReport(Collections.singletonList(clusterPartitionReportEntry));
    return new TaskExecutorPartitionTrackerImpl(shuffleEnvironment) {

        @Override
        public ClusterPartitionReport createClusterPartitionReport() {
            return clusterPartitionReport;
        }
    };
}

Example 2 with TaskExecutorPartitionTrackerImpl

use of org.apache.flink.runtime.io.network.partition.TaskExecutorPartitionTrackerImpl in project flink by apache.

the class TaskManagerRunner method startTaskManager.

public static TaskExecutor startTaskManager(Configuration configuration, ResourceID resourceID, RpcService rpcService, HighAvailabilityServices highAvailabilityServices, HeartbeatServices heartbeatServices, MetricRegistry metricRegistry, TaskExecutorBlobService taskExecutorBlobService, boolean localCommunicationOnly, ExternalResourceInfoProvider externalResourceInfoProvider, WorkingDirectory workingDirectory, FatalErrorHandler fatalErrorHandler) throws Exception {
    checkNotNull(configuration);
    checkNotNull(resourceID);
    checkNotNull(rpcService);
    checkNotNull(highAvailabilityServices);
    LOG.info("Starting TaskManager with ResourceID: {}", resourceID.getStringWithMetadata());
    String externalAddress = rpcService.getAddress();
    final TaskExecutorResourceSpec taskExecutorResourceSpec = TaskExecutorResourceUtils.resourceSpecFromConfig(configuration);
    TaskManagerServicesConfiguration taskManagerServicesConfiguration = TaskManagerServicesConfiguration.fromConfiguration(configuration, resourceID, externalAddress, localCommunicationOnly, taskExecutorResourceSpec, workingDirectory);
    Tuple2<TaskManagerMetricGroup, MetricGroup> taskManagerMetricGroup = MetricUtils.instantiateTaskManagerMetricGroup(metricRegistry, externalAddress, resourceID, taskManagerServicesConfiguration.getSystemResourceMetricsProbingInterval());
    final ExecutorService ioExecutor = Executors.newFixedThreadPool(taskManagerServicesConfiguration.getNumIoThreads(), new ExecutorThreadFactory("flink-taskexecutor-io"));
    TaskManagerServices taskManagerServices = TaskManagerServices.fromConfiguration(taskManagerServicesConfiguration, taskExecutorBlobService.getPermanentBlobService(), taskManagerMetricGroup.f1, ioExecutor, fatalErrorHandler, workingDirectory);
    MetricUtils.instantiateFlinkMemoryMetricGroup(taskManagerMetricGroup.f1, taskManagerServices.getTaskSlotTable(), taskManagerServices::getManagedMemorySize);
    TaskManagerConfiguration taskManagerConfiguration = TaskManagerConfiguration.fromConfiguration(configuration, taskExecutorResourceSpec, externalAddress, workingDirectory.getTmpDirectory());
    String metricQueryServiceAddress = metricRegistry.getMetricQueryServiceGatewayRpcAddress();
    return new TaskExecutor(rpcService, taskManagerConfiguration, highAvailabilityServices, taskManagerServices, externalResourceInfoProvider, heartbeatServices, taskManagerMetricGroup.f0, metricQueryServiceAddress, taskExecutorBlobService, fatalErrorHandler, new TaskExecutorPartitionTrackerImpl(taskManagerServices.getShuffleEnvironment()));
}

Example 3 with TaskExecutorPartitionTrackerImpl

use of org.apache.flink.runtime.io.network.partition.TaskExecutorPartitionTrackerImpl in project flink by apache.

the class TaskExecutorPartitionLifecycleTest method testBlockingLocalPartitionReleaseDoesNotBlockTaskExecutor.

@Test
public void testBlockingLocalPartitionReleaseDoesNotBlockTaskExecutor() throws Exception {
    BlockerSync sync = new BlockerSync();
    ResultPartitionManager blockingResultPartitionManager = new ResultPartitionManager() {

        @Override
        public void releasePartition(ResultPartitionID partitionId, Throwable cause) {
            sync.blockNonInterruptible();
            super.releasePartition(partitionId, cause);
        }
    };
    NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().setResultPartitionManager(blockingResultPartitionManager).setIoExecutor(TEST_EXECUTOR_SERVICE_RESOURCE.getExecutor()).build();
    final CompletableFuture<ResultPartitionID> startTrackingFuture = new CompletableFuture<>();
    final TaskExecutorPartitionTracker partitionTracker = new TaskExecutorPartitionTrackerImpl(shuffleEnvironment) {

        @Override
        public void startTrackingPartition(JobID producingJobId, TaskExecutorPartitionInfo partitionInfo) {
            super.startTrackingPartition(producingJobId, partitionInfo);
            startTrackingFuture.complete(partitionInfo.getResultPartitionId());
        }
    };
    try {
        internalTestPartitionRelease(partitionTracker, shuffleEnvironment, startTrackingFuture, (jobId, resultPartitionDeploymentDescriptor, taskExecutor, taskExecutorGateway) -> {
            final IntermediateDataSetID dataSetId = resultPartitionDeploymentDescriptor.getResultId();
            taskExecutorGateway.releaseClusterPartitions(Collections.singleton(dataSetId), timeout);
            // execute some operation to check whether the TaskExecutor is blocked
            taskExecutorGateway.canBeReleased().get(5, TimeUnit.SECONDS);
        });
    } finally {
        sync.releaseBlocker();
    }
}