Examples with RpcResultPartitionConsumableNotifier org.apache.flink.runtime.taskexecutor.rpc.RpcResultPartitionConsumableNotifier used on opensource projects

Example 1 with RpcResultPartitionConsumableNotifier

use of org.apache.flink.runtime.taskexecutor.rpc.RpcResultPartitionConsumableNotifier in project flink by apache.

the class TaskExecutor method associateWithJobManager.

private JobTable.Connection associateWithJobManager(JobTable.Job job, ResourceID resourceID, JobMasterGateway jobMasterGateway) {
    checkNotNull(resourceID);
    checkNotNull(jobMasterGateway);
    TaskManagerActions taskManagerActions = new TaskManagerActionsImpl(jobMasterGateway);
    CheckpointResponder checkpointResponder = new RpcCheckpointResponder(jobMasterGateway);
    GlobalAggregateManager aggregateManager = new RpcGlobalAggregateManager(jobMasterGateway);
    ResultPartitionConsumableNotifier resultPartitionConsumableNotifier = new RpcResultPartitionConsumableNotifier(jobMasterGateway, getRpcService().getScheduledExecutor(), taskManagerConfiguration.getRpcTimeout());
    PartitionProducerStateChecker partitionStateChecker = new RpcPartitionStateChecker(jobMasterGateway);
    registerQueryableState(job.getJobId(), jobMasterGateway);
    return job.connect(resourceID, jobMasterGateway, taskManagerActions, checkpointResponder, aggregateManager, resultPartitionConsumableNotifier, partitionStateChecker);
}

Example 2 with RpcResultPartitionConsumableNotifier

use of org.apache.flink.runtime.taskexecutor.rpc.RpcResultPartitionConsumableNotifier in project flink by apache.

the class TaskSubmissionTestEnvironment method registerJobMasterConnection.

static void registerJobMasterConnection(JobTable jobTable, JobID jobId, RpcService testingRpcService, JobMasterGateway jobMasterGateway, TaskManagerActions taskManagerActions, Time timeout, MainThreadExecutable mainThreadExecutable) {
    mainThreadExecutable.runAsync(() -> {
        final JobTable.Job job = jobTable.getOrCreateJob(jobId, () -> TestingJobServices.newBuilder().build());
        job.connect(ResourceID.generate(), jobMasterGateway, taskManagerActions, new TestCheckpointResponder(), new TestGlobalAggregateManager(), new RpcResultPartitionConsumableNotifier(jobMasterGateway, testingRpcService.getScheduledExecutor(), timeout), TestingPartitionProducerStateChecker.newBuilder().setPartitionProducerStateFunction((jobID, intermediateDataSetID, resultPartitionID) -> CompletableFuture.completedFuture(ExecutionState.RUNNING)).build());
    });
}

Example 3 with RpcResultPartitionConsumableNotifier

use of org.apache.flink.runtime.taskexecutor.rpc.RpcResultPartitionConsumableNotifier in project flink by apache.

the class TaskExecutor method associateWithJobManager.

private JobManagerConnection associateWithJobManager(JobID jobID, ResourceID resourceID, JobMasterGateway jobMasterGateway, UUID jobManagerLeaderId, int blobPort) {
    Preconditions.checkNotNull(jobID);
    Preconditions.checkNotNull(resourceID);
    Preconditions.checkNotNull(jobManagerLeaderId);
    Preconditions.checkNotNull(jobMasterGateway);
    Preconditions.checkArgument(blobPort > 0 || blobPort < MAX_BLOB_PORT, "Blob server port is out of range.");
    TaskManagerActions taskManagerActions = new TaskManagerActionsImpl(jobManagerLeaderId, jobMasterGateway);
    CheckpointResponder checkpointResponder = new RpcCheckpointResponder(jobMasterGateway);
    InetSocketAddress blobServerAddress = new InetSocketAddress(jobMasterGateway.getHostname(), blobPort);
    final LibraryCacheManager libraryCacheManager;
    try {
        final BlobCache blobCache = new BlobCache(blobServerAddress, taskManagerConfiguration.getConfiguration(), haServices);
        libraryCacheManager = new BlobLibraryCacheManager(blobCache, taskManagerConfiguration.getCleanupInterval());
    } catch (IOException e) {
        // Can't pass the IOException up - we need a RuntimeException anyway
        // two levels up where this is run asynchronously. Also, we don't
        // know whether this is caught in the thread running this method.
        final String message = "Could not create BLOB cache or library cache.";
        log.error(message, e);
        throw new RuntimeException(message, e);
    }
    ResultPartitionConsumableNotifier resultPartitionConsumableNotifier = new RpcResultPartitionConsumableNotifier(jobManagerLeaderId, jobMasterGateway, getRpcService().getExecutor(), taskManagerConfiguration.getTimeout());
    PartitionProducerStateChecker partitionStateChecker = new RpcPartitionStateChecker(jobManagerLeaderId, jobMasterGateway);
    return new JobManagerConnection(jobID, resourceID, jobMasterGateway, jobManagerLeaderId, taskManagerActions, checkpointResponder, libraryCacheManager, resultPartitionConsumableNotifier, partitionStateChecker);
}