Search in sources :

Example 1 with HighAvailabilityServices

use of org.apache.flink.runtime.highavailability.HighAvailabilityServices in project flink by apache.

the class JobManagerRunnerMockTest method setUp.

@Before
public void setUp() throws Exception {
    RpcService mockRpc = mock(RpcService.class);
    when(mockRpc.getAddress()).thenReturn("localhost");
    jobManager = mock(JobMaster.class);
    jobManagerGateway = mock(JobMasterGateway.class);
    when(jobManager.getSelf()).thenReturn(jobManagerGateway);
    when(jobManager.getRpcService()).thenReturn(mockRpc);
    PowerMockito.whenNew(JobMaster.class).withAnyArguments().thenReturn(jobManager);
    jobCompletion = new TestingOnCompletionActions();
    leaderElectionService = mock(LeaderElectionService.class);
    when(leaderElectionService.hasLeadership()).thenReturn(true);
    SubmittedJobGraphStore submittedJobGraphStore = mock(SubmittedJobGraphStore.class);
    blobStore = mock(BlobStore.class);
    HighAvailabilityServices haServices = mock(HighAvailabilityServices.class);
    when(haServices.getJobManagerLeaderElectionService(any(JobID.class))).thenReturn(leaderElectionService);
    when(haServices.getSubmittedJobGraphStore()).thenReturn(submittedJobGraphStore);
    when(haServices.createBlobStore()).thenReturn(blobStore);
    when(haServices.getRunningJobsRegistry()).thenReturn(runningJobsRegistry);
    HeartbeatServices heartbeatServices = mock(HeartbeatServices.class);
    runner = PowerMockito.spy(new JobManagerRunner(ResourceID.generate(), new JobGraph("test", new JobVertex("vertex")), mock(Configuration.class), mockRpc, haServices, heartbeatServices, JobManagerServices.fromConfiguration(new Configuration(), haServices), new MetricRegistry(MetricRegistryConfiguration.defaultMetricRegistryConfiguration()), jobCompletion, jobCompletion));
}
Also used : HeartbeatServices(org.apache.flink.runtime.heartbeat.HeartbeatServices) SubmittedJobGraphStore(org.apache.flink.runtime.jobmanager.SubmittedJobGraphStore) MetricRegistryConfiguration(org.apache.flink.runtime.metrics.MetricRegistryConfiguration) Configuration(org.apache.flink.configuration.Configuration) MetricRegistry(org.apache.flink.runtime.metrics.MetricRegistry) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) JobVertex(org.apache.flink.runtime.jobgraph.JobVertex) HighAvailabilityServices(org.apache.flink.runtime.highavailability.HighAvailabilityServices) RpcService(org.apache.flink.runtime.rpc.RpcService) LeaderElectionService(org.apache.flink.runtime.leaderelection.LeaderElectionService) BlobStore(org.apache.flink.runtime.blob.BlobStore) JobID(org.apache.flink.api.common.JobID) Before(org.junit.Before)

Example 2 with HighAvailabilityServices

use of org.apache.flink.runtime.highavailability.HighAvailabilityServices in project flink by apache.

the class TaskExecutorTest method testTaskSubmission.

/**
	 * Tests that we can submit a task to the TaskManager given that we've allocated a slot there.
	 */
@Test(timeout = 1000L)
public void testTaskSubmission() throws Exception {
    final Configuration configuration = new Configuration();
    final TestingSerialRpcService rpc = new TestingSerialRpcService();
    final TaskManagerConfiguration taskManagerConfiguration = TaskManagerConfiguration.fromConfiguration(configuration);
    final JobID jobId = new JobID();
    final AllocationID allocationId = new AllocationID();
    final UUID jobManagerLeaderId = UUID.randomUUID();
    final JobVertexID jobVertexId = new JobVertexID();
    JobInformation jobInformation = new JobInformation(jobId, name.getMethodName(), new SerializedValue<>(new ExecutionConfig()), new Configuration(), Collections.<BlobKey>emptyList(), Collections.<URL>emptyList());
    TaskInformation taskInformation = new TaskInformation(jobVertexId, "test task", 1, 1, TestInvokable.class.getName(), new Configuration());
    SerializedValue<JobInformation> serializedJobInformation = new SerializedValue<>(jobInformation);
    SerializedValue<TaskInformation> serializedJobVertexInformation = new SerializedValue<>(taskInformation);
    final TaskDeploymentDescriptor tdd = new TaskDeploymentDescriptor(serializedJobInformation, serializedJobVertexInformation, new ExecutionAttemptID(), allocationId, 0, 0, 0, null, Collections.<ResultPartitionDeploymentDescriptor>emptyList(), Collections.<InputGateDeploymentDescriptor>emptyList());
    final LibraryCacheManager libraryCacheManager = mock(LibraryCacheManager.class);
    when(libraryCacheManager.getClassLoader(eq(jobId))).thenReturn(getClass().getClassLoader());
    final JobManagerConnection jobManagerConnection = new JobManagerConnection(jobId, ResourceID.generate(), mock(JobMasterGateway.class), jobManagerLeaderId, mock(TaskManagerActions.class), mock(CheckpointResponder.class), libraryCacheManager, mock(ResultPartitionConsumableNotifier.class), mock(PartitionProducerStateChecker.class));
    final JobManagerTable jobManagerTable = new JobManagerTable();
    jobManagerTable.put(jobId, jobManagerConnection);
    final TaskSlotTable taskSlotTable = mock(TaskSlotTable.class);
    when(taskSlotTable.existsActiveSlot(eq(jobId), eq(allocationId))).thenReturn(true);
    when(taskSlotTable.addTask(any(Task.class))).thenReturn(true);
    final NetworkEnvironment networkEnvironment = mock(NetworkEnvironment.class);
    when(networkEnvironment.createKvStateTaskRegistry(eq(jobId), eq(jobVertexId))).thenReturn(mock(TaskKvStateRegistry.class));
    final TaskManagerMetricGroup taskManagerMetricGroup = mock(TaskManagerMetricGroup.class);
    when(taskManagerMetricGroup.addTaskForJob(any(JobID.class), anyString(), any(JobVertexID.class), any(ExecutionAttemptID.class), anyString(), anyInt(), anyInt())).thenReturn(mock(TaskMetricGroup.class));
    final HighAvailabilityServices haServices = mock(HighAvailabilityServices.class);
    when(haServices.getResourceManagerLeaderRetriever()).thenReturn(mock(LeaderRetrievalService.class));
    try {
        final TestingFatalErrorHandler testingFatalErrorHandler = new TestingFatalErrorHandler();
        TaskExecutor taskManager = new TaskExecutor(taskManagerConfiguration, mock(TaskManagerLocation.class), rpc, mock(MemoryManager.class), mock(IOManager.class), networkEnvironment, haServices, mock(HeartbeatServices.class, RETURNS_MOCKS), mock(MetricRegistry.class), taskManagerMetricGroup, mock(BroadcastVariableManager.class), mock(FileCache.class), taskSlotTable, jobManagerTable, mock(JobLeaderService.class), testingFatalErrorHandler);
        taskManager.start();
        taskManager.submitTask(tdd, jobManagerLeaderId);
        Future<Boolean> completionFuture = TestInvokable.completableFuture;
        completionFuture.get();
        // check if a concurrent error occurred
        testingFatalErrorHandler.rethrowError();
    } finally {
        rpc.stopService();
    }
}
Also used : Task(org.apache.flink.runtime.taskmanager.Task) Configuration(org.apache.flink.configuration.Configuration) JobVertexID(org.apache.flink.runtime.jobgraph.JobVertexID) TaskKvStateRegistry(org.apache.flink.runtime.query.TaskKvStateRegistry) ExecutionConfig(org.apache.flink.api.common.ExecutionConfig) JobMasterGateway(org.apache.flink.runtime.jobmaster.JobMasterGateway) TaskManagerActions(org.apache.flink.runtime.taskmanager.TaskManagerActions) BroadcastVariableManager(org.apache.flink.runtime.broadcast.BroadcastVariableManager) TestingSerialRpcService(org.apache.flink.runtime.rpc.TestingSerialRpcService) TaskDeploymentDescriptor(org.apache.flink.runtime.deployment.TaskDeploymentDescriptor) PartitionProducerStateChecker(org.apache.flink.runtime.io.network.netty.PartitionProducerStateChecker) UUID(java.util.UUID) ResultPartitionConsumableNotifier(org.apache.flink.runtime.io.network.partition.ResultPartitionConsumableNotifier) TestingFatalErrorHandler(org.apache.flink.runtime.util.TestingFatalErrorHandler) HeartbeatServices(org.apache.flink.runtime.heartbeat.HeartbeatServices) JobInformation(org.apache.flink.runtime.executiongraph.JobInformation) TaskInformation(org.apache.flink.runtime.executiongraph.TaskInformation) ExecutionAttemptID(org.apache.flink.runtime.executiongraph.ExecutionAttemptID) IOManager(org.apache.flink.runtime.io.disk.iomanager.IOManager) CheckpointResponder(org.apache.flink.runtime.taskmanager.CheckpointResponder) TaskMetricGroup(org.apache.flink.runtime.metrics.groups.TaskMetricGroup) TaskManagerLocation(org.apache.flink.runtime.taskmanager.TaskManagerLocation) AllocationID(org.apache.flink.runtime.clusterframework.types.AllocationID) TaskManagerMetricGroup(org.apache.flink.runtime.metrics.groups.TaskManagerMetricGroup) MetricRegistry(org.apache.flink.runtime.metrics.MetricRegistry) LibraryCacheManager(org.apache.flink.runtime.execution.librarycache.LibraryCacheManager) SerializedValue(org.apache.flink.util.SerializedValue) MemoryManager(org.apache.flink.runtime.memory.MemoryManager) FileCache(org.apache.flink.runtime.filecache.FileCache) TaskSlotTable(org.apache.flink.runtime.taskexecutor.slot.TaskSlotTable) HighAvailabilityServices(org.apache.flink.runtime.highavailability.HighAvailabilityServices) TestingHighAvailabilityServices(org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices) LeaderRetrievalService(org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService) TestingLeaderRetrievalService(org.apache.flink.runtime.leaderelection.TestingLeaderRetrievalService) NetworkEnvironment(org.apache.flink.runtime.io.network.NetworkEnvironment) JobID(org.apache.flink.api.common.JobID) Test(org.junit.Test)

Aggregations

JobID (org.apache.flink.api.common.JobID)2 Configuration (org.apache.flink.configuration.Configuration)2 HeartbeatServices (org.apache.flink.runtime.heartbeat.HeartbeatServices)2 HighAvailabilityServices (org.apache.flink.runtime.highavailability.HighAvailabilityServices)2 MetricRegistry (org.apache.flink.runtime.metrics.MetricRegistry)2 UUID (java.util.UUID)1 ExecutionConfig (org.apache.flink.api.common.ExecutionConfig)1 BlobStore (org.apache.flink.runtime.blob.BlobStore)1 BroadcastVariableManager (org.apache.flink.runtime.broadcast.BroadcastVariableManager)1 AllocationID (org.apache.flink.runtime.clusterframework.types.AllocationID)1 TaskDeploymentDescriptor (org.apache.flink.runtime.deployment.TaskDeploymentDescriptor)1 LibraryCacheManager (org.apache.flink.runtime.execution.librarycache.LibraryCacheManager)1 ExecutionAttemptID (org.apache.flink.runtime.executiongraph.ExecutionAttemptID)1 JobInformation (org.apache.flink.runtime.executiongraph.JobInformation)1 TaskInformation (org.apache.flink.runtime.executiongraph.TaskInformation)1 FileCache (org.apache.flink.runtime.filecache.FileCache)1 TestingHighAvailabilityServices (org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices)1 IOManager (org.apache.flink.runtime.io.disk.iomanager.IOManager)1 NetworkEnvironment (org.apache.flink.runtime.io.network.NetworkEnvironment)1 PartitionProducerStateChecker (org.apache.flink.runtime.io.network.netty.PartitionProducerStateChecker)1