Examples with JobResult org.apache.flink.runtime.jobmaster.JobResult used on opensource projects

Example 51 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class ApplicationDispatcherBootstrap method getJobResult.

private CompletableFuture<JobResult> getJobResult(final DispatcherGateway dispatcherGateway, final JobID jobId, final ScheduledExecutor scheduledExecutor, final boolean tolerateMissingResult) {
    final Time timeout = Time.milliseconds(configuration.get(ClientOptions.CLIENT_TIMEOUT).toMillis());
    final Time retryPeriod = Time.milliseconds(configuration.get(ClientOptions.CLIENT_RETRY_PERIOD).toMillis());
    final CompletableFuture<JobResult> jobResultFuture = JobStatusPollingUtils.getJobResult(dispatcherGateway, jobId, scheduledExecutor, timeout, retryPeriod);
    if (tolerateMissingResult) {
        // Return "unknown" job result if dispatcher no longer knows the actual result.
        return FutureUtils.handleException(jobResultFuture, FlinkJobNotFoundException.class, exception -> new JobResult.Builder().jobId(jobId).applicationStatus(ApplicationStatus.UNKNOWN).netRuntime(Long.MAX_VALUE).build());
    }
    return jobResultFuture;
}

Example 52 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class JobGraphRunningUtil method execute.

public static void execute(JobGraph jobGraph, Configuration configuration, int numTaskManagers, int numSlotsPerTaskManager) throws Exception {
    configuration.set(TaskManagerOptions.TOTAL_FLINK_MEMORY, MemorySize.parse("1g"));
    configuration.setString(RestOptions.BIND_PORT, "0");
    final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder().setConfiguration(configuration).setNumTaskManagers(numTaskManagers).setNumSlotsPerTaskManager(numSlotsPerTaskManager).build();
    try (MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration)) {
        miniCluster.start();
        MiniClusterClient miniClusterClient = new MiniClusterClient(configuration, miniCluster);
        // wait for the submission to succeed
        JobID jobID = miniClusterClient.submitJob(jobGraph).get();
        JobResult jobResult = miniClusterClient.requestJobResult(jobID).get();
        if (jobResult.getSerializedThrowable().isPresent()) {
            throw new AssertionError(jobResult.getSerializedThrowable().get());
        }
    }
}

Example 53 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class ZooKeeperLeaderElectionITCase method testJobExecutionOnClusterWithLeaderChange.

/**
 * Tests that a job can be executed after a new leader has been elected. For all except for the
 * last leader, the job is blocking. The JobManager will be terminated while executing the
 * blocking job. Once only one JobManager is left, it is checked that a non-blocking can be
 * successfully executed.
 */
@Test
@Ignore("FLINK-25235")
public void testJobExecutionOnClusterWithLeaderChange() throws Exception {
    final int numDispatchers = 3;
    final int numTMs = 2;
    final int numSlotsPerTM = 2;
    final Configuration configuration = ZooKeeperTestUtils.createZooKeeperHAConfig(zkServer.getConnectString(), tempFolder.newFolder().getAbsolutePath());
    // speed up refused registration retries
    configuration.setLong(ClusterOptions.REFUSED_REGISTRATION_DELAY, 50L);
    final TestingMiniClusterConfiguration miniClusterConfiguration = TestingMiniClusterConfiguration.newBuilder().setConfiguration(configuration).setNumberDispatcherResourceManagerComponents(numDispatchers).setNumTaskManagers(numTMs).setNumSlotsPerTaskManager(numSlotsPerTM).build();
    final Deadline timeout = Deadline.fromNow(TEST_TIMEOUT);
    try (TestingMiniCluster miniCluster = TestingMiniCluster.newBuilder(miniClusterConfiguration).build();
        final CuratorFrameworkWithUnhandledErrorListener curatorFramework = ZooKeeperUtils.startCuratorFramework(configuration, exception -> fail("Fatal error in curator framework."))) {
        // We need to watch for resource manager leader changes to avoid race conditions.
        final DefaultLeaderRetrievalService resourceManagerLeaderRetrieval = ZooKeeperUtils.createLeaderRetrievalService(curatorFramework.asCuratorFramework(), ZooKeeperUtils.getLeaderPathForResourceManager(), configuration);
        @SuppressWarnings("unchecked") final CompletableFuture<String>[] resourceManagerLeaderFutures = (CompletableFuture<String>[]) new CompletableFuture[numDispatchers];
        for (int i = 0; i < numDispatchers; i++) {
            resourceManagerLeaderFutures[i] = new CompletableFuture<>();
        }
        resourceManagerLeaderRetrieval.start(new TestLeaderRetrievalListener(resourceManagerLeaderFutures));
        miniCluster.start();
        final int parallelism = numTMs * numSlotsPerTM;
        JobGraph jobGraph = createJobGraph(parallelism);
        miniCluster.submitJob(jobGraph).get();
        String previousLeaderAddress = null;
        for (int i = 0; i < numDispatchers - 1; i++) {
            final DispatcherGateway leaderDispatcherGateway = getNextLeadingDispatcherGateway(miniCluster, previousLeaderAddress, timeout);
            // Make sure resource manager has also changed leadership.
            resourceManagerLeaderFutures[i].get();
            previousLeaderAddress = leaderDispatcherGateway.getAddress();
            awaitRunningStatus(leaderDispatcherGateway, jobGraph, timeout);
            leaderDispatcherGateway.shutDownCluster();
        }
        final DispatcherGateway leaderDispatcherGateway = getNextLeadingDispatcherGateway(miniCluster, previousLeaderAddress, timeout);
        // Make sure resource manager has also changed leadership.
        resourceManagerLeaderFutures[numDispatchers - 1].get();
        awaitRunningStatus(leaderDispatcherGateway, jobGraph, timeout);
        CompletableFuture<JobResult> jobResultFuture = leaderDispatcherGateway.requestJobResult(jobGraph.getJobID(), RPC_TIMEOUT);
        BlockingOperator.unblock();
        assertThat(jobResultFuture.get().isSuccess(), is(true));
        resourceManagerLeaderRetrieval.stop();
    }
}

Example 54 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class PipelinedRegionSchedulingITCase method executeSchedulingTest.

private JobResult executeSchedulingTest(JobGraph jobGraph, int numSlots, Configuration configuration) throws Exception {
    configuration.setString(RestOptions.BIND_PORT, "0");
    configuration.setLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT, 5000L);
    final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder().setConfiguration(configuration).setNumTaskManagers(1).setNumSlotsPerTaskManager(numSlots).build();
    try (MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration)) {
        miniCluster.start();
        final MiniClusterClient miniClusterClient = new MiniClusterClient(configuration, miniCluster);
        // wait for the submission to succeed
        final JobID jobID = miniClusterClient.submitJob(jobGraph).get();
        final CompletableFuture<JobResult> resultFuture = miniClusterClient.requestJobResult(jobID);
        final JobResult jobResult = resultFuture.get();
        return jobResult;
    }
}

Example 55 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class PipelinedRegionSchedulingITCase method testSuccessWithSlotsNoFewerThanTheMaxRegionRequired.

@Test
public void testSuccessWithSlotsNoFewerThanTheMaxRegionRequired() throws Exception {
    final JobResult jobResult = executeSchedulingTest(2);
    assertThat(jobResult.getSerializedThrowable().isPresent(), is(false));
}