Examples with JobResult org.apache.flink.runtime.jobmaster.JobResult used on opensource projects

Example 46 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class MiniCluster method executeJobBlocking.

/**
 * This method runs a job in blocking mode. The method returns only after the job completed
 * successfully, or after it failed terminally.
 *
 * @param job The Flink job to execute
 * @return The result of the job execution
 * @throws JobExecutionException Thrown if anything went amiss during initial job launch, or if
 *     the job terminally failed.
 */
public JobExecutionResult executeJobBlocking(JobGraph job) throws JobExecutionException, InterruptedException {
    checkNotNull(job, "job is null");
    final CompletableFuture<JobSubmissionResult> submissionFuture = submitJob(job);
    final CompletableFuture<JobResult> jobResultFuture = submissionFuture.thenCompose((JobSubmissionResult ignored) -> requestJobResult(job.getJobID()));
    final JobResult jobResult;
    try {
        jobResult = jobResultFuture.get();
    } catch (ExecutionException e) {
        throw new JobExecutionException(job.getJobID(), "Could not retrieve JobResult.", ExceptionUtils.stripExecutionException(e));
    }
    try {
        return jobResult.toJobExecutionResult(Thread.currentThread().getContextClassLoader());
    } catch (IOException | ClassNotFoundException e) {
        throw new JobExecutionException(job.getJobID(), e);
    }
}

Example 47 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class MiniClusterITCase method testCallFinalizeOnMasterBeforeJobCompletes.

@Test
public void testCallFinalizeOnMasterBeforeJobCompletes() throws Exception {
    final int parallelism = 11;
    final MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder().setNumTaskManagers(1).setNumSlotsPerTaskManager(2 * parallelism).setConfiguration(getDefaultConfiguration()).build();
    try (final MiniCluster miniCluster = new MiniCluster(cfg)) {
        miniCluster.start();
        final JobVertex source = new JobVertex("Source");
        source.setInvokableClass(WaitingNoOpInvokable.class);
        source.setParallelism(parallelism);
        WaitOnFinalizeJobVertex.resetFinalizedOnMaster();
        final WaitOnFinalizeJobVertex sink = new WaitOnFinalizeJobVertex("Sink", 20L);
        sink.setInvokableClass(NoOpInvokable.class);
        sink.setParallelism(parallelism);
        sink.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
        final JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(source, sink);
        final CompletableFuture<JobSubmissionResult> submissionFuture = miniCluster.submitJob(jobGraph);
        final CompletableFuture<JobResult> jobResultFuture = submissionFuture.thenCompose((JobSubmissionResult ignored) -> miniCluster.requestJobResult(jobGraph.getJobID()));
        jobResultFuture.get().toJobExecutionResult(getClass().getClassLoader());
        assertTrue(WaitOnFinalizeJobVertex.finalizedOnMaster.get());
    }
}

Example 48 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class MiniClusterITCase method testOutOfMemoryErrorMessageEnrichmentInJobVertexInitialization.

@Test
public void testOutOfMemoryErrorMessageEnrichmentInJobVertexInitialization() throws Exception {
    final int parallelism = 1;
    final MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder().setNumTaskManagers(1).setNumSlotsPerTaskManager(parallelism).setConfiguration(getDefaultConfiguration()).build();
    try (final MiniCluster miniCluster = new MiniCluster(cfg)) {
        miniCluster.start();
        final JobVertex failingJobVertex = new OutOfMemoryInInitializationVertex();
        failingJobVertex.setInvokableClass(NoOpInvokable.class);
        failingJobVertex.setParallelism(parallelism);
        final JobGraph jobGraph = JobGraphTestUtils.streamingJobGraph(failingJobVertex);
        final CompletableFuture<JobSubmissionResult> submissionFuture = miniCluster.submitJob(jobGraph);
        final CompletableFuture<JobResult> jobResultFuture = submissionFuture.thenCompose((JobSubmissionResult ignored) -> miniCluster.requestJobResult(jobGraph.getJobID()));
        try {
            jobResultFuture.get();
        } catch (ExecutionException e) {
            assertThat(e, FlinkMatchers.containsCause(OutOfMemoryError.class));
            assertThat(e, FlinkMatchers.containsMessage("Java heap space. A heap space-related out-of-memory error has occurred."));
        }
    }
}

Example 49 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class JobResultDeserializer method deserialize.

@Override
public JobResult deserialize(final JsonParser p, final DeserializationContext ctxt) throws IOException {
    JobID jobId = null;
    ApplicationStatus applicationStatus = ApplicationStatus.UNKNOWN;
    long netRuntime = -1;
    SerializedThrowable serializedThrowable = null;
    Map<String, SerializedValue<OptionalFailure<Object>>> accumulatorResults = null;
    while (true) {
        final JsonToken jsonToken = p.nextToken();
        assertNotEndOfInput(p, jsonToken);
        if (jsonToken == JsonToken.END_OBJECT) {
            break;
        }
        final String fieldName = p.getValueAsString();
        switch(fieldName) {
            case JobResultSerializer.FIELD_NAME_JOB_ID:
                assertNextToken(p, JsonToken.VALUE_STRING);
                jobId = jobIdDeserializer.deserialize(p, ctxt);
                break;
            case JobResultSerializer.FIELD_NAME_APPLICATION_STATUS:
                assertNextToken(p, JsonToken.VALUE_STRING);
                applicationStatus = ApplicationStatus.valueOf(p.getValueAsString().toUpperCase());
                break;
            case JobResultSerializer.FIELD_NAME_NET_RUNTIME:
                assertNextToken(p, JsonToken.VALUE_NUMBER_INT);
                netRuntime = p.getLongValue();
                break;
            case JobResultSerializer.FIELD_NAME_ACCUMULATOR_RESULTS:
                assertNextToken(p, JsonToken.START_OBJECT);
                accumulatorResults = parseAccumulatorResults(p, ctxt);
                break;
            case JobResultSerializer.FIELD_NAME_FAILURE_CAUSE:
                assertNextToken(p, JsonToken.START_OBJECT);
                serializedThrowable = serializedThrowableDeserializer.deserialize(p, ctxt);
                break;
            default:
        }
    }
    try {
        return new JobResult.Builder().jobId(jobId).applicationStatus(applicationStatus).netRuntime(netRuntime).accumulatorResults(accumulatorResults).serializedThrowable(serializedThrowable).build();
    } catch (final RuntimeException e) {
        throw new JsonMappingException(null, "Could not deserialize " + JobResult.class.getSimpleName(), e);
    }
}

Example 50 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class RestClusterClientTest method testSubmitJobAndWaitForExecutionResult.

@Test
public void testSubmitJobAndWaitForExecutionResult() throws Exception {
    final TestJobExecutionResultHandler testJobExecutionResultHandler = new TestJobExecutionResultHandler(new RestHandlerException("should trigger retry", HttpResponseStatus.SERVICE_UNAVAILABLE), JobExecutionResultResponseBody.inProgress(), // On an UNKNOWN JobResult it should be retried
    JobExecutionResultResponseBody.created(new JobResult.Builder().applicationStatus(ApplicationStatus.UNKNOWN).jobId(jobId).netRuntime(Long.MAX_VALUE).accumulatorResults(Collections.singletonMap("testName", new SerializedValue<>(OptionalFailure.of(1.0)))).build()), JobExecutionResultResponseBody.created(new JobResult.Builder().applicationStatus(ApplicationStatus.SUCCEEDED).jobId(jobId).netRuntime(Long.MAX_VALUE).accumulatorResults(Collections.singletonMap("testName", new SerializedValue<>(OptionalFailure.of(1.0)))).build()), JobExecutionResultResponseBody.created(new JobResult.Builder().applicationStatus(ApplicationStatus.FAILED).jobId(jobId).netRuntime(Long.MAX_VALUE).serializedThrowable(new SerializedThrowable(new RuntimeException("expected"))).build()));
    // Fail the first JobExecutionResult HTTP polling attempt, which should not be a problem
    // because of the retries.
    final AtomicBoolean firstExecutionResultPollFailed = new AtomicBoolean(false);
    // Fail the first JobSubmit HTTP request, which should not be a problem because of the
    // retries.
    final AtomicBoolean firstSubmitRequestFailed = new AtomicBoolean(false);
    failHttpRequest = (messageHeaders, messageParameters, requestBody) -> {
        if (messageHeaders instanceof JobExecutionResultHeaders) {
            return !firstExecutionResultPollFailed.getAndSet(true);
        }
        if (messageHeaders instanceof JobSubmitHeaders) {
            return !firstSubmitRequestFailed.getAndSet(true);
        }
        return false;
    };
    try (TestRestServerEndpoint restServerEndpoint = createRestServerEndpoint(testJobExecutionResultHandler, new TestJobSubmitHandler())) {
        try (RestClusterClient<?> restClusterClient = createRestClusterClient(restServerEndpoint.getServerAddress().getPort())) {
            final JobExecutionResult jobExecutionResult = restClusterClient.submitJob(jobGraph).thenCompose(restClusterClient::requestJobResult).get().toJobExecutionResult(ClassLoader.getSystemClassLoader());
            assertTrue(firstExecutionResultPollFailed.get());
            assertTrue(firstSubmitRequestFailed.get());
            assertThat(jobExecutionResult.getJobID(), equalTo(jobId));
            assertThat(jobExecutionResult.getNetRuntime(), equalTo(Long.MAX_VALUE));
            assertThat(jobExecutionResult.getAllAccumulatorResults(), equalTo(Collections.singletonMap("testName", 1.0)));
            try {
                restClusterClient.submitJob(jobGraph).thenCompose(restClusterClient::requestJobResult).get().toJobExecutionResult(ClassLoader.getSystemClassLoader());
                fail("Expected exception not thrown.");
            } catch (final Exception e) {
                final Optional<RuntimeException> cause = ExceptionUtils.findThrowable(e, RuntimeException.class);
                assertThat(cause.isPresent(), is(true));
                assertThat(cause.get().getMessage(), equalTo("expected"));
            }
        }
    }
}