Examples with JobResult org.apache.flink.runtime.jobmaster.JobResult used on opensource projects

Example 26 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class AbstractQueryableStateTestBase method testDuplicateRegistrationFailsJob.

/**
 * Tests that duplicate query registrations fail the job at the JobManager.
 */
@Test(timeout = 60_000)
public void testDuplicateRegistrationFailsJob() throws Exception {
    final int numKeys = 256;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStateBackend(stateBackend);
    env.setParallelism(maxParallelism);
    // Very important, because cluster is shared between tests and we
    // don't explicitly check that all slots are available before
    // submitting.
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 1000L));
    DataStream<Tuple2<Integer, Long>> source = env.addSource(new TestKeyRangeSource(numKeys));
    // Reducing state
    ReducingStateDescriptor<Tuple2<Integer, Long>> reducingState = new ReducingStateDescriptor<>("any-name", new SumReduce(), source.getType());
    final String queryName = "duplicate-me";
    final QueryableStateStream<Integer, Tuple2<Integer, Long>> queryableState = source.keyBy(new KeySelector<Tuple2<Integer, Long>, Integer>() {

        private static final long serialVersionUID = -4126824763829132959L;

        @Override
        public Integer getKey(Tuple2<Integer, Long> value) {
            return value.f0;
        }
    }).asQueryableState(queryName, reducingState);
    final QueryableStateStream<Integer, Tuple2<Integer, Long>> duplicate = source.keyBy(new KeySelector<Tuple2<Integer, Long>, Integer>() {

        private static final long serialVersionUID = -6265024000462809436L;

        @Override
        public Integer getKey(Tuple2<Integer, Long> value) {
            return value.f0;
        }
    }).asQueryableState(queryName);
    // Submit the job graph
    final JobGraph jobGraph = env.getStreamGraph().getJobGraph();
    clusterClient.submitJob(jobGraph).thenCompose(clusterClient::requestJobResult).thenApply(JobResult::getSerializedThrowable).thenAccept(serializedThrowable -> {
        assertTrue(serializedThrowable.isPresent());
        final Throwable t = serializedThrowable.get().deserializeError(getClass().getClassLoader());
        final String failureCause = ExceptionUtils.stringifyException(t);
        assertThat(failureCause, containsString("KvState with name '" + queryName + "' has already been registered by another operator"));
    }).get();
}

Example 27 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class JobStatusPollingUtilsTest method testHappyPath.

@Test
public void testHappyPath() throws ExecutionException, InterruptedException {
    final int maxAttemptCounter = 1;
    final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
    try {
        final ScheduledExecutor scheduledExecutor = new ScheduledExecutorServiceAdapter(executor);
        final CallCountingJobStatusSupplier jobStatusSupplier = new CallCountingJobStatusSupplier(maxAttemptCounter);
        final CompletableFuture<JobResult> result = JobStatusPollingUtils.pollJobResultAsync(jobStatusSupplier, () -> CompletableFuture.completedFuture(createSuccessfulJobResult(new JobID(0, 0))), scheduledExecutor, 10);
        result.join();
        assertThat(jobStatusSupplier.getAttemptCounter(), is(equalTo(maxAttemptCounter)));
        assertTrue(result.isDone() && result.get().isSuccess());
    } finally {
        ExecutorUtils.gracefulShutdown(5, TimeUnit.SECONDS, executor);
    }
}

Example 28 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class DefaultDispatcherGatewayServiceFactory method create.

@Override
public AbstractDispatcherLeaderProcess.DispatcherGatewayService create(DispatcherId fencingToken, Collection<JobGraph> recoveredJobs, Collection<JobResult> recoveredDirtyJobResults, JobGraphWriter jobGraphWriter, JobResultStore jobResultStore) {
    final Dispatcher dispatcher;
    try {
        dispatcher = dispatcherFactory.createDispatcher(rpcService, fencingToken, recoveredJobs, recoveredDirtyJobResults, (dispatcherGateway, scheduledExecutor, errorHandler) -> new NoOpDispatcherBootstrap(), PartialDispatcherServicesWithJobPersistenceComponents.from(partialDispatcherServices, jobGraphWriter, jobResultStore));
    } catch (Exception e) {
        throw new FlinkRuntimeException("Could not create the Dispatcher rpc endpoint.", e);
    }
    dispatcher.start();
    return DefaultDispatcherGatewayService.from(dispatcher);
}

Example 29 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class AdaptiveSchedulerSlotSharingITCase method runJob.

private void runJob() throws Exception {
    final MiniCluster miniCluster = MINI_CLUSTER_RESOURCE.getMiniCluster();
    final JobGraph jobGraph = createJobGraphWithSlotSharingGroup();
    miniCluster.submitJob(jobGraph).join();
    final JobResult jobResult = miniCluster.requestJobResult(jobGraph.getJobID()).join();
    // this throws an exception if the job failed
    jobResult.toJobExecutionResult(getClass().getClassLoader());
    assertTrue(jobResult.isSuccess());
}

Example 30 with JobResult

use of org.apache.flink.runtime.jobmaster.JobResult in project flink by apache.

the class AdaptiveSchedulerClusterITCase method testAutomaticScaleUp.

@Test
public void testAutomaticScaleUp() throws Exception {
    final MiniCluster miniCluster = miniClusterResource.getMiniCluster();
    int initialInstanceCount = NUMBER_SLOTS_PER_TASK_MANAGER * NUMBER_TASK_MANAGERS;
    int targetInstanceCount = initialInstanceCount + NUMBER_SLOTS_PER_TASK_MANAGER;
    final JobGraph jobGraph = createBlockingJobGraph(targetInstanceCount);
    log.info("Submitting job with parallelism of " + targetInstanceCount + ", to a cluster with only one TM.");
    miniCluster.submitJob(jobGraph).join();
    final CompletableFuture<JobResult> jobResultFuture = miniCluster.requestJobResult(jobGraph.getJobID());
    waitUntilParallelismForVertexReached(jobGraph.getJobID(), JOB_VERTEX_ID, initialInstanceCount);
    log.info("Start additional TaskManager to scale up to the full parallelism.");
    miniCluster.startTaskManager();
    log.info("Waiting until Invokable is running with higher parallelism");
    waitUntilParallelismForVertexReached(jobGraph.getJobID(), JOB_VERTEX_ID, targetInstanceCount);
    OnceBlockingNoOpInvokable.unblock();
    assertTrue(jobResultFuture.join().isSuccess());
}