Examples with Deadline org.apache.flink.api.common.time.Deadline used on opensource projects

Example 56 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class PulsarPartitionSplitReaderBase method fetch.

@Override
public RecordsWithSplitIds<PulsarMessage<OUT>> fetch() throws IOException {
    RecordsBySplits.Builder<PulsarMessage<OUT>> builder = new RecordsBySplits.Builder<>();
    // Return when no split registered to this reader.
    if (pulsarConsumer == null || registeredSplit == null) {
        return builder.build();
    }
    // Set wakeup to false for start consuming.
    wakeup.compareAndSet(true, false);
    StopCursor stopCursor = registeredSplit.getStopCursor();
    String splitId = registeredSplit.splitId();
    PulsarMessageCollector<OUT> collector = new PulsarMessageCollector<>(splitId, builder);
    Deadline deadline = Deadline.fromNow(sourceConfiguration.getMaxFetchTime());
    // Consume message from pulsar until it was woke up by flink reader.
    for (int messageNum = 0; messageNum < sourceConfiguration.getMaxFetchRecords() && deadline.hasTimeLeft() && isNotWakeup(); messageNum++) {
        try {
            Duration timeout = deadline.timeLeftIfAny();
            Message<byte[]> message = pollMessage(timeout);
            if (message == null) {
                break;
            }
            // Deserialize message.
            collector.setMessage(message);
            deserializationSchema.deserialize(message, collector);
            // Acknowledge message if need.
            finishedPollMessage(message);
            if (stopCursor.shouldStop(message)) {
                builder.addFinishedSplit(splitId);
                break;
            }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            break;
        } catch (TimeoutException e) {
            break;
        } catch (ExecutionException e) {
            LOG.error("Error in polling message from pulsar consumer.", e);
            break;
        } catch (Exception e) {
            throw new IOException(e);
        }
    }
    return builder.build();
}

Example 57 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class JMXJobManagerMetricTest method testJobManagerJMXMetricAccess.

/**
 * Tests that metrics registered on the JobManager are actually accessible via JMX.
 */
@Test
void testJobManagerJMXMetricAccess(@InjectClusterClient ClusterClient<?> client) throws Exception {
    Deadline deadline = Deadline.now().plus(Duration.ofMinutes(2));
    try {
        JobVertex sourceJobVertex = new JobVertex("Source");
        sourceJobVertex.setInvokableClass(BlockingInvokable.class);
        sourceJobVertex.setParallelism(1);
        final JobCheckpointingSettings jobCheckpointingSettings = new JobCheckpointingSettings(new CheckpointCoordinatorConfiguration(500, 500, 50, 5, CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION, true, false, 0, 0), null);
        final JobGraph jobGraph = JobGraphBuilder.newStreamingJobGraphBuilder().setJobName("TestingJob").addJobVertex(sourceJobVertex).setJobCheckpointingSettings(jobCheckpointingSettings).build();
        client.submitJob(jobGraph).get();
        FutureUtils.retrySuccessfulWithDelay(() -> client.getJobStatus(jobGraph.getJobID()), Time.milliseconds(10), deadline, status -> status == JobStatus.RUNNING, TestingUtils.defaultScheduledExecutor()).get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
        MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer();
        Set<ObjectName> nameSet = mBeanServer.queryNames(new ObjectName("org.apache.flink.jobmanager.job.lastCheckpointSize:job_name=TestingJob,*"), null);
        assertThat(nameSet).hasSize(1);
        assertThat(mBeanServer.getAttribute(nameSet.iterator().next(), "Value")).isEqualTo(-1L);
        BlockingInvokable.unblock();
    } finally {
        BlockingInvokable.unblock();
    }
}

Example 58 with Deadline

use of org.apache.flink.api.common.time.Deadline in project beam by apache.

the class FlinkPortableClientEntryPoint method startJobService.

private void startJobService() throws Exception {
    jobInvokerFactory = new DetachedJobInvokerFactory();
    jobServer = FlinkJobServerDriver.fromConfig(FlinkJobServerDriver.parseArgs(new String[] { "--job-port=" + jobPort, "--artifact-port=0", "--expansion-port=0" }), jobInvokerFactory);
    jobServerThread = new Thread(jobServer);
    jobServerThread.start();
    Deadline deadline = Deadline.fromNow(JOB_SERVICE_STARTUP_TIMEOUT);
    while (jobServer.getJobServerUrl() == null && deadline.hasTimeLeft()) {
        try {
            Thread.sleep(500);
        } catch (InterruptedException interruptEx) {
            Thread.currentThread().interrupt();
            throw new RuntimeException(interruptEx);
        }
    }
    if (!jobServerThread.isAlive()) {
        throw new IllegalStateException("Job service thread is not alive");
    }
    if (jobServer.getJobServerUrl() == null) {
        String msg = String.format("Timeout of %s waiting for job service to start.", deadline);
        throw new TimeoutException(msg);
    }
}

Example 59 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class JobMasterTest method waitUntilAllExecutionsAreScheduledOrDeployed.

private void waitUntilAllExecutionsAreScheduledOrDeployed(final JobMasterGateway jobMasterGateway) throws Exception {
    final Duration duration = Duration.ofMillis(testingTimeout.toMilliseconds());
    final Deadline deadline = Deadline.fromNow(duration);
    CommonTestUtils.waitUntilCondition(() -> {
        final Collection<AccessExecution> executions = getExecutions(jobMasterGateway);
        return !executions.isEmpty() && executions.stream().allMatch(execution -> execution.getState() == ExecutionState.SCHEDULED || execution.getState() == ExecutionState.DEPLOYING);
    }, deadline);
}

Example 60 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class MiniClusterResource method cancelAllJobs.

private void cancelAllJobs(boolean waitUntilSlotsAreFreed) {
    try {
        final long shutdownTimeout = miniClusterResourceConfiguration.getShutdownTimeout().toMilliseconds();
        final Deadline jobCancellationDeadline = Deadline.fromNow(Duration.ofMillis(shutdownTimeout));
        final List<CompletableFuture<Acknowledge>> jobCancellationFutures = miniCluster.listJobs().get(jobCancellationDeadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS).stream().filter(status -> !status.getJobState().isGloballyTerminalState()).map(status -> miniCluster.cancelJob(status.getJobId())).collect(Collectors.toList());
        FutureUtils.waitForAll(jobCancellationFutures).get(jobCancellationDeadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
        CommonTestUtils.waitUntilCondition(() -> {
            final long unfinishedJobs = miniCluster.listJobs().get(jobCancellationDeadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS).stream().filter(status -> !status.getJobState().isGloballyTerminalState()).count();
            return unfinishedJobs == 0;
        }, jobCancellationDeadline);
        if (waitUntilSlotsAreFreed) {
            CommonTestUtils.waitUntilCondition(() -> {
                final ResourceOverview resourceOverview = miniCluster.getResourceOverview().get();
                return resourceOverview.getNumberRegisteredSlots() == resourceOverview.getNumberFreeSlots();
            }, jobCancellationDeadline);
        }
    } catch (Exception e) {
        log.warn("Exception while shutting down remaining jobs.", e);
    }
}