Examples with Deadline org.apache.flink.api.common.time.Deadline used on opensource projects

Example 6 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class KinesisExampleTest method main.

public static void main(String[] args) throws Exception {
    LOG.info("System properties: {}", System.getProperties());
    final ParameterTool parameterTool = ParameterTool.fromArgs(args);
    String inputStream = parameterTool.getRequired("input-stream");
    String outputStream = parameterTool.getRequired("output-stream");
    KinesisPubsubClient pubsub = new KinesisPubsubClient(parameterTool.getProperties());
    pubsub.createTopic(inputStream, 2, parameterTool.getProperties());
    pubsub.createTopic(outputStream, 2, parameterTool.getProperties());
    // The example job needs to start after streams are created and run in parallel to the
    // validation logic.
    // The thread that runs the job won't terminate, we don't have a job reference to cancel it.
    // Once results are validated, the driver main thread will exit; job/cluster will be
    // terminated from script.
    final AtomicReference<Exception> executeException = new AtomicReference<>();
    Thread executeThread = new Thread(() -> {
        try {
            KinesisExample.main(args);
            // this message won't appear in the log,
            // job is terminated when shutting down cluster
            LOG.info("executed program");
        } catch (Exception e) {
            executeException.set(e);
        }
    });
    executeThread.start();
    // generate input
    String[] messages = { "elephant,5,45218", "squirrel,12,46213", "bee,3,51348", "squirrel,22,52444", "bee,10,53412", "elephant,9,54867" };
    for (String msg : messages) {
        pubsub.sendMessage(inputStream, msg);
    }
    LOG.info("generated records");
    Deadline deadline = Deadline.fromNow(Duration.ofSeconds(60));
    List<String> results = pubsub.readAllMessages(outputStream);
    while (deadline.hasTimeLeft() && executeException.get() == null && results.size() < messages.length) {
        LOG.info("waiting for results..");
        Thread.sleep(1000);
        results = pubsub.readAllMessages(outputStream);
    }
    if (executeException.get() != null) {
        throw executeException.get();
    }
    LOG.info("results: {}", results);
    Assert.assertEquals("Results received from '" + outputStream + "': " + results, messages.length, results.size());
    String[] expectedResults = { "elephant,5,45218", "elephant,14,54867", "squirrel,12,46213", "squirrel,34,52444", "bee,3,51348", "bee,13,53412" };
    for (String expectedResult : expectedResults) {
        Assert.assertTrue(expectedResult, results.contains(expectedResult));
    }
    // TODO: main thread needs to create job or CLI fails with:
    // "The program didn't contain a Flink job. Perhaps you forgot to call execute() on the
    // execution environment."
    System.out.println("test finished");
    System.exit(0);
}

Example 7 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class ComponentClosingUtils method shutdownExecutorForcefully.

/**
 * Shutdown the given executor forcefully within the given timeout.
 *
 * @param executor the executor to shut down.
 * @param timeout the timeout duration.
 * @param interruptable when set to true, the method can be interrupted. Each interruption to
 *     the thread results in another {@code ExecutorService.shutdownNow()} call to the shutting
 *     down executor.
 * @return true if the given executor is terminated, false otherwise.
 */
public static boolean shutdownExecutorForcefully(ExecutorService executor, Duration timeout, boolean interruptable) {
    Deadline deadline = Deadline.fromNowWithClock(timeout, clock);
    boolean isInterrupted = false;
    do {
        executor.shutdownNow();
        try {
            executor.awaitTermination(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
        } catch (InterruptedException e) {
            isInterrupted = interruptable;
        }
    } while (!isInterrupted && deadline.hasTimeLeft() && !executor.isTerminated());
    return executor.isTerminated();
}

Example 8 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class ApplicationDispatcherBootstrapITCase method testDispatcherRecoversAfterLosingAndRegainingLeadership.

@Test
public void testDispatcherRecoversAfterLosingAndRegainingLeadership() throws Exception {
    final String blockId = UUID.randomUUID().toString();
    final Deadline deadline = Deadline.fromNow(TIMEOUT);
    final Configuration configuration = new Configuration();
    configuration.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());
    configuration.set(DeploymentOptions.TARGET, EmbeddedExecutor.NAME);
    configuration.set(ClientOptions.CLIENT_RETRY_PERIOD, Duration.ofMillis(100));
    final TestingMiniClusterConfiguration clusterConfiguration = TestingMiniClusterConfiguration.newBuilder().setConfiguration(configuration).build();
    final EmbeddedHaServicesWithLeadershipControl haServices = new EmbeddedHaServicesWithLeadershipControl(TestingUtils.defaultExecutor());
    final TestingMiniCluster.Builder clusterBuilder = TestingMiniCluster.newBuilder(clusterConfiguration).setHighAvailabilityServicesSupplier(() -> haServices).setDispatcherResourceManagerComponentFactorySupplier(createApplicationModeDispatcherResourceManagerComponentFactorySupplier(clusterConfiguration.getConfiguration(), BlockingJob.getProgram(blockId)));
    try (final MiniCluster cluster = clusterBuilder.build()) {
        // start mini cluster and submit the job
        cluster.start();
        // wait until job is running
        awaitJobStatus(cluster, ApplicationDispatcherBootstrap.ZERO_JOB_ID, JobStatus.RUNNING, deadline);
        // make sure the operator is actually running
        BlockingJob.awaitRunning(blockId);
        final CompletableFuture<JobResult> firstJobResult = cluster.requestJobResult(ApplicationDispatcherBootstrap.ZERO_JOB_ID);
        haServices.revokeDispatcherLeadership();
        // make sure the leadership is revoked to avoid race conditions
        assertThat(firstJobResult.get()).extracting(JobResult::getApplicationStatus).isEqualTo(ApplicationStatus.UNKNOWN);
        haServices.grantDispatcherLeadership();
        // job is suspended, wait until it's running
        awaitJobStatus(cluster, ApplicationDispatcherBootstrap.ZERO_JOB_ID, JobStatus.RUNNING, deadline);
        // unblock processing so the job can finish
        BlockingJob.unblock(blockId);
        // and wait for it to actually finish
        final JobResult secondJobResult = cluster.requestJobResult(ApplicationDispatcherBootstrap.ZERO_JOB_ID).get();
        assertThat(secondJobResult.isSuccess()).isTrue();
        assertThat(secondJobResult.getApplicationStatus()).isEqualTo(ApplicationStatus.SUCCEEDED);
        // the cluster should shut down automatically once the application completes
        awaitClusterStopped(cluster, deadline);
    } finally {
        BlockingJob.cleanUp(blockId);
    }
}

Example 9 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class HadoopRecoverableFsDataOutputStream method waitUntilLeaseIsRevoked.

/**
 * Called when resuming execution after a failure and waits until the lease of the file we are
 * resuming is free.
 *
 * <p>The lease of the file we are resuming writing/committing to may still belong to the
 * process that failed previously and whose state we are recovering.
 *
 * @param path The path to the file we want to resume writing to.
 */
private static boolean waitUntilLeaseIsRevoked(final FileSystem fs, final Path path) throws IOException {
    Preconditions.checkState(fs instanceof DistributedFileSystem);
    final DistributedFileSystem dfs = (DistributedFileSystem) fs;
    dfs.recoverLease(path);
    final Deadline deadline = Deadline.now().plus(Duration.ofMillis(LEASE_TIMEOUT));
    boolean isClosed = dfs.isFileClosed(path);
    while (!isClosed && deadline.hasTimeLeft()) {
        try {
            Thread.sleep(500L);
        } catch (InterruptedException e1) {
            throw new IOException("Recovering the lease failed: ", e1);
        }
        isClosed = dfs.isFileClosed(path);
    }
    return isClosed;
}

Example 10 with Deadline

use of org.apache.flink.api.common.time.Deadline in project flink by apache.

the class GlueSchemaRegistryAvroKinesisITCase method testGSRAvroGenericFormatWithFlink.

@Test
public void testGSRAvroGenericFormatWithFlink() throws Exception {
    List<GenericRecord> messages = getRecords();
    for (GenericRecord msg : messages) {
        kinesisClient.sendMessage(getSchema().toString(), INPUT_STREAM, msg);
    }
    log.info("generated records");
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    DataStream<GenericRecord> input = env.addSource(createSource());
    input.addSink(createSink());
    env.executeAsync();
    Deadline deadline = Deadline.fromNow(Duration.ofSeconds(60));
    List<Object> results = kinesisClient.readAllMessages(OUTPUT_STREAM);
    while (deadline.hasTimeLeft() && results.size() < messages.size()) {
        log.info("waiting for results..");
        Thread.sleep(1000);
        results = kinesisClient.readAllMessages(OUTPUT_STREAM);
    }
    log.info("results: {}", results);
    assertThat(results).containsExactlyInAnyOrderElementsOf(messages);
}