Examples with LocalFlinkMiniCluster org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster used on opensource projects

Example 21 with LocalFlinkMiniCluster

use of org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster in project flink by apache.

the class ManualExactlyOnceTest method main.

public static void main(String[] args) throws Exception {
    final ParameterTool pt = ParameterTool.fromArgs(args);
    LOG.info("Starting exactly once test");
    final String streamName = "flink-test-" + UUID.randomUUID().toString();
    final String accessKey = pt.getRequired("accessKey");
    final String secretKey = pt.getRequired("secretKey");
    final String region = pt.getRequired("region");
    Properties configProps = new Properties();
    configProps.setProperty(AWSConfigConstants.AWS_ACCESS_KEY_ID, accessKey);
    configProps.setProperty(AWSConfigConstants.AWS_SECRET_ACCESS_KEY, secretKey);
    configProps.setProperty(AWSConfigConstants.AWS_REGION, region);
    AmazonKinesisClient client = AWSUtil.createKinesisClient(configProps);
    // create a stream for the test:
    client.createStream(streamName, 1);
    // wait until stream has been created
    DescribeStreamResult status = client.describeStream(streamName);
    LOG.info("status {}", status);
    while (!status.getStreamDescription().getStreamStatus().equals("ACTIVE")) {
        status = client.describeStream(streamName);
        LOG.info("Status of stream {}", status);
        Thread.sleep(1000);
    }
    final Configuration flinkConfig = new Configuration();
    flinkConfig.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
    flinkConfig.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 8);
    flinkConfig.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 16);
    flinkConfig.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s");
    LocalFlinkMiniCluster flink = new LocalFlinkMiniCluster(flinkConfig, false);
    flink.start();
    final int flinkPort = flink.getLeaderRPCPort();
    try {
        final AtomicReference<Throwable> producerError = new AtomicReference<>();
        Thread producerThread = KinesisEventsGeneratorProducerThread.create(TOTAL_EVENT_COUNT, 2, accessKey, secretKey, region, streamName, producerError, flinkPort, flinkConfig);
        producerThread.start();
        final AtomicReference<Throwable> consumerError = new AtomicReference<>();
        Thread consumerThread = ExactlyOnceValidatingConsumerThread.create(TOTAL_EVENT_COUNT, 200, 2, 500, 500, accessKey, secretKey, region, streamName, consumerError, flinkPort, flinkConfig);
        consumerThread.start();
        boolean deadlinePassed = false;
        // wait at most for two minutes
        long deadline = System.currentTimeMillis() + (1000 * 2 * 60);
        // wait until both producer and consumer finishes, or an unexpected error is thrown
        while ((consumerThread.isAlive() || producerThread.isAlive()) && (producerError.get() == null && consumerError.get() == null)) {
            Thread.sleep(1000);
            if (System.currentTimeMillis() >= deadline) {
                LOG.warn("Deadline passed");
                deadlinePassed = true;
                // enough waiting
                break;
            }
        }
        if (producerThread.isAlive()) {
            producerThread.interrupt();
        }
        if (consumerThread.isAlive()) {
            consumerThread.interrupt();
        }
        if (producerError.get() != null) {
            LOG.info("+++ TEST failed! +++");
            throw new RuntimeException("Producer failed", producerError.get());
        }
        if (consumerError.get() != null) {
            LOG.info("+++ TEST failed! +++");
            throw new RuntimeException("Consumer failed", consumerError.get());
        }
        if (!deadlinePassed) {
            LOG.info("+++ TEST passed! +++");
        } else {
            LOG.info("+++ TEST failed! +++");
        }
    } finally {
        client.deleteStream(streamName);
        client.shutdown();
        // stopping flink
        flink.stop();
    }
}

Example 22 with LocalFlinkMiniCluster

use of org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster in project flink by apache.

the class KafkaTestBase method startClusters.

protected static void startClusters(boolean secureMode) throws ClassNotFoundException {
    // dynamically load the implementation for the test
    Class<?> clazz = Class.forName("org.apache.flink.streaming.connectors.kafka.KafkaTestEnvironmentImpl");
    kafkaServer = (KafkaTestEnvironment) InstantiationUtil.instantiate(clazz);
    LOG.info("Starting KafkaTestBase.prepare() for Kafka " + kafkaServer.getVersion());
    kafkaServer.prepare(NUMBER_OF_KAFKA_SERVERS, secureMode);
    standardProps = kafkaServer.getStandardProperties();
    brokerConnectionStrings = kafkaServer.getBrokerConnectionString();
    if (secureMode) {
        if (!kafkaServer.isSecureRunSupported()) {
            throw new IllegalStateException("Attempting to test in secure mode but secure mode not supported by the KafkaTestEnvironment.");
        }
        secureProps = kafkaServer.getSecureProperties();
    }
    // start also a re-usable Flink mini cluster
    flink = new LocalFlinkMiniCluster(getFlinkConfiguration(), false);
    flink.start();
    flinkPort = flink.getLeaderRPCPort();
}

Example 23 with LocalFlinkMiniCluster

use of org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster in project flink by apache.

the class FlinkLocalCluster method submitTopologyWithOpts.

@SuppressWarnings("rawtypes")
public void submitTopologyWithOpts(final String topologyName, final Map conf, final FlinkTopology topology, final SubmitOptions submitOpts) throws Exception {
    LOG.info("Running Storm topology on FlinkLocalCluster");
    boolean submitBlocking = false;
    if (conf != null) {
        Object blockingFlag = conf.get(SUBMIT_BLOCKING);
        if (blockingFlag != null && blockingFlag instanceof Boolean) {
            submitBlocking = ((Boolean) blockingFlag).booleanValue();
        }
    }
    FlinkClient.addStormConfigToTopology(topology, conf);
    StreamGraph streamGraph = topology.getExecutionEnvironment().getStreamGraph();
    streamGraph.setJobName(topologyName);
    JobGraph jobGraph = streamGraph.getJobGraph();
    if (this.flink == null) {
        Configuration configuration = new Configuration();
        configuration.addAll(jobGraph.getJobConfiguration());
        configuration.setLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1L);
        configuration.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, jobGraph.getMaximumParallelism());
        this.flink = new LocalFlinkMiniCluster(configuration, true);
        this.flink.start();
    }
    if (submitBlocking) {
        this.flink.submitJobAndWait(jobGraph, false);
    } else {
        this.flink.submitJobDetached(jobGraph);
    }
}

Example 24 with LocalFlinkMiniCluster

use of org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster in project flink by apache.

the class LocalExecutor method start.

// --------------------------------------------------------------------------------------------
@Override
public void start() throws Exception {
    synchronized (lock) {
        if (flink == null) {
            // create the embedded runtime
            Configuration configuration = createConfiguration();
            if (this.configuration != null) {
                configuration.addAll(this.configuration);
            }
            // start it up
            flink = new LocalFlinkMiniCluster(configuration, true);
            this.flink.start();
        } else {
            throw new IllegalStateException("The local executor was already started.");
        }
    }
}

Example 25 with LocalFlinkMiniCluster

use of org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster in project flink by apache.

the class KafkaShortRetentionTestBase method prepare.

@BeforeClass
public static void prepare() throws IOException, ClassNotFoundException {
    LOG.info("-------------------------------------------------------------------------");
    LOG.info("    Starting KafkaShortRetentionTestBase ");
    LOG.info("-------------------------------------------------------------------------");
    Configuration flinkConfig = new Configuration();
    // dynamically load the implementation for the test
    Class<?> clazz = Class.forName("org.apache.flink.streaming.connectors.kafka.KafkaTestEnvironmentImpl");
    kafkaServer = (KafkaTestEnvironment) InstantiationUtil.instantiate(clazz);
    LOG.info("Starting KafkaTestBase.prepare() for Kafka " + kafkaServer.getVersion());
    if (kafkaServer.isSecureRunSupported()) {
        secureProps = kafkaServer.getSecureProperties();
    }
    Properties specificProperties = new Properties();
    specificProperties.setProperty("log.retention.hours", "0");
    specificProperties.setProperty("log.retention.minutes", "0");
    specificProperties.setProperty("log.retention.ms", "250");
    specificProperties.setProperty("log.retention.check.interval.ms", "100");
    kafkaServer.prepare(1, specificProperties, false);
    standardProps = kafkaServer.getStandardProperties();
    // start also a re-usable Flink mini cluster
    flinkConfig.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
    flinkConfig.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 8);
    flinkConfig.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 16);
    flinkConfig.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s");
    flink = new LocalFlinkMiniCluster(flinkConfig, false);
    flink.start();
}