Search in sources :

Example 1 with LocalFlinkMiniCluster

use of org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster in project flink by apache.

the class ReduceOnEdgesWithExceptionITCase method setupCluster.

@BeforeClass
public static void setupCluster() {
    try {
        Configuration config = new Configuration();
        config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, PARALLELISM);
        cluster = new LocalFlinkMiniCluster(config, false);
        cluster.start();
    } catch (Exception e) {
        e.printStackTrace();
        fail("Error starting test cluster: " + e.getMessage());
    }
}
Also used : Configuration(org.apache.flink.configuration.Configuration) LocalFlinkMiniCluster(org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster) BeforeClass(org.junit.BeforeClass)

Example 2 with LocalFlinkMiniCluster

use of org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster in project flink by apache.

the class FastFailuresITCase method testThis.

@Test
public void testThis() {
    Configuration config = new Configuration();
    config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
    config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 2);
    LocalFlinkMiniCluster cluster = new LocalFlinkMiniCluster(config, false);
    cluster.start();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
    env.getConfig().disableSysoutLogging();
    env.setParallelism(4);
    env.enableCheckpointing(1000);
    env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(210, 0));
    DataStream<Tuple2<Integer, Integer>> input = env.addSource(new RichSourceFunction<Tuple2<Integer, Integer>>() {

        @Override
        public void open(Configuration parameters) {
            if (FAILURES_SO_FAR.incrementAndGet() <= NUM_FAILURES) {
                throw new RuntimeException("fail");
            }
        }

        @Override
        public void run(SourceContext<Tuple2<Integer, Integer>> ctx) {
        }

        @Override
        public void cancel() {
        }
    });
    input.keyBy(0).map(new MapFunction<Tuple2<Integer, Integer>, Integer>() {

        @Override
        public Integer map(Tuple2<Integer, Integer> value) {
            return value.f0;
        }
    }).addSink(new SinkFunction<Integer>() {

        @Override
        public void invoke(Integer value) {
        }
    });
    try {
        env.execute();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}
Also used : Configuration(org.apache.flink.configuration.Configuration) MapFunction(org.apache.flink.api.common.functions.MapFunction) LocalFlinkMiniCluster(org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Tuple2(org.apache.flink.api.java.tuple.Tuple2) StreamExecutionEnvironment(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment) Test(org.junit.Test)

Example 3 with LocalFlinkMiniCluster

use of org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster in project flink by apache.

the class SimpleRecoveryFailureRateStrategyITBase method setupCluster.

@BeforeClass
public static void setupCluster() {
    Configuration config = new Configuration();
    config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
    config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 2);
    config.setString(ConfigConstants.RESTART_STRATEGY, "failure-rate");
    config.setInteger(ConfigConstants.RESTART_STRATEGY_FAILURE_RATE_MAX_FAILURES_PER_INTERVAL, 1);
    config.setString(ConfigConstants.RESTART_STRATEGY_FAILURE_RATE_FAILURE_RATE_INTERVAL, "1 second");
    config.setString(ConfigConstants.RESTART_STRATEGY_FAILURE_RATE_DELAY, "100 ms");
    cluster = new LocalFlinkMiniCluster(config, false);
    cluster.start();
}
Also used : Configuration(org.apache.flink.configuration.Configuration) LocalFlinkMiniCluster(org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster) BeforeClass(org.junit.BeforeClass)

Example 4 with LocalFlinkMiniCluster

use of org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster in project flink by apache.

the class SimpleRecoveryFixedDelayRestartStrategyITBase method setupCluster.

@BeforeClass
public static void setupCluster() {
    Configuration config = new Configuration();
    config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
    config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 2);
    config.setString(ConfigConstants.RESTART_STRATEGY, "fixed-delay");
    config.setInteger(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 1);
    config.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "100 ms");
    cluster = new LocalFlinkMiniCluster(config, false);
    cluster.start();
}
Also used : Configuration(org.apache.flink.configuration.Configuration) LocalFlinkMiniCluster(org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster) BeforeClass(org.junit.BeforeClass)

Example 5 with LocalFlinkMiniCluster

use of org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster in project flink by apache.

the class IPv6HostnamesITCase method testClusterWithIPv6host.

@Test
public void testClusterWithIPv6host() {
    final Inet6Address ipv6address = getLocalIPv6Address();
    if (ipv6address == null) {
        System.err.println("--- Cannot find a non-loopback local IPv6 address that Akka/Netty can bind to; skipping IPv6HostnamesITCase");
        return;
    }
    LocalFlinkMiniCluster flink = null;
    try {
        final String addressString = ipv6address.getHostAddress();
        log.info("Test will use IPv6 address " + addressString + " for connection tests");
        Configuration conf = new Configuration();
        conf.setString(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY, addressString);
        conf.setString(ConfigConstants.TASK_MANAGER_HOSTNAME_KEY, addressString);
        conf.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
        conf.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 2);
        conf.setInteger(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, 16);
        flink = new LocalFlinkMiniCluster(conf, false);
        flink.start();
        ExecutionEnvironment env = ExecutionEnvironment.createRemoteEnvironment(addressString, flink.getLeaderRPCPort());
        env.setParallelism(4);
        env.getConfig().disableSysoutLogging();
        // get input data
        DataSet<String> text = env.fromElements(WordCountData.TEXT.split("\n"));
        DataSet<Tuple2<String, Integer>> counts = text.flatMap(new FlatMapFunction<String, Tuple2<String, Integer>>() {

            @Override
            public void flatMap(String value, Collector<Tuple2<String, Integer>> out) throws Exception {
                for (String token : value.toLowerCase().split("\\W+")) {
                    if (token.length() > 0) {
                        out.collect(new Tuple2<String, Integer>(token, 1));
                    }
                }
            }
        }).groupBy(0).sum(1);
        List<Tuple2<String, Integer>> result = counts.collect();
        TestBaseUtils.compareResultAsText(result, WordCountData.COUNTS_AS_TUPLES);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        if (flink != null) {
            flink.shutdown();
        }
    }
}
Also used : ExecutionEnvironment(org.apache.flink.api.java.ExecutionEnvironment) Configuration(org.apache.flink.configuration.Configuration) Inet6Address(java.net.Inet6Address) IOException(java.io.IOException) LocalFlinkMiniCluster(org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster) Tuple2(org.apache.flink.api.java.tuple.Tuple2) Test(org.junit.Test)

Aggregations

LocalFlinkMiniCluster (org.apache.flink.runtime.minicluster.LocalFlinkMiniCluster)39 Configuration (org.apache.flink.configuration.Configuration)36 BeforeClass (org.junit.BeforeClass)19 Test (org.junit.Test)9 JobGraph (org.apache.flink.runtime.jobgraph.JobGraph)5 ProgramInvocationException (org.apache.flink.client.program.ProgramInvocationException)4 ActorGateway (org.apache.flink.runtime.instance.ActorGateway)4 File (java.io.File)3 IOException (java.io.IOException)3 Properties (java.util.Properties)3 AtomicReference (java.util.concurrent.atomic.AtomicReference)3 JobVertex (org.apache.flink.runtime.jobgraph.JobVertex)3 JobManagerMessages (org.apache.flink.runtime.messages.JobManagerMessages)3 FiniteDuration (scala.concurrent.duration.FiniteDuration)3 ActorSystem (akka.actor.ActorSystem)2 AmazonKinesisClient (com.amazonaws.services.kinesis.AmazonKinesisClient)2 DescribeStreamResult (com.amazonaws.services.kinesis.model.DescribeStreamResult)2 HashSet (java.util.HashSet)2 ExecutionEnvironment (org.apache.flink.api.java.ExecutionEnvironment)2 Tuple2 (org.apache.flink.api.java.tuple.Tuple2)2