Examples with StreamGraph org.apache.flink.streaming.api.graph.StreamGraph used on opensource projects

Example 11 with StreamGraph

use of org.apache.flink.streaming.api.graph.StreamGraph in project flink by apache.

the class FlinkLocalCluster method submitTopologyWithOpts.

@SuppressWarnings("rawtypes")
public void submitTopologyWithOpts(final String topologyName, final Map conf, final FlinkTopology topology, final SubmitOptions submitOpts) throws Exception {
    LOG.info("Running Storm topology on FlinkLocalCluster");
    boolean submitBlocking = false;
    if (conf != null) {
        Object blockingFlag = conf.get(SUBMIT_BLOCKING);
        if (blockingFlag != null && blockingFlag instanceof Boolean) {
            submitBlocking = ((Boolean) blockingFlag).booleanValue();
        }
    }
    FlinkClient.addStormConfigToTopology(topology, conf);
    StreamGraph streamGraph = topology.getExecutionEnvironment().getStreamGraph();
    streamGraph.setJobName(topologyName);
    JobGraph jobGraph = streamGraph.getJobGraph();
    if (this.flink == null) {
        Configuration configuration = new Configuration();
        configuration.addAll(jobGraph.getJobConfiguration());
        configuration.setLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1L);
        configuration.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, jobGraph.getMaximumParallelism());
        this.flink = new LocalFlinkMiniCluster(configuration, true);
        this.flink.start();
    }
    if (submitBlocking) {
        this.flink.submitJobAndWait(jobGraph, false);
    } else {
        this.flink.submitJobDetached(jobGraph);
    }
}

Example 12 with StreamGraph

use of org.apache.flink.streaming.api.graph.StreamGraph in project flink by apache.

the class StreamingJobGraphGeneratorNodeHashTest method testUserProvidedHashing.

@Test
public void testUserProvidedHashing() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
    List<String> userHashes = Arrays.asList("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb");
    env.addSource(new NoOpSourceFunction(), "src").setUidHash(userHashes.get(0)).map(new NoOpMapFunction()).filter(new NoOpFilterFunction()).keyBy(new NoOpKeySelector()).reduce(new NoOpReduceFunction()).name("reduce").setUidHash(userHashes.get(1));
    StreamGraph streamGraph = env.getStreamGraph();
    int idx = 1;
    for (JobVertex jobVertex : streamGraph.getJobGraph().getVertices()) {
        Assert.assertEquals(jobVertex.getIdAlternatives().get(1).toString(), userHashes.get(idx));
        --idx;
    }
}

Example 13 with StreamGraph

use of org.apache.flink.streaming.api.graph.StreamGraph in project flink by apache.

the class SavepointITCase method testSavepointForJobWithIteration.

@Test
public void testSavepointForJobWithIteration() throws Exception {
    for (int i = 0; i < ITER_TEST_PARALLELISM; ++i) {
        ITER_TEST_SNAPSHOT_WAIT[i] = new OneShotLatch();
        ITER_TEST_RESTORE_WAIT[i] = new OneShotLatch();
        ITER_TEST_CHECKPOINT_VERIFY[i] = 0;
    }
    TemporaryFolder folder = new TemporaryFolder();
    folder.create();
    // Temporary directory for file state backend
    final File tmpDir = folder.newFolder();
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    final IntegerStreamSource source = new IntegerStreamSource();
    IterativeStream<Integer> iteration = env.addSource(source).flatMap(new RichFlatMapFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void flatMap(Integer in, Collector<Integer> clctr) throws Exception {
            clctr.collect(in);
        }
    }).setParallelism(ITER_TEST_PARALLELISM).keyBy(new KeySelector<Integer, Object>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Object getKey(Integer value) throws Exception {
            return value;
        }
    }).flatMap(new DuplicateFilter()).setParallelism(ITER_TEST_PARALLELISM).iterate();
    DataStream<Integer> iterationBody = iteration.map(new MapFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Integer map(Integer value) throws Exception {
            return value;
        }
    }).setParallelism(ITER_TEST_PARALLELISM);
    iteration.closeWith(iterationBody);
    StreamGraph streamGraph = env.getStreamGraph();
    streamGraph.setJobName("Test");
    JobGraph jobGraph = streamGraph.getJobGraph();
    Configuration config = new Configuration();
    config.addAll(jobGraph.getJobConfiguration());
    config.setLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1L);
    config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 2 * jobGraph.getMaximumParallelism());
    final File checkpointDir = new File(tmpDir, "checkpoints");
    final File savepointDir = new File(tmpDir, "savepoints");
    if (!checkpointDir.mkdir() || !savepointDir.mkdirs()) {
        fail("Test setup failed: failed to create temporary directories.");
    }
    config.setString(CoreOptions.STATE_BACKEND, "filesystem");
    config.setString(FsStateBackendFactory.CHECKPOINT_DIRECTORY_URI_CONF_KEY, checkpointDir.toURI().toString());
    config.setString(FsStateBackendFactory.MEMORY_THRESHOLD_CONF_KEY, "0");
    config.setString(ConfigConstants.SAVEPOINT_DIRECTORY_KEY, savepointDir.toURI().toString());
    TestingCluster cluster = new TestingCluster(config, false);
    String savepointPath = null;
    try {
        cluster.start();
        cluster.submitJobDetached(jobGraph);
        for (OneShotLatch latch : ITER_TEST_SNAPSHOT_WAIT) {
            latch.await();
        }
        savepointPath = cluster.triggerSavepoint(jobGraph.getJobID());
        source.cancel();
        jobGraph = streamGraph.getJobGraph();
        jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath));
        cluster.submitJobDetached(jobGraph);
        for (OneShotLatch latch : ITER_TEST_RESTORE_WAIT) {
            latch.await();
        }
        source.cancel();
    } finally {
        if (null != savepointPath) {
            cluster.disposeSavepoint(savepointPath);
        }
        cluster.stop();
        cluster.awaitTermination();
    }
}

Example 14 with StreamGraph

use of org.apache.flink.streaming.api.graph.StreamGraph in project flink by apache.

the class Flip6LocalStreamEnvironment method execute.

/**
	 * Executes the JobGraph of the on a mini cluster of CLusterUtil with a user
	 * specified name.
	 * 
	 * @param jobName
	 *            name of the job
	 * @return The result of the job execution, containing elapsed time and accumulators.
	 */
@Override
public JobExecutionResult execute(String jobName) throws Exception {
    // transform the streaming program into a JobGraph
    StreamGraph streamGraph = getStreamGraph();
    streamGraph.setJobName(jobName);
    // TODO - temp fix to enforce restarts due to a bug in the allocation protocol
    streamGraph.getExecutionConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 5));
    JobGraph jobGraph = streamGraph.getJobGraph();
    jobGraph.setAllowQueuedScheduling(true);
    Configuration configuration = new Configuration();
    configuration.addAll(jobGraph.getJobConfiguration());
    configuration.setLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1L);
    // add (and override) the settings with what the user defined
    configuration.addAll(this.conf);
    MiniClusterConfiguration cfg = new MiniClusterConfiguration(configuration);
    // Currently we do not reuse slot anymore,
    // so we need to sum up the parallelism of all vertices
    int slotsCount = 0;
    for (JobVertex jobVertex : jobGraph.getVertices()) {
        slotsCount += jobVertex.getParallelism();
    }
    cfg.setNumTaskManagerSlots(slotsCount);
    if (LOG.isInfoEnabled()) {
        LOG.info("Running job on local embedded Flink mini cluster");
    }
    MiniCluster miniCluster = new MiniCluster(cfg);
    try {
        miniCluster.start();
        miniCluster.waitUntilTaskManagerRegistrationsComplete();
        return miniCluster.runJobBlocking(jobGraph);
    } finally {
        transformations.clear();
        miniCluster.shutdown();
    }
}

Example 15 with StreamGraph

use of org.apache.flink.streaming.api.graph.StreamGraph in project flink by apache.

the class LocalStreamEnvironment method execute.

/**
	 * Executes the JobGraph of the on a mini cluster of CLusterUtil with a user
	 * specified name.
	 * 
	 * @param jobName
	 *            name of the job
	 * @return The result of the job execution, containing elapsed time and accumulators.
	 */
@Override
public JobExecutionResult execute(String jobName) throws Exception {
    // transform the streaming program into a JobGraph
    StreamGraph streamGraph = getStreamGraph();
    streamGraph.setJobName(jobName);
    JobGraph jobGraph = streamGraph.getJobGraph();
    Configuration configuration = new Configuration();
    configuration.addAll(jobGraph.getJobConfiguration());
    configuration.setLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1L);
    configuration.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, jobGraph.getMaximumParallelism());
    // add (and override) the settings with what the user defined
    configuration.addAll(this.conf);
    if (LOG.isInfoEnabled()) {
        LOG.info("Running job on local embedded Flink mini cluster");
    }
    LocalFlinkMiniCluster exec = new LocalFlinkMiniCluster(configuration, true);
    try {
        exec.start();
        return exec.submitJobAndWait(jobGraph, getConfig().isSysoutLoggingEnabled());
    } finally {
        transformations.clear();
        exec.stop();
    }
}