Examples with JobGraph org.apache.flink.runtime.jobgraph.JobGraph used on opensource projects

Example 76 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class Flip6LocalStreamEnvironment method execute.

/**
	 * Executes the JobGraph of the on a mini cluster of CLusterUtil with a user
	 * specified name.
	 * 
	 * @param jobName
	 *            name of the job
	 * @return The result of the job execution, containing elapsed time and accumulators.
	 */
@Override
public JobExecutionResult execute(String jobName) throws Exception {
    // transform the streaming program into a JobGraph
    StreamGraph streamGraph = getStreamGraph();
    streamGraph.setJobName(jobName);
    // TODO - temp fix to enforce restarts due to a bug in the allocation protocol
    streamGraph.getExecutionConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 5));
    JobGraph jobGraph = streamGraph.getJobGraph();
    jobGraph.setAllowQueuedScheduling(true);
    Configuration configuration = new Configuration();
    configuration.addAll(jobGraph.getJobConfiguration());
    configuration.setLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1L);
    // add (and override) the settings with what the user defined
    configuration.addAll(this.conf);
    MiniClusterConfiguration cfg = new MiniClusterConfiguration(configuration);
    // Currently we do not reuse slot anymore,
    // so we need to sum up the parallelism of all vertices
    int slotsCount = 0;
    for (JobVertex jobVertex : jobGraph.getVertices()) {
        slotsCount += jobVertex.getParallelism();
    }
    cfg.setNumTaskManagerSlots(slotsCount);
    if (LOG.isInfoEnabled()) {
        LOG.info("Running job on local embedded Flink mini cluster");
    }
    MiniCluster miniCluster = new MiniCluster(cfg);
    try {
        miniCluster.start();
        miniCluster.waitUntilTaskManagerRegistrationsComplete();
        return miniCluster.runJobBlocking(jobGraph);
    } finally {
        transformations.clear();
        miniCluster.shutdown();
    }
}

Example 77 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class LocalStreamEnvironment method execute.

/**
	 * Executes the JobGraph of the on a mini cluster of CLusterUtil with a user
	 * specified name.
	 * 
	 * @param jobName
	 *            name of the job
	 * @return The result of the job execution, containing elapsed time and accumulators.
	 */
@Override
public JobExecutionResult execute(String jobName) throws Exception {
    // transform the streaming program into a JobGraph
    StreamGraph streamGraph = getStreamGraph();
    streamGraph.setJobName(jobName);
    JobGraph jobGraph = streamGraph.getJobGraph();
    Configuration configuration = new Configuration();
    configuration.addAll(jobGraph.getJobConfiguration());
    configuration.setLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1L);
    configuration.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, jobGraph.getMaximumParallelism());
    // add (and override) the settings with what the user defined
    configuration.addAll(this.conf);
    if (LOG.isInfoEnabled()) {
        LOG.info("Running job on local embedded Flink mini cluster");
    }
    LocalFlinkMiniCluster exec = new LocalFlinkMiniCluster(configuration, true);
    try {
        exec.start();
        return exec.submitJobAndWait(jobGraph, getConfig().isSysoutLoggingEnabled());
    } finally {
        transformations.clear();
        exec.stop();
    }
}

Example 78 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class SlotAllocationTest method testTwoPipelines.

@Test
public void testTwoPipelines() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    FilterFunction<Long> dummyFilter = new FilterFunction<Long>() {

        @Override
        public boolean filter(Long value) {
            return false;
        }
    };
    env.generateSequence(1, 10).filter(dummyFilter).slotSharingGroup("isolated").filter(dummyFilter).slotSharingGroup("default").disableChaining().filter(dummyFilter).slotSharingGroup("group 1").filter(dummyFilter).startNewChain().print().disableChaining();
    // verify that a second pipeline does not inherit the groups from the first pipeline
    env.generateSequence(1, 10).filter(dummyFilter).slotSharingGroup("isolated-2").filter(dummyFilter).slotSharingGroup("default").disableChaining().filter(dummyFilter).slotSharingGroup("group 2").filter(dummyFilter).startNewChain().print().disableChaining();
    JobGraph jobGraph = env.getStreamGraph().getJobGraph();
    List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();
    assertEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(3).getSlotSharingGroup());
    assertNotEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(2).getSlotSharingGroup());
    assertNotEquals(vertices.get(3).getSlotSharingGroup(), vertices.get(4).getSlotSharingGroup());
    assertEquals(vertices.get(4).getSlotSharingGroup(), vertices.get(5).getSlotSharingGroup());
    assertEquals(vertices.get(5).getSlotSharingGroup(), vertices.get(6).getSlotSharingGroup());
    int pipelineStart = 6;
    assertEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(pipelineStart + 2).getSlotSharingGroup());
    assertNotEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(pipelineStart + 1).getSlotSharingGroup());
    assertNotEquals(vertices.get(pipelineStart + 2).getSlotSharingGroup(), vertices.get(pipelineStart + 3).getSlotSharingGroup());
    assertEquals(vertices.get(pipelineStart + 3).getSlotSharingGroup(), vertices.get(pipelineStart + 4).getSlotSharingGroup());
    assertEquals(vertices.get(pipelineStart + 4).getSlotSharingGroup(), vertices.get(pipelineStart + 5).getSlotSharingGroup());
}

Example 79 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class SlotAllocationTest method testInheritOverride.

@Test
public void testInheritOverride() {
    // verify that we can explicitly disable inheritance of the input slot sharing groups
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    FilterFunction<Long> dummyFilter = new FilterFunction<Long>() {

        @Override
        public boolean filter(Long value) {
            return false;
        }
    };
    DataStream<Long> src1 = env.generateSequence(1, 10).slotSharingGroup("group-1");
    DataStream<Long> src2 = env.generateSequence(1, 10).slotSharingGroup("group-1");
    // this should not inherit group but be in "default"
    src1.union(src2).filter(dummyFilter).slotSharingGroup("default");
    JobGraph jobGraph = env.getStreamGraph().getJobGraph();
    List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();
    assertEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(1).getSlotSharingGroup());
    assertNotEquals(vertices.get(0).getSlotSharingGroup(), vertices.get(2).getSlotSharingGroup());
    assertNotEquals(vertices.get(1).getSlotSharingGroup(), vertices.get(2).getSlotSharingGroup());
}

Example 80 with JobGraph

use of org.apache.flink.runtime.jobgraph.JobGraph in project flink by apache.

the class StreamingJobGraphGeneratorTest method testChainStartEndSetting.

/**
	 * Verifies that the chain start/end is correctly set.
	 */
@Test
public void testChainStartEndSetting() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    // fromElements -> CHAIN(Map -> Print)
    env.fromElements(1, 2, 3).map(new MapFunction<Integer, Integer>() {

        @Override
        public Integer map(Integer value) throws Exception {
            return value;
        }
    }).print();
    JobGraph jobGraph = new StreamingJobGraphGenerator(env.getStreamGraph(), 1).createJobGraph();
    List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
    JobVertex sourceVertex = verticesSorted.get(0);
    JobVertex mapPrintVertex = verticesSorted.get(1);
    assertEquals(ResultPartitionType.PIPELINED_BOUNDED, sourceVertex.getProducedDataSets().get(0).getResultType());
    assertEquals(ResultPartitionType.PIPELINED_BOUNDED, mapPrintVertex.getInputs().get(0).getSource().getResultType());
    StreamConfig sourceConfig = new StreamConfig(sourceVertex.getConfiguration());
    StreamConfig mapConfig = new StreamConfig(mapPrintVertex.getConfiguration());
    Map<Integer, StreamConfig> chainedConfigs = mapConfig.getTransitiveChainedTaskConfigs(getClass().getClassLoader());
    StreamConfig printConfig = chainedConfigs.values().iterator().next();
    assertTrue(sourceConfig.isChainStart());
    assertTrue(sourceConfig.isChainEnd());
    assertTrue(mapConfig.isChainStart());
    assertFalse(mapConfig.isChainEnd());
    assertFalse(printConfig.isChainStart());
    assertTrue(printConfig.isChainEnd());
}