Examples with ExecutionConfig org.apache.flink.api.common.ExecutionConfig used on opensource projects

Example 81 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class PointwisePatternTest method test3NToN.

@Test
public void test3NToN() throws Exception {
    final int N = 17;
    JobVertex v1 = new JobVertex("vertex1");
    JobVertex v2 = new JobVertex("vertex2");
    v1.setParallelism(3 * N);
    v2.setParallelism(N);
    v1.setInvokableClass(AbstractInvokable.class);
    v2.setInvokableClass(AbstractInvokable.class);
    v2.connectNewDataSetAsInput(v1, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
    List<JobVertex> ordered = new ArrayList<JobVertex>(Arrays.asList(v1, v2));
    ExecutionGraph eg = new ExecutionGraph(TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), jobId, jobName, cfg, new SerializedValue<>(new ExecutionConfig()), AkkaUtils.getDefaultTimeout(), new NoRestartStrategy(), new Scheduler(TestingUtils.defaultExecutionContext()));
    try {
        eg.attachJobGraph(ordered);
    } catch (JobException e) {
        e.printStackTrace();
        fail("Job failed with exception: " + e.getMessage());
    }
    ExecutionJobVertex target = eg.getAllVertices().get(v2.getID());
    for (ExecutionVertex ev : target.getTaskVertices()) {
        assertEquals(1, ev.getNumberOfInputs());
        ExecutionEdge[] inEdges = ev.getInputEdges(0);
        assertEquals(3, inEdges.length);
        assertEquals(ev.getParallelSubtaskIndex() * 3, inEdges[0].getSource().getPartitionNumber());
        assertEquals(ev.getParallelSubtaskIndex() * 3 + 1, inEdges[1].getSource().getPartitionNumber());
        assertEquals(ev.getParallelSubtaskIndex() * 3 + 2, inEdges[2].getSource().getPartitionNumber());
    }
}

Example 82 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class ExecutionStateProgressTest method testAccumulatedStateFinished.

@Test
public void testAccumulatedStateFinished() {
    try {
        final JobID jid = new JobID();
        final JobVertexID vid = new JobVertexID();
        JobVertex ajv = new JobVertex("TestVertex", vid);
        ajv.setParallelism(3);
        ajv.setInvokableClass(mock(AbstractInvokable.class).getClass());
        ExecutionGraph graph = new ExecutionGraph(TestingUtils.defaultExecutor(), TestingUtils.defaultExecutor(), jid, "test job", new Configuration(), new SerializedValue<>(new ExecutionConfig()), AkkaUtils.getDefaultTimeout(), new NoRestartStrategy(), new Scheduler(TestingUtils.defaultExecutionContext()));
        graph.attachJobGraph(Collections.singletonList(ajv));
        setGraphStatus(graph, JobStatus.RUNNING);
        ExecutionJobVertex ejv = graph.getJobVertex(vid);
        // mock resources and mock taskmanager
        for (ExecutionVertex ee : ejv.getTaskVertices()) {
            SimpleSlot slot = getInstance(new ActorTaskManagerGateway(new SimpleActorGateway(TestingUtils.defaultExecutionContext()))).allocateSimpleSlot(jid);
            ee.deployToSlot(slot);
        }
        // finish all
        for (ExecutionVertex ee : ejv.getTaskVertices()) {
            ee.executionFinished();
        }
        assertTrue(ejv.isInFinalState());
        assertEquals(JobStatus.FINISHED, graph.getState());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 83 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class GroupReduceDriverTest method testAllReduceDriverAccumulatingImmutable.

@Test
public void testAllReduceDriverAccumulatingImmutable() {
    try {
        TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>> context = new TestTaskContext<GroupReduceFunction<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>, Tuple2<StringValue, IntValue>>();
        List<Tuple2<StringValue, IntValue>> data = DriverTestData.createReduceMutableData();
        TupleTypeInfo<Tuple2<StringValue, IntValue>> typeInfo = (TupleTypeInfo<Tuple2<StringValue, IntValue>>) TypeExtractor.getForObject(data.get(0));
        MutableObjectIterator<Tuple2<StringValue, IntValue>> input = new RegularToMutableObjectIterator<Tuple2<StringValue, IntValue>>(data.iterator(), typeInfo.createSerializer(new ExecutionConfig()));
        TypeComparator<Tuple2<StringValue, IntValue>> comparator = typeInfo.createComparator(new int[] { 0 }, new boolean[] { true }, 0, new ExecutionConfig());
        GatheringCollector<Tuple2<StringValue, IntValue>> result = new GatheringCollector<Tuple2<StringValue, IntValue>>(typeInfo.createSerializer(new ExecutionConfig()));
        context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
        context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
        context.setComparator1(comparator);
        context.setCollector(result);
        context.setUdf(new ConcatSumMutableAccumulatingReducer());
        context.setMutableObjectMode(false);
        GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>> driver = new GroupReduceDriver<Tuple2<StringValue, IntValue>, Tuple2<StringValue, IntValue>>();
        driver.setup(context);
        driver.prepare();
        driver.run();
        Object[] res = result.getList().toArray();
        Object[] expected = DriverTestData.createReduceMutableDataGroupedResult().toArray();
        DriverTestData.compareTupleArrays(expected, res);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 84 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class GroupReduceDriverTest method testAllReduceDriverImmutableEmpty.

@Test
public void testAllReduceDriverImmutableEmpty() {
    try {
        TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context = new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>>();
        List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
        TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
        MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
        TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[] { 0 }, new boolean[] { true }, 0, new ExecutionConfig());
        context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
        GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String, Integer>>(typeInfo.createSerializer(new ExecutionConfig()));
        context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
        context.setComparator1(comparator);
        context.setCollector(result);
        GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>> driver = new GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>>();
        driver.setup(context);
        driver.prepare();
        driver.run();
        Assert.assertTrue(result.getList().isEmpty());
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 85 with ExecutionConfig

use of org.apache.flink.api.common.ExecutionConfig in project flink by apache.

the class GroupReduceDriverTest method testAllReduceDriverImmutable.

@Test
public void testAllReduceDriverImmutable() {
    try {
        TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context = new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>>();
        List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
        TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
        MutableObjectIterator<Tuple2<String, Integer>> input = new RegularToMutableObjectIterator<Tuple2<String, Integer>>(data.iterator(), typeInfo.createSerializer(new ExecutionConfig()));
        TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[] { 0 }, new boolean[] { true }, 0, new ExecutionConfig());
        GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String, Integer>>(typeInfo.createSerializer(new ExecutionConfig()));
        context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
        context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
        context.setCollector(result);
        context.setComparator1(comparator);
        context.setUdf(new ConcatSumReducer());
        GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>> driver = new GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>>();
        driver.setup(context);
        driver.prepare();
        driver.run();
        Object[] res = result.getList().toArray();
        Object[] expected = DriverTestData.createReduceImmutableDataGroupedResult().toArray();
        DriverTestData.compareTupleArrays(expected, res);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}