Examples with Configuration org.apache.flink.configuration.Configuration used on opensource projects

Example 31 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class CoGroupOnConflictingPartitioningsTest method testRejectCoGroupOnHashAndRangePartitioning.

@Test
public void testRejectCoGroupOnHashAndRangePartitioning() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        DataSet<Tuple2<Long, Long>> input = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
        Configuration cfg = new Configuration();
        cfg.setString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH);
        cfg.setString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT, Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE);
        input.coGroup(input).where(0).equalTo(0).with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>()).withParameters(cfg).output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>());
        Plan p = env.createProgramPlan();
        try {
            compileNoStats(p);
            fail("This should fail with an exception");
        } catch (CompilerException e) {
        // expected
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 32 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class JobGraphGeneratorTest method testResourcesForChainedOperators.

/**
	 * Verifies that the resources are merged correctly for chained operators when
	 * generating job graph
	 */
@Test
public void testResourcesForChainedOperators() throws Exception {
    ResourceSpec resource1 = new ResourceSpec(0.1, 100);
    ResourceSpec resource2 = new ResourceSpec(0.2, 200);
    ResourceSpec resource3 = new ResourceSpec(0.3, 300);
    ResourceSpec resource4 = new ResourceSpec(0.4, 400);
    ResourceSpec resource5 = new ResourceSpec(0.5, 500);
    ResourceSpec resource6 = new ResourceSpec(0.6, 600);
    ResourceSpec resource7 = new ResourceSpec(0.7, 700);
    Method opMethod = Operator.class.getDeclaredMethod("setResources", ResourceSpec.class);
    opMethod.setAccessible(true);
    Method sinkMethod = DataSink.class.getDeclaredMethod("setResources", ResourceSpec.class);
    sinkMethod.setAccessible(true);
    MapFunction<Long, Long> mapFunction = new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return value;
        }
    };
    FilterFunction<Long> filterFunction = new FilterFunction<Long>() {

        @Override
        public boolean filter(Long value) throws Exception {
            return false;
        }
    };
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Long> input = env.fromElements(1L, 2L, 3L);
    opMethod.invoke(input, resource1);
    DataSet<Long> map1 = input.map(mapFunction);
    opMethod.invoke(map1, resource2);
    // CHAIN(Source -> Map -> Filter)
    DataSet<Long> filter1 = map1.filter(filterFunction);
    opMethod.invoke(filter1, resource3);
    IterativeDataSet<Long> startOfIteration = filter1.iterate(10);
    opMethod.invoke(startOfIteration, resource4);
    DataSet<Long> map2 = startOfIteration.map(mapFunction);
    opMethod.invoke(map2, resource5);
    // CHAIN(Map -> Filter)
    DataSet<Long> feedback = map2.filter(filterFunction);
    opMethod.invoke(feedback, resource6);
    DataSink<Long> sink = startOfIteration.closeWith(feedback).output(new DiscardingOutputFormat<Long>());
    sinkMethod.invoke(sink, resource7);
    Plan plan = env.createProgramPlan();
    Optimizer pc = new Optimizer(new Configuration());
    OptimizedPlan op = pc.compile(plan);
    JobGraphGenerator jgg = new JobGraphGenerator();
    JobGraph jobGraph = jgg.compileJobGraph(op);
    JobVertex sourceMapFilterVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(0);
    JobVertex iterationHeadVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
    JobVertex feedbackVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(2);
    JobVertex sinkVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(3);
    JobVertex iterationSyncVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(4);
    assertTrue(sourceMapFilterVertex.getMinResources().equals(resource1.merge(resource2).merge(resource3)));
    assertTrue(iterationHeadVertex.getPreferredResources().equals(resource4));
    assertTrue(feedbackVertex.getMinResources().equals(resource5.merge(resource6)));
    assertTrue(sinkVertex.getPreferredResources().equals(resource7));
    assertTrue(iterationSyncVertex.getMinResources().equals(resource4));
}

Example 33 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class TempInIterationsTest method testTempInIterationTest.

/*
	 * Tests whether temps barriers are correctly set in within iterations
	 */
@Test
public void testTempInIterationTest() throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple2<Long, Long>> input = env.readCsvFile("file:///does/not/exist").types(Long.class, Long.class);
    DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = input.iterateDelta(input, 1, 0);
    DataSet<Tuple2<Long, Long>> update = iteration.getWorkset().join(iteration.getSolutionSet()).where(0).equalTo(0).with(new DummyFlatJoinFunction<Tuple2<Long, Long>>());
    iteration.closeWith(update, update).output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
    Plan plan = env.createProgramPlan();
    OptimizedPlan oPlan = (new Optimizer(new Configuration())).compile(plan);
    JobGraphGenerator jgg = new JobGraphGenerator();
    JobGraph jg = jgg.compileJobGraph(oPlan);
    boolean solutionSetUpdateChecked = false;
    for (JobVertex v : jg.getVertices()) {
        if (v.getName().equals("SolutionSet Delta")) {
            // check if input of solution set delta is temped
            TaskConfig tc = new TaskConfig(v.getConfiguration());
            assertTrue(tc.isInputAsynchronouslyMaterialized(0));
            solutionSetUpdateChecked = true;
        }
    }
    assertTrue(solutionSetUpdateChecked);
}

Example 34 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class CoGroupNodeTest method testGetSemanticProperties.

@Test
public void testGetSemanticProperties() {
    DualInputSemanticProperties origProps = new DualInputSemanticProperties();
    // props for first input
    origProps.addForwardedField(0, 0, 1);
    origProps.addForwardedField(0, 2, 2);
    origProps.addForwardedField(0, 3, 4);
    origProps.addForwardedField(0, 6, 0);
    origProps.addReadFields(0, new FieldSet(0, 2, 4, 7));
    // props for second input
    origProps.addForwardedField(1, 1, 2);
    origProps.addForwardedField(1, 2, 8);
    origProps.addForwardedField(1, 3, 7);
    origProps.addForwardedField(1, 6, 6);
    origProps.addReadFields(1, new FieldSet(1, 3, 4));
    CoGroupOperatorBase<?, ?, ?, ?> op = mock(CoGroupOperatorBase.class);
    when(op.getSemanticProperties()).thenReturn(origProps);
    when(op.getKeyColumns(0)).thenReturn(new int[] { 3, 2 });
    when(op.getKeyColumns(1)).thenReturn(new int[] { 6, 3 });
    when(op.getParameters()).thenReturn(new Configuration());
    CoGroupNode node = new CoGroupNode(op);
    SemanticProperties filteredProps = node.getSemanticPropertiesForLocalPropertyFiltering();
    // check first input props
    assertTrue(filteredProps.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(filteredProps.getForwardingTargetFields(0, 2).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(0, 2).contains(2));
    assertTrue(filteredProps.getForwardingTargetFields(0, 3).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(0, 3).contains(4));
    assertTrue(filteredProps.getForwardingTargetFields(0, 6).size() == 0);
    assertTrue(filteredProps.getForwardingSourceField(0, 1) < 0);
    assertTrue(filteredProps.getForwardingSourceField(0, 2) == 2);
    assertTrue(filteredProps.getForwardingSourceField(0, 4) == 3);
    assertTrue(filteredProps.getForwardingSourceField(0, 0) < 0);
    // check second input props
    assertTrue(filteredProps.getReadFields(0).size() == 4);
    assertTrue(filteredProps.getReadFields(0).contains(0));
    assertTrue(filteredProps.getReadFields(0).contains(2));
    assertTrue(filteredProps.getReadFields(0).contains(4));
    assertTrue(filteredProps.getReadFields(0).contains(7));
    assertTrue(filteredProps.getForwardingTargetFields(1, 1).size() == 0);
    assertTrue(filteredProps.getForwardingTargetFields(1, 2).size() == 0);
    assertTrue(filteredProps.getForwardingTargetFields(1, 3).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(1, 3).contains(7));
    assertTrue(filteredProps.getForwardingTargetFields(1, 6).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(1, 6).contains(6));
    assertTrue(filteredProps.getForwardingSourceField(1, 2) < 0);
    assertTrue(filteredProps.getForwardingSourceField(1, 8) < 0);
    assertTrue(filteredProps.getForwardingSourceField(1, 7) == 3);
    assertTrue(filteredProps.getForwardingSourceField(1, 6) == 6);
    assertTrue(filteredProps.getReadFields(1).size() == 3);
    assertTrue(filteredProps.getReadFields(1).contains(1));
    assertTrue(filteredProps.getReadFields(1).contains(3));
    assertTrue(filteredProps.getReadFields(1).contains(4));
}

Example 35 with Configuration

use of org.apache.flink.configuration.Configuration in project flink by apache.

the class GroupCombineNodeTest method testGetSemanticProperties.

@Test
public void testGetSemanticProperties() {
    SingleInputSemanticProperties origProps = new SingleInputSemanticProperties();
    origProps.addForwardedField(0, 1);
    origProps.addForwardedField(2, 2);
    origProps.addForwardedField(3, 4);
    origProps.addForwardedField(6, 0);
    origProps.addReadFields(new FieldSet(0, 2, 4, 7));
    GroupCombineOperatorBase<?, ?, ?> op = mock(GroupCombineOperatorBase.class);
    when(op.getSemanticProperties()).thenReturn(origProps);
    when(op.getKeyColumns(0)).thenReturn(new int[] { 3, 2 });
    when(op.getParameters()).thenReturn(new Configuration());
    GroupCombineNode node = new GroupCombineNode(op);
    SemanticProperties filteredProps = node.getSemanticPropertiesForLocalPropertyFiltering();
    assertTrue(filteredProps.getForwardingTargetFields(0, 0).size() == 0);
    assertTrue(filteredProps.getForwardingTargetFields(0, 2).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(0, 2).contains(2));
    assertTrue(filteredProps.getForwardingTargetFields(0, 3).size() == 1);
    assertTrue(filteredProps.getForwardingTargetFields(0, 3).contains(4));
    assertTrue(filteredProps.getForwardingTargetFields(0, 6).size() == 0);
    assertTrue(filteredProps.getForwardingSourceField(0, 1) < 0);
    assertTrue(filteredProps.getForwardingSourceField(0, 2) == 2);
    assertTrue(filteredProps.getForwardingSourceField(0, 4) == 3);
    assertTrue(filteredProps.getForwardingSourceField(0, 0) < 0);
    assertTrue(filteredProps.getReadFields(0).size() == 4);
    assertTrue(filteredProps.getReadFields(0).contains(0));
    assertTrue(filteredProps.getReadFields(0).contains(2));
    assertTrue(filteredProps.getReadFields(0).contains(4));
    assertTrue(filteredProps.getReadFields(0).contains(7));
}