Examples with Tuple3 org.apache.flink.api.java.tuple.Tuple3 used on opensource projects

Example 16 with Tuple3

use of org.apache.flink.api.java.tuple.Tuple3 in project flink by apache.

the class WindowTranslationTest method testFoldProcessingTime.

@Test
@SuppressWarnings("rawtypes")
public void testFoldProcessingTime() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));
    DataStream<Tuple3<String, String, Integer>> window = source.keyBy(new TupleKeySelector()).window(SlidingProcessingTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))).fold(new Tuple3<>("", "", 0), new DummyFolder());
    OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window.getTransformation();
    OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
    Assert.assertTrue(operator instanceof WindowOperator);
    WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
    Assert.assertTrue(winOperator.getTrigger() instanceof ProcessingTimeTrigger);
    Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingProcessingTimeWindows);
    Assert.assertTrue(winOperator.getStateDescriptor() instanceof FoldingStateDescriptor);
    processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}

Example 17 with Tuple3

use of org.apache.flink.api.java.tuple.Tuple3 in project flink by apache.

the class CoGroupSortTranslationTest method testGroupSortTuples.

@Test
public void testGroupSortTuples() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(0L, 0L));
        DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(0L, 0L, 0L));
        input1.coGroup(input2).where(1).equalTo(2).sortFirstGroup(0, Order.DESCENDING).sortSecondGroup(1, Order.ASCENDING).sortSecondGroup(0, Order.DESCENDING).with(new CoGroupFunction<Tuple2<Long, Long>, Tuple3<Long, Long, Long>, Long>() {

            @Override
            public void coGroup(Iterable<Tuple2<Long, Long>> first, Iterable<Tuple3<Long, Long, Long>> second, Collector<Long> out) {
            }
        }).output(new DiscardingOutputFormat<Long>());
        Plan p = env.createProgramPlan();
        GenericDataSinkBase<?> sink = p.getDataSinks().iterator().next();
        CoGroupOperatorBase<?, ?, ?, ?> coGroup = (CoGroupOperatorBase<?, ?, ?, ?>) sink.getInput();
        assertNotNull(coGroup.getGroupOrderForInputOne());
        assertNotNull(coGroup.getGroupOrderForInputTwo());
        assertEquals(1, coGroup.getGroupOrderForInputOne().getNumberOfFields());
        assertEquals(0, coGroup.getGroupOrderForInputOne().getFieldNumber(0).intValue());
        assertEquals(Order.DESCENDING, coGroup.getGroupOrderForInputOne().getOrder(0));
        assertEquals(2, coGroup.getGroupOrderForInputTwo().getNumberOfFields());
        assertEquals(1, coGroup.getGroupOrderForInputTwo().getFieldNumber(0).intValue());
        assertEquals(0, coGroup.getGroupOrderForInputTwo().getFieldNumber(1).intValue());
        assertEquals(Order.ASCENDING, coGroup.getGroupOrderForInputTwo().getOrder(0));
        assertEquals(Order.DESCENDING, coGroup.getGroupOrderForInputTwo().getOrder(1));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 18 with Tuple3

use of org.apache.flink.api.java.tuple.Tuple3 in project flink by apache.

the class DeltaIterationTranslationTest method testCorrectTranslation.

@Test
public void testCorrectTranslation() {
    try {
        final String JOB_NAME = "Test JobName";
        final String ITERATION_NAME = "Test Name";
        final String BEFORE_NEXT_WORKSET_MAP = "Some Mapper";
        final String AGGREGATOR_NAME = "AggregatorName";
        final int[] ITERATION_KEYS = new int[] { 2 };
        final int NUM_ITERATIONS = 13;
        final int DEFAULT_parallelism = 133;
        final int ITERATION_parallelism = 77;
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        // ------------ construct the test program ------------------
        {
            env.setParallelism(DEFAULT_parallelism);
            @SuppressWarnings("unchecked") DataSet<Tuple3<Double, Long, String>> initialSolutionSet = env.fromElements(new Tuple3<Double, Long, String>(3.44, 5L, "abc"));
            @SuppressWarnings("unchecked") DataSet<Tuple2<Double, String>> initialWorkSet = env.fromElements(new Tuple2<Double, String>(1.23, "abc"));
            DeltaIteration<Tuple3<Double, Long, String>, Tuple2<Double, String>> iteration = initialSolutionSet.iterateDelta(initialWorkSet, NUM_ITERATIONS, ITERATION_KEYS);
            iteration.name(ITERATION_NAME).parallelism(ITERATION_parallelism);
            iteration.registerAggregator(AGGREGATOR_NAME, new LongSumAggregator());
            // test that multiple workset consumers are supported
            DataSet<Tuple2<Double, String>> worksetSelfJoin = iteration.getWorkset().map(new IdentityMapper<Tuple2<Double, String>>()).join(iteration.getWorkset()).where(1).equalTo(1).projectFirst(0, 1);
            DataSet<Tuple3<Double, Long, String>> joined = worksetSelfJoin.join(iteration.getSolutionSet()).where(1).equalTo(2).with(new SolutionWorksetJoin());
            DataSet<Tuple3<Double, Long, String>> result = iteration.closeWith(joined, joined.map(new NextWorksetMapper()).name(BEFORE_NEXT_WORKSET_MAP));
            result.output(new DiscardingOutputFormat<Tuple3<Double, Long, String>>());
            result.writeAsText("/dev/null");
        }
        Plan p = env.createProgramPlan(JOB_NAME);
        // ------------- validate the plan ----------------
        assertEquals(JOB_NAME, p.getJobName());
        assertEquals(DEFAULT_parallelism, p.getDefaultParallelism());
        // validate the iteration
        GenericDataSinkBase<?> sink1, sink2;
        {
            Iterator<? extends GenericDataSinkBase<?>> sinks = p.getDataSinks().iterator();
            sink1 = sinks.next();
            sink2 = sinks.next();
        }
        DeltaIterationBase<?, ?> iteration = (DeltaIterationBase<?, ?>) sink1.getInput();
        // check that multi consumer translation works for iterations
        assertEquals(iteration, sink2.getInput());
        // check the basic iteration properties
        assertEquals(NUM_ITERATIONS, iteration.getMaximumNumberOfIterations());
        assertArrayEquals(ITERATION_KEYS, iteration.getSolutionSetKeyFields());
        assertEquals(ITERATION_parallelism, iteration.getParallelism());
        assertEquals(ITERATION_NAME, iteration.getName());
        MapOperatorBase<?, ?, ?> nextWorksetMapper = (MapOperatorBase<?, ?, ?>) iteration.getNextWorkset();
        InnerJoinOperatorBase<?, ?, ?, ?> solutionSetJoin = (InnerJoinOperatorBase<?, ?, ?, ?>) iteration.getSolutionSetDelta();
        InnerJoinOperatorBase<?, ?, ?, ?> worksetSelfJoin = (InnerJoinOperatorBase<?, ?, ?, ?>) solutionSetJoin.getFirstInput();
        MapOperatorBase<?, ?, ?> worksetMapper = (MapOperatorBase<?, ?, ?>) worksetSelfJoin.getFirstInput();
        assertEquals(IdentityMapper.class, worksetMapper.getUserCodeWrapper().getUserCodeClass());
        assertEquals(NextWorksetMapper.class, nextWorksetMapper.getUserCodeWrapper().getUserCodeClass());
        if (solutionSetJoin.getUserCodeWrapper().getUserCodeObject() instanceof WrappingFunction) {
            WrappingFunction<?> wf = (WrappingFunction<?>) solutionSetJoin.getUserCodeWrapper().getUserCodeObject();
            assertEquals(SolutionWorksetJoin.class, wf.getWrappedFunction().getClass());
        } else {
            assertEquals(SolutionWorksetJoin.class, solutionSetJoin.getUserCodeWrapper().getUserCodeClass());
        }
        assertEquals(BEFORE_NEXT_WORKSET_MAP, nextWorksetMapper.getName());
        assertEquals(AGGREGATOR_NAME, iteration.getAggregators().getAllRegisteredAggregators().iterator().next().getName());
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 19 with Tuple3

use of org.apache.flink.api.java.tuple.Tuple3 in project flink by apache.

the class DeltaIterationTranslationTest method testRejectWhenSolutionSetKeysDontMatchCoGroup.

@Test
public void testRejectWhenSolutionSetKeysDontMatchCoGroup() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        @SuppressWarnings("unchecked") DataSet<Tuple3<Double, Long, String>> initialSolutionSet = env.fromElements(new Tuple3<Double, Long, String>(3.44, 5L, "abc"));
        @SuppressWarnings("unchecked") DataSet<Tuple2<Double, String>> initialWorkSet = env.fromElements(new Tuple2<Double, String>(1.23, "abc"));
        DeltaIteration<Tuple3<Double, Long, String>, Tuple2<Double, String>> iteration = initialSolutionSet.iterateDelta(initialWorkSet, 10, 1);
        try {
            iteration.getWorkset().coGroup(iteration.getSolutionSet()).where(1).equalTo(2).with(new SolutionWorksetCoGroup1());
            fail("Accepted invalid program.");
        } catch (InvalidProgramException e) {
        // all good!
        }
        try {
            iteration.getSolutionSet().coGroup(iteration.getWorkset()).where(2).equalTo(1).with(new SolutionWorksetCoGroup2());
            fail("Accepted invalid program.");
        } catch (InvalidProgramException e) {
        // all good!
        }
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 20 with Tuple3

use of org.apache.flink.api.java.tuple.Tuple3 in project flink by apache.

the class TriangleEnumeratorITCase method testTriangleEnumerator.

@Test
public void testTriangleEnumerator() throws Exception {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    Graph<Long, NullValue, NullValue> graph = Graph.fromDataSet(TriangleCountData.getDefaultEdgeDataSet(env), env);
    List<Tuple3<Long, Long, Long>> actualOutput = graph.run(new TriangleEnumerator<Long, NullValue, NullValue>()).collect();
    List<Tuple3<Long, Long, Long>> expectedResult = TriangleCountData.getListOfTriangles();
    Assert.assertEquals(expectedResult.size(), actualOutput.size());
    for (Tuple3<Long, Long, Long> resultTriangle : actualOutput) {
        Assert.assertTrue(expectedResult.indexOf(resultTriangle) >= 0);
    }
}