Examples with JoinOperator org.apache.flink.api.java.operators.JoinOperator used on opensource projects

Example 1 with JoinOperator

use of org.apache.flink.api.java.operators.JoinOperator in project flink by apache.

the class WorksetIterationsRecordApiCompilerTest method getTestPlan.

private Plan getTestPlan(boolean joinPreservesSolutionSet, boolean mapBeforeSolutionDelta) {
    // construct the plan
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(DEFAULT_PARALLELISM);
    DataSet<Tuple2<Long, Long>> solSetInput = env.readCsvFile("/tmp/sol.csv").types(Long.class, Long.class).name("Solution Set");
    DataSet<Tuple2<Long, Long>> workSetInput = env.readCsvFile("/tmp/sol.csv").types(Long.class, Long.class).name("Workset");
    DataSet<Tuple2<Long, Long>> invariantInput = env.readCsvFile("/tmp/sol.csv").types(Long.class, Long.class).name("Invariant Input");
    DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> deltaIt = solSetInput.iterateDelta(workSetInput, 100, 0).name(ITERATION_NAME);
    DataSet<Tuple2<Long, Long>> join1 = deltaIt.getWorkset().join(invariantInput).where(0).equalTo(0).with(new IdentityJoiner<Tuple2<Long, Long>>()).withForwardedFieldsFirst("*").name(JOIN_WITH_INVARIANT_NAME);
    DataSet<Tuple2<Long, Long>> join2 = deltaIt.getSolutionSet().join(join1).where(0).equalTo(0).with(new IdentityJoiner<Tuple2<Long, Long>>()).name(JOIN_WITH_SOLUTION_SET);
    if (joinPreservesSolutionSet) {
        ((JoinOperator<?, ?, ?>) join2).withForwardedFieldsFirst("*");
    }
    DataSet<Tuple2<Long, Long>> nextWorkset = join2.groupBy(0).reduceGroup(new IdentityGroupReducer<Tuple2<Long, Long>>()).withForwardedFields("*").name(NEXT_WORKSET_REDUCER_NAME);
    if (mapBeforeSolutionDelta) {
        DataSet<Tuple2<Long, Long>> mapper = join2.map(new IdentityMapper<Tuple2<Long, Long>>()).withForwardedFields("*").name(SOLUTION_DELTA_MAPPER_NAME);
        deltaIt.closeWith(mapper, nextWorkset).output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
    } else {
        deltaIt.closeWith(join2, nextWorkset).output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
    }
    return env.createProgramPlan();
}