Examples with DualInputSemanticProperties org.apache.flink.api.common.operators.DualInputSemanticProperties used on opensource projects

Example 11 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class SemanticPropertiesTranslationTest method testBinaryForwardedAnnotationInLineMixed1.

@Test
public void testBinaryForwardedAnnotationInLineMixed1() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3l, 4l));
    @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3l, 2l));
    input1.join(input2).where(0).equalTo(0).with(new ForwardedFirstAnnotationJoin<Long>()).withForwardedFieldsSecond("1").output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>());
    Plan plan = env.createProgramPlan();
    GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
    InnerJoinOperatorBase<?, ?, ?, ?> join = (InnerJoinOperatorBase<?, ?, ?, ?>) sink.getInput();
    DualInputSemanticProperties semantics = join.getSemanticProperties();
    assertNotNull(semantics.getForwardingTargetFields(0, 1));
    assertNotNull(semantics.getForwardingTargetFields(1, 0));
    assertNotNull(semantics.getForwardingTargetFields(0, 0));
    assertNotNull(semantics.getForwardingTargetFields(1, 1));
    assertEquals(1, semantics.getForwardingTargetFields(0, 0).size());
    assertEquals(1, semantics.getForwardingTargetFields(1, 1).size());
    assertTrue(semantics.getForwardingTargetFields(0, 0).contains(2));
    assertTrue(semantics.getForwardingTargetFields(1, 1).contains(1));
    assertEquals(0, semantics.getForwardingTargetFields(0, 1).size());
    assertEquals(0, semantics.getForwardingTargetFields(1, 0).size());
}

Example 12 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class SemanticPropertiesTranslationTest method testBinaryAllForwardedExceptAnnotation.

@Test
public void testBinaryAllForwardedExceptAnnotation() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    @SuppressWarnings("unchecked") DataSet<Tuple3<Long, Long, Long>> input1 = env.fromElements(new Tuple3<Long, Long, Long>(3l, 4l, 5l));
    @SuppressWarnings("unchecked") DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(3l, 2l, 1l));
    input1.join(input2).where(0).equalTo(0).with(new AllForwardedExceptJoin<Long>()).output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>());
    Plan plan = env.createProgramPlan();
    GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
    InnerJoinOperatorBase<?, ?, ?, ?> join = (InnerJoinOperatorBase<?, ?, ?, ?>) sink.getInput();
    DualInputSemanticProperties semantics = join.getSemanticProperties();
    assertNotNull(semantics.getForwardingTargetFields(0, 0));
    assertNotNull(semantics.getForwardingTargetFields(0, 2));
    assertNotNull(semantics.getForwardingTargetFields(1, 0));
    assertNotNull(semantics.getForwardingTargetFields(1, 1));
    assertEquals(1, semantics.getForwardingTargetFields(0, 1).size());
    assertEquals(1, semantics.getForwardingTargetFields(1, 2).size());
    assertTrue(semantics.getForwardingTargetFields(0, 1).contains(1));
    assertTrue(semantics.getForwardingTargetFields(1, 2).contains(2));
    assertEquals(0, semantics.getForwardingTargetFields(0, 0).size());
    assertEquals(0, semantics.getForwardingTargetFields(0, 2).size());
    assertEquals(0, semantics.getForwardingTargetFields(1, 0).size());
    assertEquals(0, semantics.getForwardingTargetFields(1, 1).size());
}

Example 13 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class SemanticPropertiesTranslationTest method testBinaryReadFieldsAnnotation.

@Test
public void testBinaryReadFieldsAnnotation() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3l, 4l));
    @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3l, 2l));
    input1.join(input2).where(0).equalTo(0).with(new ReadSetJoin<Long>()).output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>());
    Plan plan = env.createProgramPlan();
    GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
    InnerJoinOperatorBase<?, ?, ?, ?> join = (InnerJoinOperatorBase<?, ?, ?, ?>) sink.getInput();
    DualInputSemanticProperties semantics = join.getSemanticProperties();
    assertNotNull(semantics.getReadFields(0));
    assertNotNull(semantics.getReadFields(1));
    assertEquals(1, semantics.getReadFields(0).size());
    assertEquals(1, semantics.getReadFields(1).size());
    assertTrue(semantics.getReadFields(0).contains(1));
    assertTrue(semantics.getReadFields(1).contains(0));
}

Example 14 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class SemanticPropUtil method getSemanticPropsDual.

public static DualInputSemanticProperties getSemanticPropsDual(Set<Annotation> set, TypeInformation<?> inType1, TypeInformation<?> inType2, TypeInformation<?> outType) {
    if (set == null) {
        return null;
    }
    Iterator<Annotation> it = set.iterator();
    String[] forwardedFirst = null;
    String[] forwardedSecond = null;
    String[] nonForwardedFirst = null;
    String[] nonForwardedSecond = null;
    String[] readFirst = null;
    String[] readSecond = null;
    while (it.hasNext()) {
        Annotation ann = it.next();
        if (ann instanceof ForwardedFieldsFirst) {
            forwardedFirst = ((ForwardedFieldsFirst) ann).value();
        } else if (ann instanceof ForwardedFieldsSecond) {
            forwardedSecond = ((ForwardedFieldsSecond) ann).value();
        } else if (ann instanceof NonForwardedFieldsFirst) {
            nonForwardedFirst = ((NonForwardedFieldsFirst) ann).value();
        } else if (ann instanceof NonForwardedFieldsSecond) {
            nonForwardedSecond = ((NonForwardedFieldsSecond) ann).value();
        } else if (ann instanceof ReadFieldsFirst) {
            readFirst = ((ReadFieldsFirst) ann).value();
        } else if (ann instanceof ReadFieldsSecond) {
            readSecond = ((ReadFieldsSecond) ann).value();
        } else if (ann instanceof ForwardedFields || ann instanceof NonForwardedFields || ann instanceof ReadFields) {
            throw new InvalidSemanticAnnotationException("Annotation " + ann.getClass() + " invalid for dual input function.");
        }
    }
    if (forwardedFirst != null || nonForwardedFirst != null || readFirst != null || forwardedSecond != null || nonForwardedSecond != null || readSecond != null) {
        DualInputSemanticProperties result = new DualInputSemanticProperties();
        getSemanticPropsDualFromString(result, forwardedFirst, forwardedSecond, nonForwardedFirst, nonForwardedSecond, readFirst, readSecond, inType1, inType2, outType);
        return result;
    }
    return null;
}

Example 15 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class CoGroupOperator method getSemanticProperties.

@Override
public DualInputSemanticProperties getSemanticProperties() {
    DualInputSemanticProperties props = super.getSemanticProperties();
    // offset semantic information by extracted key fields
    if (props != null && (this.keys1 instanceof SelectorFunctionKeys || this.keys2 instanceof SelectorFunctionKeys)) {
        int numFields1 = this.getInput1Type().getTotalFields();
        int numFields2 = this.getInput2Type().getTotalFields();
        int offset1 = (this.keys1 instanceof SelectorFunctionKeys) ? ((SelectorFunctionKeys<?, ?>) this.keys1).getKeyType().getTotalFields() : 0;
        int offset2 = (this.keys2 instanceof SelectorFunctionKeys) ? ((SelectorFunctionKeys<?, ?>) this.keys2).getKeyType().getTotalFields() : 0;
        props = SemanticPropUtil.addSourceFieldOffsets(props, numFields1, numFields2, offset1, offset2);
    }
    return props;
}