Examples with DualInputSemanticProperties org.apache.flink.api.common.operators.DualInputSemanticProperties used on opensource projects

Example 21 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class SemanticPropertiesTranslationTest method testBinaryForwardedAnnotationInLineMixed2.

@Test
public void testBinaryForwardedAnnotationInLineMixed2() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3l, 4l));
    @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3l, 2l));
    input1.join(input2).where(0).equalTo(0).with(new ForwardedSecondAnnotationJoin<Long>()).withForwardedFieldsFirst("0->1").output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>());
    Plan plan = env.createProgramPlan();
    GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
    InnerJoinOperatorBase<?, ?, ?, ?> join = (InnerJoinOperatorBase<?, ?, ?, ?>) sink.getInput();
    DualInputSemanticProperties semantics = join.getSemanticProperties();
    assertNotNull(semantics.getForwardingTargetFields(0, 1));
    assertNotNull(semantics.getForwardingTargetFields(1, 0));
    assertNotNull(semantics.getForwardingTargetFields(0, 0));
    assertNotNull(semantics.getForwardingTargetFields(1, 1));
    assertEquals(1, semantics.getForwardingTargetFields(0, 0).size());
    assertEquals(1, semantics.getForwardingTargetFields(1, 1).size());
    assertTrue(semantics.getForwardingTargetFields(0, 0).contains(1));
    assertTrue(semantics.getForwardingTargetFields(1, 1).contains(2));
    assertEquals(0, semantics.getForwardingTargetFields(0, 1).size());
    assertEquals(0, semantics.getForwardingTargetFields(1, 0).size());
}

Example 22 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class SemanticPropertiesTranslationTest method testBinaryForwardedInLine1.

@Test
public void testBinaryForwardedInLine1() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3l, 4l));
    @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3l, 2l));
    input1.join(input2).where(0).equalTo(0).with(new NoAnnotationJoin<Long>()).withForwardedFieldsFirst("0->1; 1->2").withForwardedFieldsSecond("1->0").output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>());
    Plan plan = env.createProgramPlan();
    GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
    InnerJoinOperatorBase<?, ?, ?, ?> join = (InnerJoinOperatorBase<?, ?, ?, ?>) sink.getInput();
    DualInputSemanticProperties semantics = join.getSemanticProperties();
    assertNotNull(semantics.getForwardingTargetFields(1, 0));
    assertEquals(1, semantics.getForwardingTargetFields(0, 0).size());
    assertEquals(1, semantics.getForwardingTargetFields(0, 1).size());
    assertEquals(1, semantics.getForwardingTargetFields(1, 1).size());
    assertTrue(semantics.getForwardingTargetFields(0, 0).contains(1));
    assertTrue(semantics.getForwardingTargetFields(0, 1).contains(2));
    assertTrue(semantics.getForwardingTargetFields(1, 1).contains(0));
    assertEquals(0, semantics.getForwardingTargetFields(1, 0).size());
}

Example 23 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class SemanticPropUtil method parseForwardedFields.

private static void parseForwardedFields(SemanticProperties sp, String[] forwardedStr, TypeInformation<?> inType, TypeInformation<?> outType, int input, boolean skipIncompatibleTypes) {
    if (forwardedStr == null) {
        return;
    }
    for (String s : forwardedStr) {
        if (s == null) {
            continue;
        }
        // remove white characters
        s = s.replaceAll("\\s", "");
        Matcher wcMatcher = PATTERN_WILDCARD.matcher(s);
        // simple wildcard
        if (wcMatcher.matches()) {
            if (!inType.equals(outType)) {
                if (skipIncompatibleTypes) {
                    continue;
                } else {
                    throw new InvalidSemanticAnnotationException("Forwarded field annotation \"" + s + "\" with wildcard only allowed for identical input and output types.");
                }
            }
            for (int i = 0; i < inType.getTotalFields(); i++) {
                if (sp instanceof SingleInputSemanticProperties) {
                    ((SingleInputSemanticProperties) sp).addForwardedField(i, i);
                } else if (sp instanceof DualInputSemanticProperties) {
                    ((DualInputSemanticProperties) sp).addForwardedField(input, i, i);
                }
            }
            return;
        }
        // check format of annotation string
        Matcher matcher = PATTERN_ANNOTATION.matcher(s);
        if (!matcher.matches()) {
            throw new InvalidSemanticAnnotationException("Invalid format of forwarded field annotation \"" + s + "\".");
        }
        // add forward annotations "->"
        Matcher forwardMatcher = PATTERN_FORWARD.matcher(s);
        while (forwardMatcher.find()) {
            String sourceStr = forwardMatcher.group(2);
            String targetStr = forwardMatcher.group(6);
            try {
                // check type compatibility
                if (!areFieldsCompatible(sourceStr, inType, targetStr, outType, !skipIncompatibleTypes)) {
                    if (skipIncompatibleTypes) {
                        continue;
                    } else {
                        throw new InvalidSemanticAnnotationException("Referenced fields of forwarded field annotation \"" + s + "\" do not match.");
                    }
                }
                List<FlatFieldDescriptor> inFFDs = getFlatFields(sourceStr, inType);
                List<FlatFieldDescriptor> outFFDs = getFlatFields(targetStr, outType);
                if (sp instanceof SingleInputSemanticProperties) {
                    for (int i = 0; i < inFFDs.size(); i++) {
                        int sourceField = inFFDs.get(i).getPosition();
                        int targetField = outFFDs.get(i).getPosition();
                        ((SingleInputSemanticProperties) sp).addForwardedField(sourceField, targetField);
                    }
                } else if (sp instanceof DualInputSemanticProperties) {
                    for (int i = 0; i < inFFDs.size(); i++) {
                        int sourceField = inFFDs.get(i).getPosition();
                        int targetField = outFFDs.get(i).getPosition();
                        ((DualInputSemanticProperties) sp).addForwardedField(input, sourceField, targetField);
                    }
                }
            } catch (InvalidFieldReferenceException ifre) {
                throw new InvalidSemanticAnnotationException("Invalid field reference in forwarded field annotation \"" + sourceStr + "->" + targetStr + "\".", ifre);
            } catch (InvalidSemanticAnnotationException isae) {
                throw new InvalidSemanticAnnotationException("Forwarded field annotation \"" + sourceStr + "->" + targetStr + "\" could not be added.", isae);
            }
        }
        // remove forward annotations
        s = forwardMatcher.replaceAll("");
        // add forwarded annotations
        Matcher listMatcher = PATTERN_LIST.matcher(s);
        while (listMatcher.find()) {
            String list = listMatcher.group();
            Matcher fieldMatcher = PATTERN_FIELD.matcher(list);
            // for each nested field
            while (fieldMatcher.find()) {
                String fieldStr = fieldMatcher.group();
                try {
                    // check if field is compatible in input and output type
                    if (!areFieldsCompatible(fieldStr, inType, fieldStr, outType, !skipIncompatibleTypes)) {
                        if (skipIncompatibleTypes) {
                            continue;
                        } else {
                            throw new InvalidSemanticAnnotationException("Referenced fields of forwarded field annotation \"" + s + "\" do not match.");
                        }
                    }
                    // add flat field positions
                    List<FlatFieldDescriptor> inFFDs = getFlatFields(fieldStr, inType);
                    List<FlatFieldDescriptor> outFFDs = getFlatFields(fieldStr, outType);
                    for (int i = 0; i < inFFDs.size(); i++) {
                        int sourcePos = inFFDs.get(i).getPosition();
                        int targetPos = outFFDs.get(i).getPosition();
                        if (sp instanceof SingleInputSemanticProperties) {
                            ((SingleInputSemanticProperties) sp).addForwardedField(sourcePos, targetPos);
                        } else if (sp instanceof DualInputSemanticProperties) {
                            ((DualInputSemanticProperties) sp).addForwardedField(input, sourcePos, targetPos);
                        }
                    }
                } catch (InvalidFieldReferenceException ifre) {
                    throw new InvalidSemanticAnnotationException("Invalid field reference in forwarded field annotation \"" + fieldStr + "\".", ifre);
                } catch (InvalidSemanticAnnotationException isae) {
                    throw new InvalidSemanticAnnotationException("Forwarded field annotation \"" + fieldStr + "\" could not be added.", isae);
                }
            }
        }
    }
}

Example 24 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class SemanticPropUtil method parseReadFields.

private static void parseReadFields(SemanticProperties sp, String[] readFieldStrings, TypeInformation<?> inType, int input) {
    if (readFieldStrings == null) {
        return;
    }
    for (String s : readFieldStrings) {
        FieldSet readFields = new FieldSet();
        // remove white characters
        s = s.replaceAll("\\s", "");
        Matcher wcMatcher = PATTERN_WILDCARD.matcher(s);
        // simple wildcard
        if (wcMatcher.matches()) {
            // add all fields
            for (int i = 0; i < inType.getTotalFields(); i++) {
                readFields = readFields.addField(i);
            }
        } else {
            // process field list
            Matcher matcher = PATTERN_LIST.matcher(s);
            if (!matcher.matches()) {
                throw new InvalidSemanticAnnotationException("Invalid format of read field annotation \"" + s + "\".");
            }
            // process field
            matcher = PATTERN_FIELD.matcher(s);
            while (matcher.find()) {
                String fieldStr = matcher.group();
                try {
                    List<FlatFieldDescriptor> ffds = getFlatFields(fieldStr, inType);
                    // get and add flat field positions
                    for (FlatFieldDescriptor ffd : ffds) {
                        readFields = readFields.addField(ffd.getPosition());
                    }
                } catch (InvalidFieldReferenceException ifre) {
                    throw new InvalidSemanticAnnotationException("Invalid field reference in read field annotation \"" + fieldStr + "\".", ifre);
                }
            }
        }
        if (sp instanceof SingleInputSemanticProperties) {
            ((SingleInputSemanticProperties) sp).addReadFields(readFields);
        } else if (sp instanceof DualInputSemanticProperties) {
            ((DualInputSemanticProperties) sp).addReadFields(input, readFields);
        }
    }
}

Example 25 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class SemanticPropUtil method parseNonForwardedFields.

private static void parseNonForwardedFields(SemanticProperties sp, String[] nonForwardedStr, TypeInformation<?> inType, TypeInformation<?> outType, int input, boolean skipIncompatibleTypes) {
    if (nonForwardedStr == null) {
        return;
    }
    FieldSet excludedFields = new FieldSet();
    for (String s : nonForwardedStr) {
        // remove white characters
        s = s.replaceAll("\\s", "");
        if (s.equals("")) {
            continue;
        }
        if (!inType.equals(outType)) {
            if (skipIncompatibleTypes) {
                continue;
            } else {
                throw new InvalidSemanticAnnotationException("Non-forwarded fields annotation only allowed for identical input and output types.");
            }
        }
        Matcher matcher = PATTERN_LIST.matcher(s);
        if (!matcher.matches()) {
            throw new InvalidSemanticAnnotationException("Invalid format of non-forwarded fields annotation \"" + s + "\".");
        }
        // process individual fields
        matcher = PATTERN_FIELD.matcher(s);
        while (matcher.find()) {
            String fieldStr = matcher.group();
            try {
                // get and add all flat field positions
                List<FlatFieldDescriptor> inFFDs = getFlatFields(fieldStr, inType);
                for (FlatFieldDescriptor ffd : inFFDs) {
                    excludedFields = excludedFields.addField(ffd.getPosition());
                }
            } catch (InvalidFieldReferenceException ifre) {
                throw new InvalidSemanticAnnotationException("Invalid field reference in non-forwarded fields annotation \"" + fieldStr + "\".", ifre);
            }
        }
    }
    for (int i = 0; i < inType.getTotalFields(); i++) {
        if (!excludedFields.contains(i)) {
            if (sp instanceof SingleInputSemanticProperties) {
                ((SingleInputSemanticProperties) sp).addForwardedField(i, i);
            } else if (sp instanceof DualInputSemanticProperties) {
                ((DualInputSemanticProperties) sp).addForwardedField(input, i, i);
            }
        }
    }
}