Search in sources :

Example 21 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class SemanticPropertiesTranslationTest method testBinaryForwardedAnnotationInLineMixed2.

@Test
public void testBinaryForwardedAnnotationInLineMixed2() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3l, 4l));
    @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3l, 2l));
    input1.join(input2).where(0).equalTo(0).with(new ForwardedSecondAnnotationJoin<Long>()).withForwardedFieldsFirst("0->1").output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>());
    Plan plan = env.createProgramPlan();
    GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
    InnerJoinOperatorBase<?, ?, ?, ?> join = (InnerJoinOperatorBase<?, ?, ?, ?>) sink.getInput();
    DualInputSemanticProperties semantics = join.getSemanticProperties();
    assertNotNull(semantics.getForwardingTargetFields(0, 1));
    assertNotNull(semantics.getForwardingTargetFields(1, 0));
    assertNotNull(semantics.getForwardingTargetFields(0, 0));
    assertNotNull(semantics.getForwardingTargetFields(1, 1));
    assertEquals(1, semantics.getForwardingTargetFields(0, 0).size());
    assertEquals(1, semantics.getForwardingTargetFields(1, 1).size());
    assertTrue(semantics.getForwardingTargetFields(0, 0).contains(1));
    assertTrue(semantics.getForwardingTargetFields(1, 1).contains(2));
    assertEquals(0, semantics.getForwardingTargetFields(0, 1).size());
    assertEquals(0, semantics.getForwardingTargetFields(1, 0).size());
}
Also used : ExecutionEnvironment(org.apache.flink.api.java.ExecutionEnvironment) Plan(org.apache.flink.api.common.Plan) DualInputSemanticProperties(org.apache.flink.api.common.operators.DualInputSemanticProperties) Tuple2(org.apache.flink.api.java.tuple.Tuple2) Tuple3(org.apache.flink.api.java.tuple.Tuple3) InnerJoinOperatorBase(org.apache.flink.api.common.operators.base.InnerJoinOperatorBase) Test(org.junit.Test)

Example 22 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class SemanticPropertiesTranslationTest method testBinaryForwardedInLine1.

@Test
public void testBinaryForwardedInLine1() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3l, 4l));
    @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3l, 2l));
    input1.join(input2).where(0).equalTo(0).with(new NoAnnotationJoin<Long>()).withForwardedFieldsFirst("0->1; 1->2").withForwardedFieldsSecond("1->0").output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>());
    Plan plan = env.createProgramPlan();
    GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next();
    InnerJoinOperatorBase<?, ?, ?, ?> join = (InnerJoinOperatorBase<?, ?, ?, ?>) sink.getInput();
    DualInputSemanticProperties semantics = join.getSemanticProperties();
    assertNotNull(semantics.getForwardingTargetFields(1, 0));
    assertEquals(1, semantics.getForwardingTargetFields(0, 0).size());
    assertEquals(1, semantics.getForwardingTargetFields(0, 1).size());
    assertEquals(1, semantics.getForwardingTargetFields(1, 1).size());
    assertTrue(semantics.getForwardingTargetFields(0, 0).contains(1));
    assertTrue(semantics.getForwardingTargetFields(0, 1).contains(2));
    assertTrue(semantics.getForwardingTargetFields(1, 1).contains(0));
    assertEquals(0, semantics.getForwardingTargetFields(1, 0).size());
}
Also used : ExecutionEnvironment(org.apache.flink.api.java.ExecutionEnvironment) Plan(org.apache.flink.api.common.Plan) DualInputSemanticProperties(org.apache.flink.api.common.operators.DualInputSemanticProperties) Tuple2(org.apache.flink.api.java.tuple.Tuple2) Tuple3(org.apache.flink.api.java.tuple.Tuple3) InnerJoinOperatorBase(org.apache.flink.api.common.operators.base.InnerJoinOperatorBase) Test(org.junit.Test)

Example 23 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class SemanticPropUtil method parseForwardedFields.

private static void parseForwardedFields(SemanticProperties sp, String[] forwardedStr, TypeInformation<?> inType, TypeInformation<?> outType, int input, boolean skipIncompatibleTypes) {
    if (forwardedStr == null) {
        return;
    }
    for (String s : forwardedStr) {
        if (s == null) {
            continue;
        }
        // remove white characters
        s = s.replaceAll("\\s", "");
        Matcher wcMatcher = PATTERN_WILDCARD.matcher(s);
        // simple wildcard
        if (wcMatcher.matches()) {
            if (!inType.equals(outType)) {
                if (skipIncompatibleTypes) {
                    continue;
                } else {
                    throw new InvalidSemanticAnnotationException("Forwarded field annotation \"" + s + "\" with wildcard only allowed for identical input and output types.");
                }
            }
            for (int i = 0; i < inType.getTotalFields(); i++) {
                if (sp instanceof SingleInputSemanticProperties) {
                    ((SingleInputSemanticProperties) sp).addForwardedField(i, i);
                } else if (sp instanceof DualInputSemanticProperties) {
                    ((DualInputSemanticProperties) sp).addForwardedField(input, i, i);
                }
            }
            return;
        }
        // check format of annotation string
        Matcher matcher = PATTERN_ANNOTATION.matcher(s);
        if (!matcher.matches()) {
            throw new InvalidSemanticAnnotationException("Invalid format of forwarded field annotation \"" + s + "\".");
        }
        // add forward annotations "->"
        Matcher forwardMatcher = PATTERN_FORWARD.matcher(s);
        while (forwardMatcher.find()) {
            String sourceStr = forwardMatcher.group(2);
            String targetStr = forwardMatcher.group(6);
            try {
                // check type compatibility
                if (!areFieldsCompatible(sourceStr, inType, targetStr, outType, !skipIncompatibleTypes)) {
                    if (skipIncompatibleTypes) {
                        continue;
                    } else {
                        throw new InvalidSemanticAnnotationException("Referenced fields of forwarded field annotation \"" + s + "\" do not match.");
                    }
                }
                List<FlatFieldDescriptor> inFFDs = getFlatFields(sourceStr, inType);
                List<FlatFieldDescriptor> outFFDs = getFlatFields(targetStr, outType);
                if (sp instanceof SingleInputSemanticProperties) {
                    for (int i = 0; i < inFFDs.size(); i++) {
                        int sourceField = inFFDs.get(i).getPosition();
                        int targetField = outFFDs.get(i).getPosition();
                        ((SingleInputSemanticProperties) sp).addForwardedField(sourceField, targetField);
                    }
                } else if (sp instanceof DualInputSemanticProperties) {
                    for (int i = 0; i < inFFDs.size(); i++) {
                        int sourceField = inFFDs.get(i).getPosition();
                        int targetField = outFFDs.get(i).getPosition();
                        ((DualInputSemanticProperties) sp).addForwardedField(input, sourceField, targetField);
                    }
                }
            } catch (InvalidFieldReferenceException ifre) {
                throw new InvalidSemanticAnnotationException("Invalid field reference in forwarded field annotation \"" + sourceStr + "->" + targetStr + "\".", ifre);
            } catch (InvalidSemanticAnnotationException isae) {
                throw new InvalidSemanticAnnotationException("Forwarded field annotation \"" + sourceStr + "->" + targetStr + "\" could not be added.", isae);
            }
        }
        // remove forward annotations
        s = forwardMatcher.replaceAll("");
        // add forwarded annotations
        Matcher listMatcher = PATTERN_LIST.matcher(s);
        while (listMatcher.find()) {
            String list = listMatcher.group();
            Matcher fieldMatcher = PATTERN_FIELD.matcher(list);
            // for each nested field
            while (fieldMatcher.find()) {
                String fieldStr = fieldMatcher.group();
                try {
                    // check if field is compatible in input and output type
                    if (!areFieldsCompatible(fieldStr, inType, fieldStr, outType, !skipIncompatibleTypes)) {
                        if (skipIncompatibleTypes) {
                            continue;
                        } else {
                            throw new InvalidSemanticAnnotationException("Referenced fields of forwarded field annotation \"" + s + "\" do not match.");
                        }
                    }
                    // add flat field positions
                    List<FlatFieldDescriptor> inFFDs = getFlatFields(fieldStr, inType);
                    List<FlatFieldDescriptor> outFFDs = getFlatFields(fieldStr, outType);
                    for (int i = 0; i < inFFDs.size(); i++) {
                        int sourcePos = inFFDs.get(i).getPosition();
                        int targetPos = outFFDs.get(i).getPosition();
                        if (sp instanceof SingleInputSemanticProperties) {
                            ((SingleInputSemanticProperties) sp).addForwardedField(sourcePos, targetPos);
                        } else if (sp instanceof DualInputSemanticProperties) {
                            ((DualInputSemanticProperties) sp).addForwardedField(input, sourcePos, targetPos);
                        }
                    }
                } catch (InvalidFieldReferenceException ifre) {
                    throw new InvalidSemanticAnnotationException("Invalid field reference in forwarded field annotation \"" + fieldStr + "\".", ifre);
                } catch (InvalidSemanticAnnotationException isae) {
                    throw new InvalidSemanticAnnotationException("Forwarded field annotation \"" + fieldStr + "\" could not be added.", isae);
                }
            }
        }
    }
}
Also used : Matcher(java.util.regex.Matcher) InvalidSemanticAnnotationException(org.apache.flink.api.common.operators.SemanticProperties.InvalidSemanticAnnotationException) SingleInputSemanticProperties(org.apache.flink.api.common.operators.SingleInputSemanticProperties) DualInputSemanticProperties(org.apache.flink.api.common.operators.DualInputSemanticProperties) FlatFieldDescriptor(org.apache.flink.api.common.typeutils.CompositeType.FlatFieldDescriptor) InvalidFieldReferenceException(org.apache.flink.api.common.typeutils.CompositeType.InvalidFieldReferenceException)

Example 24 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class SemanticPropUtil method parseReadFields.

private static void parseReadFields(SemanticProperties sp, String[] readFieldStrings, TypeInformation<?> inType, int input) {
    if (readFieldStrings == null) {
        return;
    }
    for (String s : readFieldStrings) {
        FieldSet readFields = new FieldSet();
        // remove white characters
        s = s.replaceAll("\\s", "");
        Matcher wcMatcher = PATTERN_WILDCARD.matcher(s);
        // simple wildcard
        if (wcMatcher.matches()) {
            // add all fields
            for (int i = 0; i < inType.getTotalFields(); i++) {
                readFields = readFields.addField(i);
            }
        } else {
            // process field list
            Matcher matcher = PATTERN_LIST.matcher(s);
            if (!matcher.matches()) {
                throw new InvalidSemanticAnnotationException("Invalid format of read field annotation \"" + s + "\".");
            }
            // process field
            matcher = PATTERN_FIELD.matcher(s);
            while (matcher.find()) {
                String fieldStr = matcher.group();
                try {
                    List<FlatFieldDescriptor> ffds = getFlatFields(fieldStr, inType);
                    // get and add flat field positions
                    for (FlatFieldDescriptor ffd : ffds) {
                        readFields = readFields.addField(ffd.getPosition());
                    }
                } catch (InvalidFieldReferenceException ifre) {
                    throw new InvalidSemanticAnnotationException("Invalid field reference in read field annotation \"" + fieldStr + "\".", ifre);
                }
            }
        }
        if (sp instanceof SingleInputSemanticProperties) {
            ((SingleInputSemanticProperties) sp).addReadFields(readFields);
        } else if (sp instanceof DualInputSemanticProperties) {
            ((DualInputSemanticProperties) sp).addReadFields(input, readFields);
        }
    }
}
Also used : FieldSet(org.apache.flink.api.common.operators.util.FieldSet) Matcher(java.util.regex.Matcher) InvalidSemanticAnnotationException(org.apache.flink.api.common.operators.SemanticProperties.InvalidSemanticAnnotationException) SingleInputSemanticProperties(org.apache.flink.api.common.operators.SingleInputSemanticProperties) DualInputSemanticProperties(org.apache.flink.api.common.operators.DualInputSemanticProperties) FlatFieldDescriptor(org.apache.flink.api.common.typeutils.CompositeType.FlatFieldDescriptor) InvalidFieldReferenceException(org.apache.flink.api.common.typeutils.CompositeType.InvalidFieldReferenceException)

Example 25 with DualInputSemanticProperties

use of org.apache.flink.api.common.operators.DualInputSemanticProperties in project flink by apache.

the class SemanticPropUtil method parseNonForwardedFields.

private static void parseNonForwardedFields(SemanticProperties sp, String[] nonForwardedStr, TypeInformation<?> inType, TypeInformation<?> outType, int input, boolean skipIncompatibleTypes) {
    if (nonForwardedStr == null) {
        return;
    }
    FieldSet excludedFields = new FieldSet();
    for (String s : nonForwardedStr) {
        // remove white characters
        s = s.replaceAll("\\s", "");
        if (s.equals("")) {
            continue;
        }
        if (!inType.equals(outType)) {
            if (skipIncompatibleTypes) {
                continue;
            } else {
                throw new InvalidSemanticAnnotationException("Non-forwarded fields annotation only allowed for identical input and output types.");
            }
        }
        Matcher matcher = PATTERN_LIST.matcher(s);
        if (!matcher.matches()) {
            throw new InvalidSemanticAnnotationException("Invalid format of non-forwarded fields annotation \"" + s + "\".");
        }
        // process individual fields
        matcher = PATTERN_FIELD.matcher(s);
        while (matcher.find()) {
            String fieldStr = matcher.group();
            try {
                // get and add all flat field positions
                List<FlatFieldDescriptor> inFFDs = getFlatFields(fieldStr, inType);
                for (FlatFieldDescriptor ffd : inFFDs) {
                    excludedFields = excludedFields.addField(ffd.getPosition());
                }
            } catch (InvalidFieldReferenceException ifre) {
                throw new InvalidSemanticAnnotationException("Invalid field reference in non-forwarded fields annotation \"" + fieldStr + "\".", ifre);
            }
        }
    }
    for (int i = 0; i < inType.getTotalFields(); i++) {
        if (!excludedFields.contains(i)) {
            if (sp instanceof SingleInputSemanticProperties) {
                ((SingleInputSemanticProperties) sp).addForwardedField(i, i);
            } else if (sp instanceof DualInputSemanticProperties) {
                ((DualInputSemanticProperties) sp).addForwardedField(input, i, i);
            }
        }
    }
}
Also used : FieldSet(org.apache.flink.api.common.operators.util.FieldSet) Matcher(java.util.regex.Matcher) InvalidSemanticAnnotationException(org.apache.flink.api.common.operators.SemanticProperties.InvalidSemanticAnnotationException) SingleInputSemanticProperties(org.apache.flink.api.common.operators.SingleInputSemanticProperties) DualInputSemanticProperties(org.apache.flink.api.common.operators.DualInputSemanticProperties) FlatFieldDescriptor(org.apache.flink.api.common.typeutils.CompositeType.FlatFieldDescriptor) InvalidFieldReferenceException(org.apache.flink.api.common.typeutils.CompositeType.InvalidFieldReferenceException)

Aggregations

DualInputSemanticProperties (org.apache.flink.api.common.operators.DualInputSemanticProperties)36 Test (org.junit.Test)24 Plan (org.apache.flink.api.common.Plan)11 ExecutionEnvironment (org.apache.flink.api.java.ExecutionEnvironment)11 FieldSet (org.apache.flink.api.common.operators.util.FieldSet)8 InnerJoinOperatorBase (org.apache.flink.api.common.operators.base.InnerJoinOperatorBase)7 Tuple2 (org.apache.flink.api.java.tuple.Tuple2)6 Tuple3 (org.apache.flink.api.java.tuple.Tuple3)6 InvalidSemanticAnnotationException (org.apache.flink.api.common.operators.SemanticProperties.InvalidSemanticAnnotationException)4 SingleInputSemanticProperties (org.apache.flink.api.common.operators.SingleInputSemanticProperties)4 Matcher (java.util.regex.Matcher)3 FlatFieldDescriptor (org.apache.flink.api.common.typeutils.CompositeType.FlatFieldDescriptor)3 InvalidFieldReferenceException (org.apache.flink.api.common.typeutils.CompositeType.InvalidFieldReferenceException)3 Annotation (java.lang.annotation.Annotation)2 SemanticProperties (org.apache.flink.api.common.operators.SemanticProperties)2 DualInputOperator (org.apache.flink.api.common.operators.DualInputOperator)1 SelectorFunctionKeys (org.apache.flink.api.common.operators.Keys.SelectorFunctionKeys)1 FunctionAnnotation (org.apache.flink.api.java.functions.FunctionAnnotation)1 ForwardedFields (org.apache.flink.api.java.functions.FunctionAnnotation.ForwardedFields)1 ForwardedFieldsFirst (org.apache.flink.api.java.functions.FunctionAnnotation.ForwardedFieldsFirst)1