Examples with Pattern org.apache.flink.cep.pattern.Pattern used on opensource projects

Example 1 with Pattern

use of org.apache.flink.cep.pattern.Pattern in project flink by apache.

the class CEPITCase method testSimpleAfterMatchSkip.

@Test
public void testSimpleAfterMatchSkip() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(envConfiguration);
    DataStream<Tuple2<Integer, String>> input = env.fromElements(new Tuple2<>(1, "a"), new Tuple2<>(2, "a"), new Tuple2<>(3, "a"), new Tuple2<>(4, "a"));
    Pattern<Tuple2<Integer, String>, ?> pattern = Pattern.<Tuple2<Integer, String>>begin("start", AfterMatchSkipStrategy.skipPastLastEvent()).where(new SimpleCondition<Tuple2<Integer, String>>() {

        @Override
        public boolean filter(Tuple2<Integer, String> rec) throws Exception {
            return rec.f1.equals("a");
        }
    }).times(2);
    PatternStream<Tuple2<Integer, String>> pStream = CEP.pattern(input, pattern).inProcessingTime();
    DataStream<Tuple2<Integer, String>> result = pStream.select(new PatternSelectFunction<Tuple2<Integer, String>, Tuple2<Integer, String>>() {

        @Override
        public Tuple2<Integer, String> select(Map<String, List<Tuple2<Integer, String>>> pattern) throws Exception {
            return pattern.get("start").get(0);
        }
    });
    List<Tuple2<Integer, String>> resultList = new ArrayList<>();
    DataStreamUtils.collect(result).forEachRemaining(resultList::add);
    resultList.sort(Comparator.comparing(tuple2 -> tuple2.toString()));
    List<Tuple2<Integer, String>> expected = Arrays.asList(Tuple2.of(1, "a"), Tuple2.of(3, "a"));
    assertEquals(expected, resultList);
}

Example 2 with Pattern

use of org.apache.flink.cep.pattern.Pattern in project flink by apache.

the class StreamExecMatch method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    final RowType inputRowType = (RowType) inputEdge.getOutputType();
    checkOrderKeys(inputRowType);
    final EventComparator<RowData> eventComparator = createEventComparator(config, inputRowType);
    final Transformation<RowData> timestampedInputTransform = translateOrder(inputTransform, inputRowType);
    final Tuple2<Pattern<RowData, RowData>, List<String>> cepPatternAndNames = translatePattern(matchSpec, config.getTableConfig(), planner.getRelBuilder(), inputRowType);
    final Pattern<RowData, RowData> cepPattern = cepPatternAndNames.f0;
    // TODO remove this once it is supported in CEP library
    if (NFACompiler.canProduceEmptyMatches(cepPattern)) {
        throw new TableException("Patterns that can produce empty matches are not supported. There must be at least one non-optional state.");
    }
    // TODO remove this once it is supported in CEP library
    if (cepPattern.getQuantifier().hasProperty(Quantifier.QuantifierProperty.GREEDY)) {
        throw new TableException("Greedy quantifiers are not allowed as the last element of a Pattern yet. " + "Finish your pattern with either a simple variable or reluctant quantifier.");
    }
    if (matchSpec.isAllRows()) {
        throw new TableException("All rows per match mode is not supported yet.");
    }
    final int[] partitionKeys = matchSpec.getPartition().getFieldIndices();
    final SortSpec.SortFieldSpec timeOrderField = matchSpec.getOrderKeys().getFieldSpec(0);
    final LogicalType timeOrderFieldType = inputRowType.getTypeAt(timeOrderField.getFieldIndex());
    final boolean isProctime = TypeCheckUtils.isProcTime(timeOrderFieldType);
    final InternalTypeInfo<RowData> inputTypeInfo = (InternalTypeInfo<RowData>) inputTransform.getOutputType();
    final TypeSerializer<RowData> inputSerializer = inputTypeInfo.createSerializer(planner.getExecEnv().getConfig());
    final NFACompiler.NFAFactory<RowData> nfaFactory = NFACompiler.compileFactory(cepPattern, false);
    final MatchCodeGenerator generator = new MatchCodeGenerator(new CodeGeneratorContext(config.getTableConfig()), planner.getRelBuilder(), // nullableInput
    false, JavaScalaConversionUtil.toScala(cepPatternAndNames.f1), JavaScalaConversionUtil.toScala(Optional.empty()), CodeGenUtils.DEFAULT_COLLECTOR_TERM());
    generator.bindInput(inputRowType, CodeGenUtils.DEFAULT_INPUT1_TERM(), JavaScalaConversionUtil.toScala(Optional.empty()));
    final PatternProcessFunctionRunner patternProcessFunction = generator.generateOneRowPerMatchExpression((RowType) getOutputType(), partitionKeys, matchSpec.getMeasures());
    final CepOperator<RowData, RowData, RowData> operator = new CepOperator<>(inputSerializer, isProctime, nfaFactory, eventComparator, cepPattern.getAfterMatchSkipStrategy(), patternProcessFunction, null);
    final OneInputTransformation<RowData, RowData> transform = ExecNodeUtil.createOneInputTransformation(timestampedInputTransform, createTransformationMeta(MATCH_TRANSFORMATION, config), operator, InternalTypeInfo.of(getOutputType()), timestampedInputTransform.getParallelism());
    final RowDataKeySelector selector = KeySelectorUtil.getRowDataSelector(partitionKeys, inputTypeInfo);
    transform.setStateKeySelector(selector);
    transform.setStateKeyType(selector.getProducedType());
    if (inputsContainSingleton()) {
        transform.setParallelism(1);
        transform.setMaxParallelism(1);
    }
    return transform;
}

Example 3 with Pattern

use of org.apache.flink.cep.pattern.Pattern in project flink by apache.

the class NFATest method testNFASerialization.

@Test
public void testNFASerialization() throws Exception {
    Pattern<Event, ?> pattern1 = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {

        private static final long serialVersionUID = 1858562682635302605L;

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("a");
        }
    }).followedByAny("middle").where(new SimpleCondition<Event>() {

        private static final long serialVersionUID = 8061969839441121955L;

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("b");
        }
    }).oneOrMore().optional().allowCombinations().followedByAny("end").where(new SimpleCondition<Event>() {

        private static final long serialVersionUID = 8061969839441121955L;

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("d");
        }
    });
    Pattern<Event, ?> pattern2 = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {

        private static final long serialVersionUID = 1858562682635302605L;

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("a");
        }
    }).notFollowedBy("not").where(new SimpleCondition<Event>() {

        private static final long serialVersionUID = -6085237016591726715L;

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("c");
        }
    }).followedByAny("middle").where(new SimpleCondition<Event>() {

        private static final long serialVersionUID = 8061969839441121955L;

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("b");
        }
    }).oneOrMore().optional().allowCombinations().followedByAny("end").where(new IterativeCondition<Event>() {

        private static final long serialVersionUID = 8061969839441121955L;

        @Override
        public boolean filter(Event value, IterativeCondition.Context<Event> ctx) throws Exception {
            double sum = 0.0;
            for (Event e : ctx.getEventsForPattern("middle")) {
                sum += e.getPrice();
            }
            return sum > 5.0;
        }
    });
    Pattern<Event, ?> pattern3 = Pattern.<Event>begin("start").notFollowedBy("not").where(new SimpleCondition<Event>() {

        private static final long serialVersionUID = -6085237016591726715L;

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("c");
        }
    }).followedByAny("middle").where(new SimpleCondition<Event>() {

        private static final long serialVersionUID = 8061969839441121955L;

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("b");
        }
    }).oneOrMore().allowCombinations().followedByAny("end").where(new SimpleCondition<Event>() {

        private static final long serialVersionUID = 8061969839441121955L;

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("d");
        }
    });
    List<Pattern<Event, ?>> patterns = new ArrayList<>();
    patterns.add(pattern1);
    patterns.add(pattern2);
    patterns.add(pattern3);
    for (Pattern<Event, ?> p : patterns) {
        NFA<Event> nfa = compile(p, false);
        Event a = new Event(40, "a", 1.0);
        Event b = new Event(41, "b", 2.0);
        Event c = new Event(42, "c", 3.0);
        Event b1 = new Event(41, "b", 3.0);
        Event b2 = new Event(41, "b", 4.0);
        Event b3 = new Event(41, "b", 5.0);
        Event d = new Event(43, "d", 4.0);
        NFAState nfaState = nfa.createInitialNFAState();
        NFATestHarness nfaTestHarness = NFATestHarness.forNFA(nfa).withNFAState(nfaState).build();
        nfaTestHarness.consumeRecord(new StreamRecord<>(a, 1));
        nfaTestHarness.consumeRecord(new StreamRecord<>(b, 2));
        nfaTestHarness.consumeRecord(new StreamRecord<>(c, 3));
        nfaTestHarness.consumeRecord(new StreamRecord<>(b1, 4));
        nfaTestHarness.consumeRecord(new StreamRecord<>(b2, 5));
        nfaTestHarness.consumeRecord(new StreamRecord<>(b3, 6));
        nfaTestHarness.consumeRecord(new StreamRecord<>(d, 7));
        nfaTestHarness.consumeRecord(new StreamRecord<>(a, 8));
        NFAStateSerializer serializer = new NFAStateSerializer();
        // serialize
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        serializer.serialize(nfaState, new DataOutputViewStreamWrapper(baos));
        baos.close();
        // copy
        ByteArrayInputStream in = new ByteArrayInputStream(baos.toByteArray());
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        serializer.duplicate().copy(new DataInputViewStreamWrapper(in), new DataOutputViewStreamWrapper(out));
        in.close();
        out.close();
        // deserialize
        ByteArrayInputStream bais = new ByteArrayInputStream(out.toByteArray());
        NFAState copy = serializer.duplicate().deserialize(new DataInputViewStreamWrapper(bais));
        bais.close();
        assertEquals(nfaState, copy);
    }
}

Example 4 with Pattern

use of org.apache.flink.cep.pattern.Pattern in project flink by apache.

the class CEPITCase method testTimeoutHandling.

@Test
public void testTimeoutHandling() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(envConfiguration);
    env.setParallelism(1);
    // (Event, timestamp)
    DataStream<Event> input = env.fromElements(Tuple2.of(new Event(1, "start", 1.0), 1L), Tuple2.of(new Event(1, "middle", 2.0), 5L), Tuple2.of(new Event(1, "start", 2.0), 4L), Tuple2.of(new Event(1, "end", 2.0), 6L)).assignTimestampsAndWatermarks(new AssignerWithPunctuatedWatermarks<Tuple2<Event, Long>>() {

        @Override
        public long extractTimestamp(Tuple2<Event, Long> element, long currentTimestamp) {
            return element.f1;
        }

        @Override
        public Watermark checkAndGetNextWatermark(Tuple2<Event, Long> lastElement, long extractedTimestamp) {
            return new Watermark(lastElement.f1 - 5);
        }
    }).map(new MapFunction<Tuple2<Event, Long>, Event>() {

        @Override
        public Event map(Tuple2<Event, Long> value) throws Exception {
            return value.f0;
        }
    });
    Pattern<Event, ?> pattern = Pattern.<Event>begin("start").where(new SimpleCondition<Event>() {

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("start");
        }
    }).followedByAny("middle").where(new SimpleCondition<Event>() {

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("middle");
        }
    }).followedByAny("end").where(new SimpleCondition<Event>() {

        @Override
        public boolean filter(Event value) throws Exception {
            return value.getName().equals("end");
        }
    }).within(Time.milliseconds(3));
    DataStream<Either<String, String>> result = CEP.pattern(input, pattern).select(new PatternTimeoutFunction<Event, String>() {

        @Override
        public String timeout(Map<String, List<Event>> pattern, long timeoutTimestamp) throws Exception {
            return pattern.get("start").get(0).getPrice() + "";
        }
    }, new PatternSelectFunction<Event, String>() {

        @Override
        public String select(Map<String, List<Event>> pattern) {
            StringBuilder builder = new StringBuilder();
            builder.append(pattern.get("start").get(0).getPrice()).append(",").append(pattern.get("middle").get(0).getPrice()).append(",").append(pattern.get("end").get(0).getPrice());
            return builder.toString();
        }
    });
    List<Either<String, String>> resultList = new ArrayList<>();
    DataStreamUtils.collect(result).forEachRemaining(resultList::add);
    resultList.sort(Comparator.comparing(either -> either.toString()));
    List<Either<String, String>> expected = Arrays.asList(Either.Left.of("1.0"), Either.Left.of("2.0"), Either.Left.of("2.0"), Either.Right.of("2.0,2.0,2.0"));
    assertEquals(expected, resultList);
}