Examples with SubEvent org.apache.flink.cep.SubEvent used on opensource projects

Example 1 with SubEvent

use of org.apache.flink.cep.SubEvent in project flink by apache.

the class CEPMigration11to13Test method testNonKeyedCEPFunctionMigration.

@Test
public void testNonKeyedCEPFunctionMigration() throws Exception {
    final Event startEvent = new Event(42, "start", 1.0);
    final SubEvent middleEvent = new SubEvent(42, "foo", 1.0, 10.0);
    final Event endEvent = new Event(42, "end", 1.0);
    // uncomment these lines for regenerating the snapshot on Flink 1.1
    /*
		OneInputStreamOperatorTestHarness<Event, Map<String, Event>> harness = new OneInputStreamOperatorTestHarness<>(
				new CEPPatternOperator<>(
						Event.createTypeSerializer(),
						false,
						new NFAFactory()));
		harness.open();
		harness.processElement(new StreamRecord<Event>(startEvent, 1));
		harness.processElement(new StreamRecord<Event>(new Event(42, "foobar", 1.0), 2));
		harness.processElement(new StreamRecord<Event>(new SubEvent(42, "barfoo", 1.0, 5.0), 3));
		harness.processWatermark(new Watermark(2));

		// simulate snapshot/restore with empty element queue but NFA state
		StreamTaskState snapshot = harness.snapshot(1, 1);
		FileOutputStream out = new FileOutputStream(
				"src/test/resources/cep-non-keyed-snapshot-1.1");
		ObjectOutputStream oos = new ObjectOutputStream(out);
		oos.writeObject(snapshot);
		out.close();
		harness.close();
		*/
    NullByteKeySelector keySelector = new NullByteKeySelector();
    OneInputStreamOperatorTestHarness<Event, Map<String, Event>> harness = new KeyedOneInputStreamOperatorTestHarness<>(new KeyedCEPPatternOperator<>(Event.createTypeSerializer(), false, keySelector, ByteSerializer.INSTANCE, new NFAFactory(), false), keySelector, BasicTypeInfo.BYTE_TYPE_INFO);
    harness.setup();
    harness.initializeStateFromLegacyCheckpoint(getResourceFilename("cep-non-keyed-snapshot-1.1"));
    harness.open();
    harness.processElement(new StreamRecord<Event>(middleEvent, 3));
    harness.processElement(new StreamRecord<>(new Event(42, "start", 1.0), 4));
    harness.processElement(new StreamRecord<>(endEvent, 5));
    harness.processWatermark(new Watermark(Long.MAX_VALUE));
    ConcurrentLinkedQueue<Object> result = harness.getOutput();
    // watermark and the result
    assertEquals(2, result.size());
    Object resultObject = result.poll();
    assertTrue(resultObject instanceof StreamRecord);
    StreamRecord<?> resultRecord = (StreamRecord<?>) resultObject;
    assertTrue(resultRecord.getValue() instanceof Map);
    @SuppressWarnings("unchecked") Map<String, Event> patternMap = (Map<String, Event>) resultRecord.getValue();
    assertEquals(startEvent, patternMap.get("start"));
    assertEquals(middleEvent, patternMap.get("middle"));
    assertEquals(endEvent, patternMap.get("end"));
    harness.close();
}

Example 2 with SubEvent

use of org.apache.flink.cep.SubEvent in project flink by apache.

the class CEPMigration11to13Test method testKeyedCEPOperatorMigratation.

@Test
public void testKeyedCEPOperatorMigratation() throws Exception {
    KeySelector<Event, Integer> keySelector = new KeySelector<Event, Integer>() {

        private static final long serialVersionUID = -4873366487571254798L;

        @Override
        public Integer getKey(Event value) throws Exception {
            return value.getId();
        }
    };
    final Event startEvent = new Event(42, "start", 1.0);
    final SubEvent middleEvent = new SubEvent(42, "foo", 1.0, 10.0);
    final Event endEvent = new Event(42, "end", 1.0);
    // uncomment these lines for regenerating the snapshot on Flink 1.1
    /*
		OneInputStreamOperatorTestHarness<Event, Map<String, Event>> harness = new OneInputStreamOperatorTestHarness<>(
				new KeyedCEPPatternOperator<>(
						Event.createTypeSerializer(),
						false,
						keySelector,
						IntSerializer.INSTANCE,
						new NFAFactory()));
		harness.configureForKeyedStream(keySelector, BasicTypeInfo.INT_TYPE_INFO);
		harness.open();
		harness.processElement(new StreamRecord<Event>(startEvent, 1));
		harness.processElement(new StreamRecord<Event>(new Event(42, "foobar", 1.0), 2));
		harness.processElement(new StreamRecord<Event>(new SubEvent(42, "barfoo", 1.0, 5.0), 3));
		harness.processWatermark(new Watermark(2));
		// simulate snapshot/restore with empty element queue but NFA state
		StreamTaskState snapshot = harness.snapshot(1, 1);
		FileOutputStream out = new FileOutputStream(
				"src/test/resources/cep-keyed-snapshot-1.1");
		ObjectOutputStream oos = new ObjectOutputStream(out);
		oos.writeObject(snapshot);
		out.close();
		harness.close();
		*/
    OneInputStreamOperatorTestHarness<Event, Map<String, Event>> harness = new KeyedOneInputStreamOperatorTestHarness<>(new KeyedCEPPatternOperator<>(Event.createTypeSerializer(), false, keySelector, IntSerializer.INSTANCE, new NFAFactory(), true), keySelector, BasicTypeInfo.INT_TYPE_INFO);
    harness.setup();
    harness.initializeStateFromLegacyCheckpoint(getResourceFilename("cep-keyed-snapshot-1.1"));
    harness.open();
    harness.processElement(new StreamRecord<Event>(middleEvent, 3));
    harness.processElement(new StreamRecord<>(new Event(42, "start", 1.0), 4));
    harness.processElement(new StreamRecord<>(endEvent, 5));
    harness.processWatermark(new Watermark(20));
    ConcurrentLinkedQueue<Object> result = harness.getOutput();
    // watermark and the result
    assertEquals(2, result.size());
    Object resultObject = result.poll();
    assertTrue(resultObject instanceof StreamRecord);
    StreamRecord<?> resultRecord = (StreamRecord<?>) resultObject;
    assertTrue(resultRecord.getValue() instanceof Map);
    @SuppressWarnings("unchecked") Map<String, Event> patternMap = (Map<String, Event>) resultRecord.getValue();
    assertEquals(startEvent, patternMap.get("start"));
    assertEquals(middleEvent, patternMap.get("middle"));
    assertEquals(endEvent, patternMap.get("end"));
    harness.close();
}

Example 3 with SubEvent

use of org.apache.flink.cep.SubEvent in project flink by apache.

the class CEPOperatorTest method testCEPOperatorComparatorProcessTime.

@Test
public void testCEPOperatorComparatorProcessTime() throws Exception {
    Event startEvent1 = new Event(42, "start", 1.0);
    Event startEvent2 = new Event(42, "start", 2.0);
    SubEvent middleEvent1 = new SubEvent(42, "foo1", 3.0, 10.0);
    SubEvent middleEvent2 = new SubEvent(42, "foo2", 4.0, 10.0);
    Event endEvent1 = new Event(42, "end", 1.0);
    Event startEventK2 = new Event(43, "start", 1.0);
    CepOperator<Event, Integer, Map<String, List<Event>>> operator = getKeyedCepOperatorWithComparator(true);
    OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(operator);
    try {
        harness.open();
        harness.setProcessingTime(0L);
        harness.processElement(new StreamRecord<>(startEvent1, 0L));
        harness.processElement(new StreamRecord<>(startEventK2, 0L));
        harness.processElement(new StreamRecord<>(new Event(42, "foobar", 1.0), 0L));
        harness.processElement(new StreamRecord<>(new SubEvent(42, "barfoo", 1.0, 5.0), 0L));
        assertTrue(!operator.hasNonEmptySharedBuffer(42));
        assertTrue(!operator.hasNonEmptySharedBuffer(43));
        harness.setProcessingTime(3L);
        assertTrue(operator.hasNonEmptySharedBuffer(42));
        assertTrue(operator.hasNonEmptySharedBuffer(43));
        harness.processElement(new StreamRecord<>(middleEvent2, 3L));
        harness.processElement(new StreamRecord<>(middleEvent1, 3L));
        harness.processElement(new StreamRecord<>(startEvent2, 3L));
        OperatorSubtaskState snapshot = harness.snapshot(0L, 0L);
        harness.close();
        CepOperator<Event, Integer, Map<String, List<Event>>> operator2 = getKeyedCepOperatorWithComparator(true);
        harness = CepOperatorTestUtilities.getCepTestHarness(operator2);
        harness.setup();
        harness.initializeState(snapshot);
        harness.open();
        harness.setProcessingTime(4L);
        harness.processElement(new StreamRecord<>(endEvent1, 5L));
        harness.setProcessingTime(5L);
        verifyPattern(harness.getOutput().poll(), startEvent1, middleEvent1, endEvent1);
        verifyPattern(harness.getOutput().poll(), startEvent1, middleEvent2, endEvent1);
        verifyPattern(harness.getOutput().poll(), startEvent2, middleEvent1, endEvent1);
        verifyPattern(harness.getOutput().poll(), startEvent2, middleEvent2, endEvent1);
    } finally {
        harness.close();
    }
}

Example 4 with SubEvent

use of org.apache.flink.cep.SubEvent in project flink by apache.

the class CEPOperatorTest method testKeyedCEPOperatorCheckpointing.

@Test
public void testKeyedCEPOperatorCheckpointing() throws Exception {
    OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = getCepTestHarness(false);
    try {
        harness.open();
        Event startEvent = new Event(42, "start", 1.0);
        SubEvent middleEvent = new SubEvent(42, "foo", 1.0, 10.0);
        Event endEvent = new Event(42, "end", 1.0);
        harness.processElement(new StreamRecord<>(startEvent, 1L));
        harness.processElement(new StreamRecord<>(new Event(42, "foobar", 1.0), 2L));
        // simulate snapshot/restore with some elements in internal sorting queue
        OperatorSubtaskState snapshot = harness.snapshot(0L, 0L);
        harness.close();
        harness = getCepTestHarness(false);
        harness.setup();
        harness.initializeState(snapshot);
        harness.open();
        harness.processWatermark(new Watermark(Long.MIN_VALUE));
        harness.processElement(new StreamRecord<Event>(new SubEvent(42, "barfoo", 1.0, 5.0), 3L));
        // if element timestamps are not correctly checkpointed/restored this will lead to
        // a pruning time underflow exception in NFA
        harness.processWatermark(new Watermark(2L));
        harness.processElement(new StreamRecord<Event>(middleEvent, 3L));
        harness.processElement(new StreamRecord<>(new Event(42, "start", 1.0), 4L));
        harness.processElement(new StreamRecord<>(endEvent, 5L));
        // simulate snapshot/restore with empty element queue but NFA state
        OperatorSubtaskState snapshot2 = harness.snapshot(1L, 1L);
        harness.close();
        harness = getCepTestHarness(false);
        harness.setup();
        harness.initializeState(snapshot2);
        harness.open();
        harness.processWatermark(new Watermark(Long.MAX_VALUE));
        // get and verify the output
        Queue<Object> result = harness.getOutput();
        assertEquals(2, result.size());
        verifyPattern(result.poll(), startEvent, middleEvent, endEvent);
        verifyWatermark(result.poll(), Long.MAX_VALUE);
    } finally {
        harness.close();
    }
}

Example 5 with SubEvent

use of org.apache.flink.cep.SubEvent in project flink by apache.

the class CEPOperatorTest method testKeyedCEPOperatorNFAUpdate.

@Test
public void testKeyedCEPOperatorNFAUpdate() throws Exception {
    CepOperator<Event, Integer, Map<String, List<Event>>> operator = CepOperatorTestUtilities.getKeyedCepOperator(true, new SimpleNFAFactory());
    OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(operator);
    try {
        harness.open();
        Event startEvent = new Event(42, "c", 1.0);
        SubEvent middleEvent = new SubEvent(42, "a", 1.0, 10.0);
        Event endEvent = new Event(42, "b", 1.0);
        harness.processElement(new StreamRecord<>(startEvent, 1L));
        // simulate snapshot/restore with some elements in internal sorting queue
        OperatorSubtaskState snapshot = harness.snapshot(0L, 0L);
        harness.close();
        operator = CepOperatorTestUtilities.getKeyedCepOperator(true, new SimpleNFAFactory());
        harness = CepOperatorTestUtilities.getCepTestHarness(operator);
        harness.setup();
        harness.initializeState(snapshot);
        harness.open();
        harness.processElement(new StreamRecord<>(new Event(42, "d", 1.0), 4L));
        OperatorSubtaskState snapshot2 = harness.snapshot(0L, 0L);
        harness.close();
        operator = CepOperatorTestUtilities.getKeyedCepOperator(true, new SimpleNFAFactory());
        harness = CepOperatorTestUtilities.getCepTestHarness(operator);
        harness.setup();
        harness.initializeState(snapshot2);
        harness.open();
        harness.processElement(new StreamRecord<Event>(middleEvent, 4L));
        harness.processElement(new StreamRecord<>(endEvent, 4L));
        // get and verify the output
        Queue<Object> result = harness.getOutput();
        assertEquals(1, result.size());
        verifyPattern(result.poll(), startEvent, middleEvent, endEvent);
    } finally {
        harness.close();
    }
}