use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.
the class CEPOperatorTest method testKeyedCEPOperatorNFAUpdate.
@Test
public void testKeyedCEPOperatorNFAUpdate() throws Exception {
CepOperator<Event, Integer, Map<String, List<Event>>> operator = CepOperatorTestUtilities.getKeyedCepOperator(true, new SimpleNFAFactory());
OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(operator);
try {
harness.open();
Event startEvent = new Event(42, "c", 1.0);
SubEvent middleEvent = new SubEvent(42, "a", 1.0, 10.0);
Event endEvent = new Event(42, "b", 1.0);
harness.processElement(new StreamRecord<>(startEvent, 1L));
// simulate snapshot/restore with some elements in internal sorting queue
OperatorSubtaskState snapshot = harness.snapshot(0L, 0L);
harness.close();
operator = CepOperatorTestUtilities.getKeyedCepOperator(true, new SimpleNFAFactory());
harness = CepOperatorTestUtilities.getCepTestHarness(operator);
harness.setup();
harness.initializeState(snapshot);
harness.open();
harness.processElement(new StreamRecord<>(new Event(42, "d", 1.0), 4L));
OperatorSubtaskState snapshot2 = harness.snapshot(0L, 0L);
harness.close();
operator = CepOperatorTestUtilities.getKeyedCepOperator(true, new SimpleNFAFactory());
harness = CepOperatorTestUtilities.getCepTestHarness(operator);
harness.setup();
harness.initializeState(snapshot2);
harness.open();
harness.processElement(new StreamRecord<Event>(middleEvent, 4L));
harness.processElement(new StreamRecord<>(endEvent, 4L));
// get and verify the output
Queue<Object> result = harness.getOutput();
assertEquals(1, result.size());
verifyPattern(result.poll(), startEvent, middleEvent, endEvent);
} finally {
harness.close();
}
}
use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.
the class CEPOperatorTest method testCEPOperatorCleanupEventTime.
@Test
public void testCEPOperatorCleanupEventTime() throws Exception {
Event startEvent1 = new Event(42, "start", 1.0);
Event startEvent2 = new Event(42, "start", 2.0);
SubEvent middleEvent1 = new SubEvent(42, "foo1", 1.0, 10.0);
SubEvent middleEvent2 = new SubEvent(42, "foo2", 1.0, 10.0);
SubEvent middleEvent3 = new SubEvent(42, "foo3", 1.0, 10.0);
Event endEvent1 = new Event(42, "end", 1.0);
Event endEvent2 = new Event(42, "end", 2.0);
Event startEventK2 = new Event(43, "start", 1.0);
CepOperator<Event, Integer, Map<String, List<Event>>> operator = getKeyedCepOperator(false);
OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(operator);
try {
harness.open();
harness.processWatermark(new Watermark(Long.MIN_VALUE));
harness.processElement(new StreamRecord<>(new Event(42, "foobar", 1.0), 2L));
harness.processElement(new StreamRecord<Event>(middleEvent1, 2L));
harness.processElement(new StreamRecord<Event>(new SubEvent(42, "barfoo", 1.0, 5.0), 3L));
harness.processElement(new StreamRecord<>(startEvent1, 1L));
harness.processElement(new StreamRecord<>(startEventK2, 1L));
// there must be 2 keys 42, 43 registered for the watermark callback
// all the seen elements must be in the priority queues but no NFA yet.
assertEquals(2L, harness.numEventTimeTimers());
assertEquals(4L, operator.getPQSize(42));
assertEquals(1L, operator.getPQSize(43));
assertTrue(!operator.hasNonEmptySharedBuffer(42));
assertTrue(!operator.hasNonEmptySharedBuffer(43));
harness.processWatermark(new Watermark(2L));
verifyWatermark(harness.getOutput().poll(), Long.MIN_VALUE);
verifyWatermark(harness.getOutput().poll(), 2L);
// still the 2 keys
// one element in PQ for 42 (the barfoo) as it arrived early
// for 43 the element entered the NFA and the PQ is empty
assertEquals(2L, harness.numEventTimeTimers());
assertTrue(operator.hasNonEmptySharedBuffer(42));
assertEquals(1L, operator.getPQSize(42));
assertTrue(operator.hasNonEmptySharedBuffer(43));
assertTrue(!operator.hasNonEmptyPQ(43));
harness.processElement(new StreamRecord<>(startEvent2, 4L));
harness.processElement(new StreamRecord<Event>(middleEvent2, 5L));
OperatorSubtaskState snapshot = harness.snapshot(0L, 0L);
harness.close();
CepOperator<Event, Integer, Map<String, List<Event>>> operator2 = getKeyedCepOperator(false);
harness = CepOperatorTestUtilities.getCepTestHarness(operator2);
harness.setup();
harness.initializeState(snapshot);
harness.open();
harness.processElement(new StreamRecord<>(endEvent1, 6L));
harness.processWatermark(11L);
harness.processWatermark(12L);
// now we have 1 key because the 43 expired and was removed.
// 42 is still there due to startEvent2
assertEquals(1L, harness.numEventTimeTimers());
assertTrue(operator2.hasNonEmptySharedBuffer(42));
assertTrue(!operator2.hasNonEmptyPQ(42));
assertTrue(!operator2.hasNonEmptySharedBuffer(43));
assertTrue(!operator2.hasNonEmptyPQ(43));
verifyPattern(harness.getOutput().poll(), startEvent1, middleEvent1, endEvent1);
verifyPattern(harness.getOutput().poll(), startEvent1, middleEvent2, endEvent1);
verifyPattern(harness.getOutput().poll(), startEvent2, middleEvent2, endEvent1);
verifyWatermark(harness.getOutput().poll(), 11L);
verifyWatermark(harness.getOutput().poll(), 12L);
// this is a late event, because timestamp(12) = last watermark(12)
harness.processElement(new StreamRecord<Event>(middleEvent3, 12L));
harness.processElement(new StreamRecord<>(endEvent2, 13L));
harness.processWatermark(20L);
harness.processWatermark(21L);
assertTrue(!operator2.hasNonEmptySharedBuffer(42));
assertTrue(!operator2.hasNonEmptyPQ(42));
assertEquals(0L, harness.numEventTimeTimers());
assertEquals(3, harness.getOutput().size());
verifyPattern(harness.getOutput().poll(), startEvent2, middleEvent2, endEvent2);
verifyWatermark(harness.getOutput().poll(), 20L);
verifyWatermark(harness.getOutput().poll(), 21L);
} finally {
harness.close();
}
}
use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.
the class CEPOperatorTest method testKeyedCEPOperatorNFAUpdateWithRocksDB.
@Test
public void testKeyedCEPOperatorNFAUpdateWithRocksDB() throws Exception {
String rocksDbPath = tempFolder.newFolder().getAbsolutePath();
RocksDBStateBackend rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend(), TernaryBoolean.FALSE);
rocksDBStateBackend.setDbStoragePath(rocksDbPath);
CepOperator<Event, Integer, Map<String, List<Event>>> operator = CepOperatorTestUtilities.getKeyedCepOperator(true, new SimpleNFAFactory());
OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(operator);
try {
harness.setStateBackend(rocksDBStateBackend);
harness.open();
Event startEvent = new Event(42, "c", 1.0);
SubEvent middleEvent = new SubEvent(42, "a", 1.0, 10.0);
Event endEvent = new Event(42, "b", 1.0);
harness.processElement(new StreamRecord<>(startEvent, 1L));
// simulate snapshot/restore with some elements in internal sorting queue
OperatorSubtaskState snapshot = harness.snapshot(0L, 0L);
harness.close();
operator = CepOperatorTestUtilities.getKeyedCepOperator(true, new SimpleNFAFactory());
harness = CepOperatorTestUtilities.getCepTestHarness(operator);
rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend());
rocksDBStateBackend.setDbStoragePath(rocksDbPath);
harness.setStateBackend(rocksDBStateBackend);
harness.setup();
harness.initializeState(snapshot);
harness.open();
harness.processElement(new StreamRecord<>(new Event(42, "d", 1.0), 4L));
OperatorSubtaskState snapshot2 = harness.snapshot(0L, 0L);
harness.close();
operator = CepOperatorTestUtilities.getKeyedCepOperator(true, new SimpleNFAFactory());
harness = CepOperatorTestUtilities.getCepTestHarness(operator);
rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend());
rocksDBStateBackend.setDbStoragePath(rocksDbPath);
harness.setStateBackend(rocksDBStateBackend);
harness.setup();
harness.initializeState(snapshot2);
harness.open();
harness.processElement(new StreamRecord<Event>(middleEvent, 4L));
harness.processElement(new StreamRecord<>(endEvent, 4L));
// get and verify the output
Queue<Object> result = harness.getOutput();
assertEquals(1, result.size());
verifyPattern(result.poll(), startEvent, middleEvent, endEvent);
} finally {
harness.close();
}
}
use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.
the class CEPOperatorTest method testCEPOperatorComparatorEventTime.
@Test
public void testCEPOperatorComparatorEventTime() throws Exception {
Event startEvent1 = new Event(42, "start", 1.0);
Event startEvent2 = new Event(42, "start", 2.0);
SubEvent middleEvent1 = new SubEvent(42, "foo1", 1.0, 10.0);
SubEvent middleEvent2 = new SubEvent(42, "foo2", 2.0, 10.0);
Event endEvent = new Event(42, "end", 1.0);
Event startEventK2 = new Event(43, "start", 1.0);
CepOperator<Event, Integer, Map<String, List<Event>>> operator = getKeyedCepOperatorWithComparator(false);
OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(operator);
try {
harness.open();
harness.processWatermark(0L);
harness.processElement(new StreamRecord<>(startEvent1, 1L));
harness.processElement(new StreamRecord<>(startEventK2, 1L));
harness.processElement(new StreamRecord<>(new Event(42, "foobar", 1.0), 2L));
harness.processElement(new StreamRecord<Event>(new SubEvent(42, "barfoo", 1.0, 5.0), 3L));
assertTrue(operator.hasNonEmptyPQ(42));
assertTrue(operator.hasNonEmptyPQ(43));
assertFalse(operator.hasNonEmptySharedBuffer(42));
assertFalse(operator.hasNonEmptySharedBuffer(43));
harness.processWatermark(3L);
assertFalse(operator.hasNonEmptyPQ(42));
assertFalse(operator.hasNonEmptyPQ(43));
assertTrue(operator.hasNonEmptySharedBuffer(42));
assertTrue(operator.hasNonEmptySharedBuffer(43));
harness.processElement(new StreamRecord<>(startEvent2, 4L));
harness.processElement(new StreamRecord<Event>(middleEvent2, 5L));
harness.processElement(new StreamRecord<Event>(middleEvent1, 5L));
OperatorSubtaskState snapshot = harness.snapshot(0L, 0L);
harness.close();
CepOperator<Event, Integer, Map<String, List<Event>>> operator2 = getKeyedCepOperatorWithComparator(false);
harness = CepOperatorTestUtilities.getCepTestHarness(operator2);
harness.setup();
harness.initializeState(snapshot);
harness.open();
harness.processElement(new StreamRecord<>(endEvent, 6L));
harness.processWatermark(6L);
verifyPattern(harness.getOutput().poll(), startEvent1, middleEvent1, endEvent);
verifyPattern(harness.getOutput().poll(), startEvent1, middleEvent2, endEvent);
verifyPattern(harness.getOutput().poll(), startEvent2, middleEvent1, endEvent);
verifyPattern(harness.getOutput().poll(), startEvent2, middleEvent2, endEvent);
verifyWatermark(harness.getOutput().poll(), 6L);
} finally {
harness.close();
}
}
use of org.apache.flink.runtime.checkpoint.OperatorSubtaskState in project flink by apache.
the class CEPMigrationTest method testRestoreStartingNewPatternAfterMigration.
@Test
public void testRestoreStartingNewPatternAfterMigration() throws Exception {
KeySelector<Event, Integer> keySelector = new KeySelector<Event, Integer>() {
private static final long serialVersionUID = -4873366487571254798L;
@Override
public Integer getKey(Event value) throws Exception {
return value.getId();
}
};
final Event startEvent1 = new Event(42, "start", 1.0);
final SubEvent middleEvent1 = new SubEvent(42, "foo1", 1.0, 10.0);
final Event startEvent2 = new Event(42, "start", 5.0);
final SubEvent middleEvent2 = new SubEvent(42, "foo2", 2.0, 10.0);
final Event endEvent = new Event(42, "end", 1.0);
OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = new KeyedOneInputStreamOperatorTestHarness<>(CepOperatorTestUtilities.getKeyedCepOperator(false, new NFAFactory()), keySelector, BasicTypeInfo.INT_TYPE_INFO);
try {
harness.setup();
harness.initializeState(OperatorSnapshotUtil.getResourceFilename("cep-migration-starting-new-pattern-flink" + migrateVersion + "-snapshot"));
harness.open();
harness.processElement(new StreamRecord<>(startEvent2, 5));
harness.processElement(new StreamRecord<Event>(middleEvent2, 6));
harness.processElement(new StreamRecord<>(endEvent, 7));
harness.processWatermark(new Watermark(20));
ConcurrentLinkedQueue<Object> result = harness.getOutput();
// watermark and 3 results
assertEquals(4, result.size());
Object resultObject1 = result.poll();
assertTrue(resultObject1 instanceof StreamRecord);
StreamRecord<?> resultRecord1 = (StreamRecord<?>) resultObject1;
assertTrue(resultRecord1.getValue() instanceof Map);
Object resultObject2 = result.poll();
assertTrue(resultObject2 instanceof StreamRecord);
StreamRecord<?> resultRecord2 = (StreamRecord<?>) resultObject2;
assertTrue(resultRecord2.getValue() instanceof Map);
Object resultObject3 = result.poll();
assertTrue(resultObject3 instanceof StreamRecord);
StreamRecord<?> resultRecord3 = (StreamRecord<?>) resultObject3;
assertTrue(resultRecord3.getValue() instanceof Map);
@SuppressWarnings("unchecked") Map<String, List<Event>> patternMap1 = (Map<String, List<Event>>) resultRecord1.getValue();
assertEquals(startEvent1, patternMap1.get("start").get(0));
assertEquals(middleEvent1, patternMap1.get("middle").get(0));
assertEquals(endEvent, patternMap1.get("end").get(0));
@SuppressWarnings("unchecked") Map<String, List<Event>> patternMap2 = (Map<String, List<Event>>) resultRecord2.getValue();
assertEquals(startEvent1, patternMap2.get("start").get(0));
assertEquals(middleEvent2, patternMap2.get("middle").get(0));
assertEquals(endEvent, patternMap2.get("end").get(0));
@SuppressWarnings("unchecked") Map<String, List<Event>> patternMap3 = (Map<String, List<Event>>) resultRecord3.getValue();
assertEquals(startEvent2, patternMap3.get("start").get(0));
assertEquals(middleEvent2, patternMap3.get("middle").get(0));
assertEquals(endEvent, patternMap3.get("end").get(0));
// and now go for a checkpoint with the new serializers
final Event startEvent3 = new Event(42, "start", 2.0);
final SubEvent middleEvent3 = new SubEvent(42, "foo", 1.0, 11.0);
final Event endEvent1 = new Event(42, "end", 2.0);
harness.processElement(new StreamRecord<Event>(startEvent3, 21));
harness.processElement(new StreamRecord<Event>(middleEvent3, 23));
// simulate snapshot/restore with some elements in internal sorting queue
OperatorSubtaskState snapshot = harness.snapshot(1L, 1L);
harness.close();
harness = new KeyedOneInputStreamOperatorTestHarness<>(CepOperatorTestUtilities.getKeyedCepOperator(false, new NFAFactory()), keySelector, BasicTypeInfo.INT_TYPE_INFO);
harness.setup();
harness.initializeState(snapshot);
harness.open();
harness.processElement(new StreamRecord<>(endEvent1, 25));
harness.processWatermark(new Watermark(50));
result = harness.getOutput();
// watermark and the result
assertEquals(2, result.size());
Object resultObject4 = result.poll();
assertTrue(resultObject4 instanceof StreamRecord);
StreamRecord<?> resultRecord4 = (StreamRecord<?>) resultObject4;
assertTrue(resultRecord4.getValue() instanceof Map);
@SuppressWarnings("unchecked") Map<String, List<Event>> patternMap4 = (Map<String, List<Event>>) resultRecord4.getValue();
assertEquals(startEvent3, patternMap4.get("start").get(0));
assertEquals(middleEvent3, patternMap4.get("middle").get(0));
assertEquals(endEvent1, patternMap4.get("end").get(0));
} finally {
harness.close();
}
}
Aggregations