Examples with KeySelector org.apache.flink.api.java.functions.KeySelector used on opensource projects

Example 21 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class CEPMigrationTest method testSinglePatternAfterMigration.

@Test
public void testSinglePatternAfterMigration() throws Exception {
    KeySelector<Event, Integer> keySelector = new KeySelector<Event, Integer>() {

        private static final long serialVersionUID = -4873366487571254798L;

        @Override
        public Integer getKey(Event value) throws Exception {
            return value.getId();
        }
    };
    final Event startEvent1 = new Event(42, "start", 1.0);
    OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = new KeyedOneInputStreamOperatorTestHarness<>(CepOperatorTestUtilities.getKeyedCepOperator(false, new SinglePatternNFAFactory()), keySelector, BasicTypeInfo.INT_TYPE_INFO);
    try {
        harness.setup();
        harness.initializeState(OperatorSnapshotUtil.getResourceFilename("cep-migration-single-pattern-afterwards-flink" + migrateVersion + "-snapshot"));
        harness.open();
        harness.processElement(new StreamRecord<>(startEvent1, 5));
        harness.processWatermark(new Watermark(20));
        ConcurrentLinkedQueue<Object> result = harness.getOutput();
        // watermark and the result
        assertEquals(2, result.size());
        Object resultObject = result.poll();
        assertTrue(resultObject instanceof StreamRecord);
        StreamRecord<?> resultRecord = (StreamRecord<?>) resultObject;
        assertTrue(resultRecord.getValue() instanceof Map);
        @SuppressWarnings("unchecked") Map<String, List<Event>> patternMap = (Map<String, List<Event>>) resultRecord.getValue();
        assertEquals(startEvent1, patternMap.get("start").get(0));
    } finally {
        harness.close();
    }
}

Example 22 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class CEPMigrationTest method testRestoreAfterBranchingPattern.

@Test
public void testRestoreAfterBranchingPattern() throws Exception {
    KeySelector<Event, Integer> keySelector = new KeySelector<Event, Integer>() {

        private static final long serialVersionUID = -4873366487571254798L;

        @Override
        public Integer getKey(Event value) throws Exception {
            return value.getId();
        }
    };
    final Event startEvent = new Event(42, "start", 1.0);
    final SubEvent middleEvent1 = new SubEvent(42, "foo1", 1.0, 10.0);
    final SubEvent middleEvent2 = new SubEvent(42, "foo2", 2.0, 10.0);
    final Event endEvent = new Event(42, "end", 1.0);
    OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = new KeyedOneInputStreamOperatorTestHarness<>(CepOperatorTestUtilities.getKeyedCepOperator(false, new NFAFactory()), keySelector, BasicTypeInfo.INT_TYPE_INFO);
    try {
        harness.setup();
        harness.initializeState(OperatorSnapshotUtil.getResourceFilename("cep-migration-after-branching-flink" + migrateVersion + "-snapshot"));
        harness.open();
        harness.processElement(new StreamRecord<>(new Event(42, "start", 1.0), 4));
        harness.processElement(new StreamRecord<>(endEvent, 5));
        harness.processWatermark(new Watermark(20));
        ConcurrentLinkedQueue<Object> result = harness.getOutput();
        // watermark and 2 results
        assertEquals(3, result.size());
        Object resultObject1 = result.poll();
        assertTrue(resultObject1 instanceof StreamRecord);
        StreamRecord<?> resultRecord1 = (StreamRecord<?>) resultObject1;
        assertTrue(resultRecord1.getValue() instanceof Map);
        Object resultObject2 = result.poll();
        assertTrue(resultObject2 instanceof StreamRecord);
        StreamRecord<?> resultRecord2 = (StreamRecord<?>) resultObject2;
        assertTrue(resultRecord2.getValue() instanceof Map);
        @SuppressWarnings("unchecked") Map<String, List<Event>> patternMap1 = (Map<String, List<Event>>) resultRecord1.getValue();
        assertEquals(startEvent, patternMap1.get("start").get(0));
        assertEquals(middleEvent1, patternMap1.get("middle").get(0));
        assertEquals(endEvent, patternMap1.get("end").get(0));
        @SuppressWarnings("unchecked") Map<String, List<Event>> patternMap2 = (Map<String, List<Event>>) resultRecord2.getValue();
        assertEquals(startEvent, patternMap2.get("start").get(0));
        assertEquals(middleEvent2, patternMap2.get("middle").get(0));
        assertEquals(endEvent, patternMap2.get("end").get(0));
        // and now go for a checkpoint with the new serializers
        final Event startEvent1 = new Event(42, "start", 2.0);
        final SubEvent middleEvent3 = new SubEvent(42, "foo", 1.0, 11.0);
        final Event endEvent1 = new Event(42, "end", 2.0);
        harness.processElement(new StreamRecord<Event>(startEvent1, 21));
        harness.processElement(new StreamRecord<Event>(middleEvent3, 23));
        // simulate snapshot/restore with some elements in internal sorting queue
        OperatorSubtaskState snapshot = harness.snapshot(1L, 1L);
        harness.close();
        harness = new KeyedOneInputStreamOperatorTestHarness<>(CepOperatorTestUtilities.getKeyedCepOperator(false, new NFAFactory()), keySelector, BasicTypeInfo.INT_TYPE_INFO);
        harness.setup();
        harness.initializeState(snapshot);
        harness.open();
        harness.processElement(new StreamRecord<>(endEvent1, 25));
        harness.processWatermark(new Watermark(50));
        result = harness.getOutput();
        // watermark and the result
        assertEquals(2, result.size());
        Object resultObject3 = result.poll();
        assertTrue(resultObject3 instanceof StreamRecord);
        StreamRecord<?> resultRecord3 = (StreamRecord<?>) resultObject3;
        assertTrue(resultRecord3.getValue() instanceof Map);
        @SuppressWarnings("unchecked") Map<String, List<Event>> patternMap3 = (Map<String, List<Event>>) resultRecord3.getValue();
        assertEquals(startEvent1, patternMap3.get("start").get(0));
        assertEquals(middleEvent3, patternMap3.get("middle").get(0));
        assertEquals(endEvent1, patternMap3.get("end").get(0));
    } finally {
        harness.close();
    }
}

Example 23 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class CEPMigrationTest method writeAfterBranchingPatternSnapshot.

/**
 * Manually run this to write binary snapshot data.
 */
@Ignore
@Test
public void writeAfterBranchingPatternSnapshot() throws Exception {
    KeySelector<Event, Integer> keySelector = new KeySelector<Event, Integer>() {

        private static final long serialVersionUID = -4873366487571254798L;

        @Override
        public Integer getKey(Event value) throws Exception {
            return value.getId();
        }
    };
    final Event startEvent = new Event(42, "start", 1.0);
    final SubEvent middleEvent1 = new SubEvent(42, "foo1", 1.0, 10.0);
    final SubEvent middleEvent2 = new SubEvent(42, "foo2", 2.0, 10.0);
    OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = new KeyedOneInputStreamOperatorTestHarness<>(CepOperatorTestUtilities.getKeyedCepOperator(false, new NFAFactory()), keySelector, BasicTypeInfo.INT_TYPE_INFO);
    try {
        harness.setup();
        harness.open();
        harness.processElement(new StreamRecord<Event>(startEvent, 1));
        harness.processElement(new StreamRecord<Event>(new Event(42, "foobar", 1.0), 2));
        harness.processElement(new StreamRecord<Event>(new SubEvent(42, "barfoo", 1.0, 5.0), 3));
        harness.processElement(new StreamRecord<Event>(middleEvent1, 2));
        harness.processElement(new StreamRecord<Event>(middleEvent2, 3));
        harness.processWatermark(new Watermark(5));
        // do snapshot and save to file
        OperatorSubtaskState snapshot = harness.snapshot(0L, 0L);
        OperatorSnapshotUtil.writeStateHandle(snapshot, "src/test/resources/cep-migration-after-branching-flink" + flinkGenerateSavepointVersion + "-snapshot");
    } finally {
        harness.close();
    }
}

Example 24 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class CEPMigrationTest method testAndOrSubtypeConditionsAfterMigration.

@Test
public void testAndOrSubtypeConditionsAfterMigration() throws Exception {
    KeySelector<Event, Integer> keySelector = new KeySelector<Event, Integer>() {

        private static final long serialVersionUID = -4873366487571254798L;

        @Override
        public Integer getKey(Event value) throws Exception {
            return value.getId();
        }
    };
    final Event startEvent1 = new SubEvent(42, "start", 1.0, 6.0);
    OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = new KeyedOneInputStreamOperatorTestHarness<>(CepOperatorTestUtilities.getKeyedCepOperator(false, new NFAComplexConditionsFactory()), keySelector, BasicTypeInfo.INT_TYPE_INFO);
    try {
        harness.setup();
        harness.initializeState(OperatorSnapshotUtil.getResourceFilename("cep-migration-conditions-flink" + migrateVersion + "-snapshot"));
        harness.open();
        final Event endEvent = new SubEvent(42, "end", 1.0, 2.0);
        harness.processElement(new StreamRecord<>(endEvent, 9));
        harness.processWatermark(new Watermark(20));
        ConcurrentLinkedQueue<Object> result = harness.getOutput();
        // watermark and the result
        assertEquals(2, result.size());
        Object resultObject = result.poll();
        assertTrue(resultObject instanceof StreamRecord);
        StreamRecord<?> resultRecord = (StreamRecord<?>) resultObject;
        assertTrue(resultRecord.getValue() instanceof Map);
        @SuppressWarnings("unchecked") Map<String, List<Event>> patternMap = (Map<String, List<Event>>) resultRecord.getValue();
        assertEquals(startEvent1, patternMap.get("start").get(0));
        assertEquals(endEvent, patternMap.get("start").get(1));
    } finally {
        harness.close();
    }
}

Example 25 with KeySelector

use of org.apache.flink.api.java.functions.KeySelector in project flink by apache.

the class CEPMigrationTest method writeStartingNewPatternAfterMigrationSnapshot.

/**
 * Manually run this to write binary snapshot data.
 */
@Ignore
@Test
public void writeStartingNewPatternAfterMigrationSnapshot() throws Exception {
    KeySelector<Event, Integer> keySelector = new KeySelector<Event, Integer>() {

        private static final long serialVersionUID = -4873366487571254798L;

        @Override
        public Integer getKey(Event value) throws Exception {
            return value.getId();
        }
    };
    final Event startEvent1 = new Event(42, "start", 1.0);
    final SubEvent middleEvent1 = new SubEvent(42, "foo1", 1.0, 10.0);
    OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = new KeyedOneInputStreamOperatorTestHarness<>(CepOperatorTestUtilities.getKeyedCepOperator(false, new NFAFactory()), keySelector, BasicTypeInfo.INT_TYPE_INFO);
    try {
        harness.setup();
        harness.open();
        harness.processElement(new StreamRecord<Event>(startEvent1, 1));
        harness.processElement(new StreamRecord<Event>(new Event(42, "foobar", 1.0), 2));
        harness.processElement(new StreamRecord<Event>(new SubEvent(42, "barfoo", 1.0, 5.0), 3));
        harness.processElement(new StreamRecord<Event>(middleEvent1, 2));
        harness.processWatermark(new Watermark(5));
        // do snapshot and save to file
        OperatorSubtaskState snapshot = harness.snapshot(0L, 0L);
        OperatorSnapshotUtil.writeStateHandle(snapshot, "src/test/resources/cep-migration-starting-new-pattern-flink" + flinkGenerateSavepointVersion + "-snapshot");
    } finally {
        harness.close();
    }
}