Examples with EventJournalMapEvent com.hazelcast.map.journal.EventJournalMapEvent used on opensource projects

Example 1 with EventJournalMapEvent

use of com.hazelcast.map.journal.EventJournalMapEvent in project hazelcast-jet by hazelcast.

the class HazelcastConnectorTest method when_streamMap_withFilterAndProjection.

@Test
public void when_streamMap_withFilterAndProjection() {
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", SourceProcessors.<Integer, Integer, Integer>streamMapP(streamSourceName, event -> event.getKey() != 0, EventJournalMapEvent::getKey, START_FROM_OLDEST, wmGenParams(i -> i, limitingLag(0), noThrottling(), 10_000)));
    Vertex sink = dag.newVertex("sink", writeListP(streamSinkName));
    dag.edge(between(source, sink));
    Job job = jetInstance.newJob(dag);
    IMapJet<Integer, Integer> sourceMap = jetInstance.getMap(streamSourceName);
    range(0, ENTRY_COUNT).forEach(i -> sourceMap.put(i, i));
    assertSizeEventually(ENTRY_COUNT - 1, jetInstance.getList(streamSinkName));
    assertFalse(jetInstance.getList(streamSinkName).contains(0));
    assertTrue(jetInstance.getList(streamSinkName).contains(1));
    job.cancel();
}

Example 2 with EventJournalMapEvent

use of com.hazelcast.map.journal.EventJournalMapEvent in project hazelcast-jet by hazelcast.

the class HazelcastRemoteConnectorTest method when_streamRemoteMap_withPredicateAndProjection.

@Test
public void when_streamRemoteMap_withPredicateAndProjection() {
    DAG dag = new DAG();
    Vertex source = dag.newVertex(SOURCE_NAME, SourceProcessors.<Integer, Integer, Integer>streamRemoteMapP(SOURCE_NAME, clientConfig, event -> event.getKey() != 0, EventJournalMapEvent::getKey, START_FROM_OLDEST, wmGenParams(i -> i, limitingLag(0), noThrottling(), 10_000)));
    Vertex sink = dag.newVertex(SINK_NAME, writeListP(SINK_NAME));
    dag.edge(between(source, sink));
    Job job = jet.newJob(dag);
    populateMap(hz.getMap(SOURCE_NAME));
    assertSizeEventually(ITEM_COUNT - 1, jet.getList(SINK_NAME));
    assertFalse(jet.getList(SINK_NAME).contains(0));
    assertTrue(jet.getList(SINK_NAME).contains(1));
    job.cancel();
}

Example 3 with EventJournalMapEvent

use of com.hazelcast.map.journal.EventJournalMapEvent in project hazelcast-jet by hazelcast.

the class StreamEventJournalPTest method setUp.

@Before
public void setUp() {
    JetConfig config = new JetConfig();
    EventJournalConfig journalConfig = new EventJournalConfig().setMapName("*").setCapacity(JOURNAL_CAPACITY).setEnabled(true);
    config.getHazelcastConfig().setProperty(PARTITION_COUNT.getName(), String.valueOf(NUM_PARTITIONS));
    config.getHazelcastConfig().addEventJournalConfig(journalConfig);
    instance = this.createJetMember(config);
    map = (MapProxyImpl<String, Integer>) instance.getHazelcastInstance().<String, Integer>getMap("test");
    List<Integer> allPartitions = IntStream.range(0, NUM_PARTITIONS).boxed().collect(toList());
    supplier = () -> new StreamEventJournalP<>(map, allPartitions, e -> true, EventJournalMapEvent::getNewValue, START_FROM_OLDEST, false, wmGenParams(Integer::intValue, limitingLag(0), suppressAll(), -1));
    key0 = generateKeyForPartition(instance.getHazelcastInstance(), 0);
    key1 = generateKeyForPartition(instance.getHazelcastInstance(), 1);
}

Example 4 with EventJournalMapEvent

use of com.hazelcast.map.journal.EventJournalMapEvent in project hazelcast-jet by hazelcast.

the class SnapshotFailureTest method when_snapshotFails_then_jobShouldNotFail.

@Test
public void when_snapshotFails_then_jobShouldNotFail() {
    int numPartitions = 2;
    int numElements = 10;
    IMapJet<Object, Object> results = instance1.getMap("results");
    DAG dag = new DAG();
    SequencesInPartitionsMetaSupplier sup = new SequencesInPartitionsMetaSupplier(numPartitions, numElements);
    Vertex generator = dag.newVertex("generator", peekOutputP(throttle(sup, 2))).localParallelism(1);
    Vertex writeMap = dag.newVertex("writeMap", writeMapP(results.getName())).localParallelism(1);
    dag.edge(between(generator, writeMap));
    JobConfig config = new JobConfig();
    config.setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE);
    config.setSnapshotIntervalMillis(100);
    Job job = instance1.newJob(dag, config);
    // let's start a second job that will watch the snapshots map and write failed
    // SnapshotRecords to a list, which we will check for presence of failed snapshot
    Pipeline p = Pipeline.create();
    p.drawFrom(Sources.mapJournal(snapshotsMapName(job.getId()), event -> event.getNewValue() instanceof SnapshotRecord && ((SnapshotRecord) event.getNewValue()).status() == SnapshotStatus.FAILED, EventJournalMapEvent::getNewValue, JournalInitialPosition.START_FROM_OLDEST)).peek().drainTo(Sinks.list("failed_snapshot_records"));
    instance1.newJob(p);
    job.join();
    assertEquals("numPartitions", numPartitions, results.size());
    assertEquals("offset partition 0", numElements - 1, results.get(0));
    assertEquals("offset partition 1", numElements - 1, results.get(1));
    assertTrue("no failure occurred in store", storeFailed);
    assertFalse("no failed snapshot appeared in snapshotsMap", instance1.getList("failed_snapshot_records").isEmpty());
}

Example 5 with EventJournalMapEvent

use of com.hazelcast.map.journal.EventJournalMapEvent in project hazelcast-jet-reference-manual by hazelcast.

the class ImdgConnectors method s12.

static void s12() {
    // tag::s12[]
    Pipeline p = Pipeline.create();
    StreamStage<EventJournalMapEvent<String, Long>> allFromMap = p.drawFrom(Sources.<EventJournalMapEvent<String, Long>, String, Long>mapJournal("inputMap", DistributedFunctions.alwaysTrue(), DistributedFunction.identity(), START_FROM_CURRENT));
    StreamStage<EventJournalCacheEvent<String, Long>> allFromCache = p.drawFrom(Sources.<EventJournalCacheEvent<String, Long>, String, Long>cacheJournal("inputMap", DistributedFunctions.alwaysTrue(), DistributedFunction.identity(), START_FROM_CURRENT));
// end::s12[]
}