Search in sources :

Example 1 with EventJournalCacheEvent

use of com.hazelcast.cache.journal.EventJournalCacheEvent in project hazelcast-jet by hazelcast.

the class HazelcastConnectorTest method when_streamCache_withFilterAndProjection.

@Test
public void when_streamCache_withFilterAndProjection() {
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", SourceProcessors.<Integer, Integer, Integer>streamCacheP(streamSourceName, event -> !event.getKey().equals(0), EventJournalCacheEvent::getKey, START_FROM_OLDEST, wmGenParams(i -> i, limitingLag(0), noThrottling(), 10_000)));
    Vertex sink = dag.newVertex("sink", writeListP(streamSinkName));
    dag.edge(between(source, sink));
    Job job = jetInstance.newJob(dag);
    ICacheJet<Integer, Integer> sourceCache = jetInstance.getCacheManager().getCache(streamSourceName);
    range(0, ENTRY_COUNT).forEach(i -> sourceCache.put(i, i));
    assertSizeEventually(ENTRY_COUNT - 1, jetInstance.getList(streamSinkName));
    assertFalse(jetInstance.getList(streamSinkName).contains(0));
    assertTrue(jetInstance.getList(streamSinkName).contains(1));
    job.cancel();
}
Also used : WatermarkPolicies.limitingLag(com.hazelcast.jet.core.WatermarkPolicies.limitingLag) IntStream.range(java.util.stream.IntStream.range) EventJournalConfig(com.hazelcast.config.EventJournalConfig) IMapJet(com.hazelcast.jet.IMapJet) SourceProcessors.readMapP(com.hazelcast.jet.core.processor.SourceProcessors.readMapP) CacheSimpleConfig(com.hazelcast.config.CacheSimpleConfig) SourceProcessors.readListP(com.hazelcast.jet.core.processor.SourceProcessors.readListP) SourceProcessors.streamCacheP(com.hazelcast.jet.core.processor.SourceProcessors.streamCacheP) ICacheJet(com.hazelcast.jet.ICacheJet) Map(java.util.Map) WatermarkEmissionPolicy.noThrottling(com.hazelcast.jet.core.WatermarkEmissionPolicy.noThrottling) SinkProcessors.writeCacheP(com.hazelcast.jet.core.processor.SinkProcessors.writeCacheP) DAG(com.hazelcast.jet.core.DAG) WatermarkGenerationParams.noWatermarks(com.hazelcast.jet.core.WatermarkGenerationParams.noWatermarks) Projections(com.hazelcast.projection.Projections) WatermarkGenerationParams.wmGenParams(com.hazelcast.jet.core.WatermarkGenerationParams.wmGenParams) SourceProcessors.readCacheP(com.hazelcast.jet.core.processor.SourceProcessors.readCacheP) EventJournalMapEvent(com.hazelcast.map.journal.EventJournalMapEvent) JetTestSupport(com.hazelcast.jet.core.JetTestSupport) START_FROM_OLDEST(com.hazelcast.jet.pipeline.JournalInitialPosition.START_FROM_OLDEST) Collectors.joining(java.util.stream.Collectors.joining) PredicateTestUtils.entry(com.hazelcast.query.impl.predicates.PredicateTestUtils.entry) JetCacheManager(com.hazelcast.jet.JetCacheManager) Assert.assertFalse(org.junit.Assert.assertFalse) HazelcastParallelClassRunner(com.hazelcast.test.HazelcastParallelClassRunner) Entry(java.util.Map.Entry) SinkProcessors.writeMapP(com.hazelcast.jet.core.processor.SinkProcessors.writeMapP) IntStream(java.util.stream.IntStream) JetInstance(com.hazelcast.jet.JetInstance) SourceProcessors(com.hazelcast.jet.core.processor.SourceProcessors) RunWith(org.junit.runner.RunWith) Job(com.hazelcast.jet.Job) Before(org.junit.Before) EventJournalCacheEvent(com.hazelcast.cache.journal.EventJournalCacheEvent) Config(com.hazelcast.config.Config) JetConfig(com.hazelcast.jet.config.JetConfig) SourceProcessors.streamMapP(com.hazelcast.jet.core.processor.SourceProcessors.streamMapP) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IListJet(com.hazelcast.jet.IListJet) Vertex(com.hazelcast.jet.core.Vertex) TruePredicate(com.hazelcast.query.TruePredicate) Collectors.toList(java.util.stream.Collectors.toList) Predicates(com.hazelcast.query.Predicates) ICache(com.hazelcast.cache.ICache) Util.mapPutEvents(com.hazelcast.jet.Util.mapPutEvents) Assert.assertEquals(org.junit.Assert.assertEquals) Edge.between(com.hazelcast.jet.core.Edge.between) SinkProcessors.writeListP(com.hazelcast.jet.core.processor.SinkProcessors.writeListP) Vertex(com.hazelcast.jet.core.Vertex) DAG(com.hazelcast.jet.core.DAG) Job(com.hazelcast.jet.Job) Test(org.junit.Test)

Example 2 with EventJournalCacheEvent

use of com.hazelcast.cache.journal.EventJournalCacheEvent in project hazelcast-jet by hazelcast.

the class HazelcastRemoteConnectorTest method when_streamRemoteCache_withPredicateAndProjection.

@Test
public void when_streamRemoteCache_withPredicateAndProjection() {
    DAG dag = new DAG();
    Vertex source = dag.newVertex(SOURCE_NAME, SourceProcessors.<Integer, Integer, Integer>streamRemoteCacheP(SOURCE_NAME, clientConfig, event -> !event.getKey().equals(0), EventJournalCacheEvent::getKey, START_FROM_OLDEST, wmGenParams(i -> i, limitingLag(0), noThrottling(), 10_000)));
    Vertex sink = dag.newVertex(SINK_NAME, writeListP(SINK_NAME));
    dag.edge(between(source, sink));
    Job job = jet.newJob(dag);
    populateCache(hz.getCacheManager().getCache(SOURCE_NAME));
    assertSizeEventually(ITEM_COUNT - 1, jet.getList(SINK_NAME));
    assertFalse(jet.getList(SINK_NAME).contains(0));
    assertTrue(jet.getList(SINK_NAME).contains(1));
    job.cancel();
}
Also used : WatermarkPolicies.limitingLag(com.hazelcast.jet.core.WatermarkPolicies.limitingLag) EventJournalConfig(com.hazelcast.config.EventJournalConfig) Address(com.hazelcast.nio.Address) SourceProcessors.readMapP(com.hazelcast.jet.core.processor.SourceProcessors.readMapP) CacheSimpleConfig(com.hazelcast.config.CacheSimpleConfig) Collectors.toMap(java.util.stream.Collectors.toMap) Map(java.util.Map) WatermarkEmissionPolicy.noThrottling(com.hazelcast.jet.core.WatermarkEmissionPolicy.noThrottling) SourceProcessors.readRemoteMapP(com.hazelcast.jet.core.processor.SourceProcessors.readRemoteMapP) DAG(com.hazelcast.jet.core.DAG) Projections(com.hazelcast.projection.Projections) WatermarkGenerationParams.wmGenParams(com.hazelcast.jet.core.WatermarkGenerationParams.wmGenParams) SourceProcessors.readCacheP(com.hazelcast.jet.core.processor.SourceProcessors.readCacheP) EventJournalMapEvent(com.hazelcast.map.journal.EventJournalMapEvent) AfterClass(org.junit.AfterClass) JetTestSupport(com.hazelcast.jet.core.JetTestSupport) START_FROM_OLDEST(com.hazelcast.jet.pipeline.JournalInitialPosition.START_FROM_OLDEST) Hazelcast(com.hazelcast.core.Hazelcast) Assert.assertFalse(org.junit.Assert.assertFalse) Entry(java.util.Map.Entry) IntStream(java.util.stream.IntStream) JetInstance(com.hazelcast.jet.JetInstance) BeforeClass(org.junit.BeforeClass) SourceProcessors.readRemoteCacheP(com.hazelcast.jet.core.processor.SourceProcessors.readRemoteCacheP) SourceProcessors(com.hazelcast.jet.core.processor.SourceProcessors) RunWith(org.junit.runner.RunWith) JetTestInstanceFactory(com.hazelcast.jet.JetTestInstanceFactory) HazelcastSerialClassRunner(com.hazelcast.test.HazelcastSerialClassRunner) ClientConfig(com.hazelcast.client.config.ClientConfig) SinkProcessors.writeRemoteMapP(com.hazelcast.jet.core.processor.SinkProcessors.writeRemoteMapP) Job(com.hazelcast.jet.Job) Before(org.junit.Before) EventJournalCacheEvent(com.hazelcast.cache.journal.EventJournalCacheEvent) Config(com.hazelcast.config.Config) HazelcastInstance(com.hazelcast.core.HazelcastInstance) JetConfig(com.hazelcast.jet.config.JetConfig) SourceProcessors.streamRemoteMapP(com.hazelcast.jet.core.processor.SourceProcessors.streamRemoteMapP) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IListJet(com.hazelcast.jet.IListJet) Vertex(com.hazelcast.jet.core.Vertex) DistributedObject(com.hazelcast.core.DistributedObject) Predicates(com.hazelcast.query.Predicates) ICache(com.hazelcast.cache.ICache) SinkProcessors.writeRemoteCacheP(com.hazelcast.jet.core.processor.SinkProcessors.writeRemoteCacheP) SourceProcessors.streamRemoteCacheP(com.hazelcast.jet.core.processor.SourceProcessors.streamRemoteCacheP) Assert.assertEquals(org.junit.Assert.assertEquals) Edge.between(com.hazelcast.jet.core.Edge.between) SinkProcessors.writeListP(com.hazelcast.jet.core.processor.SinkProcessors.writeListP) Vertex(com.hazelcast.jet.core.Vertex) DAG(com.hazelcast.jet.core.DAG) Job(com.hazelcast.jet.Job) Test(org.junit.Test)

Example 3 with EventJournalCacheEvent

use of com.hazelcast.cache.journal.EventJournalCacheEvent in project hazelcast-jet-reference-manual by hazelcast.

the class ImdgConnectors method s12.

static void s12() {
    // tag::s12[]
    Pipeline p = Pipeline.create();
    StreamStage<EventJournalMapEvent<String, Long>> allFromMap = p.drawFrom(Sources.<EventJournalMapEvent<String, Long>, String, Long>mapJournal("inputMap", DistributedFunctions.alwaysTrue(), DistributedFunction.identity(), START_FROM_CURRENT));
    StreamStage<EventJournalCacheEvent<String, Long>> allFromCache = p.drawFrom(Sources.<EventJournalCacheEvent<String, Long>, String, Long>cacheJournal("inputMap", DistributedFunctions.alwaysTrue(), DistributedFunction.identity(), START_FROM_CURRENT));
// end::s12[]
}
Also used : EventJournalCacheEvent(com.hazelcast.cache.journal.EventJournalCacheEvent) EventJournalMapEvent(com.hazelcast.map.journal.EventJournalMapEvent) Pipeline(com.hazelcast.jet.pipeline.Pipeline)

Aggregations

EventJournalCacheEvent (com.hazelcast.cache.journal.EventJournalCacheEvent)3 EventJournalMapEvent (com.hazelcast.map.journal.EventJournalMapEvent)3 ICache (com.hazelcast.cache.ICache)2 CacheSimpleConfig (com.hazelcast.config.CacheSimpleConfig)2 Config (com.hazelcast.config.Config)2 EventJournalConfig (com.hazelcast.config.EventJournalConfig)2 IListJet (com.hazelcast.jet.IListJet)2 JetInstance (com.hazelcast.jet.JetInstance)2 Job (com.hazelcast.jet.Job)2 JetConfig (com.hazelcast.jet.config.JetConfig)2 DAG (com.hazelcast.jet.core.DAG)2 Edge.between (com.hazelcast.jet.core.Edge.between)2 JetTestSupport (com.hazelcast.jet.core.JetTestSupport)2 Vertex (com.hazelcast.jet.core.Vertex)2 WatermarkEmissionPolicy.noThrottling (com.hazelcast.jet.core.WatermarkEmissionPolicy.noThrottling)2 WatermarkGenerationParams.wmGenParams (com.hazelcast.jet.core.WatermarkGenerationParams.wmGenParams)2 WatermarkPolicies.limitingLag (com.hazelcast.jet.core.WatermarkPolicies.limitingLag)2 SinkProcessors.writeListP (com.hazelcast.jet.core.processor.SinkProcessors.writeListP)2 SourceProcessors (com.hazelcast.jet.core.processor.SourceProcessors)2 SourceProcessors.readCacheP (com.hazelcast.jet.core.processor.SourceProcessors.readCacheP)2