Search in sources :

Example 6 with EventJournalMapEvent

use of com.hazelcast.map.EventJournalMapEvent in project hazelcast by hazelcast.

the class HazelcastConnectorTest method when_streamMap_withFilterAndProjection.

@Test
public void when_streamMap_withFilterAndProjection() {
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", SourceProcessors.<Integer, Integer, Integer>streamMapP(streamSourceName, event -> event.getKey() != 0, EventJournalMapEvent::getKey, START_FROM_OLDEST, eventTimePolicy(i -> i, limitingLag(0), 1, 0, 10_000)));
    Vertex sink = dag.newVertex("sink", writeListP(streamSinkName));
    dag.edge(between(source, sink));
    Job job = instance().getJet().newJob(dag);
    IMap<Integer, Integer> sourceMap = instance().getMap(streamSourceName);
    range(0, ENTRY_COUNT).forEach(i -> sourceMap.put(i, i));
    assertSizeEventually(ENTRY_COUNT - 1, instance().getList(streamSinkName));
    assertFalse(instance().getList(streamSinkName).contains(0));
    assertTrue(instance().getList(streamSinkName).contains(1));
    job.cancel();
}
Also used : ParallelJVMTest(com.hazelcast.test.annotation.ParallelJVMTest) IntStream.range(java.util.stream.IntStream.range) QuickTest(com.hazelcast.test.annotation.QuickTest) SourceProcessors.readMapP(com.hazelcast.jet.core.processor.SourceProcessors.readMapP) SourceProcessors.readListP(com.hazelcast.jet.core.processor.SourceProcessors.readListP) SourceProcessors.streamCacheP(com.hazelcast.jet.core.processor.SourceProcessors.streamCacheP) EventJournalCacheEvent(com.hazelcast.cache.EventJournalCacheEvent) TruePredicate(com.hazelcast.query.impl.predicates.TruePredicate) Map(java.util.Map) SinkProcessors.writeCacheP(com.hazelcast.jet.core.processor.SinkProcessors.writeCacheP) DAG(com.hazelcast.jet.core.DAG) ICacheManager(com.hazelcast.core.ICacheManager) Projections(com.hazelcast.projection.Projections) SourceProcessors.readCacheP(com.hazelcast.jet.core.processor.SourceProcessors.readCacheP) SimpleTestInClusterSupport(com.hazelcast.jet.SimpleTestInClusterSupport) EventTimePolicy.noEventTime(com.hazelcast.jet.core.EventTimePolicy.noEventTime) START_FROM_OLDEST(com.hazelcast.jet.pipeline.JournalInitialPosition.START_FROM_OLDEST) Category(org.junit.experimental.categories.Category) Collectors.joining(java.util.stream.Collectors.joining) List(java.util.List) NearCacheConfig(com.hazelcast.config.NearCacheConfig) EventJournalMapEvent(com.hazelcast.map.EventJournalMapEvent) Assert.assertFalse(org.junit.Assert.assertFalse) SinkProcessors(com.hazelcast.jet.core.processor.SinkProcessors) Entry(java.util.Map.Entry) SinkProcessors.writeMapP(com.hazelcast.jet.core.processor.SinkProcessors.writeMapP) IntStream(java.util.stream.IntStream) NearCachedMapProxyImpl(com.hazelcast.map.impl.proxy.NearCachedMapProxyImpl) BeforeClass(org.junit.BeforeClass) SourceProcessors(com.hazelcast.jet.core.processor.SourceProcessors) TestProcessors(com.hazelcast.jet.core.TestProcessors) Util.entry(com.hazelcast.jet.Util.entry) Job(com.hazelcast.jet.Job) Before(org.junit.Before) IList(com.hazelcast.collection.IList) Config(com.hazelcast.config.Config) SourceProcessors.streamMapP(com.hazelcast.jet.core.processor.SourceProcessors.streamMapP) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) WatermarkPolicy.limitingLag(com.hazelcast.jet.core.WatermarkPolicy.limitingLag) Vertex(com.hazelcast.jet.core.Vertex) Collectors.toList(java.util.stream.Collectors.toList) Predicates(com.hazelcast.query.Predicates) ICache(com.hazelcast.cache.ICache) EventTimePolicy.eventTimePolicy(com.hazelcast.jet.core.EventTimePolicy.eventTimePolicy) Util.mapPutEvents(com.hazelcast.jet.Util.mapPutEvents) Assert.assertEquals(org.junit.Assert.assertEquals) IMap(com.hazelcast.map.IMap) Edge.between(com.hazelcast.jet.core.Edge.between) SinkProcessors.writeListP(com.hazelcast.jet.core.processor.SinkProcessors.writeListP) Vertex(com.hazelcast.jet.core.Vertex) DAG(com.hazelcast.jet.core.DAG) Job(com.hazelcast.jet.Job) ParallelJVMTest(com.hazelcast.test.annotation.ParallelJVMTest) QuickTest(com.hazelcast.test.annotation.QuickTest) Test(org.junit.Test)

Example 7 with EventJournalMapEvent

use of com.hazelcast.map.EventJournalMapEvent in project hazelcast by hazelcast.

the class StreamEventJournalPTest method setUp.

@Before
public void setUp() {
    Config config = smallInstanceConfig();
    config.setProperty(PARTITION_COUNT.getName(), String.valueOf(NUM_PARTITIONS));
    config.getMapConfig("*").getEventJournalConfig().setEnabled(true).setCapacity(JOURNAL_CAPACITY);
    instance = createHazelcastInstance(config);
    map = (MapProxyImpl<String, Integer>) instance.<String, Integer>getMap("test");
    List<Integer> allPartitions = IntStream.range(0, NUM_PARTITIONS).boxed().collect(toList());
    supplier = () -> new StreamEventJournalP<>(map, allPartitions, e -> true, EventJournalMapEvent::getNewValue, START_FROM_OLDEST, false, noEventTime());
    key0 = generateKeyForPartition(instance, 0);
    key1 = generateKeyForPartition(instance, 1);
}
Also used : IntStream(java.util.stream.IntStream) ParallelJVMTest(com.hazelcast.test.annotation.ParallelJVMTest) Arrays(java.util.Arrays) QuickTest(com.hazelcast.test.annotation.QuickTest) PARTITION_COUNT(com.hazelcast.spi.properties.ClusterProperty.PARTITION_COUNT) RunWith(org.junit.runner.RunWith) Processor(com.hazelcast.jet.core.Processor) TestOutbox(com.hazelcast.jet.core.test.TestOutbox) TestProcessorContext(com.hazelcast.jet.core.test.TestProcessorContext) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) MapProxyImpl(com.hazelcast.map.impl.proxy.MapProxyImpl) DAG(com.hazelcast.jet.core.DAG) JobStatus(com.hazelcast.jet.core.JobStatus) Job(com.hazelcast.jet.Job) Before(org.junit.Before) TestInbox(com.hazelcast.jet.core.test.TestInbox) EventTimePolicy.noEventTime(com.hazelcast.jet.core.EventTimePolicy.noEventTime) Config(com.hazelcast.config.Config) HazelcastInstance(com.hazelcast.core.HazelcastInstance) SourceProcessors.streamMapP(com.hazelcast.jet.core.processor.SourceProcessors.streamMapP) JetTestSupport(com.hazelcast.jet.core.JetTestSupport) EXACTLY_ONCE(com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE) JobConfig(com.hazelcast.jet.config.JobConfig) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) START_FROM_OLDEST(com.hazelcast.jet.pipeline.JournalInitialPosition.START_FROM_OLDEST) Category(org.junit.experimental.categories.Category) Collectors(java.util.stream.Collectors) SupplierEx(com.hazelcast.function.SupplierEx) Vertex(com.hazelcast.jet.core.Vertex) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) SAME_ITEMS_ANY_ORDER(com.hazelcast.jet.core.test.TestSupport.SAME_ITEMS_ANY_ORDER) TestSupport(com.hazelcast.jet.core.test.TestSupport) JournalInitialPosition(com.hazelcast.jet.pipeline.JournalInitialPosition) EventJournalMapEvent(com.hazelcast.map.EventJournalMapEvent) Assert.assertFalse(org.junit.Assert.assertFalse) HazelcastParallelClassRunner(com.hazelcast.test.HazelcastParallelClassRunner) Entry(java.util.Map.Entry) Assert.assertEquals(org.junit.Assert.assertEquals) Config(com.hazelcast.config.Config) JobConfig(com.hazelcast.jet.config.JobConfig) Before(org.junit.Before)

Example 8 with EventJournalMapEvent

use of com.hazelcast.map.EventJournalMapEvent in project hazelcast by hazelcast.

the class HazelcastRemoteConnectorTest method when_streamRemoteMap_withPredicateAndProjection.

@Test
public void when_streamRemoteMap_withPredicateAndProjection() {
    DAG dag = new DAG();
    Vertex source = dag.newVertex(SOURCE_NAME, SourceProcessors.<Integer, Integer, Integer>streamRemoteMapP(SOURCE_NAME, clientConfig, event -> event.getKey() != 0, EventJournalMapEvent::getKey, START_FROM_OLDEST, eventTimePolicy(i -> i, limitingLag(0), 1, 0, 10_000)));
    Vertex sink = dag.newVertex(SINK_NAME, writeListP(SINK_NAME));
    dag.edge(between(source, sink));
    Job job = localHz.getJet().newJob(dag);
    populateMap(remoteHz.getMap(SOURCE_NAME));
    assertSizeEventually(ITEM_COUNT - 1, localHz.getList(SINK_NAME));
    assertFalse(localHz.getList(SINK_NAME).contains(0));
    assertTrue(localHz.getList(SINK_NAME).contains(1));
    job.cancel();
}
Also used : Address(com.hazelcast.cluster.Address) QuickTest(com.hazelcast.test.annotation.QuickTest) SourceProcessors.readMapP(com.hazelcast.jet.core.processor.SourceProcessors.readMapP) CacheSimpleConfig(com.hazelcast.config.CacheSimpleConfig) MapConfig(com.hazelcast.config.MapConfig) EventJournalCacheEvent(com.hazelcast.cache.EventJournalCacheEvent) Collectors.toMap(java.util.stream.Collectors.toMap) Map(java.util.Map) SourceProcessors.readRemoteMapP(com.hazelcast.jet.core.processor.SourceProcessors.readRemoteMapP) DAG(com.hazelcast.jet.core.DAG) Projections(com.hazelcast.projection.Projections) SourceProcessors.readCacheP(com.hazelcast.jet.core.processor.SourceProcessors.readCacheP) AfterClass(org.junit.AfterClass) JetTestSupport(com.hazelcast.jet.core.JetTestSupport) START_FROM_OLDEST(com.hazelcast.jet.pipeline.JournalInitialPosition.START_FROM_OLDEST) Category(org.junit.experimental.categories.Category) Hazelcast(com.hazelcast.core.Hazelcast) EventJournalMapEvent(com.hazelcast.map.EventJournalMapEvent) Assert.assertFalse(org.junit.Assert.assertFalse) Entry(java.util.Map.Entry) IntStream(java.util.stream.IntStream) BeforeClass(org.junit.BeforeClass) SourceProcessors.readRemoteCacheP(com.hazelcast.jet.core.processor.SourceProcessors.readRemoteCacheP) HazelcastClient(com.hazelcast.client.HazelcastClient) HazelcastInstanceFactory(com.hazelcast.instance.impl.HazelcastInstanceFactory) SourceProcessors(com.hazelcast.jet.core.processor.SourceProcessors) RunWith(org.junit.runner.RunWith) HazelcastSerialClassRunner(com.hazelcast.test.HazelcastSerialClassRunner) ClientConfig(com.hazelcast.client.config.ClientConfig) TestHazelcastFactory(com.hazelcast.client.test.TestHazelcastFactory) SinkProcessors.writeRemoteMapP(com.hazelcast.jet.core.processor.SinkProcessors.writeRemoteMapP) Job(com.hazelcast.jet.Job) Before(org.junit.Before) IList(com.hazelcast.collection.IList) Config(com.hazelcast.config.Config) HazelcastInstance(com.hazelcast.core.HazelcastInstance) SourceProcessors.streamRemoteMapP(com.hazelcast.jet.core.processor.SourceProcessors.streamRemoteMapP) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) WatermarkPolicy.limitingLag(com.hazelcast.jet.core.WatermarkPolicy.limitingLag) Vertex(com.hazelcast.jet.core.Vertex) DistributedObject(com.hazelcast.core.DistributedObject) Predicates(com.hazelcast.query.Predicates) ICache(com.hazelcast.cache.ICache) EventTimePolicy.eventTimePolicy(com.hazelcast.jet.core.EventTimePolicy.eventTimePolicy) SinkProcessors.writeRemoteCacheP(com.hazelcast.jet.core.processor.SinkProcessors.writeRemoteCacheP) SourceProcessors.streamRemoteCacheP(com.hazelcast.jet.core.processor.SourceProcessors.streamRemoteCacheP) Assert.assertEquals(org.junit.Assert.assertEquals) Edge.between(com.hazelcast.jet.core.Edge.between) SinkProcessors.writeListP(com.hazelcast.jet.core.processor.SinkProcessors.writeListP) Vertex(com.hazelcast.jet.core.Vertex) DAG(com.hazelcast.jet.core.DAG) Job(com.hazelcast.jet.Job) QuickTest(com.hazelcast.test.annotation.QuickTest) Test(org.junit.Test)

Aggregations

Config (com.hazelcast.config.Config)8 Job (com.hazelcast.jet.Job)8 DAG (com.hazelcast.jet.core.DAG)8 Vertex (com.hazelcast.jet.core.Vertex)8 START_FROM_OLDEST (com.hazelcast.jet.pipeline.JournalInitialPosition.START_FROM_OLDEST)8 EventJournalMapEvent (com.hazelcast.map.EventJournalMapEvent)8 QuickTest (com.hazelcast.test.annotation.QuickTest)8 IList (com.hazelcast.collection.IList)7 Edge.between (com.hazelcast.jet.core.Edge.between)7 EventTimePolicy.eventTimePolicy (com.hazelcast.jet.core.EventTimePolicy.eventTimePolicy)7 SourceProcessors.streamMapP (com.hazelcast.jet.core.processor.SourceProcessors.streamMapP)7 ParallelJVMTest (com.hazelcast.test.annotation.ParallelJVMTest)7 IntStream (java.util.stream.IntStream)7 Assert.assertEquals (org.junit.Assert.assertEquals)7 Before (org.junit.Before)7 Test (org.junit.Test)7 Category (org.junit.experimental.categories.Category)7 SimpleTestInClusterSupport (com.hazelcast.jet.SimpleTestInClusterSupport)6 JobConfig (com.hazelcast.jet.config.JobConfig)6 EXACTLY_ONCE (com.hazelcast.jet.config.ProcessingGuarantee.EXACTLY_ONCE)6