Examples with START_FROM_OLDEST com.hazelcast.jet.pipeline.JournalInitialPosition.START_FROM_OLDEST used on opensource projects

Example 11 with START_FROM_OLDEST

use of com.hazelcast.jet.pipeline.JournalInitialPosition.START_FROM_OLDEST in project hazelcast by hazelcast.

the class Sources_withEventJournalTest method remoteCacheJournal_withUnknownValueClass.

@Test
public void remoteCacheJournal_withUnknownValueClass() throws Exception {
    // Given
    URL jarResource = Thread.currentThread().getContextClassLoader().getResource("deployment/sample-pojo-1.0-car.jar");
    assertNotNull("jar not found", jarResource);
    ClassLoader cl = new URLClassLoader(new URL[] { jarResource });
    Class<?> carClz = cl.loadClass("com.sample.pojo.car.Car");
    Object car = carClz.getConstructor(String.class, String.class).newInstance("make", "model");
    String cacheName = JOURNALED_CACHE_PREFIX + randomName();
    ICache<String, Object> cache = remoteHz.getCacheManager().getCache(cacheName);
    // the class of the value is unknown to the remote IMDG member, it will be only known to Jet
    cache.put("key", car);
    // When
    StreamSource<Entry<Object, Object>> source = Sources.remoteCacheJournal(cacheName, clientConfig, START_FROM_OLDEST);
    // Then
    p.readFrom(source).withoutTimestamps().map(en -> en.getValue().toString()).writeTo(sink);
    JobConfig jobConfig = new JobConfig();
    jobConfig.addJar(jarResource);
    Job job = hz().getJet().newJob(p, jobConfig);
    List<Object> expected = singletonList(car.toString());
    assertTrueEventually(() -> assertEquals(expected, new ArrayList<>(sinkList)), 10);
    job.cancel();
}

Example 12 with START_FROM_OLDEST

use of com.hazelcast.jet.pipeline.JournalInitialPosition.START_FROM_OLDEST in project hazelcast by hazelcast.

the class HazelcastConnectorTest method when_streamCache_withFilterAndProjection.

@Test
public void when_streamCache_withFilterAndProjection() {
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", SourceProcessors.<Integer, Integer, Integer>streamCacheP(streamSourceName, event -> !event.getKey().equals(0), EventJournalCacheEvent::getKey, START_FROM_OLDEST, eventTimePolicy(i -> i, limitingLag(0), 1, 0, 10_000)));
    Vertex sink = dag.newVertex("sink", writeListP(streamSinkName));
    dag.edge(between(source, sink));
    Job job = instance().getJet().newJob(dag);
    ICache<Integer, Integer> sourceCache = instance().getCacheManager().getCache(streamSourceName);
    range(0, ENTRY_COUNT).forEach(i -> sourceCache.put(i, i));
    assertSizeEventually(ENTRY_COUNT - 1, instance().getList(streamSinkName));
    assertFalse(instance().getList(streamSinkName).contains(0));
    assertTrue(instance().getList(streamSinkName).contains(1));
    job.cancel();
}

Example 13 with START_FROM_OLDEST

use of com.hazelcast.jet.pipeline.JournalInitialPosition.START_FROM_OLDEST in project hazelcast by hazelcast.

the class HazelcastConnectorTest method when_streamMap_withFilterAndProjection.

@Test
public void when_streamMap_withFilterAndProjection() {
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", SourceProcessors.<Integer, Integer, Integer>streamMapP(streamSourceName, event -> event.getKey() != 0, EventJournalMapEvent::getKey, START_FROM_OLDEST, eventTimePolicy(i -> i, limitingLag(0), 1, 0, 10_000)));
    Vertex sink = dag.newVertex("sink", writeListP(streamSinkName));
    dag.edge(between(source, sink));
    Job job = instance().getJet().newJob(dag);
    IMap<Integer, Integer> sourceMap = instance().getMap(streamSourceName);
    range(0, ENTRY_COUNT).forEach(i -> sourceMap.put(i, i));
    assertSizeEventually(ENTRY_COUNT - 1, instance().getList(streamSinkName));
    assertFalse(instance().getList(streamSinkName).contains(0));
    assertTrue(instance().getList(streamSinkName).contains(1));
    job.cancel();
}

Example 14 with START_FROM_OLDEST

use of com.hazelcast.jet.pipeline.JournalInitialPosition.START_FROM_OLDEST in project hazelcast by hazelcast.

the class StreamEventJournalPTest method setUp.

@Before
public void setUp() {
    Config config = smallInstanceConfig();
    config.setProperty(PARTITION_COUNT.getName(), String.valueOf(NUM_PARTITIONS));
    config.getMapConfig("*").getEventJournalConfig().setEnabled(true).setCapacity(JOURNAL_CAPACITY);
    instance = createHazelcastInstance(config);
    map = (MapProxyImpl<String, Integer>) instance.<String, Integer>getMap("test");
    List<Integer> allPartitions = IntStream.range(0, NUM_PARTITIONS).boxed().collect(toList());
    supplier = () -> new StreamEventJournalP<>(map, allPartitions, e -> true, EventJournalMapEvent::getNewValue, START_FROM_OLDEST, false, noEventTime());
    key0 = generateKeyForPartition(instance, 0);
    key1 = generateKeyForPartition(instance, 1);
}

Example 15 with START_FROM_OLDEST

use of com.hazelcast.jet.pipeline.JournalInitialPosition.START_FROM_OLDEST in project hazelcast by hazelcast.

the class Sources_withEventJournalTest method remoteMapJournal_withUnknownValueClass.

@Test
public void remoteMapJournal_withUnknownValueClass() throws Exception {
    // Given
    URL jarResource = Thread.currentThread().getContextClassLoader().getResource("deployment/sample-pojo-1.0-car.jar");
    assertNotNull("jar not found", jarResource);
    ClassLoader cl = new URLClassLoader(new URL[] { jarResource });
    Class<?> carClz = cl.loadClass("com.sample.pojo.car.Car");
    Object car = carClz.getConstructor(String.class, String.class).newInstance("make", "model");
    IMap<String, Object> map = remoteHz.getMap(srcName);
    // the class of the value is unknown to the remote IMDG member, it will be only known to Jet
    map.put("key", car);
    // When
    StreamSource<Entry<Object, Object>> source = Sources.remoteMapJournal(srcName, clientConfig, START_FROM_OLDEST);
    // Then
    p.readFrom(source).withoutTimestamps().map(en -> en.getValue().toString()).writeTo(sink);
    JobConfig jobConfig = new JobConfig();
    jobConfig.addJar(jarResource);
    Job job = hz().getJet().newJob(p, jobConfig);
    List<Object> expected = singletonList(car.toString());
    assertTrueEventually(() -> assertEquals(expected, new ArrayList<>(sinkList)), 10);
    job.cancel();
}