Search in sources :

Example 1 with JetInstance

use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.

the class StreamKafkaPTest method integrationTest.

private void integrationTest(ProcessingGuarantee guarantee) throws Exception {
    int messageCount = 20;
    JetInstance[] instances = new JetInstance[2];
    Arrays.setAll(instances, i -> createJetMember());
    Pipeline p = Pipeline.create();
    p.drawFrom(KafkaSources.kafka(properties, topic1Name, topic2Name)).drainTo(Sinks.list("sink"));
    JobConfig config = new JobConfig();
    config.setProcessingGuarantee(guarantee);
    config.setSnapshotIntervalMillis(500);
    Job job = instances[0].newJob(p, config);
    sleepAtLeastSeconds(3);
    for (int i = 0; i < messageCount; i++) {
        produce(topic1Name, i, Integer.toString(i));
        produce(topic2Name, i - messageCount, Integer.toString(i - messageCount));
    }
    IList<Object> list = instances[0].getList("sink");
    assertTrueEventually(() -> {
        assertEquals(messageCount * 2, list.size());
        for (int i = 0; i < messageCount; i++) {
            Entry<Integer, String> entry1 = createEntry(i);
            Entry<Integer, String> entry2 = createEntry(i - messageCount);
            assertTrue("missing entry: " + entry1, list.contains(entry1));
            assertTrue("missing entry: " + entry2, list.contains(entry2));
        }
    }, 5);
    if (guarantee != ProcessingGuarantee.NONE) {
        // wait until the items are consumed and a new snapshot appears
        assertTrueEventually(() -> assertTrue(list.size() == messageCount * 2));
        IMapJet<Long, Object> snapshotsMap = instances[0].getMap(SnapshotRepository.snapshotsMapName(job.getId()));
        Long currentMax = maxSuccessfulSnapshot(snapshotsMap);
        assertTrueEventually(() -> {
            Long newMax = maxSuccessfulSnapshot(snapshotsMap);
            assertTrue("no snapshot produced", newMax != null && !newMax.equals(currentMax));
            System.out.println("snapshot " + newMax + " found, previous was " + currentMax);
        });
        // Bring down one member. Job should restart and drain additional items (and maybe
        // some of the previous duplicately).
        instances[1].shutdown();
        Thread.sleep(500);
        for (int i = messageCount; i < 2 * messageCount; i++) {
            produce(topic1Name, i, Integer.toString(i));
            produce(topic2Name, i - messageCount, Integer.toString(i - messageCount));
        }
        assertTrueEventually(() -> {
            assertTrue("Not all messages were received", list.size() >= messageCount * 4);
            for (int i = 0; i < 2 * messageCount; i++) {
                Entry<Integer, String> entry1 = createEntry(i);
                Entry<Integer, String> entry2 = createEntry(i - messageCount);
                assertTrue("missing entry: " + entry1.toString(), list.contains(entry1));
                assertTrue("missing entry: " + entry2.toString(), list.contains(entry2));
            }
        }, 10);
    }
    assertFalse(job.getFuture().isDone());
    // cancel the job
    job.cancel();
    assertTrueEventually(() -> assertTrue(job.getFuture().isDone()));
}
Also used : JetInstance(com.hazelcast.jet.JetInstance) JobConfig(com.hazelcast.jet.config.JobConfig) Pipeline(com.hazelcast.jet.pipeline.Pipeline) Job(com.hazelcast.jet.Job)

Example 2 with JetInstance

use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.

the class DetermineLocalParallelismTest method before.

@Before
public void before() {
    JetConfig cfg = new JetConfig();
    cfg.getInstanceConfig().setCooperativeThreadCount(DEFAULT_PARALLELISM);
    JetInstance jet = createJetMember(cfg);
    nodeEngine = getNode(jet.getHazelcastInstance()).getNodeEngine();
}
Also used : JetInstance(com.hazelcast.jet.JetInstance) JetConfig(com.hazelcast.jet.config.JetConfig) Before(org.junit.Before)

Example 3 with JetInstance

use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.

the class AggregateTransform_IntegrationTest method test_aggregate3_with_aggBuilder.

@Test
public void test_aggregate3_with_aggBuilder() {
    // Given
    JetInstance instance = createJetMember();
    IListJet<Integer> list = instance.getList("list");
    list.add(1);
    list.add(2);
    list.add(3);
    IListJet<String> list1 = instance.getList("list1");
    list1.add("a");
    list1.add("b");
    list1.add("c");
    IListJet<Double> list2 = instance.getList("list2");
    list2.add(6.0d);
    list2.add(7.0d);
    list2.add(8.0d);
    // When
    Pipeline p = Pipeline.create();
    BatchStage<Integer> stage0 = p.drawFrom(Sources.list("list"));
    BatchStage<String> stage1 = p.drawFrom(Sources.list("list1"));
    BatchStage<Double> stage2 = p.drawFrom(Sources.list("list2"));
    AggregateBuilder<Integer> builder = stage0.aggregateBuilder();
    Tag<Integer> tag0 = builder.tag0();
    Tag<String> tag1 = builder.add(stage1);
    Tag<Double> tag2 = builder.add(stage2);
    BatchStage<ThreeBags> resultStage = builder.build(AggregateOperation.withCreate(ThreeBags::threeBags).andAccumulate(tag0, (acc, item0) -> acc.bag0().add(item0)).andAccumulate(tag1, (acc, item1) -> acc.bag1().add(item1)).andAccumulate(tag2, (acc, item2) -> acc.bag2().add(item2)).andCombine(ThreeBags::combineWith).andDeduct(ThreeBags::deduct).andFinish(ThreeBags::finish));
    resultStage.drainTo(Sinks.list("sink"));
    instance.newJob(p).join();
    // Then
    ThreeBags threeBags = (ThreeBags) instance.getHazelcastInstance().getList("sink").iterator().next();
    assertNotNull(threeBags);
    sort(threeBags);
    assertEquals(threeBags(list, list1, list2), threeBags);
}
Also used : JetInstance(com.hazelcast.jet.JetInstance) RunWith(org.junit.runner.RunWith) ThreeBags.threeBags(com.hazelcast.jet.datamodel.ThreeBags.threeBags) TestUtil.set(com.hazelcast.jet.core.TestUtil.set) Collections.singletonList(java.util.Collections.singletonList) BatchStage(com.hazelcast.jet.pipeline.BatchStage) ThreeBags(com.hazelcast.jet.datamodel.ThreeBags) AggregateOperation(com.hazelcast.jet.aggregate.AggregateOperation) Util.entry(com.hazelcast.jet.Util.entry) AggregateOperations.toThreeBags(com.hazelcast.jet.aggregate.AggregateOperations.toThreeBags) Pipeline(com.hazelcast.jet.pipeline.Pipeline) JetTestSupport(com.hazelcast.jet.core.JetTestSupport) AggregateOperations.toTwoBags(com.hazelcast.jet.aggregate.AggregateOperations.toTwoBags) Assert.assertNotNull(org.junit.Assert.assertNotNull) AggregateOperations.toSet(com.hazelcast.jet.aggregate.AggregateOperations.toSet) Tag(com.hazelcast.jet.datamodel.Tag) Sinks(com.hazelcast.jet.pipeline.Sinks) Test(org.junit.Test) IListJet(com.hazelcast.jet.IListJet) AggregateBuilder(com.hazelcast.jet.pipeline.AggregateBuilder) ParallelTest(com.hazelcast.test.annotation.ParallelTest) Category(org.junit.experimental.categories.Category) Sources(com.hazelcast.jet.pipeline.Sources) List(java.util.List) IMap(com.hazelcast.core.IMap) HazelcastParallelClassRunner(com.hazelcast.test.HazelcastParallelClassRunner) TwoBags(com.hazelcast.jet.datamodel.TwoBags) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) TestSupport.listToString(com.hazelcast.jet.core.test.TestSupport.listToString) JetInstance(com.hazelcast.jet.JetInstance) TestSupport.listToString(com.hazelcast.jet.core.test.TestSupport.listToString) Pipeline(com.hazelcast.jet.pipeline.Pipeline) ThreeBags(com.hazelcast.jet.datamodel.ThreeBags) AggregateOperations.toThreeBags(com.hazelcast.jet.aggregate.AggregateOperations.toThreeBags) Test(org.junit.Test) ParallelTest(com.hazelcast.test.annotation.ParallelTest)

Example 4 with JetInstance

use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.

the class AggregateTransform_IntegrationTest method test_aggregate3.

@Test
public void test_aggregate3() {
    // Given
    JetInstance instance = createJetMember();
    IListJet<Integer> list = instance.getList("list");
    list.add(1);
    list.add(2);
    list.add(3);
    IListJet<String> list1 = instance.getList("list1");
    list1.add("a");
    list1.add("b");
    list1.add("c");
    IListJet<Double> list2 = instance.getList("list2");
    list2.add(6.0d);
    list2.add(7.0d);
    list2.add(8.0d);
    // When
    Pipeline p = Pipeline.create();
    BatchStage<String> stage1 = p.drawFrom(Sources.list("list1"));
    BatchStage<Double> stage2 = p.drawFrom(Sources.list("list2"));
    p.drawFrom(Sources.list("list")).aggregate3(stage1, stage2, toThreeBags()).drainTo(Sinks.list("sink"));
    instance.newJob(p).join();
    // Then
    ThreeBags threeBags = (ThreeBags) instance.getHazelcastInstance().getList("sink").iterator().next();
    assertNotNull(threeBags);
    sort(threeBags);
    assertEquals(threeBags(list, list1, list2), threeBags);
}
Also used : ThreeBags(com.hazelcast.jet.datamodel.ThreeBags) AggregateOperations.toThreeBags(com.hazelcast.jet.aggregate.AggregateOperations.toThreeBags) JetInstance(com.hazelcast.jet.JetInstance) TestSupport.listToString(com.hazelcast.jet.core.test.TestSupport.listToString) Pipeline(com.hazelcast.jet.pipeline.Pipeline) Test(org.junit.Test) ParallelTest(com.hazelcast.test.annotation.ParallelTest)

Example 5 with JetInstance

use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.

the class ExceptionUtilTest method test_serializationFromNodeToClient.

@Test
public void test_serializationFromNodeToClient() {
    // create one member and one client
    createJetMember();
    JetInstance client = createJetClient();
    RuntimeException exc = new RuntimeException("myException");
    try {
        DAG dag = new DAG();
        dag.newVertex("source", () -> new MockP().setCompleteError(exc)).localParallelism(1);
        client.newJob(dag).join();
    } catch (Exception caught) {
        assertThat(caught.toString(), containsString(exc.toString()));
        TestUtil.assertExceptionInCauses(exc, caught);
    } finally {
        shutdownFactory();
    }
}
Also used : JetInstance(com.hazelcast.jet.JetInstance) MockP(com.hazelcast.jet.core.TestProcessors.MockP) DAG(com.hazelcast.jet.core.DAG) JetException(com.hazelcast.jet.JetException) ExecutionException(java.util.concurrent.ExecutionException) ExpectedException(org.junit.rules.ExpectedException) Test(org.junit.Test)

Aggregations

JetInstance (com.hazelcast.jet.JetInstance)84 Test (org.junit.Test)45 Job (com.hazelcast.jet.Job)32 JetConfig (com.hazelcast.jet.config.JetConfig)22 MockPS (com.hazelcast.jet.core.TestProcessors.MockPS)14 Pipeline (com.hazelcast.jet.pipeline.Pipeline)14 JobConfig (com.hazelcast.jet.config.JobConfig)13 StuckProcessor (com.hazelcast.jet.core.TestProcessors.StuckProcessor)12 DAG (com.hazelcast.jet.core.DAG)10 ExpectedException (org.junit.rules.ExpectedException)10 Jet (com.hazelcast.jet.Jet)9 Assert.assertEquals (org.junit.Assert.assertEquals)9 JetService (com.hazelcast.jet.impl.JetService)8 Map (java.util.Map)8 CountDownLatch (java.util.concurrent.CountDownLatch)8 JobRepository (com.hazelcast.jet.impl.JobRepository)7 Sources (com.hazelcast.jet.pipeline.Sources)7 CancellationException (java.util.concurrent.CancellationException)7 Assert.assertNotNull (org.junit.Assert.assertNotNull)7 Before (org.junit.Before)7