use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.
the class StreamKafkaPTest method integrationTest.
private void integrationTest(ProcessingGuarantee guarantee) throws Exception {
int messageCount = 20;
JetInstance[] instances = new JetInstance[2];
Arrays.setAll(instances, i -> createJetMember());
Pipeline p = Pipeline.create();
p.drawFrom(KafkaSources.kafka(properties, topic1Name, topic2Name)).drainTo(Sinks.list("sink"));
JobConfig config = new JobConfig();
config.setProcessingGuarantee(guarantee);
config.setSnapshotIntervalMillis(500);
Job job = instances[0].newJob(p, config);
sleepAtLeastSeconds(3);
for (int i = 0; i < messageCount; i++) {
produce(topic1Name, i, Integer.toString(i));
produce(topic2Name, i - messageCount, Integer.toString(i - messageCount));
}
IList<Object> list = instances[0].getList("sink");
assertTrueEventually(() -> {
assertEquals(messageCount * 2, list.size());
for (int i = 0; i < messageCount; i++) {
Entry<Integer, String> entry1 = createEntry(i);
Entry<Integer, String> entry2 = createEntry(i - messageCount);
assertTrue("missing entry: " + entry1, list.contains(entry1));
assertTrue("missing entry: " + entry2, list.contains(entry2));
}
}, 5);
if (guarantee != ProcessingGuarantee.NONE) {
// wait until the items are consumed and a new snapshot appears
assertTrueEventually(() -> assertTrue(list.size() == messageCount * 2));
IMapJet<Long, Object> snapshotsMap = instances[0].getMap(SnapshotRepository.snapshotsMapName(job.getId()));
Long currentMax = maxSuccessfulSnapshot(snapshotsMap);
assertTrueEventually(() -> {
Long newMax = maxSuccessfulSnapshot(snapshotsMap);
assertTrue("no snapshot produced", newMax != null && !newMax.equals(currentMax));
System.out.println("snapshot " + newMax + " found, previous was " + currentMax);
});
// Bring down one member. Job should restart and drain additional items (and maybe
// some of the previous duplicately).
instances[1].shutdown();
Thread.sleep(500);
for (int i = messageCount; i < 2 * messageCount; i++) {
produce(topic1Name, i, Integer.toString(i));
produce(topic2Name, i - messageCount, Integer.toString(i - messageCount));
}
assertTrueEventually(() -> {
assertTrue("Not all messages were received", list.size() >= messageCount * 4);
for (int i = 0; i < 2 * messageCount; i++) {
Entry<Integer, String> entry1 = createEntry(i);
Entry<Integer, String> entry2 = createEntry(i - messageCount);
assertTrue("missing entry: " + entry1.toString(), list.contains(entry1));
assertTrue("missing entry: " + entry2.toString(), list.contains(entry2));
}
}, 10);
}
assertFalse(job.getFuture().isDone());
// cancel the job
job.cancel();
assertTrueEventually(() -> assertTrue(job.getFuture().isDone()));
}
use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.
the class DetermineLocalParallelismTest method before.
@Before
public void before() {
JetConfig cfg = new JetConfig();
cfg.getInstanceConfig().setCooperativeThreadCount(DEFAULT_PARALLELISM);
JetInstance jet = createJetMember(cfg);
nodeEngine = getNode(jet.getHazelcastInstance()).getNodeEngine();
}
use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.
the class AggregateTransform_IntegrationTest method test_aggregate3_with_aggBuilder.
@Test
public void test_aggregate3_with_aggBuilder() {
// Given
JetInstance instance = createJetMember();
IListJet<Integer> list = instance.getList("list");
list.add(1);
list.add(2);
list.add(3);
IListJet<String> list1 = instance.getList("list1");
list1.add("a");
list1.add("b");
list1.add("c");
IListJet<Double> list2 = instance.getList("list2");
list2.add(6.0d);
list2.add(7.0d);
list2.add(8.0d);
// When
Pipeline p = Pipeline.create();
BatchStage<Integer> stage0 = p.drawFrom(Sources.list("list"));
BatchStage<String> stage1 = p.drawFrom(Sources.list("list1"));
BatchStage<Double> stage2 = p.drawFrom(Sources.list("list2"));
AggregateBuilder<Integer> builder = stage0.aggregateBuilder();
Tag<Integer> tag0 = builder.tag0();
Tag<String> tag1 = builder.add(stage1);
Tag<Double> tag2 = builder.add(stage2);
BatchStage<ThreeBags> resultStage = builder.build(AggregateOperation.withCreate(ThreeBags::threeBags).andAccumulate(tag0, (acc, item0) -> acc.bag0().add(item0)).andAccumulate(tag1, (acc, item1) -> acc.bag1().add(item1)).andAccumulate(tag2, (acc, item2) -> acc.bag2().add(item2)).andCombine(ThreeBags::combineWith).andDeduct(ThreeBags::deduct).andFinish(ThreeBags::finish));
resultStage.drainTo(Sinks.list("sink"));
instance.newJob(p).join();
// Then
ThreeBags threeBags = (ThreeBags) instance.getHazelcastInstance().getList("sink").iterator().next();
assertNotNull(threeBags);
sort(threeBags);
assertEquals(threeBags(list, list1, list2), threeBags);
}
use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.
the class AggregateTransform_IntegrationTest method test_aggregate3.
@Test
public void test_aggregate3() {
// Given
JetInstance instance = createJetMember();
IListJet<Integer> list = instance.getList("list");
list.add(1);
list.add(2);
list.add(3);
IListJet<String> list1 = instance.getList("list1");
list1.add("a");
list1.add("b");
list1.add("c");
IListJet<Double> list2 = instance.getList("list2");
list2.add(6.0d);
list2.add(7.0d);
list2.add(8.0d);
// When
Pipeline p = Pipeline.create();
BatchStage<String> stage1 = p.drawFrom(Sources.list("list1"));
BatchStage<Double> stage2 = p.drawFrom(Sources.list("list2"));
p.drawFrom(Sources.list("list")).aggregate3(stage1, stage2, toThreeBags()).drainTo(Sinks.list("sink"));
instance.newJob(p).join();
// Then
ThreeBags threeBags = (ThreeBags) instance.getHazelcastInstance().getList("sink").iterator().next();
assertNotNull(threeBags);
sort(threeBags);
assertEquals(threeBags(list, list1, list2), threeBags);
}
use of com.hazelcast.jet.JetInstance in project hazelcast-jet by hazelcast.
the class ExceptionUtilTest method test_serializationFromNodeToClient.
@Test
public void test_serializationFromNodeToClient() {
// create one member and one client
createJetMember();
JetInstance client = createJetClient();
RuntimeException exc = new RuntimeException("myException");
try {
DAG dag = new DAG();
dag.newVertex("source", () -> new MockP().setCompleteError(exc)).localParallelism(1);
client.newJob(dag).join();
} catch (Exception caught) {
assertThat(caught.toString(), containsString(exc.toString()));
TestUtil.assertExceptionInCauses(exc, caught);
} finally {
shutdownFactory();
}
}
Aggregations