Examples with JobConfig com.hazelcast.jet.config.JobConfig used on opensource projects

Example 26 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast by hazelcast.

the class KinesisIntegrationTest method restart_staticStream.

private void restart_staticStream(boolean graceful) {
    HELPER.createStream(3);
    JobConfig jobConfig = new JobConfig().setProcessingGuarantee(ProcessingGuarantee.AT_LEAST_ONCE).setSnapshotIntervalMillis(SECONDS.toMillis(1));
    Job job = hz().getJet().newJob(getPipeline(kinesisSource().build()), jobConfig);
    Map<String, List<String>> expectedMessages = sendMessages();
    // wait for some data to start coming out of the pipeline
    assertTrueEventually(() -> assertFalse(results.isEmpty()));
    ((JobProxy) job).restart(graceful);
    assertMessages(expectedMessages, true, !graceful);
}

Example 27 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast by hazelcast.

the class KinesisIntegrationTest method restart_dynamicStream.

private void restart_dynamicStream(boolean graceful) {
    HELPER.createStream(3);
    JobConfig jobConfig = new JobConfig().setProcessingGuarantee(ProcessingGuarantee.AT_LEAST_ONCE).setSnapshotIntervalMillis(SECONDS.toMillis(1));
    Job job = hz().getJet().newJob(getPipeline(kinesisSource().build()), jobConfig);
    Map<String, List<String>> expectedMessages = sendMessages();
    // wait for some data to start coming out of the pipeline
    assertTrueEventually(() -> assertFalse(results.isEmpty()));
    List<Shard> openShards = listOpenShards();
    Shard shard1 = openShards.get(0);
    Shard shard2 = openShards.get(1);
    Shard shard3 = openShards.get(2);
    splitShard(shard1);
    HELPER.waitForStreamToActivate();
    assertOpenShards(4, shard1);
    mergeShards(shard2, shard3);
    HELPER.waitForStreamToActivate();
    assertOpenShards(3, shard2, shard3);
    ((JobProxy) job).restart(graceful);
    assertMessages(expectedMessages, false, !graceful);
}

Example 28 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast by hazelcast.

the class JobSerializerTest method when_serializerIsNotRegistered_then_throwsException.

@Test
public void when_serializerIsNotRegistered_then_throwsException() {
    String listName = "list-1";
    List<Person> list = client().getList(listName);
    list.add(Person.newBuilder().setName("Joe").setAge(33).build());
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(Sources.<Person>list(listName)).map(Person::getName).writeTo(Sinks.logger());
    assertThatThrownBy(() -> client().getJet().newJob(pipeline, new JobConfig()).join()).hasCauseInstanceOf(JetException.class);
}

Example 29 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast by hazelcast.

the class JobSerializerTest method when_serializerIsRegisteredForDistributedJob_then_itIsAvailableForAllStages.

@Test
public void when_serializerIsRegisteredForDistributedJob_then_itIsAvailableForAllStages() {
    List<String> input = IntStream.range(0, 10_000).boxed().map(t -> Integer.toString(t)).collect(toList());
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(TestSources.items(input)).map(name -> Person.newBuilder().setName(name).build()).groupingKey(identity()).filterUsingService(sharedService(ctx -> null), (s, k, v) -> true).map(person -> person.getName()).writeTo(AssertionSinks.assertAnyOrder(input));
    client().getJet().newJob(pipeline, new JobConfig().registerSerializer(Person.class, PersonSerializer.class)).join();
}

Example 30 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast by hazelcast.

the class AbstractDeploymentTest method testDeployment_whenZipAddedAsResource_thenClassesFromAllJarsAvailableOnClassLoader.

@Test
public void testDeployment_whenZipAddedAsResource_thenClassesFromAllJarsAvailableOnClassLoader() throws Throwable {
    DAG dag = new DAG();
    List<String> onClasspath = new ArrayList<>();
    onClasspath.add("com.sample.pojo.person.Person$Appereance");
    onClasspath.add("com.sample.pojo.car.Car");
    List<String> notOnClasspath = new ArrayList<>();
    notOnClasspath.add("com.sample.pojo.address.Address");
    dag.newVertex("load class", () -> new LoadClassesIsolated(onClasspath, notOnClasspath, true));
    JobConfig jobConfig = new JobConfig();
    jobConfig.addJarsInZip(this.getClass().getResource("/zip-resources/person-car-jar.zip"));
    executeAndPeel(getJet().newJob(dag, jobConfig));
}