Examples with DAG com.hazelcast.jet.core.DAG used on opensource projects

Example 46 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.

the class WriteFilePTest method when_slowSource_then_fileFlushedAfterEachItem.

@Test
public void when_slowSource_then_fileFlushedAfterEachItem() {
    // Given
    int numItems = 10;
    DAG dag = new DAG();
    Vertex source = dag.newVertex("source", () -> new SlowSourceP(semaphore, numItems)).localParallelism(1);
    Vertex sink = dag.newVertex("sink", writeFileP(directory.toString(), StandardCharsets.UTF_8, null, DISABLE_ROLLING, true, Object::toString)).localParallelism(1);
    dag.edge(between(source, sink));
    Job job = instance().getJet().newJob(dag);
    for (int i = 0; i < numItems; i++) {
        // When
        semaphore.release();
        int finalI = i;
        // Then
        assertTrueEventually(() -> checkFileContents(0, finalI + 1, false, false, true), 5);
    }
    // wait for the job to finish
    job.join();
}

Example 47 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.

the class AbstractDeploymentTest method testDeployment_whenZipAddedAsResource_thenClassesFromAllJarsAvailableOnClassLoader.

@Test
public void testDeployment_whenZipAddedAsResource_thenClassesFromAllJarsAvailableOnClassLoader() throws Throwable {
    DAG dag = new DAG();
    List<String> onClasspath = new ArrayList<>();
    onClasspath.add("com.sample.pojo.person.Person$Appereance");
    onClasspath.add("com.sample.pojo.car.Car");
    List<String> notOnClasspath = new ArrayList<>();
    notOnClasspath.add("com.sample.pojo.address.Address");
    dag.newVertex("load class", () -> new LoadClassesIsolated(onClasspath, notOnClasspath, true));
    JobConfig jobConfig = new JobConfig();
    jobConfig.addJarsInZip(this.getClass().getResource("/zip-resources/person-car-jar.zip"));
    executeAndPeel(getJet().newJob(dag, jobConfig));
}

Example 48 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.

the class AbstractDeploymentTest method testDeployment_whenJarAddedAsResource_thenClassesAvailableOnClassLoader.

@Test
public void testDeployment_whenJarAddedAsResource_thenClassesAvailableOnClassLoader() throws Throwable {
    DAG dag = new DAG();
    dag.newVertex("load class", () -> new LoadClassesIsolated(true));
    JobConfig jobConfig = new JobConfig();
    jobConfig.addJar(this.getClass().getResource("/deployment/sample-pojo-1.0-person.jar"));
    executeAndPeel(getJet().newJob(dag, jobConfig));
}

Example 49 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.

the class AbstractDeploymentTest method testDeployment_whenAddClass_thenNestedClassesAreAddedAsWell.

@Test
public void testDeployment_whenAddClass_thenNestedClassesAreAddedAsWell() throws Throwable {
    DAG dag = new DAG();
    dag.newVertex("executes lambda from a nested class", NestedClassIsLoaded::new);
    JobConfig jobConfig = new JobConfig();
    URL classUrl = new File(CLASS_DIRECTORY).toURI().toURL();
    URLClassLoader urlClassLoader = new URLClassLoader(new URL[] { classUrl }, null);
    Class<?> worker = urlClassLoader.loadClass("com.sample.lambda.Worker");
    jobConfig.addClass(worker);
    executeAndPeel(getJet().newJob(dag, jobConfig));
}

Example 50 with DAG

use of com.hazelcast.jet.core.DAG in project hazelcast by hazelcast.

the class JobMetrics_MiscTest method when_jobSuspended_andMetricsNotStored_then_onlyPeriodicMetricsReturned.

@Test
public void when_jobSuspended_andMetricsNotStored_then_onlyPeriodicMetricsReturned() throws Throwable {
    DAG dag = new DAG();
    Vertex v1 = dag.newVertex("v1", TestProcessors.MockP::new);
    Vertex v2 = dag.newVertex("v2", (SupplierEx<Processor>) TestProcessors.NoOutputSourceP::new);
    dag.edge(between(v1, v2));
    // init
    JobConfig config = new JobConfig().setMetricsEnabled(// enable metric collection
    true).setStoreMetricsAfterJobCompletion(// disable metric saving on completion
    false);
    Job job = hz().getJet().newJob(dag, config);
    // when
    TestProcessors.NoOutputSourceP.executionStarted.await();
    // then
    assertJobStatusEventually(job, JobStatus.RUNNING);
    assertTrueEventually(() -> assertJobHasMetrics(job, false));
    // when
    job.suspend();
    // then
    assertJobStatusEventually(job, SUSPENDED);
    assertTrueEventually(() -> assertEmptyJobMetrics(job, false));
    // when
    job.resume();
    // then
    assertJobStatusEventually(job, RUNNING);
    assertTrueEventually(() -> assertJobHasMetrics(job, false));
    // when
    TestProcessors.NoOutputSourceP.proceedLatch.countDown();
    job.join();
    // then
    assertJobStatusEventually(job, JobStatus.COMPLETED);
    assertEmptyJobMetrics(job, false);
}