Examples with JobConfig com.hazelcast.jet.config.JobConfig used on opensource projects

Example 11 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast by hazelcast.

the class Hz3SourcesTest method when_readFromInstanceDown_then_shouldThrowJetException.

@Test
public void when_readFromInstanceDown_then_shouldThrowJetException() {
    HazelcastInstance hz = createHazelcastInstance();
    Pipeline p = Pipeline.create();
    BatchSource<Map.Entry<Integer, String>> source = Hz3Sources.remoteMap("test-map", HZ3_DOWN_CLIENT_CONFIG);
    p.readFrom(source).map(Map.Entry::getValue).writeTo(Sinks.list("test-result"));
    JobConfig config = getJobConfig(source.name());
    Job job = hz.getJet().newJob(p, config);
    assertThatThrownBy(() -> job.join()).hasStackTraceContaining(JetException.class.getName()).hasStackTraceContaining("Unable to connect to any cluster");
}

Example 12 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast by hazelcast.

the class Hz3SourcesTest method readFromMapManyItems.

@Test
public void readFromMapManyItems() {
    IMap<Integer, String> map = hz3.getMap("test-map");
    Map<Integer, String> items = new HashMap<>();
    for (int i = 0; i < 10_000; i++) {
        items.put(i, "item " + i);
    }
    map.putAll(items);
    HazelcastInstance hz = createHazelcastInstance();
    Pipeline p = Pipeline.create();
    BatchSource<Map.Entry<Integer, String>> source = Hz3Sources.remoteMap("test-map", HZ3_CLIENT_CONFIG);
    p.readFrom(source).writeTo(Sinks.map("test-result"));
    JobConfig config = getJobConfig(source.name());
    Job job = hz.getJet().newJob(p, config);
    job.join();
    IMap<Integer, String> result = hz.getMap("test-result");
    assertThat(result.entrySet()).isEqualTo(items.entrySet());
}

Example 13 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast by hazelcast.

the class Hz3SourcesTest method readFromEmptyMap.

@Test
public void readFromEmptyMap() {
    // make sure the map exists
    hz3.getMap("test-map");
    HazelcastInstance hz = createHazelcastInstance();
    Pipeline p = Pipeline.create();
    BatchSource<Map.Entry<Integer, String>> source = Hz3Sources.remoteMap("test-map", HZ3_CLIENT_CONFIG);
    p.readFrom(source).map(Map.Entry::getValue).writeTo(Sinks.list("test-result"));
    JobConfig config = getJobConfig(source.name());
    Job job = hz.getJet().newJob(p, config);
    job.join();
    IList<String> result = hz.getList("test-result");
    assertThat(result).isEmpty();
}

Example 14 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast by hazelcast.

the class PlanExecutor method execute.

SqlResult execute(SelectPlan plan, QueryId queryId, List<Object> arguments, long timeout) {
    List<Object> args = prepareArguments(plan.getParameterMetadata(), arguments);
    JobConfig jobConfig = new JobConfig().setArgument(SQL_ARGUMENTS_KEY_NAME, args).setArgument(KEY_SQL_QUERY_TEXT, plan.getQuery()).setArgument(KEY_SQL_UNBOUNDED, plan.isStreaming()).setTimeoutMillis(timeout);
    QueryResultProducerImpl queryResultProducer = new QueryResultProducerImpl(!plan.isStreaming());
    AbstractJetInstance<?> jet = (AbstractJetInstance<?>) hazelcastInstance.getJet();
    long jobId = jet.newJobId();
    Object oldValue = resultRegistry.store(jobId, queryResultProducer);
    assert oldValue == null : oldValue;
    try {
        Job job = jet.newLightJob(jobId, plan.getDag(), jobConfig);
        job.getFuture().whenComplete((r, t) -> {
            // make sure the queryResultProducer is cleaned up after the job completes. This normally
            // takes effect when the job fails before the QRP is removed by the RootResultConsumerSink
            resultRegistry.remove(jobId);
            if (t != null) {
                int errorCode = findQueryExceptionCode(t);
                String errorMessage = findQueryExceptionMessage(t);
                queryResultProducer.onError(QueryException.error(errorCode, "The Jet SQL job failed: " + errorMessage, t));
            }
        });
    } catch (Throwable e) {
        resultRegistry.remove(jobId);
        throw e;
    }
    return new SqlResultImpl(queryId, queryResultProducer, plan.getRowMetadata(), plan.isStreaming());
}

Example 15 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast by hazelcast.

the class PostgresCdcIntegrationTest method cdcMapSink.

@Test
@Category(NightlyTest.class)
public void cdcMapSink() throws Exception {
    // given
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source("customers")).withNativeTimestamps(0).writeTo(CdcSinks.map("cache", r -> r.key().toMap().get("id"), r -> r.value().toObject(Customer.class).toString()));
    // when
    HazelcastInstance hz = createHazelcastInstances(2)[0];
    JobConfig jobConfig = new JobConfig().setProcessingGuarantee(ProcessingGuarantee.AT_LEAST_ONCE);
    Job job = hz.getJet().newJob(pipeline, jobConfig);
    JetTestSupport.assertJobStatusEventually(job, JobStatus.RUNNING);
    // then
    assertEqualsEventually(() -> mapResultsToSortedList(hz.getMap("cache")), Arrays.asList("1001:Customer {id=1001, firstName=Sally, lastName=Thomas, email=sally.thomas@acme.com}", "1002:Customer {id=1002, firstName=George, lastName=Bailey, email=gbailey@foobar.com}", "1003:Customer {id=1003, firstName=Edward, lastName=Walker, email=ed@walker.com}", "1004:Customer {id=1004, firstName=Anne, lastName=Kretchmar, email=annek@noanswer.org}"));
    // when
    job.restart();
    JetTestSupport.assertJobStatusEventually(job, JobStatus.RUNNING);
    executeBatch("UPDATE customers SET first_name='Anne Marie' WHERE id=1004", "INSERT INTO customers VALUES (1005, 'Jason', 'Bourne', 'jason@bourne.org')");
    // then
    assertEqualsEventually(() -> mapResultsToSortedList(hz.getMap("cache")), Arrays.asList("1001:Customer {id=1001, firstName=Sally, lastName=Thomas, email=sally.thomas@acme.com}", "1002:Customer {id=1002, firstName=George, lastName=Bailey, email=gbailey@foobar.com}", "1003:Customer {id=1003, firstName=Edward, lastName=Walker, email=ed@walker.com}", "1004:Customer {id=1004, firstName=Anne Marie, lastName=Kretchmar, email=annek@noanswer.org}", "1005:Customer {id=1005, firstName=Jason, lastName=Bourne, email=jason@bourne.org}"));
    // when
    executeBatch("DELETE FROM customers WHERE id=1005");
    // then
    assertEqualsEventually(() -> mapResultsToSortedList(hz.getMap("cache")), Arrays.asList("1001:Customer {id=1001, firstName=Sally, lastName=Thomas, email=sally.thomas@acme.com}", "1002:Customer {id=1002, firstName=George, lastName=Bailey, email=gbailey@foobar.com}", "1003:Customer {id=1003, firstName=Edward, lastName=Walker, email=ed@walker.com}", "1004:Customer {id=1004, firstName=Anne Marie, lastName=Kretchmar, email=annek@noanswer.org}"));
}