Examples with JobConfig com.hazelcast.jet.config.JobConfig used on opensource projects

Example 16 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast by hazelcast.

the class MySqlCdcIntegrationTest method restart.

@Test
@Category(NightlyTest.class)
public void restart() throws Exception {
    // given
    List<String> expectedRecords = Arrays.asList("1004/1:UPDATE:Customer {id=1004, firstName=Anne Marie, lastName=Kretchmar, email=annek@noanswer.org}", "1005/0:INSERT:Customer {id=1005, firstName=Jason, lastName=Bourne, email=jason@bourne.org}", "1005/1:DELETE:Customer {id=1005, firstName=Jason, lastName=Bourne, email=jason@bourne.org}");
    Pipeline pipeline = Pipeline.create();
    pipeline.readFrom(source("customers")).withNativeTimestamps(0).<ChangeRecord>customTransform("filter_timestamps", filterTimestampsProcessorSupplier()).groupingKey(record -> (Integer) record.key().toMap().get("id")).mapStateful(LongAccumulator::new, (accumulator, customerId, record) -> {
        long count = accumulator.get();
        accumulator.add(1);
        Operation operation = record.operation();
        RecordPart value = record.value();
        Customer customer = value.toObject(Customer.class);
        return entry(customerId + "/" + count, operation + ":" + customer);
    }).setLocalParallelism(1).writeTo(Sinks.map("results"));
    // when
    HazelcastInstance hz = createHazelcastInstances(2)[0];
    JobConfig jobConfig = new JobConfig().setProcessingGuarantee(ProcessingGuarantee.AT_LEAST_ONCE);
    Job job = hz.getJet().newJob(pipeline, jobConfig);
    JetTestSupport.assertJobStatusEventually(job, JobStatus.RUNNING);
    assertEqualsEventually(() -> hz.getMap("results").size(), 4);
    // then
    hz.getMap("results").destroy();
    // when
    assertEqualsEventually(() -> hz.getMap("results").size(), 0);
    // then
    job.restart();
    // when
    JetTestSupport.assertJobStatusEventually(job, JobStatus.RUNNING);
    // then update a record
    try (Connection connection = getConnection(mysql, "inventory")) {
        Statement statement = connection.createStatement();
        statement.addBatch("UPDATE customers SET first_name='Anne Marie' WHERE id=1004");
        statement.addBatch("INSERT INTO customers VALUES (1005, 'Jason', 'Bourne', 'jason@bourne.org')");
        statement.addBatch("DELETE FROM customers WHERE id=1005");
        statement.executeBatch();
    }
    // then
    try {
        assertEqualsEventually(() -> mapResultsToSortedList(hz.getMap("results")), expectedRecords);
    } finally {
        job.cancel();
        assertJobStatusEventually(job, JobStatus.FAILED);
    }
}

Example 17 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast by hazelcast.

the class JobClassLoaderService method createProcessorClassLoaders.

private Map<String, ClassLoader> createProcessorClassLoaders(long jobId, JobConfig jobConfig, ClassLoader parent) {
    logger.fine("Create processor classloader map for job " + idToString(jobId));
    String customLibDir = nodeEngine.getProperties().getString(ClusterProperty.PROCESSOR_CUSTOM_LIB_DIR);
    Map<String, ClassLoader> classLoaderMap = new HashMap<>();
    for (Entry<String, List<String>> entry : jobConfig.getCustomClassPaths().entrySet()) {
        List<URL> list = entry.getValue().stream().map(jar -> {
            try {
                Path path = Paths.get(customLibDir, jar);
                return path.toUri().toURL();
            } catch (MalformedURLException e) {
                throw new JetException(e);
            }
        }).collect(Collectors.toList());
        URL[] urls = list.toArray(new URL[] {});
        classLoaderMap.put(entry.getKey(), new ChildFirstClassLoader(urls, parent));
    }
    return unmodifiableMap(classLoaderMap);
}

Example 18 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast by hazelcast.

the class JobRepository method completeJob.

/**
 * Puts a JobResult for the given job and deletes the JobRecord.
 *
 * @throws JobNotFoundException  if the JobRecord is not found
 * @throws IllegalStateException if the JobResult is already present
 */
void completeJob(@Nonnull MasterContext masterContext, @Nullable List<RawJobMetrics> terminalMetrics, @Nullable Throwable error, long completionTime) {
    long jobId = masterContext.jobId();
    JobConfig config = masterContext.jobRecord().getConfig();
    long creationTime = masterContext.jobRecord().getCreationTime();
    JobResult jobResult = new JobResult(jobId, config, creationTime, completionTime, toErrorMsg(error));
    if (terminalMetrics != null) {
        try {
            List<RawJobMetrics> prevMetrics = jobMetrics.get().put(jobId, terminalMetrics);
            if (prevMetrics != null) {
                logger.warning("Overwriting job metrics for job " + jobResult);
            }
        } catch (Exception e) {
            logger.warning("Storing the job metrics failed, ignoring: " + e, e);
        }
    }
    for (; ; ) {
        // keep trying to store the JobResult until it succeeds
        try {
            jobResults.get().set(jobId, jobResult);
            break;
        } catch (Exception e) {
            // if the local instance was shut down, re-throw the error
            LifecycleService lifecycleService = instance.getLifecycleService();
            if (e instanceof HazelcastInstanceNotActiveException && (!lifecycleService.isRunning())) {
                throw e;
            }
            // retry otherwise, after a delay
            long retryTimeoutSeconds = 1;
            logger.warning("Failed to store JobResult, will retry in " + retryTimeoutSeconds + " seconds: " + e, e);
            LockSupport.parkNanos(SECONDS.toNanos(retryTimeoutSeconds));
        }
    }
    deleteJob(jobId);
}

Example 19 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast by hazelcast.

the class Util method copyMapUsingJob.

// used in jet-enterprise
@SuppressWarnings("WeakerAccess")
public static CompletableFuture<Void> copyMapUsingJob(HazelcastInstance instance, int queueSize, String sourceMap, String targetMap) {
    DAG dag = new DAG();
    Vertex source = dag.newVertex("readMap(" + sourceMap + ')', readMapP(sourceMap));
    Vertex sink = dag.newVertex("writeMap(" + targetMap + ')', writeMapP(targetMap));
    dag.edge(between(source, sink).setConfig(new EdgeConfig().setQueueSize(queueSize)));
    JobConfig jobConfig = new JobConfig().setName("copy-" + sourceMap + "-to-" + targetMap);
    return instance.getJet().newJob(dag, jobConfig).getFuture();
}

Example 20 with JobConfig

use of com.hazelcast.jet.config.JobConfig in project hazelcast by hazelcast.

the class MapIndexScanPTest method test_whenFilterAndSpecificProjectionExists_sorted.

@Test
public void test_whenFilterAndSpecificProjectionExists_sorted() {
    List<JetSqlRow> expected = new ArrayList<>();
    for (int i = count; i > 0; i--) {
        map.put(i, new Person("value-" + i, i));
        if (i > count / 2) {
            expected.add(jetRow((count - i + 1), "value-" + (count - i + 1), (count - i + 1)));
        }
    }
    IndexConfig indexConfig = new IndexConfig(IndexType.SORTED, "age").setName(randomName());
    map.addIndex(indexConfig);
    IndexFilter filter = new IndexRangeFilter(intValue(0), true, intValue(count / 2), true);
    MapIndexScanMetadata metadata = metadata(indexConfig.getName(), filter, 0, false);
    TestSupport.verifyProcessor(adaptSupplier(MapIndexScanP.readMapIndexSupplier(metadata))).hazelcastInstance(instance()).jobConfig(new JobConfig().setArgument(SQL_ARGUMENTS_KEY_NAME, emptyList())).outputChecker(LENIENT_SAME_ITEMS_IN_ORDER).disableSnapshots().disableProgressAssertion().expectOutput(expected);
}