Examples with JobLauncher org.apache.gobblin.runtime.JobLauncher used on opensource projects

Example 1 with JobLauncher

use of org.apache.gobblin.runtime.JobLauncher in project incubator-gobblin by apache.

the class GobblinHelixJobScheduler method scheduleJobImmediately.

public Future<?> scheduleJobImmediately(Properties jobProps, JobListener jobListener) {
    RetriggeringJobCallable retriggeringJob = new RetriggeringJobCallable(jobProps, jobListener);
    final Future<?> future = this.jobExecutor.submit(retriggeringJob);
    return new Future() {

        @Override
        public boolean cancel(boolean mayInterruptIfRunning) {
            if (!GobblinHelixJobScheduler.this.isCancelRequested()) {
                return false;
            }
            boolean result = true;
            try {
                JobLauncher jobLauncher = retriggeringJob.getCurrentJobLauncher();
                if (jobLauncher != null) {
                    jobLauncher.cancelJob(jobListener);
                }
            } catch (JobException e) {
                LOGGER.error("Failed to cancel job " + jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY), e);
                result = false;
            }
            if (mayInterruptIfRunning) {
                result &= future.cancel(true);
            }
            return result;
        }

        @Override
        public boolean isCancelled() {
            return future.isCancelled();
        }

        @Override
        public boolean isDone() {
            return future.isDone();
        }

        @Override
        public Object get() throws InterruptedException, ExecutionException {
            return future.get();
        }

        @Override
        public Object get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException {
            return future.get(timeout, unit);
        }
    };
}

Example 2 with JobLauncher

use of org.apache.gobblin.runtime.JobLauncher in project incubator-gobblin by apache.

the class Kafka08DataWriterIntegrationTest method testErrors.

@Test
public void testErrors() throws Exception {
    log.warn("Process id = " + ManagementFactory.getRuntimeMXBean().getName());
    int numRecordsPerExtract = 5;
    int numParallel = 2;
    int errorEvery = 2000;
    int totalRecords = numRecordsPerExtract * numParallel;
    int totalSuccessful = totalRecords / errorEvery + totalRecords % errorEvery;
    {
        Closer closer = Closer.create();
        try {
            kafkaTestHelper.provisionTopic(TOPIC);
            jobProps.setProperty("source.numRecordsPerExtract", "" + numRecordsPerExtract);
            jobProps.setProperty("source.numParallelism", "" + numParallel);
            jobProps.setProperty("writer.kafka.producerConfig.flaky.errorType", "regex");
            // all records from partition 0 will be dropped.
            jobProps.setProperty("writer.kafka.producerConfig.flaky.regexPattern", ":index:0.*");
            jobProps.setProperty("job.commit.policy", "partial");
            jobProps.setProperty("publish.at.job.level", "false");
            // number of records in partition 1
            totalSuccessful = 5;
            JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(gobblinProps, jobProps));
            jobLauncher.launchJob(null);
        } catch (Exception e) {
            log.error("Failed to run job with exception ", e);
            Assert.fail("Should not throw exception on running the job");
        } finally {
            closer.close();
        }
        // test records written
        testRecordsWritten(totalSuccessful, TOPIC);
    }
    boolean trySecond = true;
    if (trySecond) {
        Closer closer = Closer.create();
        try {
            jobProps.setProperty("source.numRecordsPerExtract", "" + numRecordsPerExtract);
            jobProps.setProperty("source.numParallelism", "" + numParallel);
            jobProps.setProperty("writer.kafka.producerConfig.flaky.errorType", "nth");
            jobProps.setProperty("writer.kafka.producerConfig.flaky.errorEvery", "" + errorEvery);
            JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(gobblinProps, jobProps));
            jobLauncher.launchJob(null);
            totalSuccessful = totalRecords / errorEvery + totalRecords % errorEvery;
        } catch (Exception e) {
            log.error("Failed to run job with exception ", e);
            Assert.fail("Should not throw exception on running the job");
        } finally {
            closer.close();
        }
    }
    // test records written
    testRecordsWritten(totalSuccessful, TOPIC);
}

Example 3 with JobLauncher

use of org.apache.gobblin.runtime.JobLauncher in project incubator-gobblin by apache.

the class CopyIntegrationTest method testTarGzCopy.

@Test
public void testTarGzCopy() throws Exception {
    Closer closer = Closer.create();
    try {
        JobLauncher jobLauncher = closer.register(JobLauncherFactory.newJobLauncher(gobblinProps, jobProps));
        jobLauncher.launchJob(null);
        String file1Path = gobblinProps.getProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR) + "/LogData/sub1/sub2/text1.txt";
        String file2Path = gobblinProps.getProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR) + "/LogData/sub1/sub2/text2.txt";
        FileSystem fs = FileSystem.getLocal(new Configuration());
        Assert.assertEquals(IOUtils.toString(closer.register(fs.open(new Path(file1Path)))), "text1");
        Assert.assertEquals(IOUtils.toString(closer.register(fs.open(new Path(file2Path)))), "text2");
    } finally {
        closer.close();
    }
}

Example 4 with JobLauncher

use of org.apache.gobblin.runtime.JobLauncher in project incubator-gobblin by apache.

the class JobLauncherExecutionDriver method create.

/**
 * Creates a new JobExecutionDriver which acts as an adapter to the legacy {@link JobLauncher} API.
 * @param sysConfig             the system/environment config
 * @param jobSpec               the JobSpec to be executed
 * @param jobLauncherType       an optional jobLauncher type; the value follows the convention of
 *        {@link JobLauncherFactory#newJobLauncher(java.util.Properties, java.util.Properties, String).
 *        If absent, {@link JobLauncherFactory#newJobLauncher(java.util.Properties, java.util.Properties)}
 *        will be used which looks for the {@link ConfigurationKeys#JOB_LAUNCHER_TYPE_KEY}
 *        in the system configuration.
 * @param jobExecStateListener  an optional listener to listen for state changes in the execution.
 * @param log                   an optional logger to be used; if none is specified, a default one
 *                              will be instantiated.
 */
public static JobLauncherExecutionDriver create(Configurable sysConfig, JobSpec jobSpec, Optional<JobLauncherFactory.JobLauncherType> jobLauncherType, Optional<Logger> log, boolean instrumentationEnabled, JobExecutionLauncher.StandardMetrics launcherMetrics, SharedResourcesBroker<GobblinScopeTypes> instanceBroker) {
    Logger actualLog = log.isPresent() ? log.get() : LoggerFactory.getLogger(JobLauncherExecutionDriver.class);
    JobExecutionStateListeners callbackDispatcher = new JobExecutionStateListeners(actualLog);
    JobExecutionUpdatable jobExec = JobExecutionUpdatable.createFromJobSpec(jobSpec);
    JobExecutionState jobState = new JobExecutionState(jobSpec, jobExec, Optional.<JobExecutionStateListener>of(callbackDispatcher));
    JobLauncher jobLauncher = createLauncher(sysConfig, jobSpec, actualLog, jobLauncherType.isPresent() ? Optional.of(jobLauncherType.get().toString()) : Optional.<String>absent(), instanceBroker);
    JobListenerToJobStateBridge bridge = new JobListenerToJobStateBridge(actualLog, jobState, instrumentationEnabled, launcherMetrics);
    DriverRunnable runnable = new DriverRunnable(jobLauncher, bridge, jobState, callbackDispatcher, jobExec);
    return new JobLauncherExecutionDriver(jobSpec, actualLog, runnable);
}