Examples with JobException org.apache.gobblin.runtime.JobException used on opensource projects

Example 6 with JobException

use of org.apache.gobblin.runtime.JobException in project incubator-gobblin by apache.

the class GobblinServiceJobScheduler method onDeleteSpec.

/**
 * {@inheritDoc}
 */
@Override
public void onDeleteSpec(URI deletedSpecURI, String deletedSpecVersion) {
    if (this.helixManager.isPresent() && !this.helixManager.get().isConnected()) {
        // Specs in store will be notified when Scheduler is added as listener to FlowCatalog, so ignore
        // .. Specs if in cluster mode and Helix is not yet initialized
        _log.info("System not yet initialized. Skipping Spec Deletion: " + deletedSpecURI);
        return;
    }
    _log.info("Spec deletion detected: " + deletedSpecURI + "/" + deletedSpecVersion);
    if (!isActive && helixManager.isPresent()) {
        _log.info("Scheduler running in slave mode, forward Spec delete via Helix message to master: " + deletedSpecURI);
        HelixUtils.sendUserDefinedMessage(ServiceConfigKeys.HELIX_FLOWSPEC_REMOVE, deletedSpecURI.toString() + ":" + deletedSpecVersion, UUID.randomUUID().toString(), InstanceType.CONTROLLER, helixManager.get(), _log);
        return;
    }
    try {
        Spec deletedSpec = this.scheduledFlowSpecs.get(deletedSpecURI.toString());
        if (null != deletedSpec) {
            this.orchestrator.remove(deletedSpec);
            this.scheduledFlowSpecs.remove(deletedSpecURI.toString());
            unscheduleJob(deletedSpecURI.toString());
        } else {
            _log.warn(String.format("Spec with URI: %s was not found in cache. May be it was cleaned, if not please " + "clean it manually", deletedSpecURI));
        }
    } catch (JobException e) {
        _log.warn(String.format("Spec with URI: %s was not unscheduled cleaning", deletedSpecURI), e);
    }
}

Example 7 with JobException

use of org.apache.gobblin.runtime.JobException in project incubator-gobblin by apache.

the class GobblinServiceJobScheduler method onAddSpec.

/**
 * {@inheritDoc}
 */
@Override
public void onAddSpec(Spec addedSpec) {
    if (this.helixManager.isPresent() && !this.helixManager.get().isConnected()) {
        // Specs in store will be notified when Scheduler is added as listener to FlowCatalog, so ignore
        // .. Specs if in cluster mode and Helix is not yet initialized
        _log.info("System not yet initialized. Skipping Spec Addition: " + addedSpec);
        return;
    }
    _log.info("New Flow Spec detected: " + addedSpec);
    if (addedSpec instanceof FlowSpec) {
        if (!isActive && helixManager.isPresent()) {
            _log.info("Scheduler running in slave mode, forward Spec add via Helix message to master: " + addedSpec);
            HelixUtils.sendUserDefinedMessage(ServiceConfigKeys.HELIX_FLOWSPEC_ADD, addedSpec.getUri().toString(), UUID.randomUUID().toString(), InstanceType.CONTROLLER, helixManager.get(), _log);
            return;
        }
        try {
            Properties jobConfig = new Properties();
            Properties flowSpecProperties = ((FlowSpec) addedSpec).getConfigAsProperties();
            jobConfig.putAll(this.properties);
            jobConfig.setProperty(ConfigurationKeys.JOB_NAME_KEY, addedSpec.getUri().toString());
            jobConfig.setProperty(ConfigurationKeys.JOB_GROUP_KEY, ((FlowSpec) addedSpec).getConfig().getValue(ConfigurationKeys.FLOW_GROUP_KEY).toString());
            jobConfig.setProperty(ConfigurationKeys.FLOW_RUN_IMMEDIATELY, ConfigUtils.getString(((FlowSpec) addedSpec).getConfig(), ConfigurationKeys.FLOW_RUN_IMMEDIATELY, "false"));
            if (flowSpecProperties.containsKey(ConfigurationKeys.JOB_SCHEDULE_KEY) && StringUtils.isNotBlank(flowSpecProperties.getProperty(ConfigurationKeys.JOB_SCHEDULE_KEY))) {
                jobConfig.setProperty(ConfigurationKeys.JOB_SCHEDULE_KEY, flowSpecProperties.getProperty(ConfigurationKeys.JOB_SCHEDULE_KEY));
            }
            this.scheduledFlowSpecs.put(addedSpec.getUri().toString(), addedSpec);
            if (jobConfig.containsKey(ConfigurationKeys.JOB_SCHEDULE_KEY)) {
                _log.info("Scheduling flow spec: " + addedSpec);
                scheduleJob(jobConfig, null);
                if (PropertiesUtils.getPropAsBoolean(jobConfig, ConfigurationKeys.FLOW_RUN_IMMEDIATELY, "false")) {
                    _log.info("RunImmediately requested, hence executing FlowSpec: " + addedSpec);
                    this.jobExecutor.execute(new NonScheduledJobRunner(jobConfig, null));
                }
            } else {
                _log.info("No FlowSpec schedule found, so running FlowSpec: " + addedSpec);
                this.jobExecutor.execute(new NonScheduledJobRunner(jobConfig, null));
            }
        } catch (JobException je) {
            _log.error("Failed to schedule or run FlowSpec " + addedSpec, je);
        }
    }
}

Example 8 with JobException

use of org.apache.gobblin.runtime.JobException in project incubator-gobblin by apache.

the class GobblinHelixJobLauncherTest method testLaunchMultipleJobs.

public void testLaunchMultipleJobs() throws Exception {
    final ConcurrentHashMap<String, Boolean> runningMap = new ConcurrentHashMap<>();
    // Job launcher(1) to test parallel job running
    final Properties properties1 = generateJobProperties(this.baseConfig, "2", "_1504201348471");
    final GobblinHelixJobLauncher gobblinHelixJobLauncher1 = this.closer.register(new GobblinHelixJobLauncher(properties1, this.helixManager, this.appWorkDir, ImmutableList.<Tag<?>>of(), runningMap));
    // Job launcher(2) to test parallel job running
    final Properties properties2 = generateJobProperties(this.baseConfig, "2", "_1504201348472");
    final GobblinHelixJobLauncher gobblinHelixJobLauncher2 = this.closer.register(new GobblinHelixJobLauncher(properties2, this.helixManager, this.appWorkDir, ImmutableList.<Tag<?>>of(), runningMap));
    CountDownLatch stg1 = new CountDownLatch(1);
    CountDownLatch stg2 = new CountDownLatch(1);
    CountDownLatch stg3 = new CountDownLatch(1);
    SuspendJobListener testListener = new SuspendJobListener(stg1, stg2);
    (new Thread(() -> {
        try {
            gobblinHelixJobLauncher1.launchJob(testListener);
            stg3.countDown();
        } catch (JobException e) {
        }
    })).start();
    // Wait for the first job to start
    stg1.await();
    // When first job is in the middle of running, launch the second job (which should do NOOP because previous job is still running)
    gobblinHelixJobLauncher2.launchJob(testListener);
    stg2.countDown();
    // Wait for the first job to finish
    stg3.await();
    Assert.assertEquals(testListener.getCompletes().get() == 1, true);
}

Example 9 with JobException

use of org.apache.gobblin.runtime.JobException in project incubator-gobblin by apache.

the class JobScheduler method runJob.

/**
 * Run a job.
 *
 * <p>
 *   This method runs the job immediately without going through the Quartz scheduler.
 *   This is particularly useful for testing.
 * </p>
 *
 * <p>
 *   This method does what {@link #runJob(Properties, JobListener)} does, and additionally it allows
 *   the caller to pass in a {@link JobLauncher} instance used to launch the job to run.
 * </p>
 *
 * @param jobProps Job configuration properties
 * @param jobListener {@link JobListener} used for callback, can be <em>null</em> if no callback is needed.
 * @param jobLauncher a {@link JobLauncher} object used to launch the job to run
 * @return If current job is a stop-early job based on {@link Source#isEarlyStopped()}
 * @throws JobException when there is anything wrong with running the job
 */
public boolean runJob(Properties jobProps, JobListener jobListener, JobLauncher jobLauncher) throws JobException {
    Preconditions.checkArgument(jobProps.containsKey(ConfigurationKeys.JOB_NAME_KEY), "A job must have a job name specified by job.name");
    String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY);
    // Check if the job has been disabled
    boolean disabled = Boolean.valueOf(jobProps.getProperty(ConfigurationKeys.JOB_DISABLED_KEY, "false"));
    if (disabled) {
        LOG.info("Skipping disabled job " + jobName);
        return false;
    }
    // Launch the job
    try (Closer closer = Closer.create()) {
        closer.register(jobLauncher).launchJob(jobListener);
        boolean runOnce = Boolean.valueOf(jobProps.getProperty(ConfigurationKeys.JOB_RUN_ONCE_KEY, "false"));
        boolean isEarlyStopped = jobLauncher.isEarlyStopped();
        if (!isEarlyStopped && runOnce && this.scheduledJobs.containsKey(jobName)) {
            this.scheduler.getScheduler().deleteJob(this.scheduledJobs.remove(jobName));
        }
        return isEarlyStopped;
    } catch (Throwable t) {
        throw new JobException("Failed to launch and run job " + jobName, t);
    }
}

Example 10 with JobException

use of org.apache.gobblin.runtime.JobException in project incubator-gobblin by apache.

the class JobScheduler method scheduleJob.

/**
 * Schedule a job.
 *
 * <p>
 *   This method does what {@link #scheduleJob(Properties, JobListener)} does, and additionally it
 *   allows the caller to pass in additional job data and the {@link Job} implementation class.
 * </p>
 *
 * @param jobProps Job configuration properties
 * @param jobListener {@link JobListener} used for callback,
 *                    can be <em>null</em> if no callback is needed.
 * @param additionalJobData additional job data in a {@link Map}
 * @param jobClass Quartz job class
 * @throws JobException when there is anything wrong
 *                      with scheduling the job
 */
public void scheduleJob(Properties jobProps, JobListener jobListener, Map<String, Object> additionalJobData, Class<? extends Job> jobClass) throws JobException {
    Preconditions.checkArgument(jobProps.containsKey(ConfigurationKeys.JOB_NAME_KEY), "A job must have a job name specified by job.name");
    String jobName = jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY);
    if (this.scheduledJobs.containsKey(jobName)) {
        LOG.warn("Job " + jobName + " has already been scheduled");
        return;
    }
    // Check if the job has been disabled
    boolean disabled = Boolean.valueOf(jobProps.getProperty(ConfigurationKeys.JOB_DISABLED_KEY, "false"));
    if (disabled) {
        LOG.info("Skipping disabled job " + jobName);
        return;
    }
    if (!jobProps.containsKey(ConfigurationKeys.JOB_SCHEDULE_KEY)) {
        // Submit the job to run
        this.jobExecutor.execute(new NonScheduledJobRunner(jobProps, jobListener));
        return;
    }
    if (jobListener != null) {
        this.jobListenerMap.put(jobName, jobListener);
    }
    // Build a data map that gets passed to the job
    JobDataMap jobDataMap = new JobDataMap();
    jobDataMap.put(JOB_SCHEDULER_KEY, this);
    jobDataMap.put(PROPERTIES_KEY, jobProps);
    jobDataMap.put(JOB_LISTENER_KEY, jobListener);
    jobDataMap.putAll(additionalJobData);
    // Build a Quartz job
    JobDetail job = JobBuilder.newJob(jobClass).withIdentity(jobName, Strings.nullToEmpty(jobProps.getProperty(ConfigurationKeys.JOB_GROUP_KEY))).withDescription(Strings.nullToEmpty(jobProps.getProperty(ConfigurationKeys.JOB_DESCRIPTION_KEY))).usingJobData(jobDataMap).build();
    try {
        // Schedule the Quartz job with a trigger built from the job configuration
        Trigger trigger = getTrigger(job.getKey(), jobProps);
        this.scheduler.getScheduler().scheduleJob(job, trigger);
        LOG.info(String.format("Scheduled job %s. Next run: %s.", job.getKey(), trigger.getNextFireTime()));
    } catch (SchedulerException se) {
        LOG.error("Failed to schedule job " + jobName, se);
        throw new JobException("Failed to schedule job " + jobName, se);
    }
    this.scheduledJobs.put(jobName, job.getKey());
}