Examples with JobState org.apache.gobblin.runtime.JobState used on opensource projects

Example 1 with JobState

use of org.apache.gobblin.runtime.JobState in project incubator-gobblin by apache.

the class ClusterEventMetadataGeneratorTest method testErrorMessage.

@Test
public void testErrorMessage() throws Exception {
    JobContext jobContext = Mockito.mock(JobContext.class);
    JobState jobState = new JobState("jobName", "1234");
    TaskState taskState1 = new TaskState();
    TaskState taskState2 = new TaskState();
    taskState1.setTaskId("1");
    taskState1.setProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY, "exception1");
    taskState2.setTaskId("2");
    taskState2.setProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY, "exception2");
    taskState2.setProp(EventMetadataUtils.TASK_FAILURE_MESSAGE_KEY, "failureMessage2");
    jobState.addTaskState(taskState1);
    jobState.addTaskState(taskState2);
    Mockito.when(jobContext.getJobState()).thenReturn(jobState);
    ClusterEventMetadataGenerator metadataGenerator = new ClusterEventMetadataGenerator();
    Map<String, String> metadata;
    // error message is not in job commit event
    metadata = metadataGenerator.getMetadata(jobContext, EventName.JOB_COMMIT);
    Assert.assertEquals(metadata.get("message"), null);
    // error message is in job failed event
    metadata = metadataGenerator.getMetadata(jobContext, EventName.JOB_FAILED);
    Assert.assertTrue(metadata.get("message").startsWith("failureMessage"));
    Assert.assertTrue(metadata.get("message").contains("exception1"));
    Assert.assertTrue(metadata.get("message").contains("exception2"));
}

Example 2 with JobState

use of org.apache.gobblin.runtime.JobState in project incubator-gobblin by apache.

the class GobblinHelixTaskTest method testPrepareTask.

@Test
public void testPrepareTask() throws IOException {
    // Serialize the JobState that will be read later in GobblinHelixTask
    Path jobStateFilePath = new Path(appWorkDir, TestHelper.TEST_JOB_ID + "." + AbstractJobLauncher.JOB_STATE_FILE_NAME);
    JobState jobState = new JobState();
    jobState.setJobName(TestHelper.TEST_JOB_NAME);
    jobState.setJobId(TestHelper.TEST_JOB_ID);
    SerializationUtils.serializeState(this.localFs, jobStateFilePath, jobState);
    // Prepare the WorkUnit
    WorkUnit workUnit = WorkUnit.createEmpty();
    prepareWorkUnit(workUnit);
    // Prepare the source Json file
    File sourceJsonFile = new File(this.appWorkDir.toString(), TestHelper.TEST_JOB_NAME + ".json");
    TestHelper.createSourceJsonFile(sourceJsonFile);
    workUnit.setProp(SimpleJsonSource.SOURCE_FILE_KEY, sourceJsonFile.getAbsolutePath());
    // Serialize the WorkUnit into a file
    // expected path is appWorkDir/_workunits/job_id/job_id.wu
    Path workUnitDirPath = new Path(this.appWorkDir, GobblinClusterConfigurationKeys.INPUT_WORK_UNIT_DIR_NAME);
    FsStateStore<WorkUnit> wuStateStore = new FsStateStore<>(this.localFs, workUnitDirPath.toString(), WorkUnit.class);
    Path workUnitFilePath = new Path(new Path(workUnitDirPath, TestHelper.TEST_JOB_ID), TestHelper.TEST_JOB_NAME + ".wu");
    wuStateStore.put(TestHelper.TEST_JOB_ID, TestHelper.TEST_JOB_NAME + ".wu", workUnit);
    Assert.assertTrue(this.localFs.exists(workUnitFilePath));
    // Prepare the GobblinHelixTask
    Map<String, String> taskConfigMap = Maps.newHashMap();
    taskConfigMap.put(GobblinClusterConfigurationKeys.WORK_UNIT_FILE_PATH, workUnitFilePath.toString());
    taskConfigMap.put(ConfigurationKeys.JOB_NAME_KEY, TestHelper.TEST_JOB_NAME);
    taskConfigMap.put(ConfigurationKeys.JOB_ID_KEY, TestHelper.TEST_JOB_ID);
    taskConfigMap.put(ConfigurationKeys.TASK_KEY_KEY, Long.toString(Id.parse(TestHelper.TEST_JOB_ID).getSequence()));
    TaskConfig taskConfig = new TaskConfig("", taskConfigMap, true);
    TaskCallbackContext taskCallbackContext = Mockito.mock(TaskCallbackContext.class);
    Mockito.when(taskCallbackContext.getTaskConfig()).thenReturn(taskConfig);
    Mockito.when(taskCallbackContext.getManager()).thenReturn(this.helixManager);
    GobblinHelixTaskFactory gobblinHelixTaskFactory = new GobblinHelixTaskFactory(Optional.<ContainerMetrics>absent(), this.taskExecutor, this.taskStateTracker, this.localFs, this.appWorkDir, ConfigFactory.empty(), this.helixManager);
    this.gobblinHelixTask = (GobblinHelixTask) gobblinHelixTaskFactory.createNewTask(taskCallbackContext);
}

Example 3 with JobState

use of org.apache.gobblin.runtime.JobState in project incubator-gobblin by apache.

the class SingleTask method getJobState.

private JobState getJobState() throws java.io.IOException {
    JobState jobState;
    // read the state from the state store if present, otherwise deserialize directly from the file
    if (_stateStores.haveJobStateStore()) {
        jobState = _stateStores.getJobStateStore().get(_jobStateFilePath.getParent().getName(), _jobStateFilePath.getName(), _jobStateFilePath.getParent().getName());
    } else {
        jobState = new JobState();
        SerializationUtils.deserializeState(_fs, _jobStateFilePath, jobState);
    }
    return jobState;
}

Example 4 with JobState

use of org.apache.gobblin.runtime.JobState in project incubator-gobblin by apache.

the class MRJobLauncher method runWorkUnits.

@Override
protected void runWorkUnits(List<WorkUnit> workUnits) throws Exception {
    String jobName = this.jobContext.getJobName();
    JobState jobState = this.jobContext.getJobState();
    try {
        prepareHadoopJob(workUnits);
        // Start the output TaskState collector service
        this.taskStateCollectorService.startAsync().awaitRunning();
        LOG.info("Launching Hadoop MR job " + this.job.getJobName());
        this.job.submit();
        this.hadoopJobSubmitted = true;
        // Set job tracking URL to the Hadoop job tracking URL if it is not set yet
        if (!jobState.contains(ConfigurationKeys.JOB_TRACKING_URL_KEY)) {
            jobState.setProp(ConfigurationKeys.JOB_TRACKING_URL_KEY, this.job.getTrackingURL());
        }
        TimingEvent mrJobRunTimer = this.eventSubmitter.getTimingEvent(TimingEvent.RunJobTimings.MR_JOB_RUN);
        LOG.info(String.format("Waiting for Hadoop MR job %s to complete", this.job.getJobID()));
        this.job.waitForCompletion(true);
        mrJobRunTimer.stop(ImmutableMap.of("hadoopMRJobId", this.job.getJobID().toString()));
        if (this.cancellationRequested) {
            // Wait for the cancellation execution if it has been requested
            synchronized (this.cancellationExecution) {
                if (this.cancellationExecuted) {
                    return;
                }
            }
        }
        // Create a metrics set for this job run from the Hadoop counters.
        // The metrics set is to be persisted to the metrics store later.
        countersToMetrics(JobMetrics.get(jobName, this.jobProps.getProperty(ConfigurationKeys.JOB_ID_KEY)));
    } finally {
        // The last iteration of output TaskState collecting will run when the collector service gets stopped
        this.taskStateCollectorService.stopAsync().awaitTerminated();
        cleanUpWorkingDirectory();
    }
}

Example 5 with JobState

use of org.apache.gobblin.runtime.JobState in project incubator-gobblin by apache.

the class CompactionSource method initJobDir.

/**
 * Create a temporary job directory based on job id or (if not available) UUID
 */
private void initJobDir(SourceState state) throws IOException {
    String tmpBase = state.getProp(MRCompactor.COMPACTION_TMP_DEST_DIR, MRCompactor.DEFAULT_COMPACTION_TMP_DEST_DIR);
    String jobId;
    if (state instanceof JobState) {
        jobId = ((JobState) state).getJobId();
    } else {
        jobId = UUID.randomUUID().toString();
    }
    this.tmpJobDir = new Path(tmpBase, jobId);
    this.fs.mkdirs(this.tmpJobDir);
    state.setProp(MRCompactor.COMPACTION_JOB_DIR, tmpJobDir.toString());
    log.info("Job dir is created under {}", this.tmpJobDir);
}