Examples with SubmitXCommand org.apache.oozie.command.wf.SubmitXCommand used on opensource projects

Example 1 with SubmitXCommand

use of org.apache.oozie.command.wf.SubmitXCommand in project oozie by apache.

the class DagEngine method submitJob.

/**
 * Submit a workflow job. <p> It validates configuration properties.
 *
 * @param conf job configuration.
 * @param startJob indicates if the job should be started or not.
 * @return the job Id.
 * @throws DagEngineException thrown if the job could not be created.
 */
@Override
public String submitJob(Configuration conf, boolean startJob) throws DagEngineException {
    validateSubmitConfiguration(conf);
    try {
        String jobId;
        SubmitXCommand submit = new SubmitXCommand(conf);
        jobId = submit.call();
        if (startJob) {
            start(jobId);
        }
        return jobId;
    } catch (CommandException ex) {
        throw new DagEngineException(ex);
    }
}

Example 2 with SubmitXCommand

use of org.apache.oozie.command.wf.SubmitXCommand in project oozie by apache.

the class TestSLAEventGeneration method testWorkflowJobSLARerun.

/**
 * Test for SLA Events generated through wf rerun
 *
 * @throws Exception
 */
@Test
public void testWorkflowJobSLARerun() throws Exception {
    SLAService slas = services.get(SLAService.class);
    String wfXml = IOUtils.getResourceAsString("wf-job-sla.xml", -1);
    Path appPath = getFsTestCaseDir();
    writeToFile(wfXml, appPath, "workflow.xml");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, appPath.toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    cal.setTime(new Date());
    // for start_miss
    cal.add(Calendar.MINUTE, -40);
    Date nominal = cal.getTime();
    String nominalTime = DateUtils.formatDateOozieTZ(nominal);
    conf.set("nominal_time", nominalTime);
    cal.setTime(nominal);
    // as per the sla xml
    cal.add(Calendar.MINUTE, 10);
    String expectedStart = DateUtils.formatDateOozieTZ(cal.getTime());
    cal.setTime(nominal);
    // as per the sla xml
    cal.add(Calendar.MINUTE, 30);
    String expectedEnd = DateUtils.formatDateOozieTZ(cal.getTime());
    // Call SubmitX
    SubmitXCommand sc = new SubmitXCommand(conf);
    String jobId = sc.call();
    SLACalcStatus slaEvent = slas.getSLACalculator().get(jobId);
    assertEquals(jobId, slaEvent.getId());
    assertEquals("test-wf-job-sla", slaEvent.getAppName());
    assertEquals(AppType.WORKFLOW_JOB, slaEvent.getAppType());
    assertEquals(nominalTime, DateUtils.formatDateOozieTZ(slaEvent.getNominalTime()));
    assertEquals(expectedStart, DateUtils.formatDateOozieTZ(slaEvent.getExpectedStart()));
    assertEquals(expectedEnd, DateUtils.formatDateOozieTZ(slaEvent.getExpectedEnd()));
    slas.runSLAWorker();
    slaEvent = (SLACalcStatus) ehs.getEventQueue().poll();
    assertEquals(SLAStatus.NOT_STARTED, slaEvent.getSLAStatus());
    assertEquals(EventStatus.START_MISS, slaEvent.getEventStatus());
    slas.getSLACalculator().clear();
    JPAService jpaService = Services.get().get(JPAService.class);
    WorkflowJobBean wfBean = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
    // set job status to succeeded, so rerun doesn't fail
    wfBean.setStatus(WorkflowJob.Status.SUCCEEDED);
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_MODTIME, wfBean);
    // change conf for rerun
    cal.setTime(new Date());
    // for start_miss
    cal.add(Calendar.MINUTE, -20);
    nominalTime = DateUtils.formatDateOozieTZ(cal.getTime());
    conf.set("nominal_time", nominalTime);
    nominal = cal.getTime();
    // as per the sla xml
    cal.add(Calendar.MINUTE, 10);
    expectedStart = DateUtils.formatDateOozieTZ(cal.getTime());
    cal.setTime(nominal);
    // as per the sla xml
    cal.add(Calendar.MINUTE, 30);
    expectedEnd = DateUtils.formatDateOozieTZ(cal.getTime());
    ReRunXCommand rerun = new ReRunXCommand(jobId, conf);
    rerun.call();
    slaEvent = slas.getSLACalculator().get(jobId);
    // assert for new conf
    assertNotNull(slaEvent);
    assertEquals(jobId, slaEvent.getId());
    assertEquals("test-wf-job-sla", slaEvent.getAppName());
    assertEquals(AppType.WORKFLOW_JOB, slaEvent.getAppType());
    // assert for new conf
    assertEquals(nominalTime, DateUtils.formatDateOozieTZ(slaEvent.getNominalTime()));
    assertEquals(expectedStart, DateUtils.formatDateOozieTZ(slaEvent.getExpectedStart()));
    assertEquals(expectedEnd, DateUtils.formatDateOozieTZ(slaEvent.getExpectedEnd()));
    // assert for values in summary bean to be reset
    SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
    assertEquals(0, slaSummary.getEventProcessed());
    assertEquals(-1, slaSummary.getActualDuration());
    assertNull(slaSummary.getActualStart());
    assertNull(slaSummary.getActualEnd());
    assertEquals("PREP", slaSummary.getJobStatus());
    assertEquals(SLAStatus.NOT_STARTED, slaSummary.getSLAStatus());
    assertNull(slaEvent.getEventStatus());
    ehs.getEventQueue().clear();
    slas.runSLAWorker();
    slaEvent = (SLACalcStatus) ehs.getEventQueue().poll();
    assertEquals(SLAStatus.IN_PROCESS, slaEvent.getSLAStatus());
    assertEquals(EventStatus.START_MISS, slaEvent.getEventStatus());
}

Example 3 with SubmitXCommand

use of org.apache.oozie.command.wf.SubmitXCommand in project oozie by apache.

the class TestSLAEventGeneration method testWorkflowActionSLARerun.

/**
 * Test for SLA Events generated through wf action rerun
 *
 * @throws Exception
 */
@Test
public void testWorkflowActionSLARerun() throws Exception {
    SLAService slas = services.get(SLAService.class);
    String wfXml = IOUtils.getResourceAsString("wf-action-sla.xml", -1);
    Path appPath = getFsTestCaseDir();
    writeToFile(wfXml, appPath, "workflow.xml");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, appPath.toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    cal.setTime(new Date());
    // for start_miss
    cal.add(Calendar.MINUTE, -20);
    Date nominal = cal.getTime();
    String nominalTime = DateUtils.formatDateOozieTZ(nominal);
    conf.set("nominal_time", nominalTime);
    // Call SubmitX
    SubmitXCommand sc = new SubmitXCommand(conf);
    String jobId = sc.call();
    String actionId = jobId + "@grouper";
    slas.getSLACalculator().clear();
    JPAService jpaService = Services.get().get(JPAService.class);
    WorkflowJobBean wfBean = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
    // set job status to succeeded, so rerun doesn't fail
    wfBean.setStatus(WorkflowJob.Status.SUCCEEDED);
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_MODTIME, wfBean);
    // change conf for rerun
    cal.setTime(new Date());
    nominalTime = DateUtils.formatDateOozieTZ(cal.getTime());
    conf.set("nominal_time", nominalTime);
    nominal = cal.getTime();
    // as per the sla xml
    cal.add(Calendar.MINUTE, 10);
    String expectedStart = DateUtils.formatDateOozieTZ(cal.getTime());
    cal.setTime(nominal);
    // as per the sla xml
    cal.add(Calendar.MINUTE, 30);
    String expectedEnd = DateUtils.formatDateOozieTZ(cal.getTime());
    ReRunXCommand rerun = new ReRunXCommand(jobId, conf);
    rerun.call();
    SLACalcStatus slaEvent = slas.getSLACalculator().get(actionId);
    assertNotNull(slaEvent);
    // assert for action configs
    assertEquals(actionId, slaEvent.getId());
    assertEquals("test-wf-action-sla", slaEvent.getAppName());
    assertEquals(AppType.WORKFLOW_ACTION, slaEvent.getAppType());
    // assert for new conf
    assertEquals(nominalTime, DateUtils.formatDateOozieTZ(slaEvent.getNominalTime()));
    assertEquals(expectedStart, DateUtils.formatDateOozieTZ(slaEvent.getExpectedStart()));
    assertEquals(expectedEnd, DateUtils.formatDateOozieTZ(slaEvent.getExpectedEnd()));
}

Example 4 with SubmitXCommand

use of org.apache.oozie.command.wf.SubmitXCommand in project oozie by apache.

the class TestMapReduceActionExecutor method testConfigDefaultPropsToAction.

public void testConfigDefaultPropsToAction() throws Exception {
    String actionXml = "<map-reduce>" + "        <prepare>" + "          <delete path=\"${nameNode}/user/${wf:user()}/mr/${outputDir}\"/>" + "        </prepare>" + "        <configuration>" + "          <property><name>bb</name><value>BB</value></property>" + "          <property><name>cc</name><value>from_action</value></property>" + "        </configuration>" + "      </map-reduce>";
    String wfXml = "<workflow-app xmlns=\"uri:oozie:workflow:0.5\" name=\"map-reduce-wf\">" + "<global>" + "<job-tracker>${jobTracker}</job-tracker>" + "<name-node>${nameNode}</name-node>" + "<configuration><property><name>aa</name><value>AA</value></property></configuration>" + "</global>" + "    <start to=\"mr-node\"/>" + "    <action name=\"mr-node\">" + actionXml + "    <ok to=\"end\"/>" + "    <error to=\"fail\"/>" + "</action>" + "<kill name=\"fail\">" + "    <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>" + "</kill>" + "<end name=\"end\"/>" + "</workflow-app>";
    Writer writer = new FileWriter(getTestCaseDir() + "/workflow.xml");
    IOUtils.copyCharStream(new StringReader(wfXml), writer);
    Configuration conf = new XConfiguration();
    conf.set("nameNode", getNameNodeUri());
    conf.set("jobTracker", getJobTrackerUri());
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set(OozieClient.APP_PATH, new File(getTestCaseDir(), "workflow.xml").toURI().toString());
    conf.set(OozieClient.LOG_TOKEN, "t");
    OutputStream os = new FileOutputStream(getTestCaseDir() + "/config-default.xml");
    XConfiguration defaultConf = new XConfiguration();
    defaultConf.set("outputDir", "default-output-dir");
    defaultConf.set("mapred.mapper.class", "MM");
    defaultConf.set("mapred.reducer.class", "RR");
    defaultConf.set("cc", "from_default");
    defaultConf.writeXml(os);
    os.close();
    String wfId = new SubmitXCommand(conf).call();
    new StartXCommand(wfId).call();
    waitForWorkflowAction(wfId + "@mr-node");
    WorkflowActionBean mrAction = WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQuery.GET_ACTION, wfId + "@mr-node");
    // check NN and JT settings
    Element eAction = XmlUtils.parseXml(mrAction.getConf());
    Element eConf = eAction.getChild("name-node", eAction.getNamespace());
    assertEquals(getNameNodeUri(), eConf.getText());
    eConf = eAction.getChild("job-tracker", eAction.getNamespace());
    assertEquals(getJobTrackerUri(), eConf.getText());
    // check other m-r settings
    eConf = eAction.getChild("configuration", eAction.getNamespace());
    Configuration actionConf = new XConfiguration(new StringReader(XmlUtils.prettyPrint(eConf).toString()));
    assertEquals("default-output-dir", actionConf.get("outputDir"));
    assertEquals("MM", actionConf.get("mapred.mapper.class"));
    assertEquals("RR", actionConf.get("mapred.reducer.class"));
    // check that default did not overwrite same property explicit in action conf
    assertEquals("from_action", actionConf.get("cc"));
    // check that original conf and from global was not deleted
    assertEquals("AA", actionConf.get("aa"));
    assertEquals("BB", actionConf.get("bb"));
    // test no infinite recursion by param referring to itself e.g. path = ${path}/sub-path
    actionXml = "<map-reduce>" + "        <prepare>" + "          <delete path=\"${nameNode}/user/${wf:user()}/mr/${outputDir}\"/>" + "        </prepare>" + "        <configuration>" + "          <property><name>cc</name><value>${cc}/action_cc</value></property>" + "        </configuration>" + "      </map-reduce>";
    wfXml = "<workflow-app xmlns=\"uri:oozie:workflow:0.5\" name=\"map-reduce-wf\">" + "<global>" + "<job-tracker>${jobTracker}</job-tracker>" + "<name-node>${nameNode}</name-node>" + "<configuration><property><name>outputDir</name><value>global-output-dir</value></property></configuration>" + "</global>" + "    <start to=\"mr-node\"/>" + "    <action name=\"mr-node\">" + actionXml + "    <ok to=\"end\"/>" + "    <error to=\"fail\"/>" + "</action>" + "<kill name=\"fail\">" + "    <message>Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>" + "</kill>" + "<end name=\"end\"/>" + "</workflow-app>";
    writer = new FileWriter(getTestCaseDir() + "/workflow.xml");
    IOUtils.copyCharStream(new StringReader(wfXml), writer);
    wfId = new SubmitXCommand(conf).call();
    new StartXCommand(wfId).call();
    waitForWorkflowAction(wfId + "@mr-node");
    mrAction = WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQuery.GET_ACTION, wfId + "@mr-node");
    // check param
    eAction = XmlUtils.parseXml(mrAction.getConf());
    eConf = eAction.getChild("configuration", eAction.getNamespace());
    actionConf = new XConfiguration(new StringReader(XmlUtils.prettyPrint(eConf).toString()));
    // action param referring to same param name given in defaults cc = ${cc}/action_cc
    assertEquals("from_default/action_cc", actionConf.get("cc"));
    // check global is retained and has precedence over config-default
    eConf = eAction.getChild("name-node", eAction.getNamespace());
    assertEquals(getNameNodeUri(), eConf.getText());
    assertEquals("global-output-dir", actionConf.get("outputDir"));
}

Example 5 with SubmitXCommand

use of org.apache.oozie.command.wf.SubmitXCommand in project oozie by apache.

the class DagEngine method submitJobFromCoordinator.

/**
 * Submit a workflow through a coordinator. It validates configuration properties.
 * @param conf job conf
 * @param parentId parent of workflow
 * @return jobId returns jobId of the launched workflow
 * @throws DagEngineException if the job can't be submitted
 */
public String submitJobFromCoordinator(Configuration conf, String parentId) throws DagEngineException {
    validateSubmitConfiguration(conf);
    try {
        String jobId;
        SubmitXCommand submit = new SubmitXCommand(conf, parentId);
        jobId = submit.call();
        start(jobId);
        return jobId;
    } catch (CommandException ex) {
        throw new DagEngineException(ex);
    }
}