Examples with OozieClient org.apache.oozie.client.OozieClient used on opensource projects

Example 1 with OozieClient

use of org.apache.oozie.client.OozieClient in project oozie by apache.

the class SubWorkflowActionExecutor method getWorkflowClient.

protected OozieClient getWorkflowClient(Context context, String oozieUri) {
    OozieClient oozieClient;
    if (oozieUri.equals(LOCAL)) {
        WorkflowJobBean workflow = (WorkflowJobBean) context.getWorkflow();
        String user = workflow.getUser();
        String group = workflow.getGroup();
        DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user);
        oozieClient = new LocalOozieClient(dagEngine);
    } else {
        // TODO we need to add authToken to the WC for the remote case
        oozieClient = new OozieClient(oozieUri);
    }
    return oozieClient;
}

Example 2 with OozieClient

use of org.apache.oozie.client.OozieClient in project oozie by apache.

the class SubWorkflowActionExecutor method start.

public void start(Context context, WorkflowAction action) throws ActionExecutorException {
    LOG.info("Starting action");
    try {
        Element eConf = XmlUtils.parseXml(action.getConf());
        Namespace ns = eConf.getNamespace();
        Element e = eConf.getChild("oozie", ns);
        String oozieUri = (e == null) ? LOCAL : e.getTextTrim();
        OozieClient oozieClient = getWorkflowClient(context, oozieUri);
        String subWorkflowId = null;
        String extId = context.getRecoveryId();
        String runningJobId = null;
        if (extId != null) {
            runningJobId = checkIfRunning(oozieClient, extId);
        }
        if (runningJobId == null) {
            String appPath = eConf.getChild("app-path", ns).getTextTrim();
            XConfiguration subWorkflowConf = new XConfiguration();
            Configuration parentConf = new XConfiguration(new StringReader(context.getWorkflow().getConf()));
            if (eConf.getChild(("propagate-configuration"), ns) != null) {
                XConfiguration.copy(parentConf, subWorkflowConf);
            }
            // Propagate coordinator and bundle info to subworkflow
            if (OozieJobInfo.isJobInfoEnabled()) {
                if (parentConf.get(OozieJobInfo.COORD_ID) != null) {
                    subWorkflowConf.set(OozieJobInfo.COORD_ID, parentConf.get(OozieJobInfo.COORD_ID));
                    subWorkflowConf.set(OozieJobInfo.COORD_NAME, parentConf.get(OozieJobInfo.COORD_NAME));
                    subWorkflowConf.set(OozieJobInfo.COORD_NOMINAL_TIME, parentConf.get(OozieJobInfo.COORD_NOMINAL_TIME));
                }
                if (parentConf.get(OozieJobInfo.BUNDLE_ID) != null) {
                    subWorkflowConf.set(OozieJobInfo.BUNDLE_ID, parentConf.get(OozieJobInfo.BUNDLE_ID));
                    subWorkflowConf.set(OozieJobInfo.BUNDLE_NAME, parentConf.get(OozieJobInfo.BUNDLE_NAME));
                }
            }
            // the proto has the necessary credentials
            Configuration protoActionConf = context.getProtoActionConf();
            XConfiguration.copy(protoActionConf, subWorkflowConf);
            subWorkflowConf.set(OozieClient.APP_PATH, appPath);
            String group = ConfigUtils.getWithDeprecatedCheck(parentConf, OozieClient.JOB_ACL, OozieClient.GROUP_NAME, null);
            if (group != null) {
                subWorkflowConf.set(OozieClient.GROUP_NAME, group);
            }
            injectInline(eConf.getChild("configuration", ns), subWorkflowConf);
            injectCallback(context, subWorkflowConf);
            injectRecovery(extId, subWorkflowConf);
            injectParent(context.getWorkflow().getId(), subWorkflowConf);
            injectSuperParent(context.getWorkflow(), parentConf, subWorkflowConf);
            verifyAndInjectSubworkflowDepth(parentConf, subWorkflowConf);
            // TODO: this has to be refactored later to be done in a single place for REST calls and this
            JobUtils.normalizeAppPath(context.getWorkflow().getUser(), context.getWorkflow().getGroup(), subWorkflowConf);
            subWorkflowConf.set(OOZIE_ACTION_YARN_TAG, getActionYarnTag(parentConf, context.getWorkflow(), action));
            // rerun again.
            if (action.getExternalId() != null && parentConf.getBoolean(OozieClient.RERUN_FAIL_NODES, false)) {
                subWorkflowConf.setBoolean(SUBWORKFLOW_RERUN, true);
                oozieClient.reRun(action.getExternalId(), subWorkflowConf.toProperties());
                subWorkflowId = action.getExternalId();
            } else {
                subWorkflowId = oozieClient.run(subWorkflowConf.toProperties());
            }
        } else {
            subWorkflowId = runningJobId;
        }
        LOG.info("Sub workflow id: [{0}]", subWorkflowId);
        WorkflowJob workflow = oozieClient.getJobInfo(subWorkflowId);
        String consoleUrl = workflow.getConsoleUrl();
        context.setStartData(subWorkflowId, oozieUri, consoleUrl);
        if (runningJobId != null) {
            check(context, action);
        }
    } catch (Exception ex) {
        LOG.error(ex);
        throw convertException(ex);
    }
}

Example 3 with OozieClient

use of org.apache.oozie.client.OozieClient in project oozie by apache.

the class SubWorkflowActionExecutor method check.

public void check(Context context, WorkflowAction action) throws ActionExecutorException {
    try {
        String subWorkflowId = action.getExternalId();
        String oozieUri = action.getTrackerUri();
        OozieClient oozieClient = getWorkflowClient(context, oozieUri);
        WorkflowJob subWorkflow = oozieClient.getJobInfo(subWorkflowId);
        WorkflowJob.Status status = subWorkflow.getStatus();
        switch(status) {
            case FAILED:
            case KILLED:
            case SUCCEEDED:
                context.setExecutionData(status.toString(), null);
                break;
            default:
                context.setExternalStatus(status.toString());
                break;
        }
    } catch (Exception ex) {
        throw convertException(ex);
    }
}

Example 4 with OozieClient

use of org.apache.oozie.client.OozieClient in project oozie by apache.

the class TestCoordUpdateXCommand method testReRunRefresh.

// test coord re-run with refresh. will use the updated coord definition.
public void testReRunRefresh() throws Exception {
    Configuration conf = new XConfiguration();
    File appPathFile1 = new File(getTestCaseDir(), "coordinator.xml");
    String jobId = setupCoord(conf, "coord-multiple-input-instance3.xml");
    sleep(1000);
    final int actionNum = 1;
    final String actionId = jobId + "@" + actionNum;
    final OozieClient coordClient = LocalOozie.getCoordClient();
    waitFor(120 * 1000, new Predicate() {

        @Override
        public boolean evaluate() throws Exception {
            CoordinatorAction bean = coordClient.getCoordActionInfo(actionId);
            return (bean.getStatus() == CoordinatorAction.Status.WAITING || bean.getStatus() == CoordinatorAction.Status.SUBMITTED);
        }
    });
    CoordinatorAction bean = coordClient.getCoordActionInfo(actionId);
    assertEquals(bean.getMissingDependencies(), "!!${coord:latest(0)}#${coord:latest(-1)}");
    CoordinatorJobBean job = getCoordJobs(jobId);
    Reader reader = IOUtils.getResourceAsReader("coord-multiple-input-instance4.xml", -1);
    Writer writer = new FileWriter(appPathFile1);
    IOUtils.copyCharStream(reader, writer);
    conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile1.toURI().toString());
    new CoordUpdateXCommand(false, conf, jobId).call();
    job = getCoordJobs(jobId);
    Element processedJobXml = XmlUtils.parseXml(job.getJobXml());
    Namespace namespace = processedJobXml.getNamespace();
    String text = ((Element) processedJobXml.getChild("input-events", namespace).getChild("data-in", namespace).getChildren("instance", namespace).get(0)).getText();
    assertEquals(text, "${coord:future(0, 1)}");
    new CoordActionsKillXCommand(jobId, RestConstants.JOB_COORD_SCOPE_ACTION, Integer.toString(actionNum)).call();
    coordClient.reRunCoord(jobId, RestConstants.JOB_COORD_SCOPE_ACTION, Integer.toString(actionNum), true, true);
    bean = coordClient.getCoordActionInfo(actionId);
    sleep(1000);
    assertEquals(bean.getMissingDependencies(), "!!${coord:future(0, 1)}");
}

Example 5 with OozieClient

use of org.apache.oozie.client.OozieClient in project oozie by apache.

the class TestSubWorkflowActionExecutor method testConfigNotPropagation.

public void testConfigNotPropagation() throws Exception {
    Path subWorkflowAppPath = getFsTestCaseDir();
    FileSystem fs = getFileSystem();
    Path workflowPath = new Path(subWorkflowAppPath, "workflow.xml");
    Writer writer = new OutputStreamWriter(fs.create(workflowPath));
    writer.write(APP1);
    writer.close();
    XConfiguration protoConf = getBaseProtoConf();
    WorkflowJobBean workflow = createBaseWorkflow(protoConf, "W");
    String defaultConf = workflow.getConf();
    XConfiguration newConf = new XConfiguration(new StringReader(defaultConf));
    newConf.set("abc", "xyz");
    workflow.setConf(newConf.toXmlString());
    final WorkflowActionBean action = (WorkflowActionBean) workflow.getActions().get(0);
    action.setConf("<sub-workflow xmlns='uri:oozie:workflow:0.1' name='subwf'>" + "      <app-path>" + workflowPath.toString() + "</app-path>" + "      <configuration>" + "        <property>" + "          <name>a</name>" + "          <value>A</value>" + "        </property>" + "      </configuration>" + "</sub-workflow>");
    SubWorkflowActionExecutor subWorkflow = new SubWorkflowActionExecutor();
    subWorkflow.start(new Context(workflow, action), action);
    final OozieClient oozieClient = subWorkflow.getWorkflowClient(new Context(workflow, action), SubWorkflowActionExecutor.LOCAL);
    waitFor(JOB_TIMEOUT, new Predicate() {

        public boolean evaluate() throws Exception {
            return oozieClient.getJobInfo(action.getExternalId()).getStatus() == WorkflowJob.Status.SUCCEEDED;
        }
    });
    assertEquals(WorkflowJob.Status.SUCCEEDED, oozieClient.getJobInfo(action.getExternalId()).getStatus());
    subWorkflow.check(new Context(workflow, action), action);
    assertEquals(WorkflowAction.Status.DONE, action.getStatus());
    subWorkflow.end(new Context(workflow, action), action);
    assertEquals(WorkflowAction.Status.OK, action.getStatus());
    WorkflowJob wf = oozieClient.getJobInfo(action.getExternalId());
    Configuration childConf = getWorkflowConfig(wf);
    assertNull(childConf.get("abc"));
    assertEquals("A", childConf.get("a"));
}