Examples with XConfiguration org.apache.oozie.util.XConfiguration used on opensource projects

Example 31 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class SubWorkflowActionExecutor method start.

public void start(Context context, WorkflowAction action) throws ActionExecutorException {
    LOG.info("Starting action");
    try {
        Element eConf = XmlUtils.parseXml(action.getConf());
        Namespace ns = eConf.getNamespace();
        Element e = eConf.getChild("oozie", ns);
        String oozieUri = (e == null) ? LOCAL : e.getTextTrim();
        OozieClient oozieClient = getWorkflowClient(context, oozieUri);
        String subWorkflowId = null;
        String extId = context.getRecoveryId();
        String runningJobId = null;
        if (extId != null) {
            runningJobId = checkIfRunning(oozieClient, extId);
        }
        if (runningJobId == null) {
            String appPath = eConf.getChild("app-path", ns).getTextTrim();
            XConfiguration subWorkflowConf = new XConfiguration();
            Configuration parentConf = new XConfiguration(new StringReader(context.getWorkflow().getConf()));
            if (eConf.getChild(("propagate-configuration"), ns) != null) {
                XConfiguration.copy(parentConf, subWorkflowConf);
            }
            // Propagate coordinator and bundle info to subworkflow
            if (OozieJobInfo.isJobInfoEnabled()) {
                if (parentConf.get(OozieJobInfo.COORD_ID) != null) {
                    subWorkflowConf.set(OozieJobInfo.COORD_ID, parentConf.get(OozieJobInfo.COORD_ID));
                    subWorkflowConf.set(OozieJobInfo.COORD_NAME, parentConf.get(OozieJobInfo.COORD_NAME));
                    subWorkflowConf.set(OozieJobInfo.COORD_NOMINAL_TIME, parentConf.get(OozieJobInfo.COORD_NOMINAL_TIME));
                }
                if (parentConf.get(OozieJobInfo.BUNDLE_ID) != null) {
                    subWorkflowConf.set(OozieJobInfo.BUNDLE_ID, parentConf.get(OozieJobInfo.BUNDLE_ID));
                    subWorkflowConf.set(OozieJobInfo.BUNDLE_NAME, parentConf.get(OozieJobInfo.BUNDLE_NAME));
                }
            }
            // the proto has the necessary credentials
            Configuration protoActionConf = context.getProtoActionConf();
            XConfiguration.copy(protoActionConf, subWorkflowConf);
            subWorkflowConf.set(OozieClient.APP_PATH, appPath);
            String group = ConfigUtils.getWithDeprecatedCheck(parentConf, OozieClient.JOB_ACL, OozieClient.GROUP_NAME, null);
            if (group != null) {
                subWorkflowConf.set(OozieClient.GROUP_NAME, group);
            }
            injectInline(eConf.getChild("configuration", ns), subWorkflowConf);
            injectCallback(context, subWorkflowConf);
            injectRecovery(extId, subWorkflowConf);
            injectParent(context.getWorkflow().getId(), subWorkflowConf);
            injectSuperParent(context.getWorkflow(), parentConf, subWorkflowConf);
            verifyAndInjectSubworkflowDepth(parentConf, subWorkflowConf);
            // TODO: this has to be refactored later to be done in a single place for REST calls and this
            JobUtils.normalizeAppPath(context.getWorkflow().getUser(), context.getWorkflow().getGroup(), subWorkflowConf);
            subWorkflowConf.set(OOZIE_ACTION_YARN_TAG, getActionYarnTag(parentConf, context.getWorkflow(), action));
            // rerun again.
            if (action.getExternalId() != null && parentConf.getBoolean(OozieClient.RERUN_FAIL_NODES, false)) {
                subWorkflowConf.setBoolean(SUBWORKFLOW_RERUN, true);
                oozieClient.reRun(action.getExternalId(), subWorkflowConf.toProperties());
                subWorkflowId = action.getExternalId();
            } else {
                subWorkflowId = oozieClient.run(subWorkflowConf.toProperties());
            }
        } else {
            subWorkflowId = runningJobId;
        }
        LOG.info("Sub workflow id: [{0}]", subWorkflowId);
        WorkflowJob workflow = oozieClient.getJobInfo(subWorkflowId);
        String consoleUrl = workflow.getConsoleUrl();
        context.setStartData(subWorkflowId, oozieUri, consoleUrl);
        if (runningJobId != null) {
            check(context, action);
        }
    } catch (Exception ex) {
        LOG.error(ex);
        throw convertException(ex);
    }
}

Example 32 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class LocalOozieClientCoord method getCoordinatorActions.

private List<CoordinatorAction> getCoordinatorActions(String jobId, String rerunType, String scope, boolean refresh, boolean noCleanup, boolean failed, Properties prop) throws OozieClientException {
    try {
        XConfiguration conf = null;
        if (prop != null) {
            conf = new XConfiguration(prop);
        }
        if (!(rerunType.equals(RestConstants.JOB_COORD_SCOPE_DATE) || rerunType.equals(RestConstants.JOB_COORD_SCOPE_ACTION))) {
            throw new CommandException(ErrorCode.E1018, "date or action expected.");
        }
        CoordinatorActionInfo coordInfo = coordEngine.reRun(jobId, rerunType, scope, Boolean.valueOf(refresh), Boolean.valueOf(noCleanup), Boolean.valueOf(failed), conf);
        List<CoordinatorActionBean> actionBeans;
        if (coordInfo != null) {
            actionBeans = coordInfo.getCoordActions();
        } else {
            actionBeans = CoordUtils.getCoordActions(rerunType, jobId, scope, false);
        }
        List<CoordinatorAction> actions = new ArrayList<CoordinatorAction>();
        for (CoordinatorActionBean actionBean : actionBeans) {
            actions.add(actionBean);
        }
        return actions;
    } catch (CommandException ce) {
        throw new OozieClientException(ce.getErrorCode().toString(), ce);
    } catch (BaseEngineException ex) {
        throw new OozieClientException(ex.getErrorCode().toString(), ex);
    }
}

Example 33 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestFutureActionsTimeOut method _testSubmitJob.

private String _testSubmitJob(String appPath) throws Exception {
    Configuration conf = new XConfiguration();
    GregorianCalendar start = new GregorianCalendar(TimeZone.getTimeZone("GMT"));
    start.add(Calendar.MINUTE, -15);
    GregorianCalendar end = new GregorianCalendar(TimeZone.getTimeZone("GMT"));
    end.add(Calendar.MINUTE, 45);
    String appXml = "<coordinator-app name=\"NAME\" frequency=\"5\" " + "start=\"" + start.get(Calendar.YEAR) + "-" + (start.get(Calendar.MONTH) + 1) + "-" + start.get(Calendar.DAY_OF_MONTH) + "T" + start.get(Calendar.HOUR_OF_DAY) + ":" + start.get(Calendar.MINUTE) + "Z\" " + "end=\"" + end.get(Calendar.YEAR) + "-" + (end.get(Calendar.MONTH) + 1) + "-" + end.get(Calendar.DAY_OF_MONTH) + "T" + end.get(Calendar.HOUR_OF_DAY) + ":" + end.get(Calendar.MINUTE) + "Z\" timezone=\"UTC\" " + "xmlns=\"uri:oozie:coordinator:0.1\"> <controls> <timeout>10</timeout> <concurrency>2</concurrency> " + "<execution>LIFO</execution> </controls> <datasets> " + "<dataset name=\"a\" frequency=\"${coord:days(7)}\" initial-instance=\"9999-02-01T01:00Z\" " + "timezone=\"UTC\"> <uri-template>" + getTestCaseFileUri("${YEAR}/${DAY}") + "</uri-template> </dataset> " + "<dataset name=\"local_a\" frequency=\"${coord:days(7)}\" initial-instance=\"9999-02-01T01:00Z\" " + "timezone=\"UTC\"> <uri-template>" + getTestCaseFileUri("${YEAR}/${DAY}") + "</uri-template> </dataset> " + "</datasets> <input-events> " + "<data-in name=\"A\" dataset=\"a\"> <instance>${coord:latest(0)}</instance> </data-in>  " + "</input-events> " + "<output-events> <data-out name=\"LOCAL_A\" dataset=\"local_a\"> " + "<instance>${coord:current(-1)}</instance> </data-out> </output-events> <action> <workflow> <app-path>hdfs:" + "///tmp/workflows/</app-path> " + "<configuration> <property> <name>inputA</name> <value>${coord:dataIn('A')}</value> </property> " + "<property> <name>inputB</name> <value>${coord:dataOut('LOCAL_A')}</value> " + "</property></configuration> </workflow> </action> </coordinator-app>";
    System.out.println(appXml);
    writeToFile(appXml, appPath);
    conf.set(OozieClient.COORDINATOR_APP_PATH, appPath);
    conf.set(OozieClient.USER_NAME, getTestUser());
    CoordinatorEngine ce = new CoordinatorEngine(getTestUser());
    String jobId = ce.submitJob(conf, true);
    assertEquals(jobId.substring(jobId.length() - 2), "-C");
    checkCoordJob(jobId);
    return jobId;
}

Example 34 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestActionErrors method _testKillNodeErrorMessage.

private WorkflowActionBean _testKillNodeErrorMessage(String workflowXmlFile) throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader(workflowXmlFile, -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);
    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("error", "end.error");
    conf.set("external-status", "FAILED/KILLED");
    conf.set("signal-value", "fail");
    final String jobId = engine.submitJob(conf, true);
    final JPAService jpaService = Services.get().get(JPAService.class);
    final WorkflowJobGetJPAExecutor wfJobGetCmd = new WorkflowJobGetJPAExecutor(jobId);
    waitFor(50000, new Predicate() {

        public boolean evaluate() throws Exception {
            WorkflowJobBean job = jpaService.execute(wfJobGetCmd);
            return (job.getWorkflowInstance().getStatus() == WorkflowInstance.Status.KILLED);
        }
    });
    WorkflowJobBean job = jpaService.execute(wfJobGetCmd);
    assertEquals(WorkflowJob.Status.KILLED, job.getStatus());
    WorkflowActionsGetForJobJPAExecutor wfActionsGetCmd = new WorkflowActionsGetForJobJPAExecutor(jobId);
    List<WorkflowActionBean> actions = jpaService.execute(wfActionsGetCmd);
    WorkflowActionBean action = null;
    WorkflowActionBean killAction = null;
    for (WorkflowActionBean bean : actions) {
        if (bean.getType().equals("test")) {
            action = bean;
        }
        if (bean.getType().equals(KillActionExecutor.TYPE)) {
            killAction = bean;
        }
    }
    assertNotNull(action);
    assertEquals("TEST_ERROR", action.getErrorCode());
    assertEquals("end", action.getErrorMessage());
    assertEquals(WorkflowAction.Status.ERROR, action.getStatus());
    assertNotNull(killAction);
    return killAction;
}

Example 35 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestActionErrors method _testNonTransient.

/**
 * Provides functionality to test non transient failures.
 *
 * @param errorType the error type. (start.non-transient, end.non-transient)
 * @param expStatus1 expected status. (START_MANUAL, END_MANUAL)
 * @param expErrorMsg expected error message.
 * @throws Exception
 */
private void _testNonTransient(String errorType, WorkflowActionBean.Status expStatus1, String expErrorMsg) throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);
    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("signal-value", "OK");
    conf.set("external-status", "ok");
    conf.set("error", errorType);
    final String jobId = engine.submitJob(conf, true);
    waitFor(5000, new Predicate() {

        public boolean evaluate() throws Exception {
            return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED);
        }
    });
    final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
    store.beginTrx();
    List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId, true);
    int n = actions.size();
    WorkflowActionBean action = actions.get(n - 1);
    assertEquals("TEST_ERROR", action.getErrorCode());
    assertEquals(expErrorMsg, action.getErrorMessage());
    assertEquals(expStatus1, action.getStatus());
    assertTrue(action.isPending() == false);
    assertTrue(engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED);
    String actionConf = action.getConf();
    String fixedActionConf = actionConf.replaceAll(errorType, "none");
    action.setConf(fixedActionConf);
    store.updateAction(action);
    store.commitTrx();
    store.closeTrx();
    engine.resume(jobId);
    waitFor(5000, new Predicate() {

        public boolean evaluate() throws Exception {
            return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUCCEEDED);
        }
    });
    assertEquals(WorkflowJob.Status.SUCCEEDED, engine.getJob(jobId).getStatus());
    final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create();
    store2.beginTrx();
    actions = store2.getActionsForWorkflow(jobId, false);
    action = actions.get(0);
    assertEquals(null, action.getErrorCode());
    assertEquals(null, action.getErrorMessage());
    assertEquals(WorkflowActionBean.Status.OK, action.getStatus());
    store2.commitTrx();
    store2.closeTrx();
}