Examples with XConfiguration org.apache.oozie.util.XConfiguration used on opensource projects

Example 6 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class ConfigurationService method getMaskedConfiguration.

/**
 * Return a configuration with all sensitive values masked.
 *
 * @return masked configuration.
 */
public Configuration getMaskedConfiguration() {
    XConfiguration maskedConf = new XConfiguration();
    Configuration conf = getConf();
    for (Map.Entry<String, String> entry : conf) {
        String name = entry.getKey();
        String value = getValue(conf, name);
        maskedConf.set(name, value);
    }
    return maskedConf;
}

Example 7 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class XLogService method extractInfoForLogWebService.

private void extractInfoForLogWebService(InputStream is) throws IOException {
    Properties props = new Properties();
    props.load(is);
    XConfiguration conf = new XConfiguration();
    conf.setRestrictSystemProperties(false);
    for (Map.Entry entry : props.entrySet()) {
        conf.set((String) entry.getKey(), (String) entry.getValue());
    }
    XLogUtil logUtil = new XLogUtil(conf, "oozie");
    logOverWS = logUtil.isLogOverEnable();
    oozieLogRotation = logUtil.getLogRotation() == 0 ? oozieLogRotation : logUtil.getLogRotation();
    oozieLogPath = logUtil.getLogPath() == null ? oozieLogPath : logUtil.getLogPath();
    oozieLogName = logUtil.getLogFileName() == null ? oozieLogName : logUtil.getLogFileName();
    logUtil = new XLogUtil(conf, "oozieError");
    errorLogEnabled = logUtil.isLogOverEnable();
    oozieErrorLogRotation = logUtil.getLogRotation() == 0 ? oozieErrorLogRotation : logUtil.getLogRotation();
    oozieErrorLogPath = logUtil.getLogPath() == null ? oozieErrorLogPath : logUtil.getLogPath();
    oozieErrorLogName = logUtil.getLogFileName() == null ? oozieErrorLogName : logUtil.getLogFileName();
    logUtil = new XLogUtil(conf, "oozieaudit");
    auditLogEnabled = logUtil.isLogOverEnable();
    oozieAuditLogRotation = logUtil.getLogRotation() == 0 ? oozieAuditLogRotation : logUtil.getLogRotation();
    oozieAuditLogPath = logUtil.getLogPath() == null ? oozieAuditLogPath : logUtil.getLogPath();
    oozieAuditLogName = logUtil.getLogFileName() == null ? oozieAuditLogName : logUtil.getLogFileName();
}

Example 8 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class RecoveryService method mergeConfig.

/**
 * Merge Bundle job config and the configuration from the coord job to pass
 * to Coord Engine
 *
 * @param coordElem the coordinator configuration
 * @return Configuration merged configuration
 * @throws CommandException thrown if failed to merge configuration
 */
private static Configuration mergeConfig(Element coordElem, BundleJobBean bundleJob) throws CommandException {
    XLog.Info.get().clear();
    XLog log = XLog.getLog("RecoveryService");
    String jobConf = bundleJob.getConf();
    // Step 1: runConf = jobConf
    Configuration runConf = null;
    try {
        runConf = new XConfiguration(new StringReader(jobConf));
    } catch (IOException e1) {
        log.warn("Configuration parse error in:" + jobConf);
        throw new CommandException(ErrorCode.E1306, e1.getMessage(), e1);
    }
    // Step 2: Merge local properties into runConf
    // extract 'property' tags under 'configuration' block in the coordElem
    // convert Element to XConfiguration
    Element localConfigElement = coordElem.getChild("configuration", coordElem.getNamespace());
    if (localConfigElement != null) {
        String strConfig = XmlUtils.prettyPrint(localConfigElement).toString();
        Configuration localConf;
        try {
            localConf = new XConfiguration(new StringReader(strConfig));
        } catch (IOException e1) {
            log.warn("Configuration parse error in:" + strConfig);
            throw new CommandException(ErrorCode.E1307, e1.getMessage(), e1);
        }
        // copy configuration properties in the coordElem to the runConf
        XConfiguration.copy(localConf, runConf);
    }
    // Step 3: Extract value of 'app-path' in coordElem, save it as a
    // new property called 'oozie.coord.application.path', and normalize.
    String appPath = coordElem.getChild("app-path", coordElem.getNamespace()).getValue();
    runConf.set(OozieClient.COORDINATOR_APP_PATH, appPath);
    // Normalize coordinator appPath here;
    try {
        JobUtils.normalizeAppPath(runConf.get(OozieClient.USER_NAME), runConf.get(OozieClient.GROUP_NAME), runConf);
    } catch (IOException e) {
        throw new CommandException(ErrorCode.E1001, runConf.get(OozieClient.COORDINATOR_APP_PATH));
    }
    return runConf;
}

Example 9 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestRecoveryService method testWorkflowActionRecoveryService.

/**
 * Tests functionality of the Recovery Service Runnable command. </p> Starts an action which behaves like an Async
 * Action (Action and Job state set to Running). Changes the action configuration to run in sync mode and updates
 * the store. Runs the recovery runnable, and ensures the state of the action and job have not changed. </p> Changes
 * the state of the action from RUNNING to PREP and updates the store. Again, runs the recovery runnable and ensures
 * the state changes to OK and the job completes successfully.
 *
 * @throws Exception
 */
public void testWorkflowActionRecoveryService() throws Exception {
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    createTestCaseSubDir("lib");
    IOUtils.copyCharStream(reader, writer);
    final DagEngine engine = new DagEngine(getTestUser());
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, getTestCaseFileUri("workflow.xml"));
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("external-status", "ok");
    conf.set("signal-value", "based_on_action_status");
    conf.set("running-mode", "async");
    // TODO CHECK, without this we get JPA concurrency exceptions, ODD
    sleep(1000);
    final String jobId = engine.submitJob(conf, true);
    // TODO CHECK, without this we get JPA concurrency exceptions, ODD
    sleep(1000);
    waitFor(5000, new Predicate() {

        public boolean evaluate() throws Exception {
            return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.RUNNING);
        }
    });
    sleep(1000);
    final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
    store.beginTrx();
    List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId, false);
    WorkflowActionBean action = null;
    for (WorkflowActionBean bean : actions) {
        if (bean.getType().equals("test")) {
            action = bean;
            break;
        }
    }
    assertNotNull(action);
    final String actionId = action.getId();
    assertEquals(WorkflowActionBean.Status.RUNNING, action.getStatus());
    String actionConf = action.getConf();
    String fixedActionConf = actionConf.replaceAll("async", "sync");
    action.setConf(fixedActionConf);
    action.setPending();
    store.updateAction(action);
    store.commitTrx();
    store.closeTrx();
    Runnable recoveryRunnable = new RecoveryRunnable(0, 60, 60);
    recoveryRunnable.run();
    sleep(3000);
    final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create();
    assertEquals(WorkflowJob.Status.RUNNING, engine.getJob(jobId).getStatus());
    store2.beginTrx();
    WorkflowActionBean action2 = store2.getAction(actionId, false);
    assertEquals(WorkflowActionBean.Status.RUNNING, action2.getStatus());
    action2.setStatus(WorkflowActionBean.Status.PREP);
    action2.setPending();
    store2.updateAction(action2);
    store2.commitTrx();
    store2.closeTrx();
    sleep(1000);
    recoveryRunnable.run();
    sleep(3000);
    waitFor(10000, new Predicate() {

        public boolean evaluate() throws Exception {
            return (engine.getWorkflowAction(actionId).getStatus() == WorkflowActionBean.Status.OK);
        }
    });
    // getPendingActions works correctly only with MYSQL - following assertsfail with hsql - to be investigated
    // assertEquals(WorkflowJob.Status.SUCCEEDED, engine.getJob(jobId).getStatus());
    final WorkflowStore store3 = Services.get().get(WorkflowStoreService.class).create();
    store3.beginTrx();
    WorkflowActionBean action3 = store3.getAction(actionId, false);
    assertEquals(WorkflowActionBean.Status.OK, action3.getStatus());
    store3.commitTrx();
    store3.closeTrx();
}

Example 10 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestRecoveryService method addRecordToActionTable.

private void addRecordToActionTable(String jobId, int actionNum, String actionId, String baseDir) throws Exception {
    CoordinatorActionBean action = new CoordinatorActionBean();
    action.setJobId(jobId);
    action.setId(actionId);
    action.setActionNumber(actionNum);
    action.setNominalTime(new Date());
    action.setLastModifiedTime(new Date());
    action.setStatus(CoordinatorAction.Status.SUBMITTED);
    String appPath = getTestCaseFileUri("one-op/workflow.xml");
    String actionXml = "<coordinator-app xmlns='uri:oozie:coordinator:0.2' xmlns:sla='uri:oozie:sla:0.1' name='NAME'" + " frequency=\"1\" start='2009-02-01T01:00Z' end='2009-02-03T23:59Z' timezone='UTC' freq_timeunit='DAY'" + " end_of_duration='NONE'  instance-number=\"1\" action-nominal-time=\"2009-02-01T01:00Z\">";
    actionXml += "<controls>";
    actionXml += "<timeout>10</timeout>";
    actionXml += "<concurrency>2</concurrency>";
    actionXml += "<execution>LIFO</execution>";
    actionXml += "</controls>";
    actionXml += "<input-events>";
    actionXml += "<data-in name='A' dataset='a'>";
    actionXml += "<dataset name='a' frequency='7' initial-instance='2009-02-01T01:00Z' timezone='UTC'" + " freq_timeunit='DAY' end_of_duration='NONE'>";
    actionXml += "<uri-template>" + getTestCaseFileUri("workflows/workflows/${YEAR}/${DAY}") + "</uri-template>";
    actionXml += "</dataset>";
    actionXml += "<instance>${coord:latest(0)}</instance>";
    actionXml += "</data-in>";
    actionXml += "</input-events>";
    actionXml += "<output-events>";
    actionXml += "<data-out name='LOCAL_A' dataset='local_a'>";
    actionXml += "<dataset name='local_a' frequency='7' initial-instance='2009-02-01T01:00Z' timezone='UTC'" + " freq_timeunit='DAY' end_of_duration='NONE'>";
    actionXml += "<uri-template>" + getTestCaseFileUri("workflows/${YEAR}/${DAY}") + "</uri-template>";
    actionXml += "</dataset>";
    actionXml += "<instance>${coord:current(-1)}</instance>";
    actionXml += "</data-out>";
    actionXml += "</output-events>";
    actionXml += "<action>";
    actionXml += "<workflow>";
    actionXml += "<app-path>" + appPath + "</app-path>";
    actionXml += "<configuration>";
    actionXml += "<property>";
    actionXml += "<name>inputA</name>";
    actionXml += "<value>" + getTestCaseFileUri("workflows/US/2009/02/") + "</value>";
    actionXml += "</property>";
    actionXml += "<property>";
    actionXml += "<name>inputB</name>";
    actionXml += "<value>" + getTestCaseFileUri("workflows/US/2009/01/") + "</value>";
    actionXml += "</property>";
    actionXml += "</configuration>";
    actionXml += "</workflow>";
    actionXml += "</action>";
    actionXml += "</coordinator-app>";
    action.setActionXml(actionXml);
    String createdConf = "<configuration> ";
    createdConf += "<property> <name>execution_order</name> <value>LIFO</value> </property>";
    createdConf += "<property> <name>user.name</name> <value>" + getTestUser() + "</value> </property>";
    createdConf += "<property> <name>group.name</name> <value>other</value> </property>";
    createdConf += "<property> <name>app-path</name> <value>" + appPath + "</value> </property>";
    createdConf += "<property> <name>jobTracker</name> ";
    createdConf += "<value>localhost:9001</value></property>";
    createdConf += "<property> <name>nameNode</name> <value>hdfs://localhost:9000</value></property>";
    createdConf += "<property> <name>queueName</name> <value>default</value></property>";
    createdConf += "</configuration> ";
    XConfiguration conf = new XConfiguration(new StringReader(createdConf));
    createdConf = conf.toXmlString(false);
    action.setCreatedConf(createdConf);
    addRecordToCoordActionTable(action, null);
    String content = "<workflow-app xmlns='uri:oozie:workflow:0.1'  xmlns:sla='uri:oozie:sla:0.1' name='one-op-wf'>";
    content += "<start to='fs1'/><action name='fs1'><fs><mkdir path='/tmp'/></fs><ok to='end'/><error to='end'/></action>";
    content += "<end name='end' /></workflow-app>";
    writeToFile(content, baseDir + "/one-op/");
}