Examples with XConfiguration org.apache.oozie.util.XConfiguration used on opensource projects

Example 61 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestCoordCommandUtils method testCoordOffset.

@Test
public void testCoordOffset() throws Exception {
    CoordinatorJobBean job = addRecordToCoordJobTableForWaiting("coord-dataset-offset.xml", CoordinatorJob.Status.RUNNING, false, true);
    Path appPath = new Path(getFsTestCaseDir(), "coord");
    String actionXml = getCoordActionXml(appPath, "coord-dataset-offset.xml");
    actionXml = actionXml.replace("-unit-", "DAY");
    actionXml = actionXml.replace("-frequency-", "1");
    CoordinatorActionBean actionBean = createCoordinatorActionBean(job);
    Configuration jobConf = new XConfiguration(new StringReader(job.getConf()));
    Element eAction = createActionElement(actionXml);
    jobConf.set("startInstance", "coord:offset(-4,DAY)");
    jobConf.set("endInstance", "coord:offset(0,DAY)");
    String output = CoordCommandUtils.materializeOneInstance("jobId", true, eAction, DateUtils.parseDateOozieTZ("2009-08-20T10:00Z"), DateUtils.parseDateOozieTZ("2009-08-20T10:00Z"), 1, jobConf, actionBean);
    eAction = XmlUtils.parseXml(output);
    Element e = (Element) ((Element) eAction.getChildren("input-events", eAction.getNamespace()).get(0)).getChildren().get(0);
    assertEquals(e.getChild("uris", e.getNamespace()).getTextTrim(), "hdfs:///tmp/workflows/2009/08/20/01;region=us#hdfs:///tmp/workflows/2009/08/19/01;region=us#" + "hdfs:///tmp/workflows/2009/08/18/01;region=us#hdfs:///tmp/workflows/2009/08/17/01;" + "region=us#hdfs:///tmp/workflows/2009/08/16/01;region=us");
    jobConf.set("startInstance", "coord:offset(-4,HOUR)");
    jobConf.set("endInstance", "coord:offset(0,HOUR)");
    actionXml = getCoordActionXml(appPath, "coord-dataset-offset.xml");
    actionXml = actionXml.replace("-unit-", "MINUTE");
    actionXml = actionXml.replace("-frequency-", "60");
    eAction = createActionElement(actionXml);
    output = CoordCommandUtils.materializeOneInstance("jobId", true, eAction, DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"), DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"), 1, jobConf, actionBean);
    eAction = XmlUtils.parseXml(output);
    e = (Element) ((Element) eAction.getChildren("input-events", eAction.getNamespace()).get(0)).getChildren().get(0);
    assertEquals(e.getChild("uris", e.getNamespace()).getTextTrim(), "hdfs:///tmp/workflows/2009/08/20/01;region=us#hdfs:///tmp/workflows/2009/08/20/00;region=us#" + "hdfs:///tmp/workflows/2009/08/19/23;region=us#hdfs:///tmp/workflows/2009/08/19/22;region=us#" + "hdfs:///tmp/workflows/2009/08/19/21;region=us");
}

Example 62 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestCoordMaterializeTransitionXCommand method testSuccessedJobSlaParseElFunctionVariableInMaterializeActions.

/**
 * Test a coordinator SLA define EL functions as variable
 *
 * @throws Exception
 */
public void testSuccessedJobSlaParseElFunctionVariableInMaterializeActions() throws Exception {
    Configuration conf = new XConfiguration();
    File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
    String coordXml = "<coordinator-app name=\"NAME\" frequency=\"0 * * * *\"" + " start=\"2017-06-12T01:00Z\" end=\"2017-06-12T02:00Z\" timezone=\"Asia/Shanghai\"" + " xmlns=\"uri:oozie:coordinator:0.4\" xmlns:sla=\"uri:oozie:sla:0.2\">" + "<controls> <execution>FIFO</execution> </controls>" + "<action>" + " <workflow> <app-path>hdfs:///tmp/workflows/</app-path> </workflow> " + " <sla:info>" + "  <sla:nominal-time>${NOMINAL_TIME}</sla:nominal-time>" + "  <sla:should-start>${SHOULD_START}</sla:should-start>" + "  <sla:should-end>${SHOULD_END}</sla:should-end>" + " </sla:info>" + "</action>" + "</coordinator-app>";
    writeToFile(coordXml, appPathFile);
    conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set("NOMINAL_TIME", "${coord:nominalTime()}");
    conf.set("SHOULD_START", "${5 * MINUTES}");
    conf.set("SHOULD_END", "${ SLA_OFFSET * HOURS}");
    conf.set("SLA_OFFSET", "10");
    CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
    String jobId = sc.call();
    new CoordMaterializeTransitionXCommand(jobId, 60).call();
}

Example 63 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestCoordMaterializeTransitionXCommand method testActionMaterEndOfWeeks.

public void testActionMaterEndOfWeeks() throws Exception {
    Configuration conf = new XConfiguration();
    File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
    String appXml = "<coordinator-app name=\"test\" frequency=\"${coord:endOfWeeks(1)}\" start=\"2016-02-03T01:00Z\" " + "end=\"2016-03-03T23:59Z\" timezone=\"UTC\" " + "xmlns=\"uri:oozie:coordinator:0.2\"> <controls> " + "<execution>LIFO</execution> </controls> <datasets> " + "<dataset name=\"a\" frequency=\"${coord:endOfWeeks(1)}\" initial-instance=\"2016-01-01T01:00Z\" " + "timezone=\"UTC\"> <uri-template>" + getTestCaseFileUri("coord/workflows/${YEAR}/${DAY}") + "</uri-template>  " + "</dataset> " + "<dataset name=\"local_a\" frequency=\"${coord:endOfWeeks(1)}\" initial-instance=\"2016-01-01T01:00Z\" " + "timezone=\"UTC\"> <uri-template>" + getTestCaseFileUri("coord/workflows/${YEAR}/${DAY}") + "</uri-template> " + " </dataset> " + "</datasets> <input-events> " + "<data-in name=\"A\" dataset=\"a\"> <instance>${coord:latest(0)}</instance> </data-in>  " + "</input-events> " + "<output-events> <data-out name=\"LOCAL_A\" dataset=\"local_a\"> " + "<instance>${coord:current(-1)}</instance> </data-out> </output-events> <action> <workflow> " + "<app-path>hdfs:///tmp/workflows/</app-path> " + "<configuration> <property> <name>inputA</name> <value>${coord:dataIn('A')}</value> </property> " + "<property> <name>inputB</name> <value>${coord:dataOut('LOCAL_A')}</value> " + "</property></configuration> </workflow> </action> </coordinator-app>";
    writeToFile(appXml, appPathFile);
    conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
    String jobId = sc.call();
    CoordinatorJobBean job = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, jobId);
    assertEquals(job.getLastActionNumber(), 0);
    job.setMatThrottling(10);
    CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job);
    new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
    job = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, job.getId());
    assertEquals(4, job.getLastActionNumber());
    String jobXml = job.getJobXml();
    Element eJob = XmlUtils.parseXml(jobXml);
    TimeZone appTz = DateUtils.getTimeZone(job.getTimeZone());
    TimeUnit endOfFlag = TimeUnit.valueOf(eJob.getAttributeValue("end_of_duration"));
    TimeUnit freqTU = TimeUnit.valueOf(job.getTimeUnitStr());
    Calendar origStart = Calendar.getInstance(appTz);
    origStart.setTime(job.getStartTimestamp());
    // Move to the End of duration, if needed.
    DateUtils.moveToEnd(origStart, endOfFlag);
    origStart.add(freqTU.getCalendarUnit(), 4 * Integer.parseInt(job.getFrequency()));
    assertEquals(origStart.getTime(), job.getNextMaterializedTime());
}

Example 64 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestSLAAlertXCommand method testBundleSLAAlertCommands.

public void testBundleSLAAlertCommands() throws Exception {
    setupSLAJobs();
    String jobIdsStr = bundle.getId();
    String actions = "1,2";
    String coords = null;
    bundleEngine.disableSLAAlert(jobIdsStr, actions, null, coords);
    checkSLAStatus(coord1.getId() + "@1", true);
    checkSLAStatus(coord1.getId() + "@2", true);
    checkSLAStatus(coord1.getId() + "@3", false);
    checkSLAStatus(coord1.getId() + "@5", false);
    checkSLAStatus(coord1.getId() + "@4", false);
    checkSLAStatus(coord2.getId() + "@1", true);
    checkSLAStatus(coord2.getId() + "@1", true);
    bundleEngine.enableSLAAlert(jobIdsStr, null, null, null);
    checkSLAStatus(coord1.getId() + "@1", false);
    checkSLAStatus(coord1.getId() + "@2", false);
    checkSLAStatus(coord1.getId() + "@3", false);
    checkSLAStatus(coord1.getId() + "@5", false);
    checkSLAStatus(coord1.getId() + "@4", false);
    checkSLAStatus(coord2.getId() + "@1", false);
    checkSLAStatus(coord2.getId() + "@2", false);
    CoordinatorJobBean job1 = CoordJobQueryExecutor.getInstance().get(CoordJobQueryExecutor.CoordJobQuery.GET_COORD_JOB, coord1.getId());
    XConfiguration xConf = new XConfiguration(new StringReader(job1.getConf()));
    assertEquals(xConf.get(OozieClient.SLA_DISABLE_ALERT), null);
    CoordinatorJobBean job2 = CoordJobQueryExecutor.getInstance().get(CoordJobQueryExecutor.CoordJobQuery.GET_COORD_JOB, coord2.getId());
    xConf = new XConfiguration(new StringReader(job2.getConf()));
    assertEquals(xConf.get(OozieClient.SLA_DISABLE_ALERT), null);
    bundleEngine.disableSLAAlert(jobIdsStr, null, null, "coord1");
    checkSLAStatus(coord1.getId() + "@1", true);
    checkSLAStatus(coord1.getId() + "@2", true);
    checkSLAStatus(coord1.getId() + "@3", true);
    checkSLAStatus(coord1.getId() + "@4", true);
    checkSLAStatus(coord1.getId() + "@5", true);
    checkSLAStatus(coord2.getId() + "@1", false);
    checkSLAStatus(coord2.getId() + "@2", false);
    job1 = CoordJobQueryExecutor.getInstance().get(CoordJobQueryExecutor.CoordJobQuery.GET_COORD_JOB, coord1.getId());
    xConf = new XConfiguration(new StringReader(job1.getConf()));
    assertEquals(xConf.get(OozieClient.SLA_DISABLE_ALERT), SLAOperations.ALL_VALUE);
    bundleEngine.disableSLAAlert(jobIdsStr, null, null, "coord2");
    // with multiple coordID.
    String dates = "2014-01-01T00:00Z::2014-01-03T00:00Z";
    bundleEngine.enableSLAAlert(jobIdsStr, null, dates, "coord1," + coord2.getId());
    checkSLAStatus(coord1.getId() + "@1", false);
    checkSLAStatus(coord1.getId() + "@2", false);
    checkSLAStatus(coord1.getId() + "@3", false);
    checkSLAStatus(coord1.getId() + "@4", true);
    checkSLAStatus(coord1.getId() + "@5", true);
    checkSLAStatus(coord2.getId() + "@1", false);
    checkSLAStatus(coord2.getId() + "@2", false);
    checkSLAStatus(coord2.getId() + "@3", false);
    checkSLAStatus(coord2.getId() + "@4", true);
    try {
        bundleEngine.disableSLAAlert(jobIdsStr, null, null, "dummy");
        fail("Should throw Exception");
    } catch (BaseEngineException e) {
        assertEquals(e.getErrorCode(), ErrorCode.E1026);
    }
}

Example 65 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestBundleSubmitXCommand method testMultipleCoordSubmit.

public void testMultipleCoordSubmit() throws Exception {
    final XConfiguration jobConf = setUpBundle();
    jobConf.set("coordName1", "coord1");
    jobConf.set("coordName2", "coord2");
    jobConf.set("coord1.starttime", "2009-02-01T00:00Z");
    BundleSubmitXCommand command = new BundleSubmitXCommand(jobConf);
    final BundleJobBean bundleBean = (BundleJobBean) command.getJob();
    bundleBean.setStartTime(new Date());
    bundleBean.setEndTime(new Date());
    final String jobId = command.call();
    sleep(2000);
    new BundleStartXCommand(jobId).call();
    waitFor(2000, new Predicate() {

        public boolean evaluate() throws Exception {
            List<BundleActionBean> actions = BundleActionQueryExecutor.getInstance().getList(BundleActionQuery.GET_BUNDLE_ACTIONS_STATUS_UNIGNORED_FOR_BUNDLE, jobId);
            return actions.get(0).getStatus().equals(Job.Status.RUNNING);
        }
    });
    List<BundleActionBean> actions = BundleActionQueryExecutor.getInstance().getList(BundleActionQuery.GET_BUNDLE_ACTIONS_STATUS_UNIGNORED_FOR_BUNDLE, jobId);
    assertEquals(actions.size(), 2);
    assertEquals(actions.get(0).getCoordName(), "coord1");
    assertEquals(actions.get(1).getCoordName(), "coord2");
    try {
        new BundleCoordSubmitXCommand(jobConf, jobId, "coord1").call();
        fail("Should fail. Coord job is already created");
    } catch (CommandException e) {
        assertEquals(e.getErrorCode(), ErrorCode.E1304);
    }
    actions = BundleActionQueryExecutor.getInstance().getList(BundleActionQuery.GET_BUNDLE_ACTIONS_STATUS_UNIGNORED_FOR_BUNDLE, jobId);
    assertEquals(actions.size(), 2);
    assertEquals(actions.get(0).getStatusStr(), "RUNNING");
    assertEquals(actions.get(1).getStatusStr(), "RUNNING");
}