Search in sources :

Example 61 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestCoordCommandUtils method testCoordOffset.

@Test
public void testCoordOffset() throws Exception {
    CoordinatorJobBean job = addRecordToCoordJobTableForWaiting("coord-dataset-offset.xml", CoordinatorJob.Status.RUNNING, false, true);
    Path appPath = new Path(getFsTestCaseDir(), "coord");
    String actionXml = getCoordActionXml(appPath, "coord-dataset-offset.xml");
    actionXml = actionXml.replace("-unit-", "DAY");
    actionXml = actionXml.replace("-frequency-", "1");
    CoordinatorActionBean actionBean = createCoordinatorActionBean(job);
    Configuration jobConf = new XConfiguration(new StringReader(job.getConf()));
    Element eAction = createActionElement(actionXml);
    jobConf.set("startInstance", "coord:offset(-4,DAY)");
    jobConf.set("endInstance", "coord:offset(0,DAY)");
    String output = CoordCommandUtils.materializeOneInstance("jobId", true, eAction, DateUtils.parseDateOozieTZ("2009-08-20T10:00Z"), DateUtils.parseDateOozieTZ("2009-08-20T10:00Z"), 1, jobConf, actionBean);
    eAction = XmlUtils.parseXml(output);
    Element e = (Element) ((Element) eAction.getChildren("input-events", eAction.getNamespace()).get(0)).getChildren().get(0);
    assertEquals(e.getChild("uris", e.getNamespace()).getTextTrim(), "hdfs:///tmp/workflows/2009/08/20/01;region=us#hdfs:///tmp/workflows/2009/08/19/01;region=us#" + "hdfs:///tmp/workflows/2009/08/18/01;region=us#hdfs:///tmp/workflows/2009/08/17/01;" + "region=us#hdfs:///tmp/workflows/2009/08/16/01;region=us");
    jobConf.set("startInstance", "coord:offset(-4,HOUR)");
    jobConf.set("endInstance", "coord:offset(0,HOUR)");
    actionXml = getCoordActionXml(appPath, "coord-dataset-offset.xml");
    actionXml = actionXml.replace("-unit-", "MINUTE");
    actionXml = actionXml.replace("-frequency-", "60");
    eAction = createActionElement(actionXml);
    output = CoordCommandUtils.materializeOneInstance("jobId", true, eAction, DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"), DateUtils.parseDateOozieTZ("2009-08-20T01:00Z"), 1, jobConf, actionBean);
    eAction = XmlUtils.parseXml(output);
    e = (Element) ((Element) eAction.getChildren("input-events", eAction.getNamespace()).get(0)).getChildren().get(0);
    assertEquals(e.getChild("uris", e.getNamespace()).getTextTrim(), "hdfs:///tmp/workflows/2009/08/20/01;region=us#hdfs:///tmp/workflows/2009/08/20/00;region=us#" + "hdfs:///tmp/workflows/2009/08/19/23;region=us#hdfs:///tmp/workflows/2009/08/19/22;region=us#" + "hdfs:///tmp/workflows/2009/08/19/21;region=us");
}
Also used : Path(org.apache.hadoop.fs.Path) CoordinatorJobBean(org.apache.oozie.CoordinatorJobBean) XConfiguration(org.apache.oozie.util.XConfiguration) XConfiguration(org.apache.oozie.util.XConfiguration) Configuration(org.apache.hadoop.conf.Configuration) CoordinatorActionBean(org.apache.oozie.CoordinatorActionBean) Element(org.jdom.Element) StringReader(java.io.StringReader) Test(org.junit.Test)

Example 62 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestCoordMaterializeTransitionXCommand method testSuccessedJobSlaParseElFunctionVariableInMaterializeActions.

/**
 * Test a coordinator SLA define EL functions as variable
 *
 * @throws Exception
 */
public void testSuccessedJobSlaParseElFunctionVariableInMaterializeActions() throws Exception {
    Configuration conf = new XConfiguration();
    File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
    String coordXml = "<coordinator-app name=\"NAME\" frequency=\"0 * * * *\"" + " start=\"2017-06-12T01:00Z\" end=\"2017-06-12T02:00Z\" timezone=\"Asia/Shanghai\"" + " xmlns=\"uri:oozie:coordinator:0.4\" xmlns:sla=\"uri:oozie:sla:0.2\">" + "<controls> <execution>FIFO</execution> </controls>" + "<action>" + " <workflow> <app-path>hdfs:///tmp/workflows/</app-path> </workflow> " + " <sla:info>" + "  <sla:nominal-time>${NOMINAL_TIME}</sla:nominal-time>" + "  <sla:should-start>${SHOULD_START}</sla:should-start>" + "  <sla:should-end>${SHOULD_END}</sla:should-end>" + " </sla:info>" + "</action>" + "</coordinator-app>";
    writeToFile(coordXml, appPathFile);
    conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set("NOMINAL_TIME", "${coord:nominalTime()}");
    conf.set("SHOULD_START", "${5 * MINUTES}");
    conf.set("SHOULD_END", "${ SLA_OFFSET * HOURS}");
    conf.set("SLA_OFFSET", "10");
    CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
    String jobId = sc.call();
    new CoordMaterializeTransitionXCommand(jobId, 60).call();
}
Also used : XConfiguration(org.apache.oozie.util.XConfiguration) XConfiguration(org.apache.oozie.util.XConfiguration) Configuration(org.apache.hadoop.conf.Configuration) File(java.io.File)

Example 63 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestCoordMaterializeTransitionXCommand method testActionMaterEndOfWeeks.

public void testActionMaterEndOfWeeks() throws Exception {
    Configuration conf = new XConfiguration();
    File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
    String appXml = "<coordinator-app name=\"test\" frequency=\"${coord:endOfWeeks(1)}\" start=\"2016-02-03T01:00Z\" " + "end=\"2016-03-03T23:59Z\" timezone=\"UTC\" " + "xmlns=\"uri:oozie:coordinator:0.2\"> <controls> " + "<execution>LIFO</execution> </controls> <datasets> " + "<dataset name=\"a\" frequency=\"${coord:endOfWeeks(1)}\" initial-instance=\"2016-01-01T01:00Z\" " + "timezone=\"UTC\"> <uri-template>" + getTestCaseFileUri("coord/workflows/${YEAR}/${DAY}") + "</uri-template>  " + "</dataset> " + "<dataset name=\"local_a\" frequency=\"${coord:endOfWeeks(1)}\" initial-instance=\"2016-01-01T01:00Z\" " + "timezone=\"UTC\"> <uri-template>" + getTestCaseFileUri("coord/workflows/${YEAR}/${DAY}") + "</uri-template> " + " </dataset> " + "</datasets> <input-events> " + "<data-in name=\"A\" dataset=\"a\"> <instance>${coord:latest(0)}</instance> </data-in>  " + "</input-events> " + "<output-events> <data-out name=\"LOCAL_A\" dataset=\"local_a\"> " + "<instance>${coord:current(-1)}</instance> </data-out> </output-events> <action> <workflow> " + "<app-path>hdfs:///tmp/workflows/</app-path> " + "<configuration> <property> <name>inputA</name> <value>${coord:dataIn('A')}</value> </property> " + "<property> <name>inputB</name> <value>${coord:dataOut('LOCAL_A')}</value> " + "</property></configuration> </workflow> </action> </coordinator-app>";
    writeToFile(appXml, appPathFile);
    conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
    String jobId = sc.call();
    CoordinatorJobBean job = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, jobId);
    assertEquals(job.getLastActionNumber(), 0);
    job.setMatThrottling(10);
    CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job);
    new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
    job = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, job.getId());
    assertEquals(4, job.getLastActionNumber());
    String jobXml = job.getJobXml();
    Element eJob = XmlUtils.parseXml(jobXml);
    TimeZone appTz = DateUtils.getTimeZone(job.getTimeZone());
    TimeUnit endOfFlag = TimeUnit.valueOf(eJob.getAttributeValue("end_of_duration"));
    TimeUnit freqTU = TimeUnit.valueOf(job.getTimeUnitStr());
    Calendar origStart = Calendar.getInstance(appTz);
    origStart.setTime(job.getStartTimestamp());
    // Move to the End of duration, if needed.
    DateUtils.moveToEnd(origStart, endOfFlag);
    origStart.add(freqTU.getCalendarUnit(), 4 * Integer.parseInt(job.getFrequency()));
    assertEquals(origStart.getTime(), job.getNextMaterializedTime());
}
Also used : XConfiguration(org.apache.oozie.util.XConfiguration) CoordinatorJobBean(org.apache.oozie.CoordinatorJobBean) TimeZone(java.util.TimeZone) XConfiguration(org.apache.oozie.util.XConfiguration) Configuration(org.apache.hadoop.conf.Configuration) Element(org.jdom.Element) Calendar(java.util.Calendar) TimeUnit(org.apache.oozie.coord.TimeUnit) File(java.io.File)

Example 64 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestSLAAlertXCommand method testBundleSLAAlertCommands.

public void testBundleSLAAlertCommands() throws Exception {
    setupSLAJobs();
    String jobIdsStr = bundle.getId();
    String actions = "1,2";
    String coords = null;
    bundleEngine.disableSLAAlert(jobIdsStr, actions, null, coords);
    checkSLAStatus(coord1.getId() + "@1", true);
    checkSLAStatus(coord1.getId() + "@2", true);
    checkSLAStatus(coord1.getId() + "@3", false);
    checkSLAStatus(coord1.getId() + "@5", false);
    checkSLAStatus(coord1.getId() + "@4", false);
    checkSLAStatus(coord2.getId() + "@1", true);
    checkSLAStatus(coord2.getId() + "@1", true);
    bundleEngine.enableSLAAlert(jobIdsStr, null, null, null);
    checkSLAStatus(coord1.getId() + "@1", false);
    checkSLAStatus(coord1.getId() + "@2", false);
    checkSLAStatus(coord1.getId() + "@3", false);
    checkSLAStatus(coord1.getId() + "@5", false);
    checkSLAStatus(coord1.getId() + "@4", false);
    checkSLAStatus(coord2.getId() + "@1", false);
    checkSLAStatus(coord2.getId() + "@2", false);
    CoordinatorJobBean job1 = CoordJobQueryExecutor.getInstance().get(CoordJobQueryExecutor.CoordJobQuery.GET_COORD_JOB, coord1.getId());
    XConfiguration xConf = new XConfiguration(new StringReader(job1.getConf()));
    assertEquals(xConf.get(OozieClient.SLA_DISABLE_ALERT), null);
    CoordinatorJobBean job2 = CoordJobQueryExecutor.getInstance().get(CoordJobQueryExecutor.CoordJobQuery.GET_COORD_JOB, coord2.getId());
    xConf = new XConfiguration(new StringReader(job2.getConf()));
    assertEquals(xConf.get(OozieClient.SLA_DISABLE_ALERT), null);
    bundleEngine.disableSLAAlert(jobIdsStr, null, null, "coord1");
    checkSLAStatus(coord1.getId() + "@1", true);
    checkSLAStatus(coord1.getId() + "@2", true);
    checkSLAStatus(coord1.getId() + "@3", true);
    checkSLAStatus(coord1.getId() + "@4", true);
    checkSLAStatus(coord1.getId() + "@5", true);
    checkSLAStatus(coord2.getId() + "@1", false);
    checkSLAStatus(coord2.getId() + "@2", false);
    job1 = CoordJobQueryExecutor.getInstance().get(CoordJobQueryExecutor.CoordJobQuery.GET_COORD_JOB, coord1.getId());
    xConf = new XConfiguration(new StringReader(job1.getConf()));
    assertEquals(xConf.get(OozieClient.SLA_DISABLE_ALERT), SLAOperations.ALL_VALUE);
    bundleEngine.disableSLAAlert(jobIdsStr, null, null, "coord2");
    // with multiple coordID.
    String dates = "2014-01-01T00:00Z::2014-01-03T00:00Z";
    bundleEngine.enableSLAAlert(jobIdsStr, null, dates, "coord1," + coord2.getId());
    checkSLAStatus(coord1.getId() + "@1", false);
    checkSLAStatus(coord1.getId() + "@2", false);
    checkSLAStatus(coord1.getId() + "@3", false);
    checkSLAStatus(coord1.getId() + "@4", true);
    checkSLAStatus(coord1.getId() + "@5", true);
    checkSLAStatus(coord2.getId() + "@1", false);
    checkSLAStatus(coord2.getId() + "@2", false);
    checkSLAStatus(coord2.getId() + "@3", false);
    checkSLAStatus(coord2.getId() + "@4", true);
    try {
        bundleEngine.disableSLAAlert(jobIdsStr, null, null, "dummy");
        fail("Should throw Exception");
    } catch (BaseEngineException e) {
        assertEquals(e.getErrorCode(), ErrorCode.E1026);
    }
}
Also used : CoordinatorJobBean(org.apache.oozie.CoordinatorJobBean) XConfiguration(org.apache.oozie.util.XConfiguration) StringReader(java.io.StringReader) BaseEngineException(org.apache.oozie.BaseEngineException)

Example 65 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestBundleSubmitXCommand method testMultipleCoordSubmit.

public void testMultipleCoordSubmit() throws Exception {
    final XConfiguration jobConf = setUpBundle();
    jobConf.set("coordName1", "coord1");
    jobConf.set("coordName2", "coord2");
    jobConf.set("coord1.starttime", "2009-02-01T00:00Z");
    BundleSubmitXCommand command = new BundleSubmitXCommand(jobConf);
    final BundleJobBean bundleBean = (BundleJobBean) command.getJob();
    bundleBean.setStartTime(new Date());
    bundleBean.setEndTime(new Date());
    final String jobId = command.call();
    sleep(2000);
    new BundleStartXCommand(jobId).call();
    waitFor(2000, new Predicate() {

        public boolean evaluate() throws Exception {
            List<BundleActionBean> actions = BundleActionQueryExecutor.getInstance().getList(BundleActionQuery.GET_BUNDLE_ACTIONS_STATUS_UNIGNORED_FOR_BUNDLE, jobId);
            return actions.get(0).getStatus().equals(Job.Status.RUNNING);
        }
    });
    List<BundleActionBean> actions = BundleActionQueryExecutor.getInstance().getList(BundleActionQuery.GET_BUNDLE_ACTIONS_STATUS_UNIGNORED_FOR_BUNDLE, jobId);
    assertEquals(actions.size(), 2);
    assertEquals(actions.get(0).getCoordName(), "coord1");
    assertEquals(actions.get(1).getCoordName(), "coord2");
    try {
        new BundleCoordSubmitXCommand(jobConf, jobId, "coord1").call();
        fail("Should fail. Coord job is already created");
    } catch (CommandException e) {
        assertEquals(e.getErrorCode(), ErrorCode.E1304);
    }
    actions = BundleActionQueryExecutor.getInstance().getList(BundleActionQuery.GET_BUNDLE_ACTIONS_STATUS_UNIGNORED_FOR_BUNDLE, jobId);
    assertEquals(actions.size(), 2);
    assertEquals(actions.get(0).getStatusStr(), "RUNNING");
    assertEquals(actions.get(1).getStatusStr(), "RUNNING");
}
Also used : CommandException(org.apache.oozie.command.CommandException) Date(java.util.Date) IOException(java.io.IOException) CommandException(org.apache.oozie.command.CommandException) UnsupportedEncodingException(java.io.UnsupportedEncodingException) XConfiguration(org.apache.oozie.util.XConfiguration) BundleJobBean(org.apache.oozie.BundleJobBean) List(java.util.List) BundleActionBean(org.apache.oozie.BundleActionBean)

Aggregations

XConfiguration (org.apache.oozie.util.XConfiguration)373 Configuration (org.apache.hadoop.conf.Configuration)241 Path (org.apache.hadoop.fs.Path)106 StringReader (java.io.StringReader)97 File (java.io.File)92 IOException (java.io.IOException)78 WorkflowJobBean (org.apache.oozie.WorkflowJobBean)75 WorkflowActionBean (org.apache.oozie.WorkflowActionBean)70 CommandException (org.apache.oozie.command.CommandException)68 Element (org.jdom.Element)66 Writer (java.io.Writer)58 Date (java.util.Date)50 FileSystem (org.apache.hadoop.fs.FileSystem)48 FileWriter (java.io.FileWriter)45 Reader (java.io.Reader)43 CoordinatorActionBean (org.apache.oozie.CoordinatorActionBean)37 CoordinatorJobBean (org.apache.oozie.CoordinatorJobBean)36 JPAExecutorException (org.apache.oozie.executor.jpa.JPAExecutorException)28 OutputStream (java.io.OutputStream)27 FileOutputStream (java.io.FileOutputStream)25