Examples with WorkflowJob org.apache.oozie.client.WorkflowJob used on opensource projects

Example 11 with WorkflowJob

use of org.apache.oozie.client.WorkflowJob in project oozie by apache.

the class TestLastModified method testWorkflowRun.

public void testWorkflowRun() throws Exception {
    String wfApp = "<workflow-app xmlns='uri:oozie:workflow:0.1' name='test-wf'>" + "    <start to='end'/>" + "    <end name='end'/>" + "</workflow-app>";
    FileSystem fs = getFileSystem();
    Path appPath = new Path(getFsTestCaseDir(), "app");
    fs.mkdirs(appPath);
    fs.mkdirs(new Path(appPath, "lib"));
    fs.mkdirs(new Path("input-data"));
    Writer inputWriter = new OutputStreamWriter(fs.create(new Path("input-data/data1.txt")));
    inputWriter.write("Hello. This is my input data set.");
    inputWriter.close();
    Path workflowPath = new Path(appPath, "workflow.xml");
    Writer writer = new OutputStreamWriter(fs.create(workflowPath));
    writer.write(wfApp);
    writer.close();
    try {
        LocalOozie.start();
        final OozieClient wc = LocalOozie.getClient();
        Properties conf = wc.createConfiguration();
        conf.setProperty(OozieClient.APP_PATH, workflowPath.toString());
        conf.setProperty(OozieClient.USER_NAME, getTestUser());
        conf.setProperty(OozieClient.GROUP_NAME, getTestGroup());
        final String jobId = wc.submit(conf);
        assertNotNull(jobId);
        WorkflowJob wf = wc.getJobInfo(jobId);
        assertNotNull(wf);
        assertEquals(WorkflowJob.Status.PREP, wf.getStatus());
        boolean dateTest = wf.getLastModifiedTime().compareTo(wf.getCreatedTime()) >= 0 ? true : false;
        assertEquals(true, dateTest);
        wc.start(jobId);
        wf = wc.getJobInfo(jobId);
        Date lastModTime = wf.getLastModifiedTime();
        wc.suspend(jobId);
        wf = wc.getJobInfo(jobId);
        dateTest = wf.getLastModifiedTime().compareTo(lastModTime) >= 0 ? true : false;
        assertEquals(true, dateTest);
        lastModTime = wf.getLastModifiedTime();
        sleep(1000);
        wc.resume(jobId);
        wf = wc.getJobInfo(jobId);
        dateTest = wf.getLastModifiedTime().compareTo(lastModTime) >= 0 ? true : false;
        assertEquals(true, dateTest);
        waitFor(600000, new Predicate() {

            public boolean evaluate() throws Exception {
                WorkflowJob wf = wc.getJobInfo(jobId);
                return wf.getStatus() == WorkflowJob.Status.SUCCEEDED;
            }
        });
        wf = wc.getJobInfo(jobId);
        assertNotNull(wf);
        assertEquals(WorkflowJob.Status.SUCCEEDED, wf.getStatus());
        dateTest = wf.getLastModifiedTime().compareTo(wf.getEndTime()) >= 0 ? true : false;
        assertEquals(true, dateTest);
    } finally {
        LocalOozie.stop();
    }
}

Example 12 with WorkflowJob

use of org.apache.oozie.client.WorkflowJob in project oozie by apache.

the class TestSignalXCommand method checkSuspendActions.

private void checkSuspendActions(WorkflowJob wf, final OozieClient oc, final String jobId, final WorkflowJob.Status status, String[] prepActions, String[] okActions) throws Exception {
    // Wait for the WF to transition to status
    waitFor(30 * 1000, new Predicate() {

        public boolean evaluate() throws Exception {
            WorkflowJob wf = oc.getJobInfo(jobId);
            return wf.getStatus() == status;
        }
    });
    wf = oc.getJobInfo(jobId);
    assertEquals(status, wf.getStatus());
    // Check the actions' statuses
    int numPrep = 0;
    int numOK = 0;
    for (WorkflowAction action : wf.getActions()) {
        boolean checked = false;
        for (String name : prepActions) {
            if (!checked && name.equals(action.getName())) {
                assertEquals("action [" + action.getName() + "] had incorrect status", WorkflowAction.Status.PREP, action.getStatus());
                numPrep++;
                checked = true;
            }
        }
        if (!checked) {
            for (String name : okActions) {
                if (!checked && name.equals(action.getName())) {
                    assertEquals("action [" + action.getName() + "] had incorrect status", WorkflowAction.Status.OK, action.getStatus());
                    numOK++;
                    checked = true;
                }
            }
        }
        if (!checked) {
            fail("Unexpected action [" + action.getName() + "] with status [" + action.getStatus() + "]");
        }
    }
    assertEquals(prepActions.length, numPrep);
    assertEquals(okActions.length, numOK);
}

Example 13 with WorkflowJob

use of org.apache.oozie.client.WorkflowJob in project oozie by apache.

the class TestSignalXCommand method _testSuspendPointsAll.

public void _testSuspendPointsAll() throws Exception {
    services.destroy();
    LocalOozie.start();
    FileSystem fs = getFileSystem();
    Path appPath = new Path(getFsTestCaseDir(), "app");
    fs.mkdirs(appPath);
    Reader reader = IOUtils.getResourceAsReader("wf-suspendpoints.xml", -1);
    Writer writer = new OutputStreamWriter(fs.create(new Path(appPath, "workflow.xml")));
    IOUtils.copyCharStream(reader, writer);
    writer.close();
    reader.close();
    final OozieClient oc = LocalOozie.getClient();
    Properties conf = oc.createConfiguration();
    conf.setProperty(OozieClient.APP_PATH, new Path(appPath, "workflow.xml").toString());
    conf.setProperty(OozieClient.USER_NAME, getTestUser());
    conf.setProperty("oozie.suspend.on.nodes", "*");
    final String jobId = oc.submit(conf);
    assertNotNull(jobId);
    WorkflowJob wf = oc.getJobInfo(jobId);
    assertEquals(WorkflowJob.Status.PREP, wf.getStatus());
    oc.start(jobId);
    checkSuspendActions(wf, oc, jobId, WorkflowJob.Status.SUSPENDED, new String[] { "action1" }, new String[] { ":start:" });
    oc.resume(jobId);
    checkSuspendActions(wf, oc, jobId, WorkflowJob.Status.SUSPENDED, new String[] { "action2" }, new String[] { ":start:", "action1" });
    oc.resume(jobId);
    checkSuspendActions(wf, oc, jobId, WorkflowJob.Status.SUSPENDED, new String[] { "decision1" }, new String[] { ":start:", "action1", "action2" });
    oc.resume(jobId);
    checkSuspendActions(wf, oc, jobId, WorkflowJob.Status.SUSPENDED, new String[] { "action3" }, new String[] { ":start:", "action1", "action2", "decision1" });
    oc.resume(jobId);
    checkSuspendActions(wf, oc, jobId, WorkflowJob.Status.SUSPENDED, new String[] { "fork1" }, new String[] { ":start:", "action1", "action2", "decision1", "action3" });
    oc.resume(jobId);
    checkSuspendActions(wf, oc, jobId, WorkflowJob.Status.SUSPENDED, new String[] { "action4a", "action4b", "action4c" }, new String[] { ":start:", "action1", "action2", "decision1", "action3", "fork1" });
    oc.resume(jobId);
    checkSuspendActions(wf, oc, jobId, WorkflowJob.Status.SUSPENDED, new String[] { "join1" }, new String[] { ":start:", "action1", "action2", "decision1", "action3", "fork1", "action4a", "action4b", "action4c" });
    oc.resume(jobId);
    checkSuspendActions(wf, oc, jobId, WorkflowJob.Status.SUSPENDED, new String[] { "end" }, new String[] { ":start:", "action1", "action2", "decision1", "action3", "fork1", "action4a", "action4b", "action4c", "join1" });
    oc.resume(jobId);
    checkSuspendActions(wf, oc, jobId, WorkflowJob.Status.SUCCEEDED, new String[] {}, new String[] { ":start:", "action1", "action2", "decision1", "action3", "fork1", "action4a", "action4b", "action4c", "join1", "end" });
    LocalOozie.stop();
}

Example 14 with WorkflowJob

use of org.apache.oozie.client.WorkflowJob in project oozie by apache.

the class HCatELFunctions method hcat_exists.

/* Workflow Parameterization EL functions */
/**
 * Return true if partitions exists or false if not.
 *
 * @param uri hcatalog partition uri.
 * @return <code>true</code> if the uri exists, <code>false</code> if it does not.
 * @throws Exception
 */
public static boolean hcat_exists(String uri) throws Exception {
    URI hcatURI = new URI(uri);
    URIHandlerService uriService = Services.get().get(URIHandlerService.class);
    URIHandler handler = uriService.getURIHandler(hcatURI);
    WorkflowJob workflow = DagELFunctions.getWorkflow();
    String user = workflow.getUser();
    return handler.exists(hcatURI, EMPTY_CONF, user);
}

Example 15 with WorkflowJob

use of org.apache.oozie.client.WorkflowJob in project oozie by apache.

the class TestJavaActionExecutor method testChildKill.

public void testChildKill() throws Exception {
    final JobConf clusterConf = createJobConf();
    FileSystem fileSystem = FileSystem.get(clusterConf);
    Path confFile = new Path("/tmp/cluster-conf.xml");
    OutputStream out = fileSystem.create(confFile);
    clusterConf.writeXml(out);
    out.close();
    String confFileName = fileSystem.makeQualified(confFile).toString() + "#core-site.xml";
    final String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<main-class> " + SleepJob.class.getName() + " </main-class>" + "<arg>-mt</arg>" + "<arg>300000</arg>" + "<archive>" + confFileName + "</archive>" + "</java>";
    final Context context = createContext(actionXml, null);
    final String runningJob = submitAction(context);
    YarnApplicationState state = waitUntilYarnAppState(runningJob, EnumSet.of(YarnApplicationState.RUNNING));
    assertEquals(YarnApplicationState.RUNNING, state);
    WorkflowJob wfJob = context.getWorkflow();
    Configuration conf = null;
    if (wfJob.getConf() != null) {
        conf = new XConfiguration(new StringReader(wfJob.getConf()));
    }
    String launcherTag = LauncherHelper.getActionYarnTag(conf, wfJob.getParentId(), context.getAction());
    JavaActionExecutor ae = new JavaActionExecutor();
    final Configuration jobConf = ae.createBaseHadoopConf(context, XmlUtils.parseXml(actionXml));
    jobConf.set(LauncherMain.CHILD_MAPREDUCE_JOB_TAGS, LauncherHelper.getTag(launcherTag));
    jobConf.setLong(LauncherMain.OOZIE_JOB_LAUNCH_TIME, context.getAction().getStartTime().getTime());
    // We have to use a proper UGI for retrieving the child apps, because the WF is
    // submitted as a test user, not as the current login user
    UserGroupInformationService ugiService = Services.get().get(UserGroupInformationService.class);
    final UserGroupInformation ugi = ugiService.getProxyUser(getTestUser());
    final Set<ApplicationId> childSet = new HashSet<>();
    // wait until we have a child MR job
    waitFor(60_000, new Predicate() {

        @Override
        public boolean evaluate() throws Exception {
            return ugi.doAs(new PrivilegedExceptionAction<Boolean>() {

                @Override
                public Boolean run() throws Exception {
                    childSet.clear();
                    childSet.addAll(LauncherMain.getChildYarnJobs(jobConf));
                    return childSet.size() > 0;
                }
            });
        }
    });
    assertEquals(1, childSet.size());
    // kill the action - based on the job tag, the SleepJob is expected to be killed too
    ae.kill(context, context.getAction());
    HadoopAccessorService hadoopAccessorService = getHadoopAccessorService();
    Configuration config = hadoopAccessorService.createConfiguration(getJobTrackerUri());
    YarnClient yarnClient = hadoopAccessorService.createYarnClient(getTestUser(), config);
    // check that both the launcher & MR job were successfully killed
    ApplicationId jobId = childSet.iterator().next();
    assertEquals(YarnApplicationState.KILLED, yarnClient.getApplicationReport(jobId).getYarnApplicationState());
    assertTrue(ae.isCompleted(context.getAction().getExternalStatus()));
    assertEquals(WorkflowAction.Status.DONE, context.getAction().getStatus());
    assertEquals(JavaActionExecutor.KILLED, context.getAction().getExternalStatus());
    assertEquals(FinalApplicationStatus.KILLED, yarnClient.getApplicationReport(ConverterUtils.toApplicationId(runningJob)).getFinalApplicationStatus());
}