Examples with DagEngine org.apache.oozie.DagEngine used on opensource projects

Example 6 with DagEngine

use of org.apache.oozie.DagEngine in project oozie by apache.

the class V0JobsServlet method submitJob.

/**
 * v0 service implementation to submit a workflow job
 */
@Override
protected JSONObject submitJob(HttpServletRequest request, Configuration conf) throws XServletException, IOException {
    JSONObject json = new JSONObject();
    try {
        String action = request.getParameter(RestConstants.ACTION_PARAM);
        if (action != null && !action.equals(RestConstants.JOB_ACTION_START)) {
            throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0303, RestConstants.ACTION_PARAM, action);
        }
        boolean startJob = (action != null);
        String user = conf.get(OozieClient.USER_NAME);
        DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user);
        String id = dagEngine.submitJob(conf, startJob);
        json.put(JsonTags.JOB_ID, id);
    } catch (DagEngineException ex) {
        throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
    }
    return json;
}

Example 7 with DagEngine

use of org.apache.oozie.DagEngine in project oozie by apache.

the class V0JobsServlet method getJobs.

/**
 * v0 service implementation to get a list of workflows, with filtering or interested windows embedded in the
 * request object
 */
@Override
protected JSONObject getJobs(HttpServletRequest request) throws XServletException, IOException {
    JSONObject json;
    try {
        String filter = request.getParameter(RestConstants.JOBS_FILTER_PARAM);
        String startStr = request.getParameter(RestConstants.OFFSET_PARAM);
        String lenStr = request.getParameter(RestConstants.LEN_PARAM);
        int start = (startStr != null) ? Integer.parseInt(startStr) : 1;
        start = (start < 1) ? 1 : start;
        int len = (lenStr != null) ? Integer.parseInt(lenStr) : 50;
        len = (len < 1) ? 50 : len;
        DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(getUser(request));
        WorkflowsInfo jobs = dagEngine.getJobs(filter, start, len);
        json = OozieJsonFactory.getWFJSONObject(jobs, "GMT");
    } catch (DagEngineException ex) {
        throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
    }
    return json;
}

Example 8 with DagEngine

use of org.apache.oozie.DagEngine in project oozie by apache.

the class TestRecoveryService method testWorkflowActionRecoveryService.

/**
 * Tests functionality of the Recovery Service Runnable command. </p> Starts an action which behaves like an Async
 * Action (Action and Job state set to Running). Changes the action configuration to run in sync mode and updates
 * the store. Runs the recovery runnable, and ensures the state of the action and job have not changed. </p> Changes
 * the state of the action from RUNNING to PREP and updates the store. Again, runs the recovery runnable and ensures
 * the state changes to OK and the job completes successfully.
 *
 * @throws Exception
 */
public void testWorkflowActionRecoveryService() throws Exception {
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    createTestCaseSubDir("lib");
    IOUtils.copyCharStream(reader, writer);
    final DagEngine engine = new DagEngine(getTestUser());
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, getTestCaseFileUri("workflow.xml"));
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("external-status", "ok");
    conf.set("signal-value", "based_on_action_status");
    conf.set("running-mode", "async");
    // TODO CHECK, without this we get JPA concurrency exceptions, ODD
    sleep(1000);
    final String jobId = engine.submitJob(conf, true);
    // TODO CHECK, without this we get JPA concurrency exceptions, ODD
    sleep(1000);
    waitFor(5000, new Predicate() {

        public boolean evaluate() throws Exception {
            return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.RUNNING);
        }
    });
    sleep(1000);
    final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
    store.beginTrx();
    List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId, false);
    WorkflowActionBean action = null;
    for (WorkflowActionBean bean : actions) {
        if (bean.getType().equals("test")) {
            action = bean;
            break;
        }
    }
    assertNotNull(action);
    final String actionId = action.getId();
    assertEquals(WorkflowActionBean.Status.RUNNING, action.getStatus());
    String actionConf = action.getConf();
    String fixedActionConf = actionConf.replaceAll("async", "sync");
    action.setConf(fixedActionConf);
    action.setPending();
    store.updateAction(action);
    store.commitTrx();
    store.closeTrx();
    Runnable recoveryRunnable = new RecoveryRunnable(0, 60, 60);
    recoveryRunnable.run();
    sleep(3000);
    final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create();
    assertEquals(WorkflowJob.Status.RUNNING, engine.getJob(jobId).getStatus());
    store2.beginTrx();
    WorkflowActionBean action2 = store2.getAction(actionId, false);
    assertEquals(WorkflowActionBean.Status.RUNNING, action2.getStatus());
    action2.setStatus(WorkflowActionBean.Status.PREP);
    action2.setPending();
    store2.updateAction(action2);
    store2.commitTrx();
    store2.closeTrx();
    sleep(1000);
    recoveryRunnable.run();
    sleep(3000);
    waitFor(10000, new Predicate() {

        public boolean evaluate() throws Exception {
            return (engine.getWorkflowAction(actionId).getStatus() == WorkflowActionBean.Status.OK);
        }
    });
    // getPendingActions works correctly only with MYSQL - following assertsfail with hsql - to be investigated
    // assertEquals(WorkflowJob.Status.SUCCEEDED, engine.getJob(jobId).getStatus());
    final WorkflowStore store3 = Services.get().get(WorkflowStoreService.class).create();
    store3.beginTrx();
    WorkflowActionBean action3 = store3.getAction(actionId, false);
    assertEquals(WorkflowActionBean.Status.OK, action3.getStatus());
    store3.commitTrx();
    store3.closeTrx();
}

Example 9 with DagEngine

use of org.apache.oozie.DagEngine in project oozie by apache.

the class TestAuthorizationService method _testAuthorizationService.

private void _testAuthorizationService(boolean useDefaultGroup) throws Exception {
    init(useDefaultGroup, true);
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);
    final DagEngine engine = new DagEngine(getTestUser());
    Configuration jobConf = new XConfiguration();
    jobConf.set(OozieClient.APP_PATH, getTestCaseFileUri("workflow.xml"));
    jobConf.set(OozieClient.USER_NAME, getTestUser());
    if (useDefaultGroup) {
        jobConf.set(OozieClient.GROUP_NAME, getTestGroup());
    } else {
        jobConf.set(OozieClient.GROUP_NAME, getTestGroup() + ",foogrp");
    }
    jobConf.set(OozieClient.LOG_TOKEN, "t");
    jobConf.set("external-status", "ok");
    jobConf.set("signal-value", "based_on_action_status");
    final String jobId = engine.submitJob(jobConf, true);
    HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
    URI uri = getFileSystem().getUri();
    Configuration fsConf = has.createConfiguration(uri.getAuthority());
    FileSystem fileSystem = has.createFileSystem(getTestUser(), uri, fsConf);
    Path path = new Path(fileSystem.getWorkingDirectory(), UUID.randomUUID().toString());
    Path fsTestDir = fileSystem.makeQualified(path);
    System.out.println(XLog.format("Setting FS testcase work dir[{0}]", fsTestDir));
    fileSystem.delete(fsTestDir, true);
    if (!fileSystem.mkdirs(path)) {
        throw new IOException(XLog.format("Could not create FS testcase dir [{0}]", fsTestDir));
    }
    String appPath = fsTestDir.toString() + "/app";
    Path jobXmlPath = new Path(appPath, "workflow.xml");
    fileSystem.create(jobXmlPath).close();
    fileSystem.setOwner(jobXmlPath, getTestUser(), getTestGroup());
    FsPermission permissions = new FsPermission(FsAction.READ_WRITE, FsAction.READ, FsAction.NONE);
    fileSystem.setPermission(jobXmlPath, permissions);
    AuthorizationService as = services.get(AuthorizationService.class);
    assertNotNull(as);
    as.authorizeForGroup(getTestUser(), getTestGroup());
    assertNotNull(as.getDefaultGroup(getTestUser()));
    as.authorizeForApp(getTestUser2(), getTestGroup(), appPath, jobConf);
    try {
        as.authorizeForApp(getTestUser3(), getTestGroup(), appPath, jobConf);
        fail();
    } catch (AuthorizationException ex) {
    }
    as.authorizeForJob(getTestUser(), jobId, false);
    as.authorizeForJob(getTestUser(), jobId, true);
    if (!useDefaultGroup) {
        as.authorizeForJob("foo", jobId, true);
    }
    try {
        as.authorizeForJob("bar", jobId, true);
        fail();
    } catch (AuthorizationException ex) {
    }
}

Example 10 with DagEngine

use of org.apache.oozie.DagEngine in project oozie by apache.

the class SubWorkflowActionExecutor method getWorkflowClient.

protected OozieClient getWorkflowClient(Context context, String oozieUri) {
    OozieClient oozieClient;
    if (oozieUri.equals(LOCAL)) {
        WorkflowJobBean workflow = (WorkflowJobBean) context.getWorkflow();
        String user = workflow.getUser();
        String group = workflow.getGroup();
        DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user);
        oozieClient = new LocalOozieClient(dagEngine);
    } else {
        // TODO we need to add authToken to the WC for the remote case
        oozieClient = new OozieClient(oozieUri);
    }
    return oozieClient;
}