Examples with WorkflowStore org.apache.oozie.store.WorkflowStore used on opensource projects

Example 1 with WorkflowStore

use of org.apache.oozie.store.WorkflowStore in project oozie by apache.

the class TestRecoveryService method testWorkflowActionRecoveryService.

/**
 * Tests functionality of the Recovery Service Runnable command. </p> Starts an action which behaves like an Async
 * Action (Action and Job state set to Running). Changes the action configuration to run in sync mode and updates
 * the store. Runs the recovery runnable, and ensures the state of the action and job have not changed. </p> Changes
 * the state of the action from RUNNING to PREP and updates the store. Again, runs the recovery runnable and ensures
 * the state changes to OK and the job completes successfully.
 *
 * @throws Exception
 */
public void testWorkflowActionRecoveryService() throws Exception {
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    createTestCaseSubDir("lib");
    IOUtils.copyCharStream(reader, writer);
    final DagEngine engine = new DagEngine(getTestUser());
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, getTestCaseFileUri("workflow.xml"));
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("external-status", "ok");
    conf.set("signal-value", "based_on_action_status");
    conf.set("running-mode", "async");
    // TODO CHECK, without this we get JPA concurrency exceptions, ODD
    sleep(1000);
    final String jobId = engine.submitJob(conf, true);
    // TODO CHECK, without this we get JPA concurrency exceptions, ODD
    sleep(1000);
    waitFor(5000, new Predicate() {

        public boolean evaluate() throws Exception {
            return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.RUNNING);
        }
    });
    sleep(1000);
    final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
    store.beginTrx();
    List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId, false);
    WorkflowActionBean action = null;
    for (WorkflowActionBean bean : actions) {
        if (bean.getType().equals("test")) {
            action = bean;
            break;
        }
    }
    assertNotNull(action);
    final String actionId = action.getId();
    assertEquals(WorkflowActionBean.Status.RUNNING, action.getStatus());
    String actionConf = action.getConf();
    String fixedActionConf = actionConf.replaceAll("async", "sync");
    action.setConf(fixedActionConf);
    action.setPending();
    store.updateAction(action);
    store.commitTrx();
    store.closeTrx();
    Runnable recoveryRunnable = new RecoveryRunnable(0, 60, 60);
    recoveryRunnable.run();
    sleep(3000);
    final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create();
    assertEquals(WorkflowJob.Status.RUNNING, engine.getJob(jobId).getStatus());
    store2.beginTrx();
    WorkflowActionBean action2 = store2.getAction(actionId, false);
    assertEquals(WorkflowActionBean.Status.RUNNING, action2.getStatus());
    action2.setStatus(WorkflowActionBean.Status.PREP);
    action2.setPending();
    store2.updateAction(action2);
    store2.commitTrx();
    store2.closeTrx();
    sleep(1000);
    recoveryRunnable.run();
    sleep(3000);
    waitFor(10000, new Predicate() {

        public boolean evaluate() throws Exception {
            return (engine.getWorkflowAction(actionId).getStatus() == WorkflowActionBean.Status.OK);
        }
    });
    // getPendingActions works correctly only with MYSQL - following assertsfail with hsql - to be investigated
    // assertEquals(WorkflowJob.Status.SUCCEEDED, engine.getJob(jobId).getStatus());
    final WorkflowStore store3 = Services.get().get(WorkflowStoreService.class).create();
    store3.beginTrx();
    WorkflowActionBean action3 = store3.getAction(actionId, false);
    assertEquals(WorkflowActionBean.Status.OK, action3.getStatus());
    store3.commitTrx();
    store3.closeTrx();
}

Example 2 with WorkflowStore

use of org.apache.oozie.store.WorkflowStore in project oozie by apache.

the class ForTestWorkflowStoreService method create.

public WorkflowStore create() throws StoreException {
    final WorkflowStore wfs = super.create();
    return new WorkflowStore() {

        public void insertWorkflow(WorkflowJobBean workflow) throws StoreException {
        // To change body of implemented methods use File | Settings | File Templates.
        }

        public WorkflowJobBean getWorkflow(String id, boolean locking) throws StoreException {
            WorkflowJobBean wf = new WorkflowJobBean();
            wf.setId(id);
            wf.setUser("u");
            wf.setGroup("g");
            return wf;
        }

        public WorkflowJobBean getWorkflowInfo(String id) throws StoreException {
            // To change body of implemented methods use File | Settings | File Templates.
            return null;
        }

        public String getWorkflowIdForExternalId(String extId) throws StoreException {
            // To change body of implemented methods use File | Settings | File Templates.
            return null;
        }

        public void updateWorkflow(WorkflowJobBean workflow) throws StoreException {
        // To change body of implemented methods use File | Settings | File Templates.
        }

        public int getWorkflowCountWithStatus(String status) throws StoreException {
            // To change body of implemented methods use File | Settings | File Templates.
            return 0;
        }

        public int getWorkflowCountWithStatusInLastNSeconds(String status, int secs) throws StoreException {
            // To change body of implemented methods use File | Settings | File Templates.
            return 0;
        }

        public void insertAction(WorkflowActionBean action) throws StoreException {
        // To change body of implemented methods use File | Settings | File Templates.
        }

        public WorkflowActionBean getAction(String id, boolean locking) throws StoreException {
            // To change body of implemented methods use File | Settings | File Templates.
            return null;
        }

        public void updateAction(WorkflowActionBean action) throws StoreException {
        // To change body of implemented methods use File | Settings | File Templates.
        }

        public void deleteAction(String id) throws StoreException {
        // To change body of implemented methods use File | Settings | File Templates.
        }

        public List<WorkflowActionBean> getActionsForWorkflow(String id, boolean locking) throws StoreException {
            // To change body of implemented methods use File | Settings | File Templates.
            return null;
        }

        public List<WorkflowActionBean> getPendingActions(long minimumPendingAgeSecs) throws StoreException {
            // To change body of implemented methods use File | Settings | File Templates.
            return null;
        }

        public List<WorkflowActionBean> getRunningActions(long checkAgeSecs) throws StoreException {
            // To change body of implemented methods use File | Settings | File Templates.
            return null;
        }

        public WorkflowsInfo getWorkflowsInfo(Map<String, List<String>> filter, int start, int len) throws StoreException {
            // To change body of implemented methods use File | Settings | File Templates.
            return null;
        }

        public void purge(long olderThanDays) throws StoreException {
        // To change body of implemented methods use File | Settings | File Templates.
        }

        public void commit() throws StoreException {
        // To change body of implemented methods use File | Settings | File Templates.
        }

        public void close() throws StoreException {
        // To change body of implemented methods use File | Settings | File Templates.
        }
    };
}

Example 3 with WorkflowStore

use of org.apache.oozie.store.WorkflowStore in project oozie by apache.

the class TestActionErrors method _testNonTransient.

/**
 * Provides functionality to test non transient failures.
 *
 * @param errorType the error type. (start.non-transient, end.non-transient)
 * @param expStatus1 expected status. (START_MANUAL, END_MANUAL)
 * @param expErrorMsg expected error message.
 * @throws Exception
 */
private void _testNonTransient(String errorType, WorkflowActionBean.Status expStatus1, String expErrorMsg) throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);
    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("signal-value", "OK");
    conf.set("external-status", "ok");
    conf.set("error", errorType);
    final String jobId = engine.submitJob(conf, true);
    waitFor(5000, new Predicate() {

        public boolean evaluate() throws Exception {
            return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED);
        }
    });
    final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
    store.beginTrx();
    List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId, true);
    int n = actions.size();
    WorkflowActionBean action = actions.get(n - 1);
    assertEquals("TEST_ERROR", action.getErrorCode());
    assertEquals(expErrorMsg, action.getErrorMessage());
    assertEquals(expStatus1, action.getStatus());
    assertTrue(action.isPending() == false);
    assertTrue(engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUSPENDED);
    String actionConf = action.getConf();
    String fixedActionConf = actionConf.replaceAll(errorType, "none");
    action.setConf(fixedActionConf);
    store.updateAction(action);
    store.commitTrx();
    store.closeTrx();
    engine.resume(jobId);
    waitFor(5000, new Predicate() {

        public boolean evaluate() throws Exception {
            return (engine.getJob(jobId).getStatus() == WorkflowJob.Status.SUCCEEDED);
        }
    });
    assertEquals(WorkflowJob.Status.SUCCEEDED, engine.getJob(jobId).getStatus());
    final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create();
    store2.beginTrx();
    actions = store2.getActionsForWorkflow(jobId, false);
    action = actions.get(0);
    assertEquals(null, action.getErrorCode());
    assertEquals(null, action.getErrorMessage());
    assertEquals(WorkflowActionBean.Status.OK, action.getStatus());
    store2.commitTrx();
    store2.closeTrx();
}

Example 4 with WorkflowStore

use of org.apache.oozie.store.WorkflowStore in project oozie by apache.

the class TestActionErrors method _testError.

/**
 * Provides functionality to test errors
 *
 * @param errorType the error type. (start.non-transient, end.non-transient)
 * @param externalStatus the external status to set.
 * @param signalValue the signal value to set.
 * @throws Exception
 */
private void _testError(String errorType, String externalStatus, String signalValue) throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);
    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("error", errorType);
    conf.set("external-status", externalStatus);
    conf.set("signal-value", signalValue);
    final String jobId = engine.submitJob(conf, true);
    final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
    store.beginTrx();
    waitFor(5000, new Predicate() {

        public boolean evaluate() throws Exception {
            WorkflowJobBean bean = store.getWorkflow(jobId, false);
            return (bean.getWorkflowInstance().getStatus() == WorkflowInstance.Status.KILLED);
        }
    });
    assertEquals(WorkflowJob.Status.KILLED, engine.getJob(jobId).getStatus());
    store.commitTrx();
    store.closeTrx();
}

Example 5 with WorkflowStore

use of org.apache.oozie.store.WorkflowStore in project oozie by apache.

the class TestActionErrors method _testTransient.

/**
 * Provides functionality to test transient failures.
 *
 * @param errorType the error type. (start.transient, end.transient)
 * @param expStatus1 expected status after the first step (START_RETRY, END_RETRY)
 * @param expStatus2 expected status after the second step (START_MANUAL, END_MANUAL)
 * @param expErrorMsg the expected error message.
 * @throws Exception
 */
private void _testTransient(String errorType, WorkflowActionBean.Status expStatus1, final WorkflowActionBean.Status expStatus2, String expErrorMsg) throws Exception {
    String workflowPath = getTestCaseFileUri("workflow.xml");
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);
    final int maxRetries = 2;
    final int retryInterval = 10;
    final DagEngine engine = new DagEngine("u");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, workflowPath);
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set(OozieClient.LOG_TOKEN, "t");
    conf.set("signal-value", "OK");
    conf.set("external-status", "ok");
    conf.set("error", errorType);
    conf.setInt(OozieClient.ACTION_MAX_RETRIES, maxRetries);
    conf.setInt(OozieClient.ACTION_RETRY_INTERVAL, retryInterval);
    final String jobId = engine.submitJob(conf, true);
    int retryCount = 1;
    WorkflowActionBean.Status expectedStatus = expStatus1;
    int expectedRetryCount = 2;
    Thread.sleep(20000);
    String aId = null;
    final WorkflowStore store = Services.get().get(WorkflowStoreService.class).create();
    store.beginTrx();
    while (retryCount <= maxRetries) {
        List<WorkflowActionBean> actions = store.getActionsForWorkflow(jobId, false);
        WorkflowActionBean action = null;
        for (WorkflowActionBean bean : actions) {
            if (bean.getType().equals("test")) {
                action = bean;
                break;
            }
        }
        assertNotNull(action);
        aId = action.getId();
        assertEquals(expectedStatus, action.getStatus());
        assertEquals(expectedRetryCount, action.getRetries());
        assertEquals("TEST_ERROR", action.getErrorCode());
        assertEquals(expErrorMsg, action.getErrorMessage());
        if (action.getRetries() == maxRetries) {
            expectedRetryCount = 0;
            expectedStatus = expStatus2;
            break;
        } else {
            expectedRetryCount++;
        }
        Thread.sleep(retryInterval * 1000);
        retryCount++;
    }
    store.commitTrx();
    store.closeTrx();
    Thread.sleep(5000);
    final String actionId = aId;
    waitFor(5000, new Predicate() {

        public boolean evaluate() throws Exception {
            return (engine.getWorkflowAction(actionId).getStatus() == expStatus2);
        }
    });
    final WorkflowStore store2 = Services.get().get(WorkflowStoreService.class).create();
    store2.beginTrx();
    WorkflowActionBean action = engine.getWorkflowAction(actionId);
    assertEquals("TEST_ERROR", action.getErrorCode());
    assertEquals(expErrorMsg, action.getErrorMessage());
    assertEquals(expStatus2, action.getStatus());
    assertTrue(action.isPending() == false);
    assertEquals(WorkflowJob.Status.SUSPENDED, engine.getJob(jobId).getStatus());
    store2.commitTrx();
    store2.closeTrx();
}