Examples with CallableQueueService org.apache.oozie.service.CallableQueueService used on opensource projects

Example 1 with CallableQueueService

use of org.apache.oozie.service.CallableQueueService in project oozie by apache.

the class CoordPushDependencyCheckXCommand method execute.

@Override
protected Void execute() throws CommandException {
    // this action should only get processed if current time > nominal time;
    // otherwise, requeue this action for delay execution;
    Date nominalTime = coordAction.getNominalTime();
    Date currentTime = new Date();
    if (nominalTime.compareTo(currentTime) > 0) {
        queue(new CoordPushDependencyCheckXCommand(coordAction.getId(), true), nominalTime.getTime() - currentTime.getTime());
        updateCoordAction(coordAction, false);
        LOG.info("[" + actionId + "]::CoordPushDependency:: nominal Time is newer than current time, so requeue and wait. Current=" + DateUtils.formatDateOozieTZ(currentTime) + ", nominal=" + DateUtils.formatDateOozieTZ(nominalTime));
        return null;
    }
    CoordInputDependency coordPushInputDependency = coordAction.getPushInputDependencies();
    CoordInputDependency coordPullInputDependency = coordAction.getPullInputDependencies();
    if (coordPushInputDependency.getMissingDependenciesAsList().size() == 0) {
        LOG.info("Nothing to check. Empty push missing dependency");
    } else {
        List<String> missingDependenciesArray = coordPushInputDependency.getMissingDependenciesAsList();
        LOG.info("First Push missing dependency is [{0}] ", missingDependenciesArray.get(0));
        LOG.trace("Push missing dependencies are [{0}] ", missingDependenciesArray);
        if (registerForNotification) {
            LOG.debug("Register for notifications is true");
        }
        try {
            Configuration actionConf = null;
            try {
                actionConf = new XConfiguration(new StringReader(coordAction.getRunConf()));
            } catch (IOException e) {
                throw new CommandException(ErrorCode.E1307, e.getMessage(), e);
            }
            boolean isChangeInDependency = true;
            boolean timeout = false;
            ActionDependency actionDependency = coordPushInputDependency.checkPushMissingDependencies(coordAction, registerForNotification);
            // CoordActionInputCheckXCommand for efficiency. listPartitions is costly.
            if (actionDependency.getMissingDependencies().size() == missingDependenciesArray.size()) {
                isChangeInDependency = false;
            } else {
                String stillMissingDeps = DependencyChecker.dependenciesAsString(actionDependency.getMissingDependencies());
                coordPushInputDependency.setMissingDependencies(stillMissingDeps);
            }
            if (coordPushInputDependency.isDependencyMet()) {
                // All push-based dependencies are available
                onAllPushDependenciesAvailable(coordPullInputDependency.isDependencyMet());
            } else {
                // Checking for timeout
                timeout = isTimeout();
                if (timeout) {
                    queue(new CoordActionTimeOutXCommand(coordAction, coordJob.getUser(), coordJob.getAppName()));
                } else {
                    queue(new CoordPushDependencyCheckXCommand(coordAction.getId()), getCoordPushCheckRequeueInterval());
                }
            }
            updateCoordAction(coordAction, isChangeInDependency || coordPushInputDependency.isDependencyMet());
            if (registerForNotification) {
                registerForNotification(coordPushInputDependency.getMissingDependenciesAsList(), actionConf);
            }
            if (removeAvailDependencies) {
                unregisterAvailableDependencies(actionDependency.getAvailableDependencies());
            }
            if (timeout) {
                unregisterMissingDependencies(coordPushInputDependency.getMissingDependenciesAsList(), actionId);
            }
        } catch (Exception e) {
            final CallableQueueService callableQueueService = Services.get().get(CallableQueueService.class);
            if (isTimeout()) {
                LOG.debug("Queueing timeout command");
                // XCommand.queue() will not work when there is a Exception
                callableQueueService.queue(new CoordActionTimeOutXCommand(coordAction, coordJob.getUser(), coordJob.getAppName()));
                unregisterMissingDependencies(missingDependenciesArray, actionId);
            } else if (coordPullInputDependency.getMissingDependenciesAsList().size() > 0) {
                // Queue again on exception as RecoveryService will not queue this again with
                // the action being updated regularly by CoordActionInputCheckXCommand
                callableQueueService.queue(new CoordPushDependencyCheckXCommand(coordAction.getId(), registerForNotification, removeAvailDependencies), Services.get().getConf().getInt(RecoveryService.CONF_COORD_OLDER_THAN, 600) * 1000);
            }
            throw new CommandException(ErrorCode.E1021, e.getMessage(), e);
        }
    }
    return null;
}

Example 2 with CallableQueueService

use of org.apache.oozie.service.CallableQueueService in project oozie by apache.

the class XCommand method call.

/**
 * Implements the XCommand life-cycle.
 *
 * @return the {link #execute} return value.
 * @throws CommandException thrown if the command could not be executed.
 */
@Override
public final T call() throws CommandException {
    setLogInfo();
    CallableQueueService callableQueueService = Services.get().get(CallableQueueService.class);
    Set<String> interruptTypes = callableQueueService.getInterruptTypes();
    if (interruptTypes.contains(this.getType()) && used.get()) {
        LOG.debug("Command [{0}] key [{1}]  already used for [{2}]", getName(), getEntityKey(), this.toString());
        return null;
    }
    commandQueue = null;
    instrumentation.incr(INSTRUMENTATION_GROUP, getName() + ".executions", 1);
    Instrumentation.Cron callCron = new Instrumentation.Cron();
    try {
        callCron.start();
        eagerLoadState();
        eagerVerifyPrecondition();
        try {
            T ret = null;
            if (isLockRequired() && !this.inInterruptMode()) {
                Instrumentation.Cron acquireLockCron = new Instrumentation.Cron();
                acquireLockCron.start();
                acquireLock();
                acquireLockCron.stop();
                instrumentation.addCron(INSTRUMENTATION_GROUP, getName() + ".acquireLock", acquireLockCron);
            }
            // executing interrupts only in case of the lock required commands
            if (lock != null) {
                this.executeInterrupts();
            }
            if (!isLockRequired() || (lock != null) || this.inInterruptMode()) {
                if (interruptTypes.contains(this.getType()) && !used.compareAndSet(false, true)) {
                    LOG.debug("Command [{0}] key [{1}]  already executed for [{2}]", getName(), getEntityKey(), this.toString());
                    return null;
                }
                LOG.trace("Load state for [{0}]", getEntityKey());
                loadState();
                LOG.trace("Precondition check for command [{0}] key [{1}]", getName(), getEntityKey());
                verifyPrecondition();
                LOG.debug("Execute command [{0}] key [{1}]", getName(), getEntityKey());
                Instrumentation.Cron executeCron = new Instrumentation.Cron();
                executeCron.start();
                ret = execute();
                executeCron.stop();
                instrumentation.addCron(INSTRUMENTATION_GROUP, getName() + ".execute", executeCron);
            }
            if (commandQueue != null) {
                for (Map.Entry<Long, List<XCommand<?>>> entry : commandQueue.entrySet()) {
                    LOG.debug("Queuing [{0}] commands with delay [{1}]ms", entry.getValue().size(), entry.getKey());
                    if (!callableQueueService.queueSerial(entry.getValue(), entry.getKey())) {
                        LOG.warn("Could not queue [{0}] commands with delay [{1}]ms, queue full", entry.getValue().size(), entry.getKey());
                    }
                }
            }
            return ret;
        } finally {
            if (isLockRequired() && !this.inInterruptMode()) {
                releaseLock();
            }
        }
    } catch (PreconditionException pex) {
        LOG.warn(pex.getMessage().toString() + ", Error Code: " + pex.getErrorCode().toString());
        instrumentation.incr(INSTRUMENTATION_GROUP, getName() + ".preconditionfailed", 1);
        return null;
    } catch (XException ex) {
        LOG.error("XException, ", ex);
        instrumentation.incr(INSTRUMENTATION_GROUP, getName() + ".xexceptions", 1);
        if (ex instanceof CommandException) {
            throw (CommandException) ex;
        } else {
            throw new CommandException(ex);
        }
    } catch (Exception ex) {
        LOG.error("Exception, ", ex);
        instrumentation.incr(INSTRUMENTATION_GROUP, getName() + ".exceptions", 1);
        throw new CommandException(ErrorCode.E0607, getName(), ex.getMessage(), ex);
    } catch (Error er) {
        LOG.error("Error, ", er);
        instrumentation.incr(INSTRUMENTATION_GROUP, getName() + ".errors", 1);
        throw er;
    } finally {
        FaultInjection.deactivate("org.apache.oozie.command.SkipCommitFaultInjection");
        callCron.stop();
        instrumentation.addCron(INSTRUMENTATION_GROUP, getName() + ".call", callCron);
    }
}

Example 3 with CallableQueueService

use of org.apache.oozie.service.CallableQueueService in project oozie by apache.

the class XCommand method executeInterrupts.

/**
 * Check for the existence of interrupts for the same lock key
 * Execute them if exist.
 */
protected void executeInterrupts() {
    CallableQueueService callableQueueService = Services.get().get(CallableQueueService.class);
    // getting all the list of interrupts to be executed
    Set<XCallable<?>> callables = callableQueueService.checkInterrupts(this.getEntityKey());
    if (callables != null) {
        // in the list
        for (XCallable<?> callable : callables) {
            LOG.trace("executing interrupt callable [{0}]", callable.getName());
            try {
                // executing the callable in interrupt mode
                callable.setInterruptMode(true);
                callable.call();
                LOG.trace("executed interrupt callable [{0}]", callable.getName());
            } catch (Exception ex) {
                LOG.warn("exception interrupt callable [{0}], {1}", callable.getName(), ex.getMessage(), ex);
            } finally {
                // reseting the interrupt mode to false after the command is
                // executed
                callable.setInterruptMode(false);
            }
        }
    }
}

Example 4 with CallableQueueService

use of org.apache.oozie.service.CallableQueueService in project oozie by apache.

the class TestCoordKillXCommand method testCoordKillXCommandUniqueness.

public void testCoordKillXCommandUniqueness() throws Exception {
    String currentDatePlusMonth = XDataTestCase.getCurrentDateafterIncrementingInMonths(1);
    Date start = DateUtils.parseDateOozieTZ(currentDatePlusMonth);
    Date end = DateUtils.parseDateOozieTZ(currentDatePlusMonth);
    CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, start, end, false, false, 0);
    final MyCoordKillXCommand callable1 = new MyCoordKillXCommand(job.getId(), 100);
    final MyCoordKillXCommand callable2 = new MyCoordKillXCommand(job.getId(), 100);
    final MyCoordKillXCommand callable3 = new MyCoordKillXCommand(job.getId(), 100);
    List<MyCoordKillXCommand> callables = Arrays.asList(callable1, callable2, callable3);
    CallableQueueService queueservice = services.get(CallableQueueService.class);
    for (MyCoordKillXCommand c : callables) {
        queueservice.queue(c);
    }
    waitFor(1000, new Predicate() {

        public boolean evaluate() throws Exception {
            return callable1.executed != 0 && callable2.executed == 0 && callable3.executed == 0;
        }
    });
    assertTrue(callable1.executed != 0);
    assertTrue(callable2.executed == 0);
    assertTrue(callable3.executed == 0);
}

Example 5 with CallableQueueService

use of org.apache.oozie.service.CallableQueueService in project oozie by apache.

the class TestCoordPushDependencyCheckXCommand method testRequeueOnException.

@Test
public void testRequeueOnException() throws Exception {
    Services.get().getConf().setInt(RecoveryService.CONF_SERVICE_INTERVAL, 6000);
    // Test timeout when table containing missing dependencies is dropped in between
    String newHCatDependency1 = "hcat://" + server + "/nodb/notable/dt=20120430;country=brazil";
    String newHCatDependency2 = "hcat://" + server + "/nodb/notable/dt=20120430;country=usa";
    String newHCatDependency = newHCatDependency1 + CoordELFunctions.INSTANCE_SEPARATOR + newHCatDependency2;
    CoordinatorJobBean job = addRecordToCoordJobTableForWaiting("coord-job-for-action-input-check.xml", CoordinatorJob.Status.RUNNING, false, true);
    CoordinatorActionBean action = addRecordToCoordActionTableForWaiting(job.getId(), 1, CoordinatorAction.Status.WAITING, "coord-action-for-action-input-check.xml", null, newHCatDependency, "Z");
    String actionId = action.getId();
    checkCoordAction(actionId, newHCatDependency, CoordinatorAction.Status.WAITING);
    try {
        new CoordPushDependencyCheckXCommand(actionId, true).call();
        fail();
    } catch (Exception e) {
        assertTrue(e.getMessage().contains("NoSuchObjectException"));
    }
    // Nothing should be queued as there are no pull dependencies
    CallableQueueService callableQueueService = Services.get().get(CallableQueueService.class);
    assertEquals(0, callableQueueService.getQueueDump().size());
    // Nothing should be queued as there are no pull missing dependencies
    // but only push missing deps are there
    new CoordActionInputCheckXCommand(actionId, job.getId()).call();
    callableQueueService = Services.get().get(CallableQueueService.class);
    assertEquals(0, callableQueueService.getQueueDump().size());
    setMissingDependencies(actionId, newHCatDependency1);
    try {
        new CoordPushDependencyCheckXCommand(actionId, true).call();
        fail();
    } catch (Exception e) {
        assertTrue(e.getMessage().contains("NoSuchObjectException"));
    }
    // Should be requeued at the recovery service interval
    final List<String> queueDump = callableQueueService.getQueueDump();
    assertEquals(1, callableQueueService.getQueueDump().size());
    assertTrue(queueDump.get(0).contains("coord_push_dep_check"));
    log.info("Queue dump is " + queueDump.toString());
    // Delay should be something like delay=599999 (ignore last three digits) or delay=600000
    assertTrue(queueDump.get(0).matches(".* delay=(599[0-9]{3}|600000)"));
}