Examples with RuleExecutorPlugin org.smartdata.model.rule.RuleExecutorPlugin used on opensource projects

Example 1 with RuleExecutorPlugin

use of org.smartdata.model.rule.RuleExecutorPlugin in project SSM by Intel-bigdata.

the class RuleExecutor method submitCmdlets.

private int submitCmdlets(RuleInfo ruleInfo, List<String> files) {
    long ruleId = ruleInfo.getId();
    if (files == null || files.size() == 0 || ruleManager.getCmdletManager() == null) {
        return 0;
    }
    int nSubmitted = 0;
    List<RuleExecutorPlugin> plugins = RuleExecutorPluginManager.getPlugins();
    String template = tr.getCmdDescriptor().toCmdletString();
    for (String file : files) {
        if (!exited) {
            try {
                CmdletDescriptor cmd = new CmdletDescriptor(template, ruleId);
                cmd.setCmdletParameter(CmdletDescriptor.HDFS_FILE_PATH, file);
                for (RuleExecutorPlugin plugin : plugins) {
                    cmd = plugin.preSubmitCmdletDescriptor(ruleInfo, tr, cmd);
                }
                long cid = ruleManager.getCmdletManager().submitCmdlet(cmd);
                // Not really submitted if cid is -1.
                if (cid != -1) {
                    nSubmitted++;
                }
            } catch (QueueFullException e) {
                break;
            } catch (IOException e) {
                // it's common here, ignore this and continue submit
                LOG.debug("Failed to submit cmdlet for file: " + file, e);
            } catch (ParseException e) {
                LOG.error("Failed to submit cmdlet for file: " + file, e);
            }
        } else {
            break;
        }
    }
    return nSubmitted;
}

Example 2 with RuleExecutorPlugin

use of org.smartdata.model.rule.RuleExecutorPlugin in project SSM by Intel-bigdata.

the class RuleInfoRepo method doLaunchExecutor.

private RuleExecutor doLaunchExecutor(RuleManager ruleManager) throws IOException {
    RuleState state = ruleInfo.getState();
    if (state == RuleState.ACTIVE || state == RuleState.DRYRUN) {
        if (executor != null && !executor.isExited()) {
            return null;
        }
        ExecutionContext ctx = new ExecutionContext();
        ctx.setRuleId(ruleInfo.getId());
        TranslationContext transCtx = new TranslationContext(ruleInfo.getId(), ruleInfo.getSubmitTime());
        TranslateResult tr = executor != null ? executor.getTranslateResult() : new SmartRuleStringParser(ruleInfo.getRuleText(), transCtx, conf).translate();
        List<RuleExecutorPlugin> plugins = RuleExecutorPluginManager.getPlugins();
        for (RuleExecutorPlugin plugin : plugins) {
            plugin.onNewRuleExecutor(ruleInfo, tr);
        }
        executor = new RuleExecutor(ruleManager, ctx, tr, ruleManager.getMetaStore());
        return executor;
    }
    return null;
}

Example 3 with RuleExecutorPlugin

use of org.smartdata.model.rule.RuleExecutorPlugin in project SSM by Intel-bigdata.

the class RuleExecutor method run.

@Override
public void run() {
    long startCheckTime = System.currentTimeMillis();
    if (exited) {
        exitSchedule();
    }
    if (!tr.getTbScheduleInfo().isExecutable(startCheckTime)) {
        return;
    }
    List<RuleExecutorPlugin> plugins = RuleExecutorPluginManager.getPlugins();
    long rid = ctx.getRuleId();
    try {
        if (ruleManager.isClosed()) {
            exitSchedule();
        }
        long endCheckTime;
        int numCmdSubmitted = 0;
        List<String> files = new ArrayList<>();
        RuleInfo info = ruleManager.getRuleInfo(rid);
        boolean doExec = true;
        for (RuleExecutorPlugin plugin : plugins) {
            doExec &= plugin.preExecution(info, tr);
            if (!doExec) {
                break;
            }
        }
        RuleState state = info.getState();
        if (exited || state == RuleState.DELETED || state == RuleState.FINISHED || state == RuleState.DISABLED) {
            exitSchedule();
        }
        TimeBasedScheduleInfo scheduleInfo = tr.getTbScheduleInfo();
        if (!scheduleInfo.isOnce() && scheduleInfo.getEndTime() != TimeBasedScheduleInfo.FOR_EVER) {
            boolean befExit = false;
            if (scheduleInfo.isOneShot()) {
                // The subScheduleTime is set in triggering time.
                if (scheduleInfo.getSubScheduleTime() > scheduleInfo.getEndTime()) {
                    befExit = true;
                }
            } else if (startCheckTime - scheduleInfo.getEndTime() > 0) {
                befExit = true;
            }
            if (befExit) {
                LOG.info("Rule " + ctx.getRuleId() + " exit rule executor due to time passed");
                ruleManager.updateRuleInfo(rid, RuleState.FINISHED, startCheckTime, 0, 0);
                exitSchedule();
            }
        }
        if (doExec) {
            files = executeFileRuleQuery();
            if (exited) {
                exitSchedule();
            }
        }
        endCheckTime = System.currentTimeMillis();
        if (doExec) {
            for (RuleExecutorPlugin plugin : plugins) {
                files = plugin.preSubmitCmdlet(info, files);
            }
            numCmdSubmitted = submitCmdlets(info, files);
        }
        ruleManager.updateRuleInfo(rid, null, startCheckTime, 1, numCmdSubmitted);
        long endProcessTime = System.currentTimeMillis();
        if (endProcessTime - startCheckTime > 2000 || LOG.isDebugEnabled()) {
            LOG.warn("Rule " + ctx.getRuleId() + " execution took " + (endProcessTime - startCheckTime) + "ms. QueryTime = " + (endCheckTime - startCheckTime) + "ms, SubmitTime = " + (endProcessTime - endCheckTime) + "ms, fileNum = " + numCmdSubmitted + ".");
        }
        if (scheduleInfo.isOneShot()) {
            ruleManager.updateRuleInfo(rid, RuleState.FINISHED, startCheckTime, 0, 0);
            exitSchedule();
        }
        if (endProcessTime + scheduleInfo.getBaseEvery() > scheduleInfo.getEndTime()) {
            LOG.info("Rule " + ctx.getRuleId() + " exit rule executor due to finished");
            ruleManager.updateRuleInfo(rid, RuleState.FINISHED, startCheckTime, 0, 0);
            exitSchedule();
        }
        if (exited) {
            exitSchedule();
        }
    } catch (IOException e) {
        LOG.error("Rule " + ctx.getRuleId() + " exception", e);
    }
}