Examples with MetaStoreException org.smartdata.metastore.MetaStoreException used on opensource projects

Example 11 with MetaStoreException

use of org.smartdata.metastore.MetaStoreException in project SSM by Intel-bigdata.

the class CmdletManager method reloadCmdletsInDB.

private void reloadCmdletsInDB() throws IOException {
    LOG.info("reloading the dispatched and pending cmdlets in DB.");
    List<CmdletInfo> cmdletInfos;
    try {
        cmdletInfos = metaStore.getCmdlets(CmdletState.DISPATCHED);
        if (cmdletInfos != null && cmdletInfos.size() != 0) {
            for (CmdletInfo cmdletInfo : cmdletInfos) {
                // track reload cmdlets
                CmdletDescriptor cmdletDescriptor = CmdletDescriptor.fromCmdletString(cmdletInfo.getParameters());
                cmdletDescriptor.setRuleId(cmdletInfo.getRid());
                tracker.track(cmdletInfo.getCid(), cmdletDescriptor);
                List<ActionInfo> actionInfos = getActions(cmdletInfo.getAids());
                for (ActionInfo actionInfo : actionInfos) {
                    actionInfo.setCreateTime(cmdletInfo.getGenerateTime());
                    actionInfo.setFinishTime(System.currentTimeMillis());
                    // Recover scheduler status according to dispatched action.
                    recoverSchedulerStatus(actionInfo);
                }
                syncCmdAction(cmdletInfo, actionInfos);
            }
        }
        cmdletInfos = metaStore.getCmdlets(CmdletState.PENDING);
        if (cmdletInfos != null && cmdletInfos.size() != 0) {
            for (CmdletInfo cmdletInfo : cmdletInfos) {
                CmdletDescriptor cmdletDescriptor = CmdletDescriptor.fromCmdletString(cmdletInfo.getParameters());
                cmdletDescriptor.setRuleId(cmdletInfo.getRid());
                // Pending task also needs to be tracked.
                tracker.track(cmdletInfo.getCid(), cmdletDescriptor);
                LOG.debug(String.format("Reload pending cmdlet: {}", cmdletInfo));
                List<ActionInfo> actionInfos = getActions(cmdletInfo.getAids());
                syncCmdAction(cmdletInfo, actionInfos);
            }
        }
    } catch (MetaStoreException e) {
        LOG.error("DB connection error occurs when ssm is reloading cmdlets!");
        return;
    } catch (ParseException pe) {
        LOG.error("Failed to parse cmdlet string for tracking task", pe);
    }
}

Example 12 with MetaStoreException

use of org.smartdata.metastore.MetaStoreException in project SSM by Intel-bigdata.

the class SmallFilePlugin method preSubmitCmdletDescriptor.

@Override
public CmdletDescriptor preSubmitCmdletDescriptor(final RuleInfo ruleInfo, TranslateResult tResult, CmdletDescriptor descriptor) {
    for (int i = 0; i < descriptor.getActionSize(); i++) {
        if (COMPACT_ACTION_NAME.equals(descriptor.getActionName(i))) {
            String smallFiles = descriptor.getActionArgs(i).get(HdfsAction.FILE_PATH);
            if (smallFiles != null && !smallFiles.isEmpty()) {
                // Check if small file list is empty
                ArrayList<String> smallFileList = new Gson().fromJson(smallFiles, new TypeToken<ArrayList<String>>() {
                }.getType());
                if (smallFileList == null || smallFileList.isEmpty()) {
                    continue;
                }
                // Get the first small file info
                String firstFile = smallFileList.get(0);
                FileInfo firstFileInfo;
                if (firstFileInfoCache.containsKey(firstFile)) {
                    firstFileInfo = firstFileInfoCache.get(firstFile);
                } else {
                    try {
                        firstFileInfo = metaStore.getFile(firstFile);
                        if (firstFileInfo == null) {
                            LOG.debug("{} is not exist!!!", firstFile);
                            continue;
                        }
                    } catch (MetaStoreException e) {
                        LOG.error(String.format("Failed to get file info of: %s.", firstFile), e);
                        continue;
                    }
                }
                // Get valid compact action arguments
                SmartFilePermission firstFilePermission = new SmartFilePermission(firstFileInfo);
                String firstFileDir = firstFile.substring(0, firstFile.lastIndexOf("/") + 1);
                CompactActionArgs args = getCompactActionArgs(ruleInfo, firstFileDir, firstFilePermission, smallFileList);
                // Set container file path and its permission, file path of this action
                descriptor.addActionArg(i, SmallFileCompactAction.CONTAINER_FILE, args.containerFile);
                descriptor.addActionArg(i, SmallFileCompactAction.CONTAINER_FILE_PERMISSION, new Gson().toJson(args.containerFilePermission));
                descriptor.addActionArg(i, HdfsAction.FILE_PATH, new Gson().toJson(args.smartFiles));
            }
        }
    }
    return descriptor;
}

Example 13 with MetaStoreException

use of org.smartdata.metastore.MetaStoreException in project SSM by Intel-bigdata.

the class SmallFilePlugin method getCompactActionArgs.

/**
 * Get valid compact action arguments.
 */
private CompactActionArgs getCompactActionArgs(RuleInfo ruleInfo, String firstFileDir, SmartFilePermission firstFilePermission, List<String> smallFileList) {
    Map<String, FileInfo> containerFileMap = containerFileInfoCache.get(ruleInfo);
    for (Iterator<Map.Entry<String, FileInfo>> iter = containerFileMap.entrySet().iterator(); iter.hasNext(); ) {
        Map.Entry<String, FileInfo> entry = iter.next();
        String containerFilePath = entry.getKey();
        FileInfo containerFileInfo = entry.getValue();
        iter.remove();
        // Get compact action arguments
        String containerFileDir = containerFilePath.substring(0, containerFilePath.lastIndexOf("/") + 1);
        if (firstFileDir.equals(containerFileDir) && firstFilePermission.equals(new SmartFilePermission(containerFileInfo))) {
            List<String> validSmallFiles;
            try {
                validSmallFiles = getValidSmallFiles(containerFileInfo, smallFileList);
            } catch (MetaStoreException e) {
                LOG.error("Failed to get file info of small files.", e);
                continue;
            }
            if (validSmallFiles != null) {
                return new CompactActionArgs(containerFilePath, null, validSmallFiles);
            }
        }
    }
    return genCompactActionArgs(firstFileDir, firstFilePermission, smallFileList);
}

Example 14 with MetaStoreException

use of org.smartdata.metastore.MetaStoreException in project SSM by Intel-bigdata.

the class RuleManager method init.

/**
 * Init RuleManager, this includes: 1. Load related data from local storage or HDFS 2. Initial
 *
 * @throws IOException
 */
@Override
public void init() throws IOException {
    LOG.info("Initializing ...");
    // Load rules table
    List<RuleInfo> rules = null;
    try {
        rules = metaStore.getRuleInfo();
    } catch (MetaStoreException e) {
        LOG.error("Can not load rules from database:\n" + e.getMessage());
    }
    for (RuleInfo rule : rules) {
        mapRules.put(rule.getId(), new RuleInfoRepo(rule, metaStore, serverContext.getConf()));
    }
    LOG.info("Initialized. Totally " + rules.size() + " rules loaded from DataBase.");
    if (LOG.isDebugEnabled()) {
        for (RuleInfo info : rules) {
            LOG.debug("\t" + info);
        }
    }
}

Example 15 with MetaStoreException

use of org.smartdata.metastore.MetaStoreException in project SSM by Intel-bigdata.

the class RuleManager method submitRule.

/**
 * Submit a rule to RuleManger.
 *
 * @param rule
 * @param initState
 * @return
 * @throws IOException
 */
public long submitRule(String rule, RuleState initState) throws IOException {
    LOG.debug("Received Rule -> [" + rule + "]");
    if (initState != RuleState.ACTIVE && initState != RuleState.DISABLED && initState != RuleState.DRYRUN) {
        throw new IOException("Invalid initState = " + initState + ", it MUST be one of [" + RuleState.ACTIVE + ", " + RuleState.DRYRUN + ", " + RuleState.DISABLED + "]");
    }
    TranslateResult tr = doCheckRule(rule, null);
    doCheckActions(tr.getCmdDescriptor());
    // check whitelist
    if (WhitelistHelper.isEnabled(serverContext.getConf())) {
        for (String path : tr.getGlobPathCheck()) {
            if (!WhitelistHelper.isInWhitelist(path, serverContext.getConf())) {
                throw new IOException("Path " + path + " is not in the whitelist.");
            }
        }
    }
    RuleInfo.Builder builder = RuleInfo.newBuilder();
    builder.setRuleText(rule).setState(initState);
    RuleInfo ruleInfo = builder.build();
    RulePluginManager.onAddingNewRule(ruleInfo, tr);
    try {
        metaStore.insertNewRule(ruleInfo);
    } catch (MetaStoreException e) {
        throw new IOException("RuleText = " + rule, e);
    }
    RuleInfoRepo infoRepo = new RuleInfoRepo(ruleInfo, metaStore, serverContext.getConf());
    mapRules.put(ruleInfo.getId(), infoRepo);
    submitRuleToScheduler(infoRepo.launchExecutor(this));
    RulePluginManager.onNewRuleAdded(ruleInfo, tr);
    return ruleInfo.getId();
}