Examples with MetaStoreException org.smartdata.metastore.MetaStoreException used on opensource projects

Example 26 with MetaStoreException

use of org.smartdata.metastore.MetaStoreException in project SSM by Intel-bigdata.

the class RuleExecutor method getAccessCountTablesDuringLast.

/**
 * @param lastInterval
 * @return
 */
private List<String> getAccessCountTablesDuringLast(long lastInterval) {
    List<String> tableNames = new ArrayList<>();
    if (ruleManager == null || ruleManager.getStatesManager() == null) {
        return tableNames;
    }
    List<AccessCountTable> accTables = null;
    try {
        accTables = ruleManager.getStatesManager().getTablesInLast(lastInterval);
    } catch (MetaStoreException e) {
        LOG.error("Rule " + ctx.getRuleId() + " get access info tables exception", e);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Rule " + ctx.getRuleId() + " got " + accTables.size() + " tables:");
        int idx = 1;
        for (AccessCountTable t : accTables) {
            LOG.debug(idx + ".  " + (t.isEphemeral() ? " [TABLE] " : "        ") + t.getTableName() + " ");
        }
    }
    if (accTables == null || accTables.size() == 0) {
        return tableNames;
    }
    for (AccessCountTable t : accTables) {
        tableNames.add(t.getTableName());
        if (t.isEphemeral()) {
            dynamicCleanups.push("DROP TABLE IF EXISTS " + t.getTableName() + ";");
        }
    }
    return tableNames;
}

Example 27 with MetaStoreException

use of org.smartdata.metastore.MetaStoreException in project SSM by Intel-bigdata.

the class SmallFilePlugin method getContainerFileInfos.

/**
 * Get container file info map from meta store.
 */
private Map<String, FileInfo> getContainerFileInfos() {
    Map<String, FileInfo> ret = new LinkedHashMap<>();
    try {
        List<String> containerFiles = metaStore.getAllContainerFiles();
        if (!containerFiles.isEmpty()) {
            List<FileInfo> fileInfos = metaStore.getFilesByPaths(containerFiles);
            // Sort file infos based on the file length
            Collections.sort(fileInfos, new Comparator<FileInfo>() {

                @Override
                public int compare(FileInfo a, FileInfo b) {
                    return Long.compare(a.getLength(), b.getLength());
                }
            });
            for (FileInfo fileInfo : fileInfos) {
                ret.put(fileInfo.getPath(), fileInfo);
            }
        }
    } catch (MetaStoreException e) {
        LOG.error("Failed to get file info of all the container files.", e);
    }
    return ret;
}

Example 28 with MetaStoreException

use of org.smartdata.metastore.MetaStoreException in project SSM by Intel-bigdata.

the class SmallFilePlugin method preSubmitCmdlet.

@Override
public List<String> preSubmitCmdlet(final RuleInfo ruleInfo, List<String> objects) {
    if (ruleInfo.getRuleText().contains(COMPACT_ACTION_NAME)) {
        if (objects == null || objects.isEmpty()) {
            LOG.debug("Objects is null or empty.");
            return objects;
        }
        // Split valid small files according to the file permission
        Map<String, FileInfo> containerFileInfoMap = getContainerFileInfos();
        Map<SmallFileStatus, List<String>> smallFileStateMap = new HashMap<>();
        for (String object : objects) {
            LOG.debug("Start handling the file: {}.", object);
            // Check if the file is container file
            if (!object.endsWith("/")) {
                String fileName = object.substring(object.lastIndexOf("/") + 1, object.length());
                if (fileName.startsWith(CONTAINER_FILE_PREFIX) || containerFileInfoMap.containsKey(object)) {
                    LOG.debug("{} is container file.", object);
                    continue;
                }
            }
            // Check file info and state
            try {
                FileInfo fileInfo = metaStore.getFile(object);
                FileState fileState = metaStore.getFileState(object);
                if (fileInfo != null && fileInfo.getLength() > 0 && fileInfo.getLength() < containerFileSizeThreshold && fileState.getFileType().equals(FileState.FileType.NORMAL) && fileState.getFileStage().equals(FileState.FileStage.DONE)) {
                    SmallFileStatus smallFileStatus = new SmallFileStatus(fileInfo);
                    if (smallFileStateMap.containsKey(smallFileStatus)) {
                        smallFileStateMap.get(smallFileStatus).add(object);
                    } else {
                        firstFileInfoCache.put(object, fileInfo);
                        List<String> list = new ArrayList<>();
                        list.add(object);
                        smallFileStateMap.put(smallFileStatus, list);
                    }
                } else {
                    LOG.debug("Invalid file {} for small file compact.", object);
                }
            } catch (MetaStoreException e) {
                LOG.error(String.format("Failed to get file info of %s.", object), e);
            }
        }
        // Split small files according to the batch size
        List<String> smallFileList = new ArrayList<>();
        for (List<String> listElement : smallFileStateMap.values()) {
            int size = listElement.size();
            for (int i = 0; i < size; i += batchSize) {
                int toIndex = (i + batchSize <= size) ? i + batchSize : size;
                String smallFiles = new Gson().toJson(listElement.subList(i, toIndex));
                smallFileList.add(smallFiles);
            }
        }
        // Update container file info cache for preSubmitCmdletDescriptor
        updateContainerFileInfoCache(ruleInfo, containerFileInfoMap);
        return smallFileList;
    } else {
        return objects;
    }
}

Example 29 with MetaStoreException

use of org.smartdata.metastore.MetaStoreException in project SSM by Intel-bigdata.

the class FileCopyDrPlugin method onNewRuleExecutor.

public void onNewRuleExecutor(final RuleInfo ruleInfo, TranslateResult tResult) {
    long ruleId = ruleInfo.getId();
    List<String> pathsCheckGlob = tResult.getGlobPathCheck();
    if (pathsCheckGlob.size() == 0) {
        pathsCheckGlob = Arrays.asList("/*");
    }
    List<String> pathsCheck = getPathMatchesList(pathsCheckGlob);
    String dirs = StringUtil.join(",", pathsCheck);
    CmdletDescriptor des = tResult.getCmdDescriptor();
    for (int i = 0; i < des.getActionSize(); i++) {
        if (des.getActionName(i).equals("sync")) {
            List<String> statements = tResult.getSqlStatements();
            String before = statements.get(statements.size() - 1);
            String after = before.replace(";", " UNION " + referenceNonExists(tResult, pathsCheck));
            statements.set(statements.size() - 1, after);
            BackUpInfo backUpInfo = new BackUpInfo();
            backUpInfo.setRid(ruleId);
            backUpInfo.setSrc(dirs);
            String dest = des.getActionArgs(i).get(SyncAction.DEST);
            if (!dest.endsWith("/")) {
                dest += "/";
                des.addActionArg(i, SyncAction.DEST, dest);
            }
            backUpInfo.setDest(dest);
            backUpInfo.setPeriod(tResult.getTbScheduleInfo().getMinimalEvery());
            des.addActionArg(i, SyncAction.SRC, dirs);
            LOG.debug("Rule executor added for sync rule {} src={}  dest={}", ruleInfo, dirs, dest);
            synchronized (backups) {
                if (!backups.containsKey(ruleId)) {
                    backups.put(ruleId, new LinkedList<BackUpInfo>());
                }
            }
            List<BackUpInfo> infos = backups.get(ruleId);
            synchronized (infos) {
                try {
                    metaStore.deleteBackUpInfo(ruleId);
                    // Add base Sync tag
                    FileDiff fileDiff = new FileDiff(FileDiffType.BASESYNC);
                    fileDiff.setSrc(backUpInfo.getSrc());
                    fileDiff.getParameters().put("-dest", backUpInfo.getDest());
                    metaStore.insertFileDiff(fileDiff);
                    metaStore.insertBackUpInfo(backUpInfo);
                    infos.add(backUpInfo);
                } catch (MetaStoreException e) {
                    LOG.error("Insert backup info error:" + backUpInfo, e);
                }
            }
            break;
        }
    }
}

Example 30 with MetaStoreException

use of org.smartdata.metastore.MetaStoreException in project SSM by Intel-bigdata.

the class RuleInfoRepo method changeRuleState.

private boolean changeRuleState(RuleState newState, boolean updateDb) throws IOException {
    RuleState oldState = ruleInfo.getState();
    if (newState == null || oldState == newState) {
        return false;
    }
    try {
        switch(newState) {
            case ACTIVE:
                if (oldState == RuleState.DISABLED || oldState == RuleState.DRYRUN) {
                    ruleInfo.setState(newState);
                    if (updateDb && metaStore != null) {
                        metaStore.updateRuleState(ruleInfo.getId(), newState);
                    }
                    return true;
                }
                break;
            case DISABLED:
                if (oldState == RuleState.ACTIVE || oldState == RuleState.DRYRUN) {
                    ruleInfo.setState(newState);
                    markWorkExit();
                    if (updateDb && metaStore != null) {
                        metaStore.updateRuleState(ruleInfo.getId(), newState);
                    }
                    return true;
                }
                break;
            case DELETED:
                ruleInfo.setState(newState);
                markWorkExit();
                if (updateDb && metaStore != null) {
                    metaStore.updateRuleState(ruleInfo.getId(), newState);
                }
                return true;
            case FINISHED:
                if (oldState == RuleState.ACTIVE || oldState == RuleState.DRYRUN) {
                    ruleInfo.setState(newState);
                    if (updateDb && metaStore != null) {
                        metaStore.updateRuleState(ruleInfo.getId(), newState);
                    }
                    return true;
                }
                break;
        }
    } catch (MetaStoreException e) {
        throw new IOException(ruleInfo.toString(), e);
    }
    throw new IOException("This rule state transition is not supported: " + oldState.name() + " -> " + // TODO: unsupported
    newState.name());
}