Examples with MetaStoreException org.smartdata.metastore.MetaStoreException used on opensource projects

Example 1 with MetaStoreException

use of org.smartdata.metastore.MetaStoreException in project SSM by Intel-bigdata.

the class SmallFileScheduler method syncMetaStore.

/**
 * Sync compact file states with meta store.
 */
private void syncMetaStore() {
    List<CompactFileState> compactFileStates = new ArrayList<>();
    // Get compact file states from compactFileStateQueue
    for (int i = 0; i < META_STORE_INSERT_BATCH_SIZE; i++) {
        CompactFileState compactFileState = compactFileStateQueue.poll();
        if (compactFileState != null) {
            try {
                FileInfo info = metaStore.getFile(compactFileState.getPath());
                if (info != null && info.getLength() == 0) {
                    LOG.debug(String.format("Ready to insert the file state of %s.", compactFileState.getPath()));
                    compactFileStates.add(compactFileState);
                } else {
                    LOG.debug(String.format("Waiting for the small file %s synced in the meta store.", compactFileState.getPath()));
                    compactFileStateQueue.offer(compactFileState);
                }
            } catch (MetaStoreException e) {
                LOG.error("Failed to get file info.", e);
                compactFileStateQueue.offer(compactFileState);
            }
        } else {
            break;
        }
    }
    // Batch insert compact file states into meta store
    try {
        if (!compactFileStates.isEmpty()) {
            metaStore.insertCompactFileStates(compactFileStates.toArray(new CompactFileState[0]));
            for (CompactFileState fileState : compactFileStates) {
                handlingSmallFileCache.remove(fileState.getPath());
            }
        }
    } catch (MetaStoreException e) {
        for (CompactFileState fileState : compactFileStates) {
            handlingSmallFileCache.remove(fileState.getPath());
        }
        LOG.error("Failed to update file state of meta store.", e);
    }
}

Example 2 with MetaStoreException

use of org.smartdata.metastore.MetaStoreException in project SSM by Intel-bigdata.

the class SmallFileScheduler method getCompactScheduleResult.

/**
 * Get compact action schedule result according to action info.
 */
private ScheduleResult getCompactScheduleResult(ActionInfo actionInfo) {
    // Get container file and small file list of this action
    String containerFilePath = getContainerFile(actionInfo);
    ArrayList<String> smallFileList = new Gson().fromJson(actionInfo.getArgs().get(HdfsAction.FILE_PATH), new TypeToken<ArrayList<String>>() {
    }.getType());
    // Check if container file is locked and retry
    if (containerFileLock.contains(containerFilePath)) {
        return ScheduleResult.RETRY;
    } else {
        // Check if the small file list is valid
        if (!checkIfValidSmallFiles(smallFileList)) {
            actionInfo.setResult("Small file list is invalid.");
            return ScheduleResult.FAIL;
        }
        // Get container file permission
        SmartFilePermission containerFilePermission;
        try {
            containerFilePermission = getContainerFilePermission(actionInfo, containerFilePath);
        } catch (MetaStoreException e1) {
            actionInfo.setResult(String.format("Failed to get file info of the container file %s for %s.", containerFilePath, e1.toString()));
            return ScheduleResult.FAIL;
        } catch (IllegalArgumentException e2) {
            actionInfo.setResult(e2.getMessage());
            return ScheduleResult.FAIL;
        }
        // Get first small file info
        FileInfo firstFileInfo;
        SmartFilePermission firstFilePermission;
        try {
            firstFileInfo = metaStore.getFile(smallFileList.get(0));
            firstFilePermission = new SmartFilePermission(firstFileInfo);
        } catch (MetaStoreException e) {
            actionInfo.setResult(String.format("Failed to get first file info: %s.", containerFilePath));
            return ScheduleResult.FAIL;
        }
        // and its permission is null
        if (containerFilePermission == null) {
            Map<String, String> args = new HashMap<>(3);
            args.put(SmallFileCompactAction.CONTAINER_FILE, getContainerFile(actionInfo));
            args.put(SmallFileCompactAction.FILE_PATH, new Gson().toJson(smallFileList));
            args.put(SmallFileCompactAction.CONTAINER_FILE_PERMISSION, new Gson().toJson(firstFilePermission));
            actionInfo.setArgs(args);
        } else {
            if (!containerFilePermission.equals(firstFilePermission)) {
                actionInfo.setResult(String.format("Container file %s has different permission with %s.", containerFilePath, firstFileInfo.getPath()));
                return ScheduleResult.FAIL;
            }
        }
        // Lock container file and small files
        containerFileLock.add(containerFilePath);
        compactSmallFileLock.addAll(smallFileList);
        afterSchedule(actionInfo);
        return ScheduleResult.SUCCESS;
    }
}

Example 3 with MetaStoreException

use of org.smartdata.metastore.MetaStoreException in project SSM by Intel-bigdata.

the class SmallFileScheduler method checkIfValidSmallFiles.

/**
 * Check if the small file list is valid.
 */
private boolean checkIfValidSmallFiles(List<String> smallFileList) {
    for (String smallFile : smallFileList) {
        if (smallFile == null || smallFile.isEmpty()) {
            LOG.error("Illegal small file path: {}", smallFile);
            return false;
        } else if (compactSmallFileLock.contains(smallFile)) {
            LOG.error(String.format("%s is locked.", smallFile));
            return false;
        } else if (handlingSmallFileCache.contains(smallFile)) {
            LOG.error(String.format("%s is being handling.", smallFile));
            return false;
        } else if (containerFileCache.contains(smallFile) || containerFileLock.contains(smallFile)) {
            LOG.error(String.format("%s is container file.", smallFile));
            return false;
        }
    }
    // Get small file info list and file state map from meta store.
    List<FileInfo> fileInfos;
    Map<String, FileState> fileStateMap;
    try {
        fileInfos = metaStore.getFilesByPaths(smallFileList);
        fileStateMap = metaStore.getFileStates(smallFileList);
    } catch (MetaStoreException e) {
        LOG.error("Failed to get file states of small files.", e);
        return false;
    }
    // Get small file info map
    Map<String, FileInfo> fileInfoMap = new HashMap<>();
    for (FileInfo fileInfo : fileInfos) {
        fileInfoMap.put(fileInfo.getPath(), fileInfo);
    }
    // Check if the permission of small file is same,
    // and all the small files exist
    FileInfo firstFileInfo = null;
    for (String smallFile : smallFileList) {
        FileInfo fileInfo = fileInfoMap.get(smallFile);
        if (fileInfo != null) {
            if (firstFileInfo == null) {
                firstFileInfo = fileInfo;
            } else {
                if (!(new SmartFilePermission(firstFileInfo)).equals(new SmartFilePermission(fileInfo))) {
                    LOG.debug(String.format("%s has different file permission with %s.", firstFileInfo.getPath(), fileInfo.getPath()));
                    return false;
                }
            }
        } else {
            LOG.debug("{} is not exist!!!", smallFile);
            return false;
        }
    }
    // Check if the state of small file is NORMAL
    for (Map.Entry<String, FileState> entry : fileStateMap.entrySet()) {
        String smallFile = entry.getKey();
        FileState.FileType smallFileType = entry.getValue().getFileType();
        if (smallFileType != FileState.FileType.NORMAL) {
            LOG.debug(String.format("%s has invalid file state %s for small file compact.", smallFile, smallFileType.toString()));
            return false;
        }
    }
    return true;
}

Example 4 with MetaStoreException

use of org.smartdata.metastore.MetaStoreException in project SSM by Intel-bigdata.

the class SmallFileScheduler method start.

@Override
public void start() throws IOException {
    executorService.scheduleAtFixedRate(new ScheduleTask(), 100, 50, TimeUnit.MILLISECONDS);
    try {
        List<String> containerFileList = metaStore.getAllContainerFiles();
        this.containerFileCache.addAll(containerFileList);
    } catch (MetaStoreException e) {
        throw new IOException(e);
    }
}

Example 5 with MetaStoreException

use of org.smartdata.metastore.MetaStoreException in project SSM by Intel-bigdata.

the class NamespaceFetcher method startFetch.

public void startFetch() throws IOException {
    try {
        init(conf);
        metaStore.deleteAllEcPolicies();
        Map<Byte, String> idToPolicyName = CompatibilityHelperLoader.getHelper().getErasureCodingPolicies(client);
        if (idToPolicyName != null) {
            ArrayList<ErasureCodingPolicyInfo> ecInfos = new ArrayList<>();
            for (Byte id : idToPolicyName.keySet()) {
                ecInfos.add(new ErasureCodingPolicyInfo(id, idToPolicyName.get(id)));
            }
            metaStore.insertEcPolicies(ecInfos);
            LOG.info("Finished fetching all EC policies!");
        }
    } catch (MetaStoreException e) {
        throw new IOException("Failed to clean and fetch EC policies!");
    }
    try {
        metaStore.deleteAllFileInfo();
    } catch (MetaStoreException e) {
        throw new IOException("Error while reset files", e);
    }
    this.fetchTaskFutures = new ScheduledFuture[ingestionTasks.length];
    for (int i = 0; i < ingestionTasks.length; i++) {
        fetchTaskFutures[i] = this.scheduledExecutorService.scheduleAtFixedRate(ingestionTasks[i], 0, fetchInterval, TimeUnit.MILLISECONDS);
    }
    this.consumerFutures = new ScheduledFuture[consumers.length];
    for (int i = 0; i < consumers.length; i++) {
        consumerFutures[i] = this.scheduledExecutorService.scheduleAtFixedRate(consumers[i], 0, fetchInterval, TimeUnit.MILLISECONDS);
    }
    LOG.info("Started.");
}