Examples with FileInfo org.smartdata.model.FileInfo used on opensource projects

Example 1 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class TestNamespaceFetcher method init.

NamespaceFetcher init(MiniDFSCluster cluster, SmartConf conf) throws IOException, InterruptedException, MissingEventsException, MetaStoreException {
    final DistributedFileSystem dfs = cluster.getFileSystem();
    dfs.mkdir(new Path("/user"), new FsPermission("777"));
    dfs.create(new Path("/user/user1"));
    dfs.create(new Path("/user/user2"));
    dfs.mkdir(new Path("/tmp"), new FsPermission("777"));
    DFSClient client = dfs.getClient();
    MetaStore adapter = Mockito.mock(MetaStore.class);
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocationOnMock) {
            try {
                Object[] objects = invocationOnMock.getArguments();
                for (FileInfo fileInfo : (FileInfo[]) objects[0]) {
                    pathesInDB.add(fileInfo.getPath());
                }
            } catch (Throwable t) {
                t.printStackTrace();
            }
            return null;
        }
    }).when(adapter).insertFiles(any(FileInfo[].class));
    NamespaceFetcher fetcher;
    if (conf != null) {
        fetcher = new NamespaceFetcher(client, adapter, 100, conf);
    } else {
        fetcher = new NamespaceFetcher(client, adapter, 100);
    }
    return fetcher;
}

Example 2 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class SmallFileScheduler method syncMetaStore.

/**
 * Sync compact file states with meta store.
 */
private void syncMetaStore() {
    List<CompactFileState> compactFileStates = new ArrayList<>();
    // Get compact file states from compactFileStateQueue
    for (int i = 0; i < META_STORE_INSERT_BATCH_SIZE; i++) {
        CompactFileState compactFileState = compactFileStateQueue.poll();
        if (compactFileState != null) {
            try {
                FileInfo info = metaStore.getFile(compactFileState.getPath());
                if (info != null && info.getLength() == 0) {
                    LOG.debug(String.format("Ready to insert the file state of %s.", compactFileState.getPath()));
                    compactFileStates.add(compactFileState);
                } else {
                    LOG.debug(String.format("Waiting for the small file %s synced in the meta store.", compactFileState.getPath()));
                    compactFileStateQueue.offer(compactFileState);
                }
            } catch (MetaStoreException e) {
                LOG.error("Failed to get file info.", e);
                compactFileStateQueue.offer(compactFileState);
            }
        } else {
            break;
        }
    }
    // Batch insert compact file states into meta store
    try {
        if (!compactFileStates.isEmpty()) {
            metaStore.insertCompactFileStates(compactFileStates.toArray(new CompactFileState[0]));
            for (CompactFileState fileState : compactFileStates) {
                handlingSmallFileCache.remove(fileState.getPath());
            }
        }
    } catch (MetaStoreException e) {
        for (CompactFileState fileState : compactFileStates) {
            handlingSmallFileCache.remove(fileState.getPath());
        }
        LOG.error("Failed to update file state of meta store.", e);
    }
}

Example 3 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class SmallFileScheduler method getContainerFilePermission.

/**
 * Get container file info according to action arguments and meta store.
 */
private SmartFilePermission getContainerFilePermission(ActionInfo actionInfo, String containerFilePath) throws MetaStoreException, IllegalArgumentException {
    // Get container file permission from the argument of this action
    String containerFilePermissionArg = actionInfo.getArgs().get(SmallFileCompactAction.CONTAINER_FILE_PERMISSION);
    SmartFilePermission containerFilePermissionFromArg = null;
    if (containerFilePermissionArg != null && !containerFilePermissionArg.isEmpty()) {
        containerFilePermissionFromArg = new Gson().fromJson(containerFilePermissionArg, new TypeToken<SmartFilePermission>() {
        }.getType());
    }
    // Get container file permission from meta store
    SmartFilePermission containerFilePermissionFromMeta = null;
    FileInfo containerFileInfo = metaStore.getFile(containerFilePath);
    if (containerFileInfo != null) {
        containerFilePermissionFromMeta = new SmartFilePermission(containerFileInfo);
    }
    // Get container file permission
    SmartFilePermission containerFilePermission;
    if (containerFilePermissionFromArg == null || containerFilePermissionFromMeta == null) {
        containerFilePermission = (containerFilePermissionFromArg == null) ? containerFilePermissionFromMeta : containerFilePermissionFromArg;
    } else {
        if (containerFilePermissionFromArg.equals(containerFilePermissionFromMeta)) {
            containerFilePermission = containerFilePermissionFromArg;
        } else {
            throw new IllegalArgumentException("Illegal container file permission argument.");
        }
    }
    return containerFilePermission;
}

Example 4 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class SmallFileScheduler method getCompactScheduleResult.

/**
 * Get compact action schedule result according to action info.
 */
private ScheduleResult getCompactScheduleResult(ActionInfo actionInfo) {
    // Get container file and small file list of this action
    String containerFilePath = getContainerFile(actionInfo);
    ArrayList<String> smallFileList = new Gson().fromJson(actionInfo.getArgs().get(HdfsAction.FILE_PATH), new TypeToken<ArrayList<String>>() {
    }.getType());
    // Check if container file is locked and retry
    if (containerFileLock.contains(containerFilePath)) {
        return ScheduleResult.RETRY;
    } else {
        // Check if the small file list is valid
        if (!checkIfValidSmallFiles(smallFileList)) {
            actionInfo.setResult("Small file list is invalid.");
            return ScheduleResult.FAIL;
        }
        // Get container file permission
        SmartFilePermission containerFilePermission;
        try {
            containerFilePermission = getContainerFilePermission(actionInfo, containerFilePath);
        } catch (MetaStoreException e1) {
            actionInfo.setResult(String.format("Failed to get file info of the container file %s for %s.", containerFilePath, e1.toString()));
            return ScheduleResult.FAIL;
        } catch (IllegalArgumentException e2) {
            actionInfo.setResult(e2.getMessage());
            return ScheduleResult.FAIL;
        }
        // Get first small file info
        FileInfo firstFileInfo;
        SmartFilePermission firstFilePermission;
        try {
            firstFileInfo = metaStore.getFile(smallFileList.get(0));
            firstFilePermission = new SmartFilePermission(firstFileInfo);
        } catch (MetaStoreException e) {
            actionInfo.setResult(String.format("Failed to get first file info: %s.", containerFilePath));
            return ScheduleResult.FAIL;
        }
        // and its permission is null
        if (containerFilePermission == null) {
            Map<String, String> args = new HashMap<>(3);
            args.put(SmallFileCompactAction.CONTAINER_FILE, getContainerFile(actionInfo));
            args.put(SmallFileCompactAction.FILE_PATH, new Gson().toJson(smallFileList));
            args.put(SmallFileCompactAction.CONTAINER_FILE_PERMISSION, new Gson().toJson(firstFilePermission));
            actionInfo.setArgs(args);
        } else {
            if (!containerFilePermission.equals(firstFilePermission)) {
                actionInfo.setResult(String.format("Container file %s has different permission with %s.", containerFilePath, firstFileInfo.getPath()));
                return ScheduleResult.FAIL;
            }
        }
        // Lock container file and small files
        containerFileLock.add(containerFilePath);
        compactSmallFileLock.addAll(smallFileList);
        afterSchedule(actionInfo);
        return ScheduleResult.SUCCESS;
    }
}

Example 5 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class SmallFileScheduler method checkIfValidSmallFiles.

/**
 * Check if the small file list is valid.
 */
private boolean checkIfValidSmallFiles(List<String> smallFileList) {
    for (String smallFile : smallFileList) {
        if (smallFile == null || smallFile.isEmpty()) {
            LOG.error("Illegal small file path: {}", smallFile);
            return false;
        } else if (compactSmallFileLock.contains(smallFile)) {
            LOG.error(String.format("%s is locked.", smallFile));
            return false;
        } else if (handlingSmallFileCache.contains(smallFile)) {
            LOG.error(String.format("%s is being handling.", smallFile));
            return false;
        } else if (containerFileCache.contains(smallFile) || containerFileLock.contains(smallFile)) {
            LOG.error(String.format("%s is container file.", smallFile));
            return false;
        }
    }
    // Get small file info list and file state map from meta store.
    List<FileInfo> fileInfos;
    Map<String, FileState> fileStateMap;
    try {
        fileInfos = metaStore.getFilesByPaths(smallFileList);
        fileStateMap = metaStore.getFileStates(smallFileList);
    } catch (MetaStoreException e) {
        LOG.error("Failed to get file states of small files.", e);
        return false;
    }
    // Get small file info map
    Map<String, FileInfo> fileInfoMap = new HashMap<>();
    for (FileInfo fileInfo : fileInfos) {
        fileInfoMap.put(fileInfo.getPath(), fileInfo);
    }
    // Check if the permission of small file is same,
    // and all the small files exist
    FileInfo firstFileInfo = null;
    for (String smallFile : smallFileList) {
        FileInfo fileInfo = fileInfoMap.get(smallFile);
        if (fileInfo != null) {
            if (firstFileInfo == null) {
                firstFileInfo = fileInfo;
            } else {
                if (!(new SmartFilePermission(firstFileInfo)).equals(new SmartFilePermission(fileInfo))) {
                    LOG.debug(String.format("%s has different file permission with %s.", firstFileInfo.getPath(), fileInfo.getPath()));
                    return false;
                }
            }
        } else {
            LOG.debug("{} is not exist!!!", smallFile);
            return false;
        }
    }
    // Check if the state of small file is NORMAL
    for (Map.Entry<String, FileState> entry : fileStateMap.entrySet()) {
        String smallFile = entry.getKey();
        FileState.FileType smallFileType = entry.getValue().getFileType();
        if (smallFileType != FileState.FileType.NORMAL) {
            LOG.debug(String.format("%s has invalid file state %s for small file compact.", smallFile, smallFileType.toString()));
            return false;
        }
    }
    return true;
}