Examples with FileDiff org.smartdata.model.FileDiff used on opensource projects

Example 11 with FileDiff

use of org.smartdata.model.FileDiff in project SSM by Intel-bigdata.

the class InotifyEventApplier method generateFileDiff.

private void generateFileDiff(Event.RenameEvent renameEvent) throws MetaStoreException {
    String src = renameEvent.getSrcPath();
    String dest = renameEvent.getDstPath();
    FileInfo info = metaStore.getFile(src);
    // TODO: consider src or dest is ignored by SSM
    if (inBackup(src)) {
        // if not, insert a delete file diff
        if (inBackup(dest)) {
            FileDiff fileDiff = new FileDiff(FileDiffType.RENAME);
            fileDiff.setSrc(src);
            fileDiff.getParameters().put("-dest", dest);
            metaStore.insertFileDiff(fileDiff);
        } else {
            insertDeleteDiff(src, info.isdir());
        }
    } else if (inBackup(dest)) {
        // tackle such case: rename file from outside into backup dir
        if (!info.isdir()) {
            FileDiff fileDiff = new FileDiff(FileDiffType.APPEND);
            fileDiff.setSrc(dest);
            fileDiff.getParameters().put("-offset", String.valueOf(0));
            fileDiff.getParameters().put("-length", String.valueOf(info.getLength()));
            metaStore.insertFileDiff(fileDiff);
        } else {
            List<FileInfo> fileInfos = metaStore.getFilesByPrefix(src.endsWith("/") ? src : src + "/");
            for (FileInfo fileInfo : fileInfos) {
                // TODO: cover subdir with no file case
                if (fileInfo.isdir()) {
                    continue;
                }
                FileDiff fileDiff = new FileDiff(FileDiffType.APPEND);
                fileDiff.setSrc(fileInfo.getPath().replaceFirst(src, dest));
                fileDiff.getParameters().put("-offset", String.valueOf(0));
                fileDiff.getParameters().put("-length", String.valueOf(fileInfo.getLength()));
                metaStore.insertFileDiff(fileDiff);
            }
        }
    }
}

Example 12 with FileDiff

use of org.smartdata.model.FileDiff in project SSM by Intel-bigdata.

the class InotifyEventApplier method insertDeleteDiff.

private void insertDeleteDiff(String path) throws MetaStoreException {
    // TODO: remove "/" appended in src or dest in backup_file table
    String pathWithSlash = path.endsWith("/") ? path : path + "/";
    if (inBackup(pathWithSlash)) {
        List<BackUpInfo> backUpInfos = metaStore.getBackUpInfoBySrc(pathWithSlash);
        for (BackUpInfo backUpInfo : backUpInfos) {
            String destPath = pathWithSlash.replaceFirst(backUpInfo.getSrc(), backUpInfo.getDest());
            try {
                // tackle root path case
                URI namenodeUri = new URI(destPath);
                String root = "hdfs://" + namenodeUri.getHost() + ":" + String.valueOf(namenodeUri.getPort());
                if (destPath.equals(root) || destPath.equals(root + "/") || destPath.equals("/")) {
                    for (String srcFilePath : getFilesUnderDir(pathWithSlash)) {
                        FileDiff fileDiff = new FileDiff(FileDiffType.DELETE);
                        fileDiff.setSrc(srcFilePath);
                        String destFilePath = srcFilePath.replaceFirst(backUpInfo.getSrc(), backUpInfo.getDest());
                        fileDiff.getParameters().put("-dest", destFilePath);
                        metaStore.insertFileDiff(fileDiff);
                    }
                } else {
                    FileDiff fileDiff = new FileDiff(FileDiffType.DELETE);
                    // use the path getting from event with no slash appended
                    fileDiff.setSrc(path);
                    // put sync's dest path in parameter for delete use
                    fileDiff.getParameters().put("-dest", destPath);
                    metaStore.insertFileDiff(fileDiff);
                }
            } catch (URISyntaxException e) {
                LOG.error("Error occurs!", e);
            }
        }
    }
}

Example 13 with FileDiff

use of org.smartdata.model.FileDiff in project SSM by Intel-bigdata.

the class InotifyEventApplier method getCreateSql.

// Todo: times and ec policy id, etc.
private String getCreateSql(Event.CreateEvent createEvent) throws IOException, MetaStoreException {
    HdfsFileStatus fileStatus = client.getFileInfo(createEvent.getPath());
    if (fileStatus == null) {
        LOG.debug("Can not get HdfsFileStatus for file " + createEvent.getPath());
        return "";
    }
    FileInfo fileInfo = HadoopUtil.convertFileStatus(fileStatus, createEvent.getPath());
    if (inBackup(fileInfo.getPath())) {
        if (!fileInfo.isdir()) {
            // ignore dir
            FileDiff fileDiff = new FileDiff(FileDiffType.APPEND);
            fileDiff.setSrc(fileInfo.getPath());
            fileDiff.getParameters().put("-offset", String.valueOf(0));
            // Note that "-length 0" means create an empty file
            fileDiff.getParameters().put("-length", String.valueOf(fileInfo.getLength()));
            // TODO add support in CopyFileAction or split into two file diffs
            // add modification_time and access_time to filediff
            fileDiff.getParameters().put("-mtime", "" + fileInfo.getModificationTime());
            // fileDiff.getParameters().put("-atime", "" + fileInfo.getAccessTime());
            // add owner to filediff
            fileDiff.getParameters().put("-owner", "" + fileInfo.getOwner());
            fileDiff.getParameters().put("-group", "" + fileInfo.getGroup());
            // add Permission to filediff
            fileDiff.getParameters().put("-permission", "" + fileInfo.getPermission());
            // add replication count to file diff
            fileDiff.getParameters().put("-replication", "" + fileInfo.getBlockReplication());
            metaStore.insertFileDiff(fileDiff);
        }
    }
    metaStore.deleteFileByPath(fileInfo.getPath());
    metaStore.deleteFileState(fileInfo.getPath());
    metaStore.insertFile(fileInfo);
    return "";
}

Example 14 with FileDiff

use of org.smartdata.model.FileDiff in project SSM by Intel-bigdata.

the class AlluxioEntryApplier method addInodeFileFromEntry.

private String addInodeFileFromEntry(InodeFileEntry inodeFileEntry) throws MetaStoreException {
    String inodePath = getPathFromInodeFile(inodeFileEntry);
    URIStatus status = null;
    try {
        status = fs.getStatus(new AlluxioURI(inodePath));
    } catch (IOException | AlluxioException e) {
        e.printStackTrace();
    }
    FileInfo fileInfo = AlluxioUtil.convertFileStatus(status);
    if (inBackup(fileInfo.getPath())) {
        FileDiff fileDiff = new FileDiff(FileDiffType.APPEND);
        fileDiff.setSrc(fileInfo.getPath());
        fileDiff.getParameters().put("-offset", String.valueOf(0));
        // Note that "-length 0" means create an empty file
        fileDiff.getParameters().put("-length", String.valueOf(fileInfo.getLength()));
        // add modification_time and access_time to filediff
        fileDiff.getParameters().put("-mtime", "" + fileInfo.getModificationTime());
        // add owner to filediff
        fileDiff.getParameters().put("-owner", "" + fileInfo.getOwner());
        fileDiff.getParameters().put("-group", "" + fileInfo.getGroup());
        // add Permission to filediff
        fileDiff.getParameters().put("-permission", "" + fileInfo.getPermission());
        metaStore.insertFileDiff(fileDiff);
    }
    metaStore.insertFile(fileInfo);
    return "";
}

Example 15 with FileDiff

use of org.smartdata.model.FileDiff in project SSM by Intel-bigdata.

the class AlluxioEntryApplier method setAttributeFromEntry.

private String setAttributeFromEntry(SetAttributeEntry setAttrEntry) throws MetaStoreException {
    String path = getPathByFileId(setAttrEntry.getId());
    FileDiff fileDiff = null;
    if (inBackup(path)) {
        fileDiff = new FileDiff(FileDiffType.METADATA);
        fileDiff.setSrc(path);
    }
    if (setAttrEntry.hasPinned()) {
        LOG.debug(String.format("File %s is pinned %s", setAttrEntry.getId(), setAttrEntry.getPinned()));
    // Todo
    } else if (setAttrEntry.hasTtl()) {
        LOG.debug(String.format("File %s has ttl %s with ttlAction %s", setAttrEntry.getId(), setAttrEntry.getTtl(), setAttrEntry.getTtlAction()));
    // Todo
    } else if (setAttrEntry.hasPersisted()) {
        LOG.debug(String.format("File %s is persisted %s", setAttrEntry.getId(), setAttrEntry.getPersisted()));
    // Todo
    } else if (setAttrEntry.hasOwner()) {
        if (fileDiff != null) {
            fileDiff.getParameters().put("-owner", "" + setAttrEntry.getOwner());
            metaStore.insertFileDiff(fileDiff);
        }
    // Todo
    } else if (setAttrEntry.hasGroup()) {
        if (fileDiff != null) {
            fileDiff.getParameters().put("-group", "" + setAttrEntry.getGroup());
            metaStore.insertFileDiff(fileDiff);
        }
    // Todo
    } else if (setAttrEntry.hasPermission()) {
        if (fileDiff != null) {
            fileDiff.getParameters().put("-permission", "" + (short) setAttrEntry.getPermission());
            metaStore.insertFileDiff(fileDiff);
        }
        return String.format("UPDATE file SET permission = %s WHERE path = '%s';", (short) setAttrEntry.getPermission(), path);
    }
    return "";
}