Examples with FileInfo org.smartdata.model.FileInfo used on opensource projects

Example 6 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class InotifyEventApplier method insertDeleteDiff.

// TODO: just insert a fileDiff for this kind of path.
// It seems that there is no need to see if path matches with one dir in FileInfo.
private void insertDeleteDiff(String path, boolean isDir) throws MetaStoreException {
    if (isDir) {
        path = path.endsWith("/") ? path.substring(0, path.length() - 1) : path;
        List<FileInfo> fileInfos = metaStore.getFilesByPrefix(path);
        for (FileInfo fileInfo : fileInfos) {
            if (fileInfo.isdir()) {
                if (path.equals(fileInfo.getPath())) {
                    insertDeleteDiff(fileInfo.getPath());
                    break;
                }
            }
        }
    } else {
        insertDeleteDiff(path);
    }
}

Example 7 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class InotifyEventApplier method getCloseSql.

// Todo: should update mtime? atime?
private String getCloseSql(Event.CloseEvent closeEvent) throws IOException, MetaStoreException {
    FileDiff fileDiff = new FileDiff(FileDiffType.APPEND);
    fileDiff.setSrc(closeEvent.getPath());
    long newLen = closeEvent.getFileSize();
    long currLen = 0l;
    // TODO make sure offset is correct
    if (inBackup(closeEvent.getPath())) {
        FileInfo fileInfo = metaStore.getFile(closeEvent.getPath());
        if (fileInfo == null) {
            // TODO add metadata
            currLen = 0;
        } else {
            currLen = fileInfo.getLength();
        }
        if (currLen != newLen) {
            fileDiff.getParameters().put("-offset", String.valueOf(currLen));
            fileDiff.getParameters().put("-length", String.valueOf(newLen - currLen));
            metaStore.insertFileDiff(fileDiff);
        }
    }
    return String.format("UPDATE file SET length = %s, modification_time = %s WHERE path = '%s';", closeEvent.getFileSize(), closeEvent.getTimestamp(), closeEvent.getPath());
}

Example 8 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class InotifyEventApplier method getUnlinkSql.

private List<String> getUnlinkSql(Event.UnlinkEvent unlinkEvent) throws MetaStoreException {
    // delete root, i.e., /
    String root = "/";
    if (root.equals(unlinkEvent.getPath())) {
        LOG.warn("Deleting root directory!!!");
        insertDeleteDiff(root, true);
        return Arrays.asList(String.format("DELETE FROM file WHERE path like '%s%%'", root), String.format("DELETE FROM file_state WHERE path like '%s%%'", root), String.format("DELETE FROM small_file WHERE path like '%s%%'", root));
    }
    String path = unlinkEvent.getPath();
    // file has no "/" appended in the metaStore
    FileInfo fileInfo = metaStore.getFile(path.endsWith("/") ? path.substring(0, path.length() - 1) : path);
    if (fileInfo == null)
        return Arrays.asList();
    if (fileInfo.isdir()) {
        insertDeleteDiff(unlinkEvent.getPath(), true);
        // delete all files in this dir from file table
        return Arrays.asList(String.format("DELETE FROM file WHERE path LIKE '%s/%%';", unlinkEvent.getPath()), String.format("DELETE FROM file WHERE path = '%s';", unlinkEvent.getPath()), String.format("DELETE FROM file_state WHERE path LIKE '%s/%%';", unlinkEvent.getPath()), String.format("DELETE FROM file_state WHERE path = '%s';", unlinkEvent.getPath()), String.format("DELETE FROM small_file WHERE path LIKE '%s/%%';", unlinkEvent.getPath()), String.format("DELETE FROM small_file WHERE path = '%s';", unlinkEvent.getPath()));
    } else {
        insertDeleteDiff(unlinkEvent.getPath(), false);
        // delete file in file table
        return Arrays.asList(String.format("DELETE FROM file WHERE path = '%s';", unlinkEvent.getPath()), String.format("DELETE FROM file_state WHERE path = '%s';", unlinkEvent.getPath()), String.format("DELETE FROM small_file WHERE path = '%s';", unlinkEvent.getPath()));
    }
}

Example 9 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class InotifyEventApplier method getRenameSql.

// Todo: should update mtime? atime?
// private String getTruncateSql(Event.TruncateEvent truncateEvent) {
// return String.format(
// "UPDATE file SET length = %s, modification_time = %s WHERE path = '%s';",
// truncateEvent.getFileSize(), truncateEvent.getTimestamp(), truncateEvent.getPath());
// }
private List<String> getRenameSql(Event.RenameEvent renameEvent) throws IOException, MetaStoreException, InterruptedException {
    String src = renameEvent.getSrcPath();
    String dest = renameEvent.getDstPath();
    List<String> ret = new ArrayList<>();
    HdfsFileStatus status = client.getFileInfo(dest);
    FileInfo info = metaStore.getFile(src);
    // For backup data to use.
    generateFileDiff(renameEvent);
    if (status == null) {
        LOG.debug("Get rename dest status failed, {} -> {}", src, dest);
    }
    // The dest path which the src is renamed to should be checked in file table
    // to avoid duplicated record for one same path.
    FileInfo destInfo = metaStore.getFile(dest);
    if (destInfo != null) {
        metaStore.deleteFileByPath(dest);
    }
    // src is not in file table because it is not fetched or other reason
    if (info == null) {
        if (status != null) {
            // info = HadoopUtil.convertFileStatus(status, dest);
            // metaStore.insertFile(info);
            namespaceFetcher.startFetch(dest);
            while (!namespaceFetcher.fetchFinished()) {
                LOG.info("Fetching the files under " + dest);
                Thread.sleep(100);
            }
            namespaceFetcher.stop();
        }
    } else {
        // TODO: tackle with file_state and small_state
        if (shouldIgnore(dest)) {
            // fuzzy matching is used to delete content under the dir
            if (info.isdir()) {
                ret.add(String.format("DELETE FROM file WHERE path LIKE '%s/%%';", src));
            }
            ret.add(String.format("DELETE FROM file WHERE path = '%s';", src));
            return ret;
        } else {
            ret.add(String.format("UPDATE file SET path = replace(path, '%s', '%s') " + "WHERE path = '%s';", src, dest, src));
            ret.add(String.format("UPDATE file_state SET path = replace(path, '%s', '%s') " + "WHERE path = '%s';", src, dest, src));
            ret.add(String.format("UPDATE small_file SET path = replace(path, '%s', '%s') " + "WHERE path = '%s';", src, dest, src));
            if (info.isdir()) {
                if (metaStore.getDbType() == DBType.MYSQL) {
                    ret.add(String.format("UPDATE file SET path = CONCAT('%s', SUBSTR(path, %d)) " + "WHERE path LIKE '%s/%%';", dest, src.length() + 1, src));
                    ret.add(String.format("UPDATE file_state SET path = CONCAT('%s', SUBSTR(path, %d)) " + "WHERE path LIKE '%s/%%';", dest, src.length() + 1, src));
                    ret.add(String.format("UPDATE small_file SET path = CONCAT('%s', SUBSTR(path, %d)) " + "WHERE path LIKE '%s/%%';", dest, src.length() + 1, src));
                } else if (metaStore.getDbType() == DBType.SQLITE) {
                    ret.add(String.format("UPDATE file SET path = '%s' || SUBSTR(path, %d) " + "WHERE path LIKE '%s/%%';", dest, src.length() + 1, src));
                    ret.add(String.format("UPDATE file_state SET path = '%s' || SUBSTR(path, %d) " + "WHERE path LIKE '%s/%%';", dest, src.length() + 1, src));
                    ret.add(String.format("UPDATE small_file SET path = '%s' || SUBSTR(path, %d) " + "WHERE path LIKE '%s/%%';", dest, src.length() + 1, src));
                }
            }
        }
    }
    return ret;
}

Example 10 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class TestAlluxioEntryApplier method testRenameApplier.

@Test
public void testRenameApplier() throws Exception {
    FileSystem fs = Mockito.mock(FileSystem.class);
    AlluxioEntryApplier entryApplier = new AlluxioEntryApplier(metaStore, fs);
    FileInfo fooFile = FileInfo.newBuilder().setFileId(50331647).setIsdir(false).setPath("/bar/foobar1").build();
    metaStore.insertFile(fooFile);
    BackUpInfo backUpInfo = new BackUpInfo(1L, "/bar/foobar1", "remote/dest/", 10);
    metaStore.insertBackUpInfo(backUpInfo);
    alluxio.wire.FileInfo info1 = new alluxio.wire.FileInfo().setFileId(50331647).setPath("/bar/foobar1").setLength(300L).setFolder(false).setBlockSizeBytes(310000).setLastModificationTimeMs(1515665270681L).setCreationTimeMs(1515665270681L).setMode(493).setOwner("user1").setGroup("group1");
    URIStatus status1 = new URIStatus(info1);
    Mockito.when(fs.getStatus(new AlluxioURI("/bar/foobar1"))).thenReturn(status1);
    RenameEntry renameEntry = RenameEntry.newBuilder().setId(50331647).setOpTimeMs(1515666148444L).setDstPath("/bar/foobar1_new").build();
    JournalEntry renameJEntry = JournalEntry.newBuilder().setRename(renameEntry).build();
    entryApplier.apply(renameJEntry);
    List<FileDiff> fileDiffs = metaStore.getFileDiffsByFileName("/bar/foobar1");
    Assert.assertTrue(fileDiffs.size() > 0);
    for (FileDiff fileDiff : fileDiffs) {
        if (fileDiff.getDiffType().equals(FileDiffType.RENAME)) {
            Assert.assertEquals("/bar/foobar1", fileDiff.getSrc());
            Assert.assertEquals("/bar/foobar1_new", fileDiff.getParameters().get("-dest"));
        }
    }
}