Examples with FileInfo org.smartdata.model.FileInfo used on opensource projects

Example 36 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class AlluxioEntryApplier method addInodeFileFromEntry.

private String addInodeFileFromEntry(InodeFileEntry inodeFileEntry) throws MetaStoreException {
    String inodePath = getPathFromInodeFile(inodeFileEntry);
    URIStatus status = null;
    try {
        status = fs.getStatus(new AlluxioURI(inodePath));
    } catch (IOException | AlluxioException e) {
        e.printStackTrace();
    }
    FileInfo fileInfo = AlluxioUtil.convertFileStatus(status);
    if (inBackup(fileInfo.getPath())) {
        FileDiff fileDiff = new FileDiff(FileDiffType.APPEND);
        fileDiff.setSrc(fileInfo.getPath());
        fileDiff.getParameters().put("-offset", String.valueOf(0));
        // Note that "-length 0" means create an empty file
        fileDiff.getParameters().put("-length", String.valueOf(fileInfo.getLength()));
        // add modification_time and access_time to filediff
        fileDiff.getParameters().put("-mtime", "" + fileInfo.getModificationTime());
        // add owner to filediff
        fileDiff.getParameters().put("-owner", "" + fileInfo.getOwner());
        fileDiff.getParameters().put("-group", "" + fileInfo.getGroup());
        // add Permission to filediff
        fileDiff.getParameters().put("-permission", "" + fileInfo.getPermission());
        metaStore.insertFileDiff(fileDiff);
    }
    metaStore.insertFile(fileInfo);
    return "";
}

Example 37 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class AlluxioEntryApplier method processEntryToSql.

private List<String> processEntryToSql(JournalEntry entry) throws IOException, MetaStoreException {
    if (entry.hasInodeDirectory()) {
        LOG.trace("entry type:" + entry.getInodeDirectory().getClass() + ", id:" + entry.getInodeDirectory().getId());
        InodeDirectoryEntry inodeDirectoryEntry = entry.getInodeDirectory();
        String inodeDir = getPathFromInodeDir(inodeDirectoryEntry);
        URIStatus dStatus = null;
        try {
            dStatus = fs.getStatus(new AlluxioURI(inodeDir));
        } catch (AlluxioException e) {
            e.printStackTrace();
        }
        FileInfo fileInfo = AlluxioUtil.convertFileStatus(dStatus);
        metaStore.insertFile(fileInfo);
        return Collections.singletonList("");
    } else if (entry.hasInodeFile()) {
        LOG.trace("entry type:" + entry.getInodeFile().getClass() + ", id:" + entry.getInodeFile().getId());
        String addSql = addInodeFileFromEntry(entry.getInodeFile());
        return Collections.singletonList(addSql);
    } else if (entry.hasInodeLastModificationTime()) {
        LOG.trace("entry type:" + entry.getInodeLastModificationTime().getClass() + ", id:" + entry.getInodeLastModificationTime().getId());
        InodeLastModificationTimeEntry modTimeEntry = entry.getInodeLastModificationTime();
        String path = getPathByFileId(modTimeEntry.getId());
        FileDiff fileDiff = null;
        if (inBackup(path)) {
            fileDiff = new FileDiff(FileDiffType.METADATA);
            fileDiff.setSrc(path);
        }
        if (fileDiff != null) {
            fileDiff.getParameters().put("-mtime", "" + modTimeEntry.getLastModificationTimeMs());
            metaStore.insertFileDiff(fileDiff);
        }
        String modifySql = String.format("UPDATE file SET modification_time = %s WHERE fid = '%s';", modTimeEntry.getLastModificationTimeMs(), modTimeEntry.getId());
        return Collections.singletonList(modifySql);
    } else if (entry.hasPersistDirectory()) {
        LOG.trace("entry type:" + entry.getPersistDirectory().getClass() + ", id:" + entry.getPersistDirectory().getId());
        PersistDirectoryEntry typedEntry = entry.getPersistDirectory();
        LOG.debug("Persist directory id " + typedEntry.getId());
        return Collections.singletonList("");
    } else if (entry.hasSetAttribute()) {
        LOG.trace("entry type:" + entry.getSetAttribute().getClass() + ", id:" + entry.getSetAttribute().getId());
        String setAttrSql = setAttributeFromEntry(entry.getSetAttribute());
        return Collections.singletonList(setAttrSql);
    } else if (entry.hasRename()) {
        LOG.trace("entry type:" + entry.getRename().getClass() + ", id:" + entry.getRename().getId());
        return renameFromEntry(entry.getRename());
    } else if (entry.hasDeleteFile()) {
        LOG.trace("entry type:" + entry.getDeleteFile().getClass() + ", id:" + entry.getDeleteFile().getId());
        String delSql = deleteFromEntry(entry.getDeleteFile());
        return Collections.singletonList(delSql);
    } else if (entry.hasAddMountPoint()) {
        LOG.trace("entry type:" + entry.getAddMountPoint().getClass() + ", alluxio path:" + entry.getAddMountPoint().getAlluxioPath() + ", ufs path:" + entry.getAddMountPoint().getUfsPath());
        return Collections.singletonList(mountFromEntry(entry.getAddMountPoint()));
    } else if (entry.hasDeleteMountPoint()) {
        LOG.trace("entry type:" + entry.getDeleteMountPoint().getClass() + ", alluxio path:" + entry.getDeleteMountPoint().getAlluxioPath());
        return Collections.singletonList(unmountFromEntry(entry.getDeleteMountPoint()));
    } else if (entry.hasAsyncPersistRequest() || entry.hasCompleteFile() || entry.hasInodeDirectoryIdGenerator() || entry.hasReinitializeFile()) {
    // Do nothing
    } else {
        throw new IOException(ExceptionMessage.UNEXPECTED_JOURNAL_ENTRY.getMessage(entry));
    }
    return Collections.emptyList();
}

Example 38 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class AlluxioEntryApplier method renameFromEntry.

private List<String> renameFromEntry(RenameEntry renameEntry) throws MetaStoreException {
    List<String> sqlist = new ArrayList<>();
    URIStatus dStatus = null;
    try {
        dStatus = fs.getStatus(new AlluxioURI(renameEntry.getDstPath()));
    } catch (IOException | AlluxioException e) {
        e.printStackTrace();
    }
    if (dStatus == null) {
        LOG.debug("Get rename dest status failed, {}", renameEntry.getDstPath());
    }
    FileInfo fileInfo = metaStore.getFile(renameEntry.getId());
    if (fileInfo == null) {
        if (dStatus != null) {
            fileInfo = AlluxioUtil.convertFileStatus(dStatus);
            metaStore.insertFile(fileInfo);
        }
    } else {
        FileDiff fileDiff = new FileDiff(FileDiffType.RENAME);
        String srcPath = fileInfo.getPath();
        if (inBackup(srcPath)) {
            fileDiff.setSrc(srcPath);
            fileDiff.getParameters().put("-dest", renameEntry.getDstPath());
            metaStore.insertFileDiff(fileDiff);
        }
        sqlist.add(String.format("UPDATE file SET path = replace(path, '%s', '%s') WHERE path = '%s';", srcPath, renameEntry.getDstPath(), srcPath));
        if (fileInfo.isdir()) {
            sqlist.add(String.format("UPDATE file SET path = replace(path, '%s', '%s') WHERE path LIKE '%s/%%';", srcPath, renameEntry.getDstPath(), srcPath));
        }
    }
    return sqlist;
}

Example 39 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class AlluxioEntryApplier method getPathFromInodeFile.

public String getPathFromInodeFile(InodeFileEntry fileEntry) throws MetaStoreException {
    long pid = fileEntry.getParentId();
    String fName = fileEntry.getName();
    FileInfo fileInfo = metaStore.getFile(pid);
    String pPath = "";
    if (fileInfo != null) {
        pPath = formatPath(fileInfo.getPath());
    }
    return pPath.concat(fName);
}

Example 40 with FileInfo

use of org.smartdata.model.FileInfo in project SSM by Intel-bigdata.

the class TestAlluxioEntryApplier method testSetAttributeApplier.

@Test
public void testSetAttributeApplier() throws Exception {
    FileSystem fs = Mockito.mock(FileSystem.class);
    AlluxioEntryApplier entryApplier = new AlluxioEntryApplier(metaStore, fs);
    FileInfo fooFile = FileInfo.newBuilder().setFileId(33554431).setIsdir(false).setPath("/foo/foobar").build();
    metaStore.insertFile(fooFile);
    BackUpInfo backUpInfo = new BackUpInfo(1L, "/foo/foobar", "remote/dest/", 10);
    metaStore.insertBackUpInfo(backUpInfo);
    alluxio.wire.FileInfo info1 = new alluxio.wire.FileInfo().setFileId(33554431).setPath("/foo/foobar").setLength(100L).setFolder(false).setBlockSizeBytes(210000).setLastModificationTimeMs(1515665470681L).setCreationTimeMs(1515665470681L).setMode(493).setOwner("user1").setGroup("group1");
    URIStatus status1 = new URIStatus(info1);
    Mockito.when(fs.getStatus(new AlluxioURI("/foo/foobar"))).thenReturn(status1);
    SetAttributeEntry setAttributeEntry = SetAttributeEntry.newBuilder().setId(33554431).setOpTimeMs(1515667208590658L).setPermission(511).build();
    JournalEntry setAttributeJEntry = JournalEntry.newBuilder().setSetAttribute(setAttributeEntry).build();
    entryApplier.apply(setAttributeJEntry);
    List<FileDiff> fileDiffs = metaStore.getFileDiffsByFileName("/foo/foobar");
    Assert.assertTrue(fileDiffs.size() > 0);
    for (FileDiff fileDiff : fileDiffs) {
        if (fileDiff.getDiffType().equals(FileDiffType.METADATA)) {
            Assert.assertEquals("511", fileDiff.getParameters().get("-permission"));
        }
    }
}