Examples with FileDiff org.smartdata.model.FileDiff used on opensource projects

Example 16 with FileDiff

use of org.smartdata.model.FileDiff in project SSM by Intel-bigdata.

the class AlluxioEntryApplier method processEntryToSql.

private List<String> processEntryToSql(JournalEntry entry) throws IOException, MetaStoreException {
    if (entry.hasInodeDirectory()) {
        LOG.trace("entry type:" + entry.getInodeDirectory().getClass() + ", id:" + entry.getInodeDirectory().getId());
        InodeDirectoryEntry inodeDirectoryEntry = entry.getInodeDirectory();
        String inodeDir = getPathFromInodeDir(inodeDirectoryEntry);
        URIStatus dStatus = null;
        try {
            dStatus = fs.getStatus(new AlluxioURI(inodeDir));
        } catch (AlluxioException e) {
            e.printStackTrace();
        }
        FileInfo fileInfo = AlluxioUtil.convertFileStatus(dStatus);
        metaStore.insertFile(fileInfo);
        return Collections.singletonList("");
    } else if (entry.hasInodeFile()) {
        LOG.trace("entry type:" + entry.getInodeFile().getClass() + ", id:" + entry.getInodeFile().getId());
        String addSql = addInodeFileFromEntry(entry.getInodeFile());
        return Collections.singletonList(addSql);
    } else if (entry.hasInodeLastModificationTime()) {
        LOG.trace("entry type:" + entry.getInodeLastModificationTime().getClass() + ", id:" + entry.getInodeLastModificationTime().getId());
        InodeLastModificationTimeEntry modTimeEntry = entry.getInodeLastModificationTime();
        String path = getPathByFileId(modTimeEntry.getId());
        FileDiff fileDiff = null;
        if (inBackup(path)) {
            fileDiff = new FileDiff(FileDiffType.METADATA);
            fileDiff.setSrc(path);
        }
        if (fileDiff != null) {
            fileDiff.getParameters().put("-mtime", "" + modTimeEntry.getLastModificationTimeMs());
            metaStore.insertFileDiff(fileDiff);
        }
        String modifySql = String.format("UPDATE file SET modification_time = %s WHERE fid = '%s';", modTimeEntry.getLastModificationTimeMs(), modTimeEntry.getId());
        return Collections.singletonList(modifySql);
    } else if (entry.hasPersistDirectory()) {
        LOG.trace("entry type:" + entry.getPersistDirectory().getClass() + ", id:" + entry.getPersistDirectory().getId());
        PersistDirectoryEntry typedEntry = entry.getPersistDirectory();
        LOG.debug("Persist directory id " + typedEntry.getId());
        return Collections.singletonList("");
    } else if (entry.hasSetAttribute()) {
        LOG.trace("entry type:" + entry.getSetAttribute().getClass() + ", id:" + entry.getSetAttribute().getId());
        String setAttrSql = setAttributeFromEntry(entry.getSetAttribute());
        return Collections.singletonList(setAttrSql);
    } else if (entry.hasRename()) {
        LOG.trace("entry type:" + entry.getRename().getClass() + ", id:" + entry.getRename().getId());
        return renameFromEntry(entry.getRename());
    } else if (entry.hasDeleteFile()) {
        LOG.trace("entry type:" + entry.getDeleteFile().getClass() + ", id:" + entry.getDeleteFile().getId());
        String delSql = deleteFromEntry(entry.getDeleteFile());
        return Collections.singletonList(delSql);
    } else if (entry.hasAddMountPoint()) {
        LOG.trace("entry type:" + entry.getAddMountPoint().getClass() + ", alluxio path:" + entry.getAddMountPoint().getAlluxioPath() + ", ufs path:" + entry.getAddMountPoint().getUfsPath());
        return Collections.singletonList(mountFromEntry(entry.getAddMountPoint()));
    } else if (entry.hasDeleteMountPoint()) {
        LOG.trace("entry type:" + entry.getDeleteMountPoint().getClass() + ", alluxio path:" + entry.getDeleteMountPoint().getAlluxioPath());
        return Collections.singletonList(unmountFromEntry(entry.getDeleteMountPoint()));
    } else if (entry.hasAsyncPersistRequest() || entry.hasCompleteFile() || entry.hasInodeDirectoryIdGenerator() || entry.hasReinitializeFile()) {
    // Do nothing
    } else {
        throw new IOException(ExceptionMessage.UNEXPECTED_JOURNAL_ENTRY.getMessage(entry));
    }
    return Collections.emptyList();
}

Example 17 with FileDiff

use of org.smartdata.model.FileDiff in project SSM by Intel-bigdata.

the class AlluxioEntryApplier method renameFromEntry.

private List<String> renameFromEntry(RenameEntry renameEntry) throws MetaStoreException {
    List<String> sqlist = new ArrayList<>();
    URIStatus dStatus = null;
    try {
        dStatus = fs.getStatus(new AlluxioURI(renameEntry.getDstPath()));
    } catch (IOException | AlluxioException e) {
        e.printStackTrace();
    }
    if (dStatus == null) {
        LOG.debug("Get rename dest status failed, {}", renameEntry.getDstPath());
    }
    FileInfo fileInfo = metaStore.getFile(renameEntry.getId());
    if (fileInfo == null) {
        if (dStatus != null) {
            fileInfo = AlluxioUtil.convertFileStatus(dStatus);
            metaStore.insertFile(fileInfo);
        }
    } else {
        FileDiff fileDiff = new FileDiff(FileDiffType.RENAME);
        String srcPath = fileInfo.getPath();
        if (inBackup(srcPath)) {
            fileDiff.setSrc(srcPath);
            fileDiff.getParameters().put("-dest", renameEntry.getDstPath());
            metaStore.insertFileDiff(fileDiff);
        }
        sqlist.add(String.format("UPDATE file SET path = replace(path, '%s', '%s') WHERE path = '%s';", srcPath, renameEntry.getDstPath(), srcPath));
        if (fileInfo.isdir()) {
            sqlist.add(String.format("UPDATE file SET path = replace(path, '%s', '%s') WHERE path LIKE '%s/%%';", srcPath, renameEntry.getDstPath(), srcPath));
        }
    }
    return sqlist;
}

Example 18 with FileDiff

use of org.smartdata.model.FileDiff in project SSM by Intel-bigdata.

the class TestAlluxioEntryApplier method testSetAttributeApplier.

@Test
public void testSetAttributeApplier() throws Exception {
    FileSystem fs = Mockito.mock(FileSystem.class);
    AlluxioEntryApplier entryApplier = new AlluxioEntryApplier(metaStore, fs);
    FileInfo fooFile = FileInfo.newBuilder().setFileId(33554431).setIsdir(false).setPath("/foo/foobar").build();
    metaStore.insertFile(fooFile);
    BackUpInfo backUpInfo = new BackUpInfo(1L, "/foo/foobar", "remote/dest/", 10);
    metaStore.insertBackUpInfo(backUpInfo);
    alluxio.wire.FileInfo info1 = new alluxio.wire.FileInfo().setFileId(33554431).setPath("/foo/foobar").setLength(100L).setFolder(false).setBlockSizeBytes(210000).setLastModificationTimeMs(1515665470681L).setCreationTimeMs(1515665470681L).setMode(493).setOwner("user1").setGroup("group1");
    URIStatus status1 = new URIStatus(info1);
    Mockito.when(fs.getStatus(new AlluxioURI("/foo/foobar"))).thenReturn(status1);
    SetAttributeEntry setAttributeEntry = SetAttributeEntry.newBuilder().setId(33554431).setOpTimeMs(1515667208590658L).setPermission(511).build();
    JournalEntry setAttributeJEntry = JournalEntry.newBuilder().setSetAttribute(setAttributeEntry).build();
    entryApplier.apply(setAttributeJEntry);
    List<FileDiff> fileDiffs = metaStore.getFileDiffsByFileName("/foo/foobar");
    Assert.assertTrue(fileDiffs.size() > 0);
    for (FileDiff fileDiff : fileDiffs) {
        if (fileDiff.getDiffType().equals(FileDiffType.METADATA)) {
            Assert.assertEquals("511", fileDiff.getParameters().get("-permission"));
        }
    }
}

Example 19 with FileDiff

use of org.smartdata.model.FileDiff in project SSM by Intel-bigdata.

the class TestFileDiffDao method testDeleteUselessRecords.

@Test
public void testDeleteUselessRecords() {
    FileDiff[] fileDiffs = new FileDiff[2];
    fileDiffs[0] = new FileDiff();
    fileDiffs[0].setDiffId(1);
    fileDiffs[0].setRuleId(1);
    fileDiffs[0].setParameters(new HashMap<String, String>());
    fileDiffs[0].setSrc("test");
    fileDiffs[0].setState(FileDiffState.PENDING);
    fileDiffs[0].setDiffType(FileDiffType.APPEND);
    fileDiffs[0].setCreateTime(1);
    fileDiffs[1] = new FileDiff();
    fileDiffs[1].setDiffId(2);
    fileDiffs[0].setRuleId(1);
    fileDiffs[1].setParameters(new HashMap<String, String>());
    fileDiffs[1].setSrc("src");
    fileDiffs[1].setState(FileDiffState.PENDING);
    fileDiffs[1].setDiffType(FileDiffType.APPEND);
    fileDiffs[1].setCreateTime(2);
    fileDiffDao.insert(fileDiffs);
    Assert.assertEquals(fileDiffDao.getUselessRecordsNum(), 0);
    fileDiffDao.update(1, FileDiffState.APPLIED);
    Assert.assertEquals(fileDiffDao.getUselessRecordsNum(), 1);
    fileDiffDao.update(2, FileDiffState.FAILED);
    Assert.assertEquals(fileDiffDao.getUselessRecordsNum(), 2);
    fileDiffDao.update(2, FileDiffState.DELETED);
    Assert.assertEquals(fileDiffDao.getUselessRecordsNum(), 2);
    fileDiffDao.deleteUselessRecords(1);
    Assert.assertEquals(fileDiffDao.getAll().size(), 1);
}

Example 20 with FileDiff

use of org.smartdata.model.FileDiff in project SSM by Intel-bigdata.

the class TestFileDiffDao method testBatchInsertAndQuery.

@Test
public void testBatchInsertAndQuery() {
    List<FileDiff> fileDiffs = new ArrayList<>();
    FileDiff fileDiff = new FileDiff();
    fileDiff.setParameters(new HashMap<String, String>());
    fileDiff.setSrc("test");
    fileDiff.setState(FileDiffState.RUNNING);
    fileDiff.setDiffType(FileDiffType.APPEND);
    fileDiff.setCreateTime(1);
    fileDiffs.add(fileDiff);
    fileDiff = new FileDiff();
    fileDiff.setParameters(new HashMap<String, String>());
    fileDiff.setSrc("src");
    fileDiff.setState(FileDiffState.PENDING);
    fileDiff.setDiffType(FileDiffType.APPEND);
    fileDiff.setCreateTime(1);
    fileDiffs.add(fileDiff);
    fileDiffDao.insert(fileDiffs);
    List<FileDiff> fileInfoList = fileDiffDao.getAll();
    for (int i = 0; i < 2; i++) {
        Assert.assertTrue(fileInfoList.get(i).equals(fileDiffs.get(i)));
    }
    List<String> paths = fileDiffDao.getSyncPath(0);
    Assert.assertTrue(paths.size() == 1);
    Assert.assertTrue(fileDiffDao.getPendingDiff("src").size() == 1);
    Assert.assertTrue(fileDiffDao.getByState("test", FileDiffState.RUNNING).size() == 1);
}