Examples with JournalEntry alluxio.proto.journal.Journal.JournalEntry used on opensource projects

Example 21 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project SSM by Intel-bigdata.

the class TestAlluxioEntryApplier method testInodeLastMTimeApplier.

@Test
public void testInodeLastMTimeApplier() throws Exception {
    FileSystem fs = Mockito.mock(FileSystem.class);
    AlluxioEntryApplier entryApplier = new AlluxioEntryApplier(metaStore, fs);
    FileInfo fooFile = FileInfo.newBuilder().setFileId(5).setIsdir(true).setPath("/baz").build();
    metaStore.insertFile(fooFile);
    BackUpInfo backUpInfo = new BackUpInfo(1L, "/baz", "remote/dest/", 10);
    metaStore.insertBackUpInfo(backUpInfo);
    alluxio.wire.FileInfo info1 = new alluxio.wire.FileInfo().setFileId(5).setPath("/baz").setLength(0L).setFolder(true).setBlockSizeBytes(1000000).setLastModificationTimeMs(1515665470681L).setCreationTimeMs(1515665470681L).setMode(493).setOwner("user1").setGroup("group1");
    URIStatus status1 = new URIStatus(info1);
    Mockito.when(fs.getStatus(new AlluxioURI("/baz"))).thenReturn(status1);
    InodeLastModificationTimeEntry inodeLastMTimeEntry = InodeLastModificationTimeEntry.newBuilder().setId(5).setLastModificationTimeMs(1515667810911L).build();
    JournalEntry inodeLastMTimeJEntry = JournalEntry.newBuilder().setInodeLastModificationTime(inodeLastMTimeEntry).build();
    entryApplier.apply(inodeLastMTimeJEntry);
    List<FileDiff> fileDiffs = metaStore.getFileDiffsByFileName("/baz");
    Assert.assertTrue(fileDiffs.size() > 0);
    for (FileDiff fileDiff : fileDiffs) {
        if (fileDiff.getDiffType().equals(FileDiffType.METADATA)) {
            Assert.assertEquals("/baz", fileDiff.getSrc());
            Assert.assertEquals("1515667810911", fileDiff.getParameters().get("-mtime"));
        }
    }
}

Example 22 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project SSM by Intel-bigdata.

the class TestAlluxioEntryApplier method testInodeFileApplier.

@Test
public void testInodeFileApplier() throws Exception {
    FileSystem fs = Mockito.mock(FileSystem.class);
    AlluxioEntryApplier entryApplier = new AlluxioEntryApplier(metaStore, fs);
    FileInfo fooDir = FileInfo.newBuilder().setFileId(6).setIsdir(true).setPath("/foo").build();
    metaStore.insertFile(fooDir);
    BackUpInfo backUpInfo = new BackUpInfo(1L, "/foo/foobar1", "remote/dest/", 10);
    metaStore.insertBackUpInfo(backUpInfo);
    alluxio.wire.FileInfo info1 = new alluxio.wire.FileInfo().setFileId(33554431).setPath("/foo/foobar1").setLength(10L).setFolder(false).setBlockSizeBytes(536870912).setLastModificationTimeMs(1515665470681L).setCreationTimeMs(1515665470681L).setMode(420).setOwner("user1").setGroup("group1");
    URIStatus status1 = new URIStatus(info1);
    Mockito.when(fs.getStatus(new AlluxioURI("/foo/foobar1"))).thenReturn(status1);
    InodeFileEntry inodeFileEntry = InodeFileEntry.newBuilder().setId(33554431).setParentId(6).setName("foobar1").setPersistenceState("NOT_PERSISTED").setPinned(false).setCreationTimeMs(1515665470681L).setBlockSizeBytes(536870912).setLength(10L).setCompleted(false).setCacheable(true).setTtl(-1L).setOwner("user1").setGroup("group1").setMode(420).setTtlAction(PTtlAction.DELETE).build();
    JournalEntry inodeFileJEntry = JournalEntry.newBuilder().setInodeFile(inodeFileEntry).build();
    entryApplier.apply(inodeFileJEntry);
    Assert.assertEquals(33554431, metaStore.getFile("/foo/foobar1").getFileId());
    Assert.assertEquals("user1", metaStore.getFile("/foo/foobar1").getOwner());
    Assert.assertEquals(536870912, metaStore.getFile("/foo/foobar1").getBlocksize());
    List<FileDiff> fileDiffs = metaStore.getFileDiffsByFileName("/foo/foobar1");
    Assert.assertTrue(fileDiffs.size() > 0);
    for (FileDiff fileDiff : fileDiffs) {
        if (fileDiff.getDiffType().equals(FileDiffType.APPEND)) {
            Assert.assertTrue(fileDiff.getParameters().get("-owner").equals("user1"));
            Assert.assertTrue(fileDiff.getParameters().get("-mtime").equals("1515665470681"));
            Assert.assertTrue(fileDiff.getParameters().get("-length").equals("10"));
        }
    }
}

Example 23 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project SSM by Intel-bigdata.

the class TestAlluxioEntryApplier method testDeleteFileApplier.

@Test
public void testDeleteFileApplier() throws Exception {
    FileSystem fs = Mockito.mock(FileSystem.class);
    AlluxioEntryApplier entryApplier = new AlluxioEntryApplier(metaStore, fs);
    FileInfo fooFile = FileInfo.newBuilder().setFileId(100663295).setIsdir(false).setPath("/foo/foobar_del").build();
    metaStore.insertFile(fooFile);
    BackUpInfo backUpInfo = new BackUpInfo(1L, "/foo/foobar_del", "remote/dest/", 10);
    metaStore.insertBackUpInfo(backUpInfo);
    alluxio.wire.FileInfo info1 = new alluxio.wire.FileInfo().setFileId(100663295).setPath("/foo/foobar_del").setLength(500L).setFolder(false).setBlockSizeBytes(510000).setLastModificationTimeMs(1515665270681L).setCreationTimeMs(1515665270681L).setMode(493).setOwner("user1").setGroup("group1");
    URIStatus status1 = new URIStatus(info1);
    Mockito.when(fs.getStatus(new AlluxioURI("/foo/foobar_del"))).thenReturn(status1);
    DeleteFileEntry deleteFileEntry = DeleteFileEntry.newBuilder().setId(100663295).setOpTimeMs(1515737580798L).setAlluxioOnly(true).setRecursive(false).build();
    JournalEntry deleteFileJEntry = JournalEntry.newBuilder().setDeleteFile(deleteFileEntry).build();
    entryApplier.apply(deleteFileJEntry);
    List<FileDiff> fileDiffs = metaStore.getFileDiffsByFileName("/foo/foobar_del");
    Assert.assertTrue(fileDiffs.size() > 0);
    for (FileDiff fileDiff : fileDiffs) {
        if (fileDiff.getDiffType().equals(FileDiffType.DELETE)) {
            Assert.assertEquals("/foo/foobar_del", fileDiff.getSrc());
        }
    }
}

Example 24 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project SSM by Intel-bigdata.

the class TestAlluxioEntryFetcher method testEntryFetcher.

@Test
public void testEntryFetcher() throws Exception {
    URI journalLocation = JournalUtils.getJournalLocation();
    SmartConf conf = new SmartConf();
    conf.set(SmartConfKeys.SMART_ALLUXIO_MASTER_JOURNAL_DIR_KEY, journalLocation.getPath());
    EntryApplierForTest entryApplierForTest = new EntryApplierForTest(metaStore, fs);
    final AlluxioEntryFetcher entryFetcher = new AlluxioEntryFetcher(fs, metaStore, Executors.newScheduledThreadPool(2), entryApplierForTest, new Callable() {

        @Override
        public Object call() throws Exception {
            // Do nothing
            return null;
        }
    }, conf);
    Assert.assertFalse(AlluxioEntryFetcher.canReadFromLastSeqNum(100L));
    /**
     * Generate such local structure
     *      ├── foo |
     *              ├── foobar1
     *              └── foobar2
     *      ├── bar |
     *              └── foobar3
     *      └── foobar4
     */
    fs.createDirectory(new AlluxioURI("/foo"));
    fs.createDirectory(new AlluxioURI("/bar"));
    FileSystemTestUtils.createByteFile(fs, "/foo/foobar1", WriteType.CACHE_THROUGH, 10);
    FileSystemTestUtils.createByteFile(fs, "/foo/foobar2", WriteType.CACHE_THROUGH, 20);
    FileSystemTestUtils.createByteFile(fs, "/bar/foobar3", WriteType.CACHE_THROUGH, 30);
    FileSystemTestUtils.createByteFile(fs, "/foobar4", WriteType.CACHE_THROUGH, 40);
    Thread thread = new Thread() {

        public void run() {
            try {
                entryFetcher.start();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    };
    thread.start();
    // need wait long enough to finish namespace fetcher
    Thread.sleep(10 * 1000);
    fs.setAttribute(new AlluxioURI("/foo/foobar1"), SetAttributeOptions.defaults().setPersisted(true));
    fs.setAttribute(new AlluxioURI("/foo/foobar2"), SetAttributeOptions.defaults().setPinned(true));
    fs.setAttribute(new AlluxioURI("/bar/foobar3"), SetAttributeOptions.defaults().setTtl(1000000L));
    String mLocalUfsRoot = mTemporaryFolder.getRoot().getAbsolutePath();
    UnderFileSystem mLocalUfs = UnderFileSystem.Factory.create(mLocalUfsRoot);
    String mountpath = PathUtils.concatPath(mLocalUfsRoot, "mtd_ufs");
    mLocalUfs.mkdirs(mountpath);
    Assert.assertTrue(new File(mountpath).exists());
    fs.mount(new AlluxioURI("/mtd_t"), new AlluxioURI(mountpath), MountOptions.defaults());
    fs.rename(new AlluxioURI("/foo/foobar1"), new AlluxioURI("/foo/foo1"), RenameOptions.defaults());
    fs.delete(new AlluxioURI("/bar/foobar3"), DeleteOptions.defaults().setRecursive(true));
    fs.createDirectory(new AlluxioURI("/baz"));
    FileSystemTestUtils.createByteFile(fs, "/baz/foobar5", WriteType.CACHE_THROUGH, 50);
    Mode mode = new Mode((short) 0755);
    fs.setAttribute(new AlluxioURI("/baz/foobar5"), SetAttributeOptions.defaults().setMode(mode));
    // free action does not generate journal entry
    fs.free(new AlluxioURI("/baz"), FreeOptions.defaults().setRecursive(true));
    while (entryApplierForTest.getEntries().size() != 16) {
        Thread.sleep(100);
    }
    List<JournalEntry> entries = entryApplierForTest.getEntries();
    Assert.assertTrue(entries.get(0).hasSetAttribute() && entries.get(0).getSetAttribute().hasPersisted());
    Assert.assertTrue(entries.get(1).hasSetAttribute() && entries.get(1).getSetAttribute().hasPinned());
    Assert.assertTrue(entries.get(2).hasSetAttribute() && entries.get(2).getSetAttribute().hasTtl());
    Assert.assertTrue(entries.get(3).hasInodeLastModificationTime());
    Assert.assertTrue(entries.get(4).hasInodeDirectoryIdGenerator());
    Assert.assertTrue(entries.get(5).hasInodeDirectory());
    Assert.assertTrue(entries.get(6).hasAddMountPoint());
    Assert.assertTrue(entries.get(7).hasRename());
    Assert.assertTrue(entries.get(8).hasDeleteFile());
    Assert.assertTrue(entries.get(9).hasInodeLastModificationTime());
    Assert.assertTrue(entries.get(10).hasInodeDirectoryIdGenerator());
    Assert.assertTrue(entries.get(11).hasInodeDirectory());
    Assert.assertTrue(entries.get(12).hasInodeLastModificationTime());
    Assert.assertTrue(entries.get(13).hasInodeFile());
    Assert.assertTrue(entries.get(14).hasCompleteFile());
    Assert.assertTrue(entries.get(15).hasSetAttribute() && entries.get(15).getSetAttribute().hasPermission());
    entryFetcher.stop();
    Assert.assertTrue(metaStore.containSystemInfo(SmartConstants.SMART_ALLUXIO_LAST_ENTRY_SN));
    Assert.assertTrue(AlluxioEntryFetcher.canReadFromLastSeqNum(Long.parseLong(metaStore.getSystemInfoByProperty(SmartConstants.SMART_ALLUXIO_LAST_ENTRY_SN).getValue())));
}

Example 25 with JournalEntry

use of alluxio.proto.journal.Journal.JournalEntry in project alluxio by Alluxio.

the class JournalSinkTest method replayEvents.

@Test
public void replayEvents() throws Exception {
    // start standby masters
    TestJournalSink standbySink = new TestJournalSink(new LinkedBlockingQueue<>());
    startMasters(new MasterRegistry(), JournalTestUtils.createJournalSystem(mJournalFolder), standbySink, false);
    int nextCreateId1 = 1;
    int nextCreateId2 = 1;
    int nextRenameId = 1;
    int nextDeleteId = 1;
    int completed = 0;
    while (completed < 5000) {
        switch(ThreadLocalRandom.current().nextInt(4)) {
            case 0:
                createFile("/file_for_rename" + nextCreateId1++);
                completed++;
                break;
            case 1:
                createFile("/file_for_delete" + nextCreateId2++);
                completed++;
                break;
            case 2:
                if (nextRenameId < nextCreateId1) {
                    mFileSystemMaster.rename(new AlluxioURI("/file_for_rename" + nextRenameId), new AlluxioURI("/renamed" + nextRenameId), RenameContext.defaults());
                    nextRenameId++;
                    completed++;
                }
                break;
            case 3:
                if (nextDeleteId < nextCreateId2) {
                    mFileSystemMaster.delete(new AlluxioURI("/file_for_delete" + nextDeleteId), DeleteContext.create(DeletePOptions.newBuilder().setRecursive(true)));
                    nextDeleteId++;
                    completed++;
                }
                break;
            default:
        }
    }
    // Stop and restart the default masters
    stopMasters(mRegistry, mJournalSystem);
    mJournalSystem.removeJournalSink(mRegistry.get(FileSystemMaster.class), mJournalSink);
    TestJournalSink restartSink = new TestJournalSink(new LinkedBlockingQueue<>());
    startMasters(mRegistry, mJournalSystem, restartSink, true);
    CommonUtils.waitFor("leader receives all entries", () -> mJournalSink.getEntries().size() == restartSink.getEntries().size(), WaitForOptions.defaults().setTimeoutMs(5000));
    CommonUtils.waitFor("standby receives all entries", () -> mJournalSink.getEntries().size() == standbySink.getEntries().size(), WaitForOptions.defaults().setTimeoutMs(5000));
    // Need to strip out the sequence number because
    List<JournalEntry> masterEntries = stripSeqNo(mJournalSink.getEntries());
    List<JournalEntry> restartEntries = stripSeqNo(restartSink.getEntries());
    List<JournalEntry> standbyEntries = stripSeqNo(standbySink.getEntries());
    assertFalse(masterEntries.isEmpty());
    assertEquals("leader restart failed to see all entries", masterEntries, restartEntries);
    assertEquals("standby failed to see all entries", masterEntries, standbyEntries);
}