Examples with AlluxioEntryFetcher org.smartdata.alluxio.metric.fetcher.AlluxioEntryFetcher used on opensource projects

Example 1 with AlluxioEntryFetcher

use of org.smartdata.alluxio.metric.fetcher.AlluxioEntryFetcher in project SSM by Intel-bigdata.

the class TestAlluxioEntryFetcher method testEntryFetcher.

@Test
public void testEntryFetcher() throws Exception {
    URI journalLocation = JournalUtils.getJournalLocation();
    SmartConf conf = new SmartConf();
    conf.set(SmartConfKeys.SMART_ALLUXIO_MASTER_JOURNAL_DIR_KEY, journalLocation.getPath());
    EntryApplierForTest entryApplierForTest = new EntryApplierForTest(metaStore, fs);
    final AlluxioEntryFetcher entryFetcher = new AlluxioEntryFetcher(fs, metaStore, Executors.newScheduledThreadPool(2), entryApplierForTest, new Callable() {

        @Override
        public Object call() throws Exception {
            // Do nothing
            return null;
        }
    }, conf);
    Assert.assertFalse(AlluxioEntryFetcher.canReadFromLastSeqNum(100L));
    /**
     * Generate such local structure
     *      ├── foo |
     *              ├── foobar1
     *              └── foobar2
     *      ├── bar |
     *              └── foobar3
     *      └── foobar4
     */
    fs.createDirectory(new AlluxioURI("/foo"));
    fs.createDirectory(new AlluxioURI("/bar"));
    FileSystemTestUtils.createByteFile(fs, "/foo/foobar1", WriteType.CACHE_THROUGH, 10);
    FileSystemTestUtils.createByteFile(fs, "/foo/foobar2", WriteType.CACHE_THROUGH, 20);
    FileSystemTestUtils.createByteFile(fs, "/bar/foobar3", WriteType.CACHE_THROUGH, 30);
    FileSystemTestUtils.createByteFile(fs, "/foobar4", WriteType.CACHE_THROUGH, 40);
    Thread thread = new Thread() {

        public void run() {
            try {
                entryFetcher.start();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    };
    thread.start();
    // need wait long enough to finish namespace fetcher
    Thread.sleep(10 * 1000);
    fs.setAttribute(new AlluxioURI("/foo/foobar1"), SetAttributeOptions.defaults().setPersisted(true));
    fs.setAttribute(new AlluxioURI("/foo/foobar2"), SetAttributeOptions.defaults().setPinned(true));
    fs.setAttribute(new AlluxioURI("/bar/foobar3"), SetAttributeOptions.defaults().setTtl(1000000L));
    String mLocalUfsRoot = mTemporaryFolder.getRoot().getAbsolutePath();
    UnderFileSystem mLocalUfs = UnderFileSystem.Factory.create(mLocalUfsRoot);
    String mountpath = PathUtils.concatPath(mLocalUfsRoot, "mtd_ufs");
    mLocalUfs.mkdirs(mountpath);
    Assert.assertTrue(new File(mountpath).exists());
    fs.mount(new AlluxioURI("/mtd_t"), new AlluxioURI(mountpath), MountOptions.defaults());
    fs.rename(new AlluxioURI("/foo/foobar1"), new AlluxioURI("/foo/foo1"), RenameOptions.defaults());
    fs.delete(new AlluxioURI("/bar/foobar3"), DeleteOptions.defaults().setRecursive(true));
    fs.createDirectory(new AlluxioURI("/baz"));
    FileSystemTestUtils.createByteFile(fs, "/baz/foobar5", WriteType.CACHE_THROUGH, 50);
    Mode mode = new Mode((short) 0755);
    fs.setAttribute(new AlluxioURI("/baz/foobar5"), SetAttributeOptions.defaults().setMode(mode));
    // free action does not generate journal entry
    fs.free(new AlluxioURI("/baz"), FreeOptions.defaults().setRecursive(true));
    while (entryApplierForTest.getEntries().size() != 16) {
        Thread.sleep(100);
    }
    List<JournalEntry> entries = entryApplierForTest.getEntries();
    Assert.assertTrue(entries.get(0).hasSetAttribute() && entries.get(0).getSetAttribute().hasPersisted());
    Assert.assertTrue(entries.get(1).hasSetAttribute() && entries.get(1).getSetAttribute().hasPinned());
    Assert.assertTrue(entries.get(2).hasSetAttribute() && entries.get(2).getSetAttribute().hasTtl());
    Assert.assertTrue(entries.get(3).hasInodeLastModificationTime());
    Assert.assertTrue(entries.get(4).hasInodeDirectoryIdGenerator());
    Assert.assertTrue(entries.get(5).hasInodeDirectory());
    Assert.assertTrue(entries.get(6).hasAddMountPoint());
    Assert.assertTrue(entries.get(7).hasRename());
    Assert.assertTrue(entries.get(8).hasDeleteFile());
    Assert.assertTrue(entries.get(9).hasInodeLastModificationTime());
    Assert.assertTrue(entries.get(10).hasInodeDirectoryIdGenerator());
    Assert.assertTrue(entries.get(11).hasInodeDirectory());
    Assert.assertTrue(entries.get(12).hasInodeLastModificationTime());
    Assert.assertTrue(entries.get(13).hasInodeFile());
    Assert.assertTrue(entries.get(14).hasCompleteFile());
    Assert.assertTrue(entries.get(15).hasSetAttribute() && entries.get(15).getSetAttribute().hasPermission());
    entryFetcher.stop();
    Assert.assertTrue(metaStore.containSystemInfo(SmartConstants.SMART_ALLUXIO_LAST_ENTRY_SN));
    Assert.assertTrue(AlluxioEntryFetcher.canReadFromLastSeqNum(Long.parseLong(metaStore.getSystemInfoByProperty(SmartConstants.SMART_ALLUXIO_LAST_ENTRY_SN).getValue())));
}

Example 2 with AlluxioEntryFetcher

use of org.smartdata.alluxio.metric.fetcher.AlluxioEntryFetcher in project SSM by Intel-bigdata.

the class AlluxioStatesUpdateService method init.

/**
 * Load configure/data to initialize.
 *
 * @return true if initialized successfully
 */
@Override
public void init() throws IOException {
    LOG.info("Initializing ...");
    SmartContext context = getContext();
    this.alluxioFs = AlluxioUtil.getAlluxioFs(context);
    this.moverIdOutStream = checkAndMarkRunning(alluxioFs);
    this.executorService = Executors.newScheduledThreadPool(4);
    this.alluxioEntryFetcher = new AlluxioEntryFetcher(alluxioFs, metaStore, executorService, new EntryFetchFinishedCallBack(), context.getConf());
    LOG.info("Initialized.");
}