Examples with UnderFileSystem alluxio.underfs.UnderFileSystem used on opensource projects

Example 41 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class AlluxioMasterProcess method initFromBackup.

private void initFromBackup(AlluxioURI backup) throws IOException {
    CloseableResource<UnderFileSystem> ufsResource;
    if (URIUtils.isLocalFilesystem(backup.toString())) {
        UnderFileSystem ufs = UnderFileSystem.Factory.create("/", UnderFileSystemConfiguration.defaults(ServerConfiguration.global()));
        ufsResource = new CloseableResource<UnderFileSystem>(ufs) {

            @Override
            public void closeResource() {
            }
        };
    } else {
        ufsResource = mUfsManager.getRoot().acquireUfsResource();
    }
    try (CloseableResource<UnderFileSystem> closeUfs = ufsResource;
        InputStream ufsIn = closeUfs.get().open(backup.getPath())) {
        LOG.info("Initializing metadata from backup {}", backup);
        mBackupManager.initFromBackup(ufsIn);
    }
}

Example 42 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DailyMetadataBackup method deleteStaleBackups.

/**
 * Deletes stale backup files to avoid consuming too many spaces.
 */
private void deleteStaleBackups() throws Exception {
    try (CloseableResource<UnderFileSystem> ufsResource = mUfsManager.getRoot().acquireUfsResource()) {
        UnderFileSystem ufs = ufsResource.get();
        UfsStatus[] statuses = ufs.listStatus(mBackupDir);
        if (statuses.length <= mRetainedFiles) {
            return;
        }
        // Sort the backup files according to create time from oldest to newest
        TreeMap<Instant, String> timeToFile = new TreeMap<>((a, b) -> (a.isBefore(b) ? -1 : a.isAfter(b) ? 1 : 0));
        for (UfsStatus status : statuses) {
            if (status.isFile()) {
                Matcher matcher = BackupManager.BACKUP_FILE_PATTERN.matcher(status.getName());
                if (matcher.matches()) {
                    timeToFile.put(Instant.ofEpochMilli(Long.parseLong(matcher.group(1))), status.getName());
                }
            }
        }
        int toDeleteFileNum = timeToFile.size() - mRetainedFiles;
        if (toDeleteFileNum <= 0) {
            return;
        }
        for (int i = 0; i < toDeleteFileNum; i++) {
            String toDeleteFile = PathUtils.concatPath(mBackupDir, timeToFile.pollFirstEntry().getValue());
            ufs.deleteExistingFile(toDeleteFile);
        }
        LOG.info("Deleted {} stale metadata backup files at {}", toDeleteFileNum, mBackupDir);
    }
}

Example 43 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class FileSystemMasterMetricsTest method testMetricsUfsCapacity.

@Test
public void testMetricsUfsCapacity() throws Exception {
    UfsManager.UfsClient client = Mockito.mock(UfsManager.UfsClient.class);
    UnderFileSystem ufs = Mockito.mock(UnderFileSystem.class);
    String ufsDataFolder = ServerConfiguration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS);
    when(ufs.getSpace(ufsDataFolder, UnderFileSystem.SpaceType.SPACE_TOTAL)).thenReturn(1000L);
    when(ufs.getSpace(ufsDataFolder, UnderFileSystem.SpaceType.SPACE_USED)).thenReturn(200L);
    when(ufs.getSpace(ufsDataFolder, UnderFileSystem.SpaceType.SPACE_FREE)).thenReturn(800L);
    when(client.acquireUfsResource()).thenReturn(new CloseableResource<UnderFileSystem>(ufs) {

        @Override
        public void closeResource() {
        }
    });
    when(mUfsManager.getRoot()).thenReturn(client);
    assertEquals(1000L, getGauge(MetricKey.CLUSTER_ROOT_UFS_CAPACITY_TOTAL.getName()));
    assertEquals(200L, getGauge(MetricKey.CLUSTER_ROOT_UFS_CAPACITY_USED.getName()));
    assertEquals(800L, getGauge(MetricKey.CLUSTER_ROOT_UFS_CAPACITY_FREE.getName()));
}

Example 44 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project SSM by Intel-bigdata.

the class TestAlluxioEntryFetcher method testEntryFetcher.

@Test
public void testEntryFetcher() throws Exception {
    URI journalLocation = JournalUtils.getJournalLocation();
    SmartConf conf = new SmartConf();
    conf.set(SmartConfKeys.SMART_ALLUXIO_MASTER_JOURNAL_DIR_KEY, journalLocation.getPath());
    EntryApplierForTest entryApplierForTest = new EntryApplierForTest(metaStore, fs);
    final AlluxioEntryFetcher entryFetcher = new AlluxioEntryFetcher(fs, metaStore, Executors.newScheduledThreadPool(2), entryApplierForTest, new Callable() {

        @Override
        public Object call() throws Exception {
            // Do nothing
            return null;
        }
    }, conf);
    Assert.assertFalse(AlluxioEntryFetcher.canReadFromLastSeqNum(100L));
    /**
     * Generate such local structure
     *      ├── foo |
     *              ├── foobar1
     *              └── foobar2
     *      ├── bar |
     *              └── foobar3
     *      └── foobar4
     */
    fs.createDirectory(new AlluxioURI("/foo"));
    fs.createDirectory(new AlluxioURI("/bar"));
    FileSystemTestUtils.createByteFile(fs, "/foo/foobar1", WriteType.CACHE_THROUGH, 10);
    FileSystemTestUtils.createByteFile(fs, "/foo/foobar2", WriteType.CACHE_THROUGH, 20);
    FileSystemTestUtils.createByteFile(fs, "/bar/foobar3", WriteType.CACHE_THROUGH, 30);
    FileSystemTestUtils.createByteFile(fs, "/foobar4", WriteType.CACHE_THROUGH, 40);
    Thread thread = new Thread() {

        public void run() {
            try {
                entryFetcher.start();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    };
    thread.start();
    // need wait long enough to finish namespace fetcher
    Thread.sleep(10 * 1000);
    fs.setAttribute(new AlluxioURI("/foo/foobar1"), SetAttributeOptions.defaults().setPersisted(true));
    fs.setAttribute(new AlluxioURI("/foo/foobar2"), SetAttributeOptions.defaults().setPinned(true));
    fs.setAttribute(new AlluxioURI("/bar/foobar3"), SetAttributeOptions.defaults().setTtl(1000000L));
    String mLocalUfsRoot = mTemporaryFolder.getRoot().getAbsolutePath();
    UnderFileSystem mLocalUfs = UnderFileSystem.Factory.create(mLocalUfsRoot);
    String mountpath = PathUtils.concatPath(mLocalUfsRoot, "mtd_ufs");
    mLocalUfs.mkdirs(mountpath);
    Assert.assertTrue(new File(mountpath).exists());
    fs.mount(new AlluxioURI("/mtd_t"), new AlluxioURI(mountpath), MountOptions.defaults());
    fs.rename(new AlluxioURI("/foo/foobar1"), new AlluxioURI("/foo/foo1"), RenameOptions.defaults());
    fs.delete(new AlluxioURI("/bar/foobar3"), DeleteOptions.defaults().setRecursive(true));
    fs.createDirectory(new AlluxioURI("/baz"));
    FileSystemTestUtils.createByteFile(fs, "/baz/foobar5", WriteType.CACHE_THROUGH, 50);
    Mode mode = new Mode((short) 0755);
    fs.setAttribute(new AlluxioURI("/baz/foobar5"), SetAttributeOptions.defaults().setMode(mode));
    // free action does not generate journal entry
    fs.free(new AlluxioURI("/baz"), FreeOptions.defaults().setRecursive(true));
    while (entryApplierForTest.getEntries().size() != 16) {
        Thread.sleep(100);
    }
    List<JournalEntry> entries = entryApplierForTest.getEntries();
    Assert.assertTrue(entries.get(0).hasSetAttribute() && entries.get(0).getSetAttribute().hasPersisted());
    Assert.assertTrue(entries.get(1).hasSetAttribute() && entries.get(1).getSetAttribute().hasPinned());
    Assert.assertTrue(entries.get(2).hasSetAttribute() && entries.get(2).getSetAttribute().hasTtl());
    Assert.assertTrue(entries.get(3).hasInodeLastModificationTime());
    Assert.assertTrue(entries.get(4).hasInodeDirectoryIdGenerator());
    Assert.assertTrue(entries.get(5).hasInodeDirectory());
    Assert.assertTrue(entries.get(6).hasAddMountPoint());
    Assert.assertTrue(entries.get(7).hasRename());
    Assert.assertTrue(entries.get(8).hasDeleteFile());
    Assert.assertTrue(entries.get(9).hasInodeLastModificationTime());
    Assert.assertTrue(entries.get(10).hasInodeDirectoryIdGenerator());
    Assert.assertTrue(entries.get(11).hasInodeDirectory());
    Assert.assertTrue(entries.get(12).hasInodeLastModificationTime());
    Assert.assertTrue(entries.get(13).hasInodeFile());
    Assert.assertTrue(entries.get(14).hasCompleteFile());
    Assert.assertTrue(entries.get(15).hasSetAttribute() && entries.get(15).getSetAttribute().hasPermission());
    entryFetcher.stop();
    Assert.assertTrue(metaStore.containSystemInfo(SmartConstants.SMART_ALLUXIO_LAST_ENTRY_SN));
    Assert.assertTrue(AlluxioEntryFetcher.canReadFromLastSeqNum(Long.parseLong(metaStore.getSystemInfoByProperty(SmartConstants.SMART_ALLUXIO_LAST_ENTRY_SN).getValue())));
}

Example 45 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class UnderFileSystemBlockReader method updateUnderFileSystemInputStream.

/**
 * Updates the UFS input stream given an offset to read.
 *
 * @param offset the read offset within the block
 */
private void updateUnderFileSystemInputStream(long offset) throws IOException {
    if ((mUnderFileSystemInputStream != null) && offset != mInStreamPos) {
        mUfsInstreamCache.release(mUnderFileSystemInputStream);
        mUnderFileSystemInputStream = null;
        mInStreamPos = -1;
    }
    if (mUnderFileSystemInputStream == null && offset < mBlockMeta.getBlockSize()) {
        UnderFileSystem ufs = mUfsResource.get();
        mUnderFileSystemInputStream = mUfsInstreamCache.acquire(ufs, mBlockMeta.getUnderFileSystemPath(), IdUtils.fileIdFromBlockId(mBlockMeta.getBlockId()), OpenOptions.defaults().setOffset(mBlockMeta.getOffset() + offset).setPositionShort(mIsPositionShort));
        mInStreamPos = offset;
    }
}