Examples with UnderFileSystem alluxio.underfs.UnderFileSystem used on opensource projects

Example 86 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class GlusterFSUnderFileSystemFactoryTest method createGlusterFS.

/**
   * Tests the {@link UnderFileSystem#create(String)} method.
   */
@Test
public void createGlusterFS() throws Exception {
    // Using Assume will mark the tests as skipped rather than passed which provides a truer
    // indication of their status
    Assume.assumeTrue(!StringUtils.isEmpty(mMount));
    Assume.assumeTrue(!StringUtils.isEmpty(mVolume));
    UnderFileSystem gfs = UnderFileSystem.Factory.get("glusterfs:///");
    Assert.assertNotNull(gfs.create("alluxio_test"));
}

Example 87 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class FileOutStream method close.

@Override
public void close() throws IOException {
    if (mClosed) {
        return;
    }
    try {
        if (mCurrentBlockOutStream != null) {
            mPreviousBlockOutStreams.add(mCurrentBlockOutStream);
        }
        CompleteFileOptions options = CompleteFileOptions.defaults();
        if (mUnderStorageType.isSyncPersist()) {
            if (mUfsDelegation) {
                mUnderStorageOutputStream.close();
                if (mCanceled) {
                    mFileSystemWorkerClient.cancelUfsFile(mUfsFileId, CancelUfsFileOptions.defaults());
                } else {
                    long len = mFileSystemWorkerClient.completeUfsFile(mUfsFileId, CompleteUfsFileOptions.defaults());
                    options.setUfsLength(len);
                }
            } else {
                UnderFileSystem ufs = UnderFileSystem.Factory.get(mUfsPath);
                if (mCanceled) {
                    // TODO(yupeng): Handle this special case in under storage integrations.
                    mUnderStorageOutputStream.close();
                    ufs.deleteFile(mUfsPath);
                } else {
                    mUnderStorageOutputStream.flush();
                    mUnderStorageOutputStream.close();
                    options.setUfsLength(ufs.getFileSize(mUfsPath));
                }
            }
        }
        if (mAlluxioStorageType.isStore()) {
            if (mCanceled) {
                for (OutputStream bos : mPreviousBlockOutStreams) {
                    outStreamCancel(bos);
                }
            } else {
                for (OutputStream bos : mPreviousBlockOutStreams) {
                    bos.close();
                }
            }
        }
        // Complete the file if it's ready to be completed.
        if (!mCanceled && (mUnderStorageType.isSyncPersist() || mAlluxioStorageType.isStore())) {
            try (CloseableResource<FileSystemMasterClient> masterClient = mContext.acquireMasterClientResource()) {
                masterClient.get().completeFile(mUri, options);
            }
        }
        if (mUnderStorageType.isAsyncPersist()) {
            scheduleAsyncPersist();
        }
    } catch (AlluxioException e) {
        throw mCloser.rethrow(new IOException(e));
    } catch (Throwable e) {
        // IOException will be thrown as-is
        throw mCloser.rethrow(e);
    } finally {
        mClosed = true;
        mCloser.close();
    }
}

Example 88 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class CheckConsistencyCommandTest method inconsistent.

/**
   * Tests the check consistency shell command correctly identifies inconsistent files in a subtree.
   */
@Test
public void inconsistent() throws Exception {
    FileSystemTestUtils.createByteFile(mFileSystem, "/testRoot/testFileA", WriteType.CACHE_THROUGH, 10);
    FileSystemTestUtils.createByteFile(mFileSystem, "/testRoot/testDir/testFileB", WriteType.CACHE_THROUGH, 20);
    String ufsPath = mFileSystem.getStatus(new AlluxioURI("/testRoot/testDir")).getUfsPath();
    UnderFileSystem ufs = UnderFileSystem.Factory.get(ufsPath);
    ufs.deleteDirectory(ufsPath, DeleteOptions.defaults().setRecursive(true));
    mFsShell.run("checkConsistency", "/testRoot");
    StringBuilder expected = new StringBuilder();
    expected.append("The following files are inconsistent:\n");
    expected.append("/testRoot/testDir\n");
    expected.append("/testRoot/testDir/testFileB\n");
    Assert.assertEquals(expected.toString(), mOutput.toString());
    mOutput.reset();
    mFsShell.run("checkConsistency", "-r", "/testRoot");
    String res = mOutput.toString();
    Assert.assertTrue(res.contains("/testRoot" + " has: " + "2 inconsistent files.\n") && res.contains("repairing path: " + "/testRoot/testDir\n") && res.contains("repairing path: " + "/testRoot/testDir/testFileB\n"));
    Assert.assertTrue(!mFileSystem.exists(new AlluxioURI("/testRoot/testDir")));
    Assert.assertTrue(!mFileSystem.exists(new AlluxioURI("/testRoot/testDir/testFileB")));
    FileSystemTestUtils.createByteFile(mFileSystem, "/testRoot/testDir/testFileB", WriteType.CACHE_THROUGH, 20);
    ufsPath = mFileSystem.getStatus(new AlluxioURI("/testRoot/testDir/testFileB")).getUfsPath();
    ufs.deleteFile(ufsPath);
    OutputStream outputStream = ufs.create(ufsPath);
    byte[] bytes = { 1, 2, 3 };
    outputStream.write(bytes);
    outputStream.close();
    mOutput.reset();
    mFsShell.run("checkConsistency", "-r", "/testRoot");
    res = mOutput.toString();
    Assert.assertTrue(res.contains("/testRoot" + " has: " + "1 inconsistent files.\n") && res.contains("repairing path: " + "/testRoot/testDir/testFileB\n"));
    Assert.assertTrue(mFileSystem.exists(new AlluxioURI("/testRoot/testDir/testFileB")));
    Assert.assertTrue(3 == mFileSystem.getStatus(new AlluxioURI("/testRoot/testDir/testFileB")).getLength());
}

Example 89 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class PersistCommandTest method persistWithAncestorPermission.

@Test
public void persistWithAncestorPermission() throws Exception {
    String ufsRoot = PathUtils.concatPath(Configuration.get(PropertyKey.UNDERFS_ADDRESS));
    UnderFileSystem ufs = UnderFileSystem.Factory.get(ufsRoot);
    // Skip non-local and non-HDFS UFSs.
    Assume.assumeTrue(UnderFileSystemUtils.isLocal(ufs) || UnderFileSystemUtils.isHdfs(ufs));
    AlluxioURI testFile = new AlluxioURI("/grand/parent/file");
    AlluxioURI grandParent = new AlluxioURI("/grand");
    Mode grandParentMode = new Mode((short) 0777);
    FileSystemTestUtils.createByteFile(mFileSystem, testFile, WriteType.MUST_CACHE, 10);
    URIStatus status = mFileSystem.getStatus(testFile);
    Assert.assertFalse(status.isPersisted());
    mFileSystem.setAttribute(grandParent, SetAttributeOptions.defaults().setMode(grandParentMode));
    int ret = mFsShell.run("persist", testFile.toString());
    Assert.assertEquals(0, ret);
    checkFilePersisted(testFile, 10);
    // Check the permission of the created file and ancestor dir are in-sync between Alluxio and UFS
    short fileMode = (short) status.getMode();
    short parentMode = (short) mFileSystem.getStatus(testFile.getParent()).getMode();
    Assert.assertEquals(fileMode, ufs.getMode(PathUtils.concatPath(ufsRoot, testFile)));
    Assert.assertEquals(parentMode, ufs.getMode(PathUtils.concatPath(ufsRoot, testFile.getParent())));
    Assert.assertEquals(grandParentMode, new Mode(ufs.getMode(PathUtils.concatPath(ufsRoot, grandParent))));
}

Example 90 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class UfsJournal method isFormatted.

@Override
public boolean isFormatted() throws IOException {
    UfsStatus[] files;
    try (UnderFileSystem ufs = UnderFileSystem.Factory.create(mLocation.toString(), UnderFileSystemConfiguration.defaults(ServerConfiguration.global()))) {
        files = ufs.listStatus(mLocation.toString());
    }
    if (files == null) {
        return false;
    }
    // Search for the format file.
    String formatFilePrefix = ServerConfiguration.getString(PropertyKey.MASTER_FORMAT_FILE_PREFIX);
    for (UfsStatus file : files) {
        if (file.getName().startsWith(formatFilePrefix)) {
            return true;
        }
    }
    return false;
}