Examples with UnderFileSystem alluxio.underfs.UnderFileSystem used on opensource projects

Example 6 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class StartupConsistencyCheckTest method inconsistent.

/**
   * Tests that an inconsistent Alluxio system's startup check correctly detects the inconsistent
   * files.
   */
@Test
public void inconsistent() throws Exception {
    String topLevelFileUfsPath = mFileSystem.getStatus(TOP_LEVEL_FILE).getUfsPath();
    String secondLevelDirUfsPath = mFileSystem.getStatus(SECOND_LEVEL_DIR).getUfsPath();
    mCluster.stopFS();
    UnderFileSystem ufs = UnderFileSystem.Factory.get(topLevelFileUfsPath);
    ufs.deleteFile(topLevelFileUfsPath);
    ufs.deleteDirectory(secondLevelDirUfsPath, DeleteOptions.defaults().setRecursive(true));
    final FileSystemMaster master = MasterTestUtils.createLeaderFileSystemMasterFromJournal();
    MasterTestUtils.waitForStartupConsistencyCheck(master);
    List expected = Lists.newArrayList(TOP_LEVEL_FILE, SECOND_LEVEL_DIR, THIRD_LEVEL_FILE);
    List result = master.getStartupConsistencyCheck().getInconsistentUris();
    Collections.sort(expected);
    Collections.sort(result);
    Assert.assertEquals(expected, result);
}

Example 7 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class JournalIntegrationTest method loadMetadata.

/**
   * Tests loading metadata.
   */
@Test
public void loadMetadata() throws Exception {
    String ufsRoot = PathUtils.concatPath(Configuration.get(PropertyKey.UNDERFS_ADDRESS));
    UnderFileSystem ufs = UnderFileSystem.Factory.get(ufsRoot);
    ufs.create(ufsRoot + "/xyz").close();
    mFileSystem.loadMetadata(new AlluxioURI("/xyz"));
    URIStatus status = mFileSystem.getStatus(new AlluxioURI("/xyz"));
    mLocalAlluxioCluster.stopFS();
    loadMetadataTestUtil(status);
    deleteFsMasterJournalLogs();
    loadMetadataTestUtil(status);
}

Example 8 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class FileSystemMaster method mountInternal.

/**
   * Updates the mount table with the specified mount point. The mount options may be updated during
   * this method.
   *
   * @param inodePath the Alluxio mount point
   * @param ufsPath the UFS endpoint to mount
   * @param replayed whether the operation is a result of replaying the journal
   * @param options the mount options (may be updated)
   * @throws FileAlreadyExistsException if the mount point already exists
   * @throws InvalidPathException if an invalid path is encountered
   * @throws IOException if an I/O exception occurs
   */
private void mountInternal(LockedInodePath inodePath, AlluxioURI ufsPath, boolean replayed, MountOptions options) throws FileAlreadyExistsException, InvalidPathException, IOException {
    AlluxioURI alluxioPath = inodePath.getUri();
    if (!replayed) {
        // Check that the ufsPath exists and is a directory
        UnderFileSystem ufs = UnderFileSystem.Factory.get(ufsPath.toString());
        ufs.setProperties(options.getProperties());
        if (!ufs.isDirectory(ufsPath.toString())) {
            throw new IOException(ExceptionMessage.UFS_PATH_DOES_NOT_EXIST.getMessage(ufsPath.getPath()));
        }
        // Check that the alluxioPath we're creating doesn't shadow a path in the default UFS
        String defaultUfsPath = Configuration.get(PropertyKey.UNDERFS_ADDRESS);
        UnderFileSystem defaultUfs = UnderFileSystem.Factory.get(defaultUfsPath);
        String shadowPath = PathUtils.concatPath(defaultUfsPath, alluxioPath.getPath());
        if (defaultUfs.exists(shadowPath)) {
            throw new IOException(ExceptionMessage.MOUNT_PATH_SHADOWS_DEFAULT_UFS.getMessage(alluxioPath));
        }
        // Configure the ufs properties, and update the mount options with the configured properties.
        ufs.configureProperties();
        options.setProperties(ufs.getProperties());
    }
    // Add the mount point. This will only succeed if we are not mounting a prefix of an existing
    // mount and no existing mount is a prefix of this mount.
    mMountTable.add(alluxioPath, ufsPath, options);
}

Example 9 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class DefaultAlluxioMaster method connectToUFS.

private void connectToUFS() throws IOException {
    String ufsAddress = Configuration.get(PropertyKey.UNDERFS_ADDRESS);
    UnderFileSystem ufs = UnderFileSystem.Factory.get(ufsAddress);
    ufs.connectFromMaster(NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC));
}

Example 10 with UnderFileSystem

use of alluxio.underfs.UnderFileSystem in project alluxio by Alluxio.

the class JournalWriterTest method rotateLogOnFlushIOException.

@Test
public void rotateLogOnFlushIOException() throws Exception {
    // Setup so that we can trigger an IOException when flush is called on the underlying stream.
    JournalWriter mockJournalWriter = PowerMockito.mock(JournalWriter.class);
    OutputStream mockOutStream = mock(OutputStream.class);
    UnderFileSystem mockUfs = mock(UnderFileSystem.class);
    doReturn(mockOutStream).when(mockUfs).create(eq(mJournal.getCurrentLogFilePath()), any(CreateOptions.class));
    EntryOutputStream entryOutStream = new EntryOutputStream(mockUfs, mJournal.getCurrentLogFilePath(), mJournal.getJournalFormatter(), mockJournalWriter);
    entryOutStream.writeEntry(JournalEntry.newBuilder().build());
    doThrow(new IOException("flush failed")).when(mockOutStream).flush();
    try {
        entryOutStream.flush();
        Assert.fail("Should have thrown an exception");
    } catch (IOException ignored) {
    // expected
    }
    // Undo the earlier mocking so that the writeEntry doesn't fail.
    doNothing().when(mockOutStream).flush();
    // The rotation happens the next time an entry is written.
    entryOutStream.writeEntry(JournalEntry.newBuilder().build());
    verify(mockJournalWriter).completeCurrentLog();
}