Examples with FileSystemMaster alluxio.master.file.FileSystemMaster used on opensource projects

Example 21 with FileSystemMaster

use of alluxio.master.file.FileSystemMaster in project alluxio by Alluxio.

the class UfsJournalIntegrationTest method loadMetadataTestUtil.

private void loadMetadataTestUtil(URIStatus status) throws Exception {
    try (FsMasterResource masterResource = createFsMasterFromJournal()) {
        FileSystemMaster fsMaster = masterResource.getRegistry().get(FileSystemMaster.class);
        long rootId = fsMaster.getFileId(mRootUri);
        Assert.assertTrue(rootId != IdUtils.INVALID_FILE_ID);
        Assert.assertEquals(1, fsMaster.listStatus(mRootUri, ListStatusContext.mergeFrom(ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER))).size());
        Assert.assertTrue(fsMaster.getFileId(new AlluxioURI("/xyz")) != IdUtils.INVALID_FILE_ID);
        FileInfo fsMasterInfo = fsMaster.getFileInfo(fsMaster.getFileId(new AlluxioURI("/xyz")));
        Assert.assertEquals(status, new URIStatus(fsMasterInfo.setMountId(status.getMountId())));
    }
}

Example 22 with FileSystemMaster

use of alluxio.master.file.FileSystemMaster in project alluxio by Alluxio.

the class FileSystemMasterRestartIntegrationTest method unavailableUfsRecursiveCreate.

@Test
public void unavailableUfsRecursiveCreate() throws Exception {
    String ufsBase = "test://test/";
    UnderFileSystemFactory mockUfsFactory = Mockito.mock(UnderFileSystemFactory.class);
    Mockito.when(mockUfsFactory.supportsPath(ArgumentMatchers.anyString(), ArgumentMatchers.any())).thenReturn(Boolean.FALSE);
    Mockito.when(mockUfsFactory.supportsPath(ArgumentMatchers.eq(ufsBase), ArgumentMatchers.any())).thenReturn(Boolean.TRUE);
    UnderFileSystem mockUfs = Mockito.mock(UnderFileSystem.class);
    UfsDirectoryStatus ufsStatus = new UfsDirectoryStatus("test", "owner", "group", (short) 511);
    Mockito.when(mockUfsFactory.create(ArgumentMatchers.eq(ufsBase), ArgumentMatchers.any())).thenReturn(mockUfs);
    Mockito.when(mockUfs.isDirectory(ufsBase)).thenReturn(true);
    Mockito.when(mockUfs.resolveUri(new AlluxioURI(ufsBase), "")).thenReturn(new AlluxioURI(ufsBase));
    Mockito.when(mockUfs.resolveUri(new AlluxioURI(ufsBase), "/dir1")).thenReturn(new AlluxioURI(ufsBase + "/dir1"));
    Mockito.when(mockUfs.getExistingDirectoryStatus(ufsBase)).thenReturn(ufsStatus);
    Mockito.when(mockUfs.mkdirs(ArgumentMatchers.eq(ufsBase + "/dir1"), ArgumentMatchers.any())).thenThrow(new IOException("ufs unavailable"));
    Mockito.when(mockUfs.getStatus(ufsBase)).thenReturn(ufsStatus);
    UnderFileSystemFactoryRegistry.register(mockUfsFactory);
    mFsMaster.mount(new AlluxioURI("/mnt"), new AlluxioURI(ufsBase), MountContext.defaults());
    AlluxioURI root = new AlluxioURI("/mnt/");
    AlluxioURI alluxioFile = new AlluxioURI("/mnt/dir1/dir2/file");
    // Create a persisted Alluxio file (but no ufs file).
    try {
        mFsMaster.createFile(alluxioFile, CreateFileContext.mergeFrom(CreateFilePOptions.newBuilder().setRecursive(true)).setWriteType(WriteType.CACHE_THROUGH));
        Assert.fail("persisted create should fail, when UFS is unavailable");
    } catch (Exception e) {
    // expected, ignore
    }
    List<FileInfo> files = mFsMaster.listStatus(root, ListStatusContext.defaults());
    Assert.assertTrue(files.isEmpty());
    try {
        // should not exist
        files = mFsMaster.listStatus(new AlluxioURI("/mnt/dir1/"), ListStatusContext.defaults());
        Assert.fail("dir should not exist, when UFS is unavailable");
    } catch (Exception e) {
    // expected, ignore
    }
    try {
        // should not exist
        mFsMaster.delete(new AlluxioURI("/mnt/dir1/"), DeleteContext.mergeFrom(DeletePOptions.newBuilder().setRecursive(true)));
        Assert.fail("cannot delete non-existing directory, when UFS is unavailable");
    } catch (Exception e) {
        // expected, ignore
        files = null;
    }
    files = mFsMaster.listStatus(new AlluxioURI("/mnt/"), ListStatusContext.defaults());
    Assert.assertTrue(files.isEmpty());
    // Stop Alluxio.
    mLocalAlluxioClusterResource.get().stopFS();
    // Create the master using the existing journal.
    try (FsMasterResource masterResource = MasterTestUtils.createLeaderFileSystemMasterFromJournal()) {
        FileSystemMaster newFsMaster = masterResource.getRegistry().get(FileSystemMaster.class);
        files = newFsMaster.listStatus(new AlluxioURI("/mnt/"), ListStatusContext.defaults());
        Assert.assertTrue(files.isEmpty());
    }
}

Example 23 with FileSystemMaster

use of alluxio.master.file.FileSystemMaster in project alluxio by Alluxio.

the class FileSystemMasterRestartIntegrationTest method syncReplay.

@Test
public void syncReplay() throws Exception {
    AlluxioURI root = new AlluxioURI("/");
    AlluxioURI alluxioFile = new AlluxioURI("/in_alluxio");
    // Create a persisted Alluxio file (but no ufs file).
    mFsMaster.createFile(alluxioFile, CreateFileContext.defaults().setWriteType(WriteType.CACHE_THROUGH));
    mFsMaster.completeFile(alluxioFile, CompleteFileContext.mergeFrom(CompleteFilePOptions.newBuilder().setUfsLength(0)).setOperationTimeMs(TEST_TIME_MS));
    // List what is in Alluxio, without syncing.
    List<FileInfo> files = mFsMaster.listStatus(root, ListStatusContext.mergeFrom(ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(-1))));
    Assert.assertEquals(1, files.size());
    Assert.assertEquals(alluxioFile.getName(), files.get(0).getName());
    // Add ufs only paths
    String ufs = ServerConfiguration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS);
    Files.createDirectory(Paths.get(ufs, "ufs_dir"));
    Files.createFile(Paths.get(ufs, "ufs_file"));
    // List with syncing, which will remove alluxio only path, and add ufs only paths.
    files = mFsMaster.listStatus(root, ListStatusContext.mergeFrom(ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(0))));
    Assert.assertEquals(2, files.size());
    Set<String> filenames = files.stream().map(FileInfo::getName).collect(Collectors.toSet());
    Assert.assertTrue(filenames.contains("ufs_dir"));
    Assert.assertTrue(filenames.contains("ufs_file"));
    // Stop Alluxio.
    mLocalAlluxioClusterResource.get().stopFS();
    // Create the master using the existing journal.
    try (FsMasterResource masterResource = createFileSystemMasterFromJournal()) {
        FileSystemMaster fsMaster = masterResource.getRegistry().get(FileSystemMaster.class);
        // List what is in Alluxio, without syncing. Should match the last state.
        files = fsMaster.listStatus(root, ListStatusContext.mergeFrom(ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(-1))));
        Assert.assertEquals(2, files.size());
        filenames = files.stream().map(FileInfo::getName).collect(Collectors.toSet());
        Assert.assertTrue(filenames.contains("ufs_dir"));
        Assert.assertTrue(filenames.contains("ufs_file"));
    }
}

Example 24 with FileSystemMaster

use of alluxio.master.file.FileSystemMaster in project alluxio by Alluxio.

the class MultiUfsMountIntegrationTest method mountWithCredentials.

@Test
public void mountWithCredentials() throws Exception {
    MountPOptions options3 = MountPOptions.newBuilder().putAllProperties(UFS_CONF3).build();
    mFileSystem.mount(mMountPoint3, new AlluxioURI(mUfsUri3), options3);
    mLocalAlluxioCluster.stopFS();
    try (FsMasterResource masterResource = MasterTestUtils.createLeaderFileSystemMasterFromJournal()) {
        FileSystemMaster fsMaster = masterResource.getRegistry().get(FileSystemMaster.class);
        Map<String, MountPointInfo> mountTable = fsMaster.getMountPointInfoSummary();
        Assert.assertTrue(mountTable.containsKey(MOUNT_POINT3));
        MountPointInfo mountPointInfo3 = mountTable.get(MOUNT_POINT3);
        Assert.assertEquals(mUfsUri3, mountPointInfo3.getUfsUri());
        Assert.assertEquals(UFS_CONF3.size(), mountPointInfo3.getProperties().size());
        Assert.assertTrue(mountPointInfo3.getProperties().containsKey(PropertyKey.Name.S3A_ACCESS_KEY));
        Assert.assertNotEquals(UFS_CONF3.get(PropertyKey.Name.S3A_ACCESS_KEY), mountPointInfo3.getProperties().get(PropertyKey.Name.S3A_ACCESS_KEY));
    }
}

Example 25 with FileSystemMaster

use of alluxio.master.file.FileSystemMaster in project alluxio by Alluxio.

the class BlockMasterIntegrityIntegrationTest method removeFileMetadata.

private void removeFileMetadata(AlluxioURI uri) throws Exception {
    FileSystemMaster fsm = mCluster.getLocalAlluxioMaster().getMasterProcess().getMaster(FileSystemMaster.class);
    InodeTree tree = Whitebox.getInternalState(fsm, "mInodeTree");
    LockedInodePath path = tree.lockInodePath(uri, LockPattern.WRITE_EDGE);
    RpcContext rpcContext = ((DefaultFileSystemMaster) fsm).createRpcContext();
    ((DefaultFileSystemMaster) fsm).deleteInternal(rpcContext, path, DeleteContext.defaults());
    path.close();
    rpcContext.close();
}