Examples with FsMasterResource alluxio.testutils.master.FsMasterResource used on opensource projects

Example 6 with FsMasterResource

use of alluxio.testutils.master.FsMasterResource in project alluxio by Alluxio.

the class UfsJournalIntegrationTest method loadMetadataTestUtil.

private void loadMetadataTestUtil(URIStatus status) throws Exception {
    try (FsMasterResource masterResource = createFsMasterFromJournal()) {
        FileSystemMaster fsMaster = masterResource.getRegistry().get(FileSystemMaster.class);
        long rootId = fsMaster.getFileId(mRootUri);
        Assert.assertTrue(rootId != IdUtils.INVALID_FILE_ID);
        Assert.assertEquals(1, fsMaster.listStatus(mRootUri, ListStatusContext.mergeFrom(ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER))).size());
        Assert.assertTrue(fsMaster.getFileId(new AlluxioURI("/xyz")) != IdUtils.INVALID_FILE_ID);
        FileInfo fsMasterInfo = fsMaster.getFileInfo(fsMaster.getFileId(new AlluxioURI("/xyz")));
        Assert.assertEquals(status, new URIStatus(fsMasterInfo.setMountId(status.getMountId())));
    }
}

Example 7 with FsMasterResource

use of alluxio.testutils.master.FsMasterResource in project alluxio by Alluxio.

the class FileSystemMasterRestartIntegrationTest method unavailableUfsRecursiveCreate.

@Test
public void unavailableUfsRecursiveCreate() throws Exception {
    String ufsBase = "test://test/";
    UnderFileSystemFactory mockUfsFactory = Mockito.mock(UnderFileSystemFactory.class);
    Mockito.when(mockUfsFactory.supportsPath(ArgumentMatchers.anyString(), ArgumentMatchers.any())).thenReturn(Boolean.FALSE);
    Mockito.when(mockUfsFactory.supportsPath(ArgumentMatchers.eq(ufsBase), ArgumentMatchers.any())).thenReturn(Boolean.TRUE);
    UnderFileSystem mockUfs = Mockito.mock(UnderFileSystem.class);
    UfsDirectoryStatus ufsStatus = new UfsDirectoryStatus("test", "owner", "group", (short) 511);
    Mockito.when(mockUfsFactory.create(ArgumentMatchers.eq(ufsBase), ArgumentMatchers.any())).thenReturn(mockUfs);
    Mockito.when(mockUfs.isDirectory(ufsBase)).thenReturn(true);
    Mockito.when(mockUfs.resolveUri(new AlluxioURI(ufsBase), "")).thenReturn(new AlluxioURI(ufsBase));
    Mockito.when(mockUfs.resolveUri(new AlluxioURI(ufsBase), "/dir1")).thenReturn(new AlluxioURI(ufsBase + "/dir1"));
    Mockito.when(mockUfs.getExistingDirectoryStatus(ufsBase)).thenReturn(ufsStatus);
    Mockito.when(mockUfs.mkdirs(ArgumentMatchers.eq(ufsBase + "/dir1"), ArgumentMatchers.any())).thenThrow(new IOException("ufs unavailable"));
    Mockito.when(mockUfs.getStatus(ufsBase)).thenReturn(ufsStatus);
    UnderFileSystemFactoryRegistry.register(mockUfsFactory);
    mFsMaster.mount(new AlluxioURI("/mnt"), new AlluxioURI(ufsBase), MountContext.defaults());
    AlluxioURI root = new AlluxioURI("/mnt/");
    AlluxioURI alluxioFile = new AlluxioURI("/mnt/dir1/dir2/file");
    // Create a persisted Alluxio file (but no ufs file).
    try {
        mFsMaster.createFile(alluxioFile, CreateFileContext.mergeFrom(CreateFilePOptions.newBuilder().setRecursive(true)).setWriteType(WriteType.CACHE_THROUGH));
        Assert.fail("persisted create should fail, when UFS is unavailable");
    } catch (Exception e) {
    // expected, ignore
    }
    List<FileInfo> files = mFsMaster.listStatus(root, ListStatusContext.defaults());
    Assert.assertTrue(files.isEmpty());
    try {
        // should not exist
        files = mFsMaster.listStatus(new AlluxioURI("/mnt/dir1/"), ListStatusContext.defaults());
        Assert.fail("dir should not exist, when UFS is unavailable");
    } catch (Exception e) {
    // expected, ignore
    }
    try {
        // should not exist
        mFsMaster.delete(new AlluxioURI("/mnt/dir1/"), DeleteContext.mergeFrom(DeletePOptions.newBuilder().setRecursive(true)));
        Assert.fail("cannot delete non-existing directory, when UFS is unavailable");
    } catch (Exception e) {
        // expected, ignore
        files = null;
    }
    files = mFsMaster.listStatus(new AlluxioURI("/mnt/"), ListStatusContext.defaults());
    Assert.assertTrue(files.isEmpty());
    // Stop Alluxio.
    mLocalAlluxioClusterResource.get().stopFS();
    // Create the master using the existing journal.
    try (FsMasterResource masterResource = MasterTestUtils.createLeaderFileSystemMasterFromJournal()) {
        FileSystemMaster newFsMaster = masterResource.getRegistry().get(FileSystemMaster.class);
        files = newFsMaster.listStatus(new AlluxioURI("/mnt/"), ListStatusContext.defaults());
        Assert.assertTrue(files.isEmpty());
    }
}

Example 8 with FsMasterResource

use of alluxio.testutils.master.FsMasterResource in project alluxio by Alluxio.

the class FileSystemMasterRestartIntegrationTest method syncReplay.

@Test
public void syncReplay() throws Exception {
    AlluxioURI root = new AlluxioURI("/");
    AlluxioURI alluxioFile = new AlluxioURI("/in_alluxio");
    // Create a persisted Alluxio file (but no ufs file).
    mFsMaster.createFile(alluxioFile, CreateFileContext.defaults().setWriteType(WriteType.CACHE_THROUGH));
    mFsMaster.completeFile(alluxioFile, CompleteFileContext.mergeFrom(CompleteFilePOptions.newBuilder().setUfsLength(0)).setOperationTimeMs(TEST_TIME_MS));
    // List what is in Alluxio, without syncing.
    List<FileInfo> files = mFsMaster.listStatus(root, ListStatusContext.mergeFrom(ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(-1))));
    Assert.assertEquals(1, files.size());
    Assert.assertEquals(alluxioFile.getName(), files.get(0).getName());
    // Add ufs only paths
    String ufs = ServerConfiguration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS);
    Files.createDirectory(Paths.get(ufs, "ufs_dir"));
    Files.createFile(Paths.get(ufs, "ufs_file"));
    // List with syncing, which will remove alluxio only path, and add ufs only paths.
    files = mFsMaster.listStatus(root, ListStatusContext.mergeFrom(ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(0))));
    Assert.assertEquals(2, files.size());
    Set<String> filenames = files.stream().map(FileInfo::getName).collect(Collectors.toSet());
    Assert.assertTrue(filenames.contains("ufs_dir"));
    Assert.assertTrue(filenames.contains("ufs_file"));
    // Stop Alluxio.
    mLocalAlluxioClusterResource.get().stopFS();
    // Create the master using the existing journal.
    try (FsMasterResource masterResource = createFileSystemMasterFromJournal()) {
        FileSystemMaster fsMaster = masterResource.getRegistry().get(FileSystemMaster.class);
        // List what is in Alluxio, without syncing. Should match the last state.
        files = fsMaster.listStatus(root, ListStatusContext.mergeFrom(ListStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(-1))));
        Assert.assertEquals(2, files.size());
        filenames = files.stream().map(FileInfo::getName).collect(Collectors.toSet());
        Assert.assertTrue(filenames.contains("ufs_dir"));
        Assert.assertTrue(filenames.contains("ufs_file"));
    }
}

Example 9 with FsMasterResource

use of alluxio.testutils.master.FsMasterResource in project alluxio by Alluxio.

the class MultiUfsMountIntegrationTest method mountWithCredentials.

@Test
public void mountWithCredentials() throws Exception {
    MountPOptions options3 = MountPOptions.newBuilder().putAllProperties(UFS_CONF3).build();
    mFileSystem.mount(mMountPoint3, new AlluxioURI(mUfsUri3), options3);
    mLocalAlluxioCluster.stopFS();
    try (FsMasterResource masterResource = MasterTestUtils.createLeaderFileSystemMasterFromJournal()) {
        FileSystemMaster fsMaster = masterResource.getRegistry().get(FileSystemMaster.class);
        Map<String, MountPointInfo> mountTable = fsMaster.getMountPointInfoSummary();
        Assert.assertTrue(mountTable.containsKey(MOUNT_POINT3));
        MountPointInfo mountPointInfo3 = mountTable.get(MOUNT_POINT3);
        Assert.assertEquals(mUfsUri3, mountPointInfo3.getUfsUri());
        Assert.assertEquals(UFS_CONF3.size(), mountPointInfo3.getProperties().size());
        Assert.assertTrue(mountPointInfo3.getProperties().containsKey(PropertyKey.Name.S3A_ACCESS_KEY));
        Assert.assertNotEquals(UFS_CONF3.get(PropertyKey.Name.S3A_ACCESS_KEY), mountPointInfo3.getProperties().get(PropertyKey.Name.S3A_ACCESS_KEY));
    }
}

Example 10 with FsMasterResource

use of alluxio.testutils.master.FsMasterResource in project alluxio by Alluxio.

the class FileSystemMasterRestartIntegrationTest method updateAccessTimeAsyncFlush.

/**
 * Tests journal is updated with access time asynchronously before master is stopped.
 */
@Test
@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.MASTER_METASTORE, "HEAP" })
public void updateAccessTimeAsyncFlush() throws Exception {
    String parentName = "d1";
    AlluxioURI parentPath = new AlluxioURI("/" + parentName);
    long parentId = mFsMaster.createDirectory(parentPath, CreateDirectoryContext.mergeFrom(CreateDirectoryPOptions.newBuilder().setRecursive(true).setMode(new Mode((short) 0700).toProto())));
    long oldAccessTime = mFsMaster.getFileInfo(parentId).getLastAccessTimeMs();
    Thread.sleep(100);
    mFsMaster.listStatus(parentPath, ListStatusContext.defaults());
    long newAccessTime = mFsMaster.getFileInfo(parentId).getLastAccessTimeMs();
    // time is changed in master
    Assert.assertNotEquals(newAccessTime, oldAccessTime);
    try (FsMasterResource masterResource = createFileSystemMasterFromJournal()) {
        FileSystemMaster fsm = masterResource.getRegistry().get(FileSystemMaster.class);
        long journaledAccessTime = fsm.getFileInfo(parentId).getLastAccessTimeMs();
        // time is not flushed to journal
        Assert.assertEquals(journaledAccessTime, oldAccessTime);
    }
    // Stop Alluxio.
    mLocalAlluxioClusterResource.get().stopFS();
    // Create the master using the existing journal.
    try (FsMasterResource masterResource = createFileSystemMasterFromJournal()) {
        FileSystemMaster fsm = masterResource.getRegistry().get(FileSystemMaster.class);
        long journaledAccessTimeAfterStop = fsm.getFileInfo(parentId).getLastAccessTimeMs();
        // time is now flushed to journal
        Assert.assertEquals(journaledAccessTimeAfterStop, newAccessTime);
    }
}