Examples with FileSystemMaster alluxio.master.file.FileSystemMaster used on opensource projects

Example 11 with FileSystemMaster

use of alluxio.master.file.FileSystemMaster in project alluxio by Alluxio.

the class JournalIntegrationTest method loadMetadataTestUtil.

private void loadMetadataTestUtil(URIStatus status) throws AccessControlException, IOException, InvalidPathException, FileDoesNotExistException {
    FileSystemMaster fsMaster = createFsMasterFromJournal();
    long rootId = fsMaster.getFileId(mRootUri);
    Assert.assertTrue(rootId != IdUtils.INVALID_FILE_ID);
    Assert.assertEquals(1, fsMaster.listStatus(mRootUri, ListStatusOptions.defaults().setLoadMetadataType(LoadMetadataType.Never)).size());
    Assert.assertTrue(fsMaster.getFileId(new AlluxioURI("/xyz")) != IdUtils.INVALID_FILE_ID);
    FileInfo fsMasterInfo = fsMaster.getFileInfo(fsMaster.getFileId(new AlluxioURI("/xyz")));
    Assert.assertEquals(status, new URIStatus(fsMasterInfo));
    fsMaster.stop();
}

Example 12 with FileSystemMaster

use of alluxio.master.file.FileSystemMaster in project alluxio by Alluxio.

the class JournalIntegrationTest method directoryTestUtil.

private void directoryTestUtil(URIStatus status) throws AccessControlException, IOException, InvalidPathException, FileDoesNotExistException {
    FileSystemMaster fsMaster = createFsMasterFromJournal();
    long rootId = fsMaster.getFileId(mRootUri);
    Assert.assertTrue(rootId != IdUtils.INVALID_FILE_ID);
    Assert.assertEquals(1, fsMaster.listStatus(mRootUri, ListStatusOptions.defaults().setLoadMetadataType(LoadMetadataType.Never)).size());
    long fileId = fsMaster.getFileId(new AlluxioURI("/xyz"));
    Assert.assertTrue(fileId != IdUtils.INVALID_FILE_ID);
    Assert.assertEquals(status, new URIStatus(fsMaster.getFileInfo(fileId)));
    fsMaster.stop();
}

Example 13 with FileSystemMaster

use of alluxio.master.file.FileSystemMaster in project alluxio by Alluxio.

the class JournalIntegrationTest method addBlockTestUtil.

private void addBlockTestUtil(URIStatus status) throws AccessControlException, IOException, InvalidPathException, FileDoesNotExistException {
    FileSystemMaster fsMaster = createFsMasterFromJournal();
    long rootId = fsMaster.getFileId(mRootUri);
    Assert.assertTrue(rootId != IdUtils.INVALID_FILE_ID);
    Assert.assertEquals(1, fsMaster.listStatus(mRootUri, ListStatusOptions.defaults().setLoadMetadataType(LoadMetadataType.Never)).size());
    long xyzId = fsMaster.getFileId(new AlluxioURI("/xyz"));
    Assert.assertTrue(xyzId != IdUtils.INVALID_FILE_ID);
    FileInfo fsMasterInfo = fsMaster.getFileInfo(xyzId);
    Assert.assertEquals(0, fsMaster.getFileInfo(xyzId).getInMemoryPercentage());
    Assert.assertEquals(status.getBlockIds(), fsMasterInfo.getBlockIds());
    Assert.assertEquals(status.getBlockSizeBytes(), fsMasterInfo.getBlockSizeBytes());
    Assert.assertEquals(status.getLength(), fsMasterInfo.getLength());
    fsMaster.stop();
}

Example 14 with FileSystemMaster

use of alluxio.master.file.FileSystemMaster in project alluxio by Alluxio.

the class TestRecomputeExecutor method recomputeLauncher.

/**
   * Tests recompute executor creates a recompute plan and launches the recompute job at heartbeat.
   *
   * @throws Exception if anything wrong happens
   */
@Test
public void recomputeLauncher() throws Exception {
    long fileId = 5L;
    // mock planner
    RecomputePlanner planner = Mockito.mock(RecomputePlanner.class);
    Job job = Mockito.mock(Job.class);
    Lineage lineage = new Lineage(1, new ArrayList<Long>(), Lists.newArrayList(fileId), job);
    Mockito.when(planner.plan()).thenReturn(new RecomputePlan(Lists.newArrayList(lineage)));
    // mock file system master
    FileSystemMaster fileSystemMaster = Mockito.mock(FileSystemMaster.class);
    Mockito.when(fileSystemMaster.getFileSystemMasterView()).thenReturn(new FileSystemMasterView(fileSystemMaster));
    Mockito.when(fileSystemMaster.getLostFiles()).thenReturn(Lists.newArrayList(fileId));
    RecomputeExecutor executor = new RecomputeExecutor(planner, fileSystemMaster);
    // wait for the executor to finish running
    executor.heartbeatWithFuture().get(5, TimeUnit.SECONDS);
    executor.close();
    Mockito.verify(fileSystemMaster).resetFile(fileId);
    Mockito.verify(job).run();
}

Example 15 with FileSystemMaster

use of alluxio.master.file.FileSystemMaster in project alluxio by Alluxio.

the class FileSystemMasterFaultToleranceIntegrationTest method partitionTolerantDeleteFile.

@Test
public void partitionTolerantDeleteFile() throws Exception {
    // Create paths for the test.
    AlluxioURI testPath1 = new AlluxioURI("/testPath1");
    // Create context1 with unique operation id.
    DeleteContext context = DeleteContext.mergeFrom(DeletePOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setOperationId(new OperationId(UUID.randomUUID()).toFsProto())));
    // Create context2 with unique operation id.
    DeleteContext context2 = DeleteContext.mergeFrom(DeletePOptions.newBuilder().setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setOperationId(new OperationId(UUID.randomUUID()).toFsProto())));
    {
        // Acquire file-system-master of leading master.
        FileSystemMaster leadingFsMaster = mMultiMasterLocalAlluxioCluster.getLocalAlluxioMaster().getMasterProcess().getMaster(FileSystemMaster.class);
        // Create the path to delete.
        leadingFsMaster.createFile(testPath1, CreateFileContext.defaults());
        leadingFsMaster.completeFile(testPath1, CompleteFileContext.defaults());
        // Delete the path the first time.
        leadingFsMaster.delete(testPath1, context);
        // Delete the path the second time with the same context.
        // It should just return successfully.
        leadingFsMaster.delete(testPath1, context);
        // Delete the path again with a different context.
        // It should fail with `FileDoesNotExistException`.
        assertThrows(FileDoesNotExistException.class, () -> leadingFsMaster.delete(testPath1, context2));
    }
    // Promote standby to be a leader and reset test state.
    mMultiMasterLocalAlluxioCluster.stopLeader();
    mMultiMasterLocalAlluxioCluster.waitForNewMaster(CLUSTER_WAIT_TIMEOUT_MS);
    mAuthenticatedUser.resetUser();
    // Run partition tolerance test on the *new* leading master.
    {
        // Acquire file-system-master of leading master.
        FileSystemMaster leadingFsMaster = mMultiMasterLocalAlluxioCluster.getLocalAlluxioMaster().getMasterProcess().getMaster(FileSystemMaster.class);
        // Deleting the file on the new leader with the original operation-id should succeed.
        leadingFsMaster.delete(testPath1, context);
        // Deleting on the new leader with a different operation-id.
        // It should fail with `FileDoesNotExistException`.
        assertThrows(FileDoesNotExistException.class, () -> leadingFsMaster.delete(testPath1, context2));
    }
}