Examples with FileSystemMaster alluxio.master.file.FileSystemMaster used on opensource projects

Example 41 with FileSystemMaster

use of alluxio.master.file.FileSystemMaster in project alluxio by Alluxio.

the class FileSystemMasterRestartIntegrationTest method updateAccessTimeAsyncFlush.

/**
 * Tests journal is updated with access time asynchronously before master is stopped.
 */
@Test
@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.MASTER_METASTORE, "HEAP" })
public void updateAccessTimeAsyncFlush() throws Exception {
    String parentName = "d1";
    AlluxioURI parentPath = new AlluxioURI("/" + parentName);
    long parentId = mFsMaster.createDirectory(parentPath, CreateDirectoryContext.mergeFrom(CreateDirectoryPOptions.newBuilder().setRecursive(true).setMode(new Mode((short) 0700).toProto())));
    long oldAccessTime = mFsMaster.getFileInfo(parentId).getLastAccessTimeMs();
    Thread.sleep(100);
    mFsMaster.listStatus(parentPath, ListStatusContext.defaults());
    long newAccessTime = mFsMaster.getFileInfo(parentId).getLastAccessTimeMs();
    // time is changed in master
    Assert.assertNotEquals(newAccessTime, oldAccessTime);
    try (FsMasterResource masterResource = createFileSystemMasterFromJournal()) {
        FileSystemMaster fsm = masterResource.getRegistry().get(FileSystemMaster.class);
        long journaledAccessTime = fsm.getFileInfo(parentId).getLastAccessTimeMs();
        // time is not flushed to journal
        Assert.assertEquals(journaledAccessTime, oldAccessTime);
    }
    // Stop Alluxio.
    mLocalAlluxioClusterResource.get().stopFS();
    // Create the master using the existing journal.
    try (FsMasterResource masterResource = createFileSystemMasterFromJournal()) {
        FileSystemMaster fsm = masterResource.getRegistry().get(FileSystemMaster.class);
        long journaledAccessTimeAfterStop = fsm.getFileInfo(parentId).getLastAccessTimeMs();
        // time is now flushed to journal
        Assert.assertEquals(journaledAccessTimeAfterStop, newAccessTime);
    }
}

Example 42 with FileSystemMaster

use of alluxio.master.file.FileSystemMaster in project alluxio by Alluxio.

the class FileSystemMasterRestartIntegrationTest method ufsModeReplay.

@Test
public void ufsModeReplay() throws Exception {
    mFsMaster.updateUfsMode(new AlluxioURI(mFsMaster.getUfsAddress()), UfsMode.NO_ACCESS);
    // Stop Alluxio.
    mLocalAlluxioClusterResource.get().stopFS();
    // Create the master using the existing journal.
    try (FsMasterResource masterResource = createFileSystemMasterFromJournal()) {
        FileSystemMaster fsMaster = masterResource.getRegistry().get(FileSystemMaster.class);
        AlluxioURI alluxioFile = new AlluxioURI("/in_alluxio");
        // Create file should throw an Exception even after restart
        mThrown.expect(AccessControlException.class);
        fsMaster.createFile(alluxioFile, CreateFileContext.defaults().setWriteType(WriteType.CACHE_THROUGH));
    }
}

Example 43 with FileSystemMaster

use of alluxio.master.file.FileSystemMaster in project alluxio by Alluxio.

the class FileSystemMasterRestartIntegrationTest method syncDirReplay.

@Test
public void syncDirReplay() throws Exception {
    AlluxioURI dir = new AlluxioURI("/dir/");
    // Add ufs nested file.
    String ufs = ServerConfiguration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS);
    Files.createDirectory(Paths.get(ufs, "dir"));
    Files.createFile(Paths.get(ufs, "dir", "file"));
    File ufsDir = new File(Paths.get(ufs, "dir").toString());
    Assert.assertTrue(ufsDir.setReadable(true, false));
    Assert.assertTrue(ufsDir.setWritable(true, false));
    Assert.assertTrue(ufsDir.setExecutable(true, false));
    // List dir with syncing
    FileInfo info = mFsMaster.getFileInfo(dir, GetStatusContext.mergeFrom(GetStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(0).build())));
    Assert.assertNotNull(info);
    Assert.assertEquals("dir", info.getName());
    // Retrieve the mode
    int mode = info.getMode();
    // Update mode of the ufs dir
    Assert.assertTrue(ufsDir.setExecutable(false, false));
    // List dir with syncing, should update the mode
    info = mFsMaster.getFileInfo(dir, GetStatusContext.mergeFrom(GetStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(0).build())));
    Assert.assertNotNull(info);
    Assert.assertEquals("dir", info.getName());
    Assert.assertNotEquals(mode, info.getMode());
    // update the expected mode
    mode = info.getMode();
    // Stop Alluxio.
    mLocalAlluxioClusterResource.get().stopFS();
    // Create the master using the existing journal.
    try (FsMasterResource masterResource = createFileSystemMasterFromJournal()) {
        FileSystemMaster fsMaster = masterResource.getRegistry().get(FileSystemMaster.class);
        // List what is in Alluxio, without syncing.
        info = fsMaster.getFileInfo(dir, GetStatusContext.mergeFrom(GetStatusPOptions.newBuilder().setLoadMetadataType(LoadMetadataPType.NEVER).setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setSyncIntervalMs(-1).build())));
        Assert.assertNotNull(info);
        Assert.assertEquals("dir", info.getName());
        Assert.assertEquals(mode, info.getMode());
    }
}

Example 44 with FileSystemMaster

use of alluxio.master.file.FileSystemMaster in project alluxio by Alluxio.

the class FileSystemMasterRestartIntegrationTest method updateAccessTimeAsyncFlushAfterDelete.

/**
 * Tests journal is not updated with access time asynchronously after delete.
 */
@Test
@LocalAlluxioClusterResource.Config(confParams = { PropertyKey.Name.MASTER_METASTORE, "HEAP" })
public void updateAccessTimeAsyncFlushAfterDelete() throws Exception {
    String parentName = "d1";
    AlluxioURI parentPath = new AlluxioURI("/" + parentName);
    long parentId = mFsMaster.createDirectory(parentPath, CreateDirectoryContext.mergeFrom(CreateDirectoryPOptions.newBuilder().setRecursive(true).setMode(new Mode((short) 0700).toProto())));
    long oldAccessTime = mFsMaster.getFileInfo(parentId).getLastAccessTimeMs();
    Thread.sleep(100);
    mFsMaster.listStatus(parentPath, ListStatusContext.defaults());
    long newAccessTime = mFsMaster.getFileInfo(parentId).getLastAccessTimeMs();
    // time is changed in master
    Assert.assertNotEquals(newAccessTime, oldAccessTime);
    try (FsMasterResource masterResource = createFileSystemMasterFromJournal()) {
        FileSystemMaster fsm = masterResource.getRegistry().get(FileSystemMaster.class);
        long journaledAccessTime = fsm.getFileInfo(parentId).getLastAccessTimeMs();
        // time is not flushed to journal
        Assert.assertEquals(journaledAccessTime, oldAccessTime);
        // delete the directory
        mFsMaster.delete(parentPath, DeleteContext.defaults());
    }
    // Stop Alluxio.
    mLocalAlluxioClusterResource.get().stopFS();
    // Create the master using the existing journal.
    try (FsMasterResource masterResource = createFileSystemMasterFromJournal()) {
        FileSystemMaster fsm = masterResource.getRegistry().get(FileSystemMaster.class);
        Assert.assertEquals(fsm.getFileId(parentPath), -1);
    }
}

Example 45 with FileSystemMaster

use of alluxio.master.file.FileSystemMaster in project alluxio by Alluxio.

the class FileSystemMasterIntegrationTest method concurrentCreateJournal.

// TODO(calvin): This test currently relies on the fact the HDFS client is a cached instance to
// avoid invalid lease exception. This should be fixed.
@Ignore
@Test
public void concurrentCreateJournal() throws Exception {
    // Makes sure the file id's are the same between a master info and the journal it creates
    for (int i = 0; i < 5; i++) {
        ConcurrentCreator concurrentCreator = new ConcurrentCreator(DEPTH, CONCURRENCY_DEPTH, ROOT_PATH);
        concurrentCreator.call();
        try (FsMasterResource masterResource = createFileSystemMasterFromJournal()) {
            FileSystemMaster fsMaster = masterResource.getRegistry().get(FileSystemMaster.class);
            for (FileInfo info : mFsMaster.listStatus(new AlluxioURI("/"), ListStatusContext.defaults())) {
                AlluxioURI path = new AlluxioURI(info.getPath());
                Assert.assertEquals(mFsMaster.getFileId(path), fsMaster.getFileId(path));
            }
        }
        before();
    }
}