Examples with FileSystem alluxio.client.file.FileSystem used on opensource projects

Example 86 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class ImpersonationIntegrationTest method checkCreateFile.

private void checkCreateFile(Subject subject, String expectedUser) throws Exception {
    FileSystemContext context = FileSystemContext.create(subject, ServerConfiguration.global());
    FileSystem fs = mLocalAlluxioClusterResource.get().getClient(context);
    fs.createFile(new AlluxioURI("/impersonation-test")).close();
    List<URIStatus> listing = fs.listStatus(new AlluxioURI("/"));
    Assert.assertEquals(1, listing.size());
    URIStatus status = listing.get(0);
    Assert.assertEquals(expectedUser, status.getOwner());
}

Example 87 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class ImpersonationIntegrationTest method impersonationNotUsed.

@Test
@LocalAlluxioClusterResource.Config(confParams = { IMPERSONATION_GROUPS_CONFIG, "*" })
public void impersonationNotUsed() throws Exception {
    ServerConfiguration.set(PropertyKey.SECURITY_LOGIN_IMPERSONATION_USERNAME, Constants.IMPERSONATION_NONE);
    FileSystemContext context = FileSystemContext.create(createHdfsSubject(), ServerConfiguration.global());
    FileSystem fs = mLocalAlluxioClusterResource.get().getClient(context);
    fs.createFile(new AlluxioURI("/impersonation-test")).close();
    List<URIStatus> listing = fs.listStatus(new AlluxioURI("/"));
    Assert.assertEquals(1, listing.size());
    URIStatus status = listing.get(0);
    Assert.assertNotEquals(IMPERSONATION_USER, status.getOwner());
}

Example 88 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class LocalFirstPolicyIntegrationTest method test.

@Test
public void test() throws Exception {
    AlluxioMasterProcess master = AlluxioMasterProcess.Factory.create();
    WorkerProcess worker1 = AlluxioWorkerProcess.Factory.create(TieredIdentityFactory.fromString("node=node1,rack=rack1", ServerConfiguration.global()));
    WorkerProcess worker2 = AlluxioWorkerProcess.Factory.create(TieredIdentityFactory.fromString("node=node2,rack=rack2", ServerConfiguration.global()));
    runProcess(mExecutor, master);
    runProcess(mExecutor, worker1);
    runProcess(mExecutor, worker2);
    TestUtils.waitForReady(master);
    TestUtils.waitForReady(worker1);
    TestUtils.waitForReady(worker2);
    FileSystem fs = FileSystem.Factory.create(ServerConfiguration.global());
    // Write to the worker in node1
    {
        Whitebox.setInternalState(TieredIdentityFactory.class, "sInstance", TieredIdentityFactory.fromString("node=node1,rack=rack1", ServerConfiguration.global()));
        try {
            FileSystemTestUtils.createByteFile(fs, "/file1", WritePType.MUST_CACHE, 100);
        } finally {
            Whitebox.setInternalState(TieredIdentityFactory.class, "sInstance", (Object) null);
        }
        BlockWorker blockWorker1 = worker1.getWorker(BlockWorker.class);
        BlockWorker blockWorker2 = worker2.getWorker(BlockWorker.class);
        assertEquals(100, blockWorker1.getStoreMeta().getUsedBytes());
        assertEquals(0, blockWorker2.getStoreMeta().getUsedBytes());
    }
    // Write to the worker in rack2
    {
        Whitebox.setInternalState(TieredIdentityFactory.class, "sInstance", TieredIdentityFactory.fromString("node=node3,rack=rack2", ServerConfiguration.global()));
        try {
            FileSystemTestUtils.createByteFile(fs, "/file2", WritePType.MUST_CACHE, 10);
        } finally {
            Whitebox.setInternalState(TieredIdentityFactory.class, "sInstance", (Object) null);
        }
        BlockWorker blockWorker1 = worker1.getWorker(BlockWorker.class);
        BlockWorker blockWorker2 = worker2.getWorker(BlockWorker.class);
        assertEquals(100, blockWorker1.getStoreMeta().getUsedBytes());
        assertEquals(10, blockWorker2.getStoreMeta().getUsedBytes());
    }
}

Example 89 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class FileSystemMasterIntegrationTest method deleteDirectoryRecursive.

@Test
public void deleteDirectoryRecursive() throws Exception {
    AlluxioURI dir = new AlluxioURI("/testFolder");
    mFsMaster.createDirectory(dir, CreateDirectoryContext.defaults());
    FileSystem fs = sLocalAlluxioClusterResource.get().getClient();
    for (int i = 0; i < 3; i++) {
        FileSystemTestUtils.createByteFile(fs, PathUtils.concatPath(dir, "file" + i), 100, CreateFilePOptions.newBuilder().setWriteType(WritePType.MUST_CACHE).build());
    }
    fs.delete(dir, DeletePOptions.newBuilder().setRecursive(true).build());
    assertFalse(fs.exists(dir));
    // Make sure that the blocks are cleaned up
    BlockMasterClient blockClient = BlockMasterClient.Factory.create(MasterClientContext.newBuilder(ClientContext.create(ServerConfiguration.global())).build());
    CommonUtils.waitFor("data to be deleted", () -> {
        try {
            return blockClient.getUsedBytes() == 0;
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }, WaitForOptions.defaults().setTimeoutMs(10 * Constants.SECOND_MS));
}

Example 90 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class FuseManager method start.

/**
 * Starts mounting the internal Fuse applications.
 */
public void start() {
    AlluxioConfiguration conf = ServerConfiguration.global();
    if (!conf.isSet(PropertyKey.WORKER_FUSE_MOUNT_POINT) || conf.getString(PropertyKey.WORKER_FUSE_MOUNT_POINT).isEmpty()) {
        LOG.error("Failed to launch worker internal Fuse application. {} should be set.", PropertyKey.WORKER_FUSE_MOUNT_POINT);
        return;
    }
    if (!conf.isSet(PropertyKey.WORKER_FUSE_MOUNT_ALLUXIO_PATH) || conf.getString(PropertyKey.WORKER_FUSE_MOUNT_ALLUXIO_PATH).isEmpty()) {
        LOG.error("Failed to launch worker internal Fuse application. {} should be set.", PropertyKey.WORKER_FUSE_MOUNT_ALLUXIO_PATH.getName());
        return;
    }
    String fuseMount = conf.getString(PropertyKey.WORKER_FUSE_MOUNT_POINT);
    String alluxioPath = conf.getString(PropertyKey.WORKER_FUSE_MOUNT_ALLUXIO_PATH);
    // create the folder if it does not exist
    try {
        String[] fuseOptsSeparated = new String[0];
        if (conf.isSet(PropertyKey.WORKER_FUSE_MOUNT_OPTIONS)) {
            String fuseOptsString = conf.getString(PropertyKey.WORKER_FUSE_MOUNT_OPTIONS);
            if (!fuseOptsString.isEmpty()) {
                fuseOptsSeparated = fuseOptsString.split(FUSE_OPTION_SEPARATOR);
            }
        }
        List<String> fuseOptions = AlluxioFuse.parseFuseOptions(fuseOptsSeparated, conf);
        FuseMountOptions options = new FuseMountOptions(fuseMount, alluxioPath, conf.getBoolean(PropertyKey.FUSE_DEBUG_ENABLED), fuseOptions);
        // TODO(lu) consider launching fuse in a separate thread as blocking operation
        // so that we can know about the fuse application status
        FileSystem fileSystem = mResourceCloser.register(FileSystem.Factory.create(mFsContext));
        mFuseUmountable = AlluxioFuse.launchFuse(fileSystem, conf, options, false);
    } catch (Throwable throwable) {
        // TODO(lu) for already mounted application, unmount first and then remount
        LOG.error("Failed to launch worker internal Fuse application", throwable);
    }
}