Examples with FileSystem alluxio.client.file.FileSystem used on opensource projects

Example 56 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class AddCommandIntegrationTest method immediatelyEffectiveForShellCommands.

@Test
public void immediatelyEffectiveForShellCommands() throws Exception {
    // Tests that after adding some path configuration, it's immediately effective for command
    // line calls afterwards.
    InstancedConfiguration conf = ServerConfiguration.global();
    try (FileSystemShell fsShell = new FileSystemShell(conf);
        FileSystemAdminShell fsAdminShell = new FileSystemAdminShell(conf)) {
        Assert.assertEquals(0, fsAdminShell.run("pathConf", "add", "--property", WRITE_TYPE_THROUGH, PATH1));
        Assert.assertEquals(0, fsAdminShell.run("pathConf", "add", "--property", WRITE_TYPE_CACHE_THROUGH, PATH2));
        FileSystem fs = sLocalAlluxioClusterResource.get().getClient();
        String file = "/file";
        FileSystemTestUtils.createByteFile(fs, file, 100, CreateFilePOptions.getDefaultInstance());
        fs.createDirectory(new AlluxioURI(PATH1), CreateDirectoryPOptions.newBuilder().setRecursive(true).build());
        fs.createDirectory(new AlluxioURI(PATH2), CreateDirectoryPOptions.newBuilder().setRecursive(true).build());
        AlluxioURI target = new AlluxioURI(PATH1 + file);
        Assert.assertEquals(0, fsShell.run("cp", file, target.toString()));
        URIStatus status = fs.getStatus(target);
        Assert.assertEquals(0, status.getInMemoryPercentage());
        Assert.assertEquals(PersistenceState.PERSISTED.toString(), status.getPersistenceState());
        target = new AlluxioURI(PATH2 + file);
        Assert.assertEquals(0, fsShell.run("cp", file, target.toString()));
        status = fs.getStatus(target);
        Assert.assertEquals(100, status.getInMemoryPercentage());
        Assert.assertEquals(PersistenceState.PERSISTED.toString(), status.getPersistenceState());
    }
}

Example 57 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class LsCommandIntegrationTest method createFiles.

// Helper function to create a set of files in the file system
private void createFiles(String user) throws Exception {
    FileSystem fs = sFileSystem;
    if (user != null) {
        fs = sLocalAlluxioCluster.getClient(FileSystemContext.create(new TestUserState(user, ServerConfiguration.global()).getSubject(), ServerConfiguration.global()));
    }
    FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileA", WritePType.MUST_CACHE, 10);
    FileSystemTestUtils.createByteFile(fs, "/testRoot/testDir/testFileB", WritePType.MUST_CACHE, 20);
    FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileC", WritePType.THROUGH, 30);
}

Example 58 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class FileSystemContextReinitIntegrationTest method configHashSyncWithOpenStream.

@Test
public void configHashSyncWithOpenStream() throws Exception {
    // will be closed when restarting local masters, which in turn will close the contexts.
    try (FileSystem client = FileSystem.Factory.create(mContext);
        FileOutStream os = client.createFile(PATH_TO_UPDATE, CreateFilePOptions.newBuilder().setRecursive(true).build())) {
        checkClusterConfBeforeUpdate();
        checkPathConfBeforeUpdate();
        updateClusterConf();
        updatePathConf();
        ExecutorService service = Executors.newSingleThreadExecutor();
        Future future = service.submit(() -> {
            mExecutor.heartbeat();
        });
        TimeUnit.SECONDS.sleep(1);
        // Stream is open, so reinitialization should block until the stream is closed.
        Assert.assertFalse(future.isDone());
        future.cancel(true);
        checkHash(false, false);
        os.close();
        // Stream is closed, reinitialization should not be blocked.
        triggerAndWaitSync();
        checkClusterConfAfterUpdate();
        checkPathConfAfterUpdate();
        checkHash(true, true);
    }
}

Example 59 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class ImpersonationIntegrationTest method impersonationArbitraryUserDisallowed.

@Test
@LocalAlluxioClusterResource.Config(confParams = { IMPERSONATION_GROUPS_CONFIG, "*" })
public void impersonationArbitraryUserDisallowed() throws Exception {
    String arbitraryUser = "arbitrary_user";
    ServerConfiguration.set(PropertyKey.SECURITY_LOGIN_IMPERSONATION_USERNAME, arbitraryUser);
    FileSystemContext context = FileSystemContext.create(createHdfsSubject(), ServerConfiguration.global());
    FileSystem fs = mLocalAlluxioClusterResource.get().getClient(context);
    fs.createFile(new AlluxioURI("/impersonation-test")).close();
    List<URIStatus> listing = fs.listStatus(new AlluxioURI("/"));
    Assert.assertEquals(1, listing.size());
    URIStatus status = listing.get(0);
    Assert.assertNotEquals(arbitraryUser, status.getOwner());
}

Example 60 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class S3ClientRestApiTest method listAllMyBuckets.

@Test
public void listAllMyBuckets() throws Exception {
    Mode mode = ModeParser.parse("777");
    SetAttributePOptions options = SetAttributePOptions.newBuilder().setMode(mode.toProto()).setRecursive(true).build();
    mFileSystem.setAttribute(new AlluxioURI("/"), options);
    Subject subject = new Subject();
    subject.getPrincipals().add(new User("user0"));
    AlluxioURI bucketPath = new AlluxioURI("/bucket0");
    FileSystem fs1 = sResource.get().getClient(FileSystemContext.create(subject, ServerConfiguration.global()));
    fs1.createDirectory(bucketPath);
    SetAttributePOptions setAttributeOptions = SetAttributePOptions.newBuilder().setOwner("user0").build();
    mFileSystem.setAttribute(new AlluxioURI("/bucket0"), setAttributeOptions);
    URIStatus bucket0Status = fs1.getStatus(bucketPath);
    subject = new Subject();
    subject.getPrincipals().add(new User("user1"));
    AlluxioURI bucket1Path = new AlluxioURI("/bucket1");
    FileSystem fs2 = sResource.get().getClient(FileSystemContext.create(subject, ServerConfiguration.global()));
    fs2.createDirectory(bucket1Path);
    setAttributeOptions = SetAttributePOptions.newBuilder().setOwner("user1").build();
    mFileSystem.setAttribute(new AlluxioURI("/bucket1"), setAttributeOptions);
    URIStatus bucket1Status = fs2.getStatus(bucket1Path);
    ListAllMyBucketsResult expected = new ListAllMyBucketsResult(Collections.emptyList());
    final TestCaseOptions requestOptions = TestCaseOptions.defaults().setContentType(TestCaseOptions.XML_CONTENT_TYPE);
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/", NO_PARAMS, HttpMethod.GET, expected, requestOptions).run();
    expected = new ListAllMyBucketsResult(Lists.newArrayList(bucket0Status));
    requestOptions.setAuthorization("AWS4-HMAC-SHA256 Credential=user0/20210631");
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/", NO_PARAMS, HttpMethod.GET, expected, requestOptions).run();
    expected = new ListAllMyBucketsResult(Lists.newArrayList(bucket1Status));
    requestOptions.setAuthorization("AWS4-HMAC-SHA256 Credential=user1/20210631");
    new TestCase(mHostname, mPort, S3_SERVICE_PREFIX + "/", NO_PARAMS, HttpMethod.GET, expected, requestOptions).run();
}