Examples with FileSystem alluxio.client.file.FileSystem used on opensource projects

Example 51 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class PinCommandMultipleMediaIntegrationTest method setPinToSpecificMedia.

@Test
public void setPinToSpecificMedia() throws Exception {
    FileSystem fileSystem = sLocalAlluxioClusterResource.get().getClient();
    FileSystemShell fsShell = new FileSystemShell(ServerConfiguration.global());
    AlluxioURI filePathA = new AlluxioURI("/testFileA");
    AlluxioURI filePathB = new AlluxioURI("/testFileB");
    int fileSize = SIZE_BYTES / 2;
    FileSystemTestUtils.createByteFile(fileSystem, filePathA, WritePType.CACHE_THROUGH, fileSize);
    assertTrue(fileSystem.exists(filePathA));
    assertEquals(0, fsShell.run("pin", filePathA.toString(), Constants.MEDIUM_SSD));
    CommonUtils.waitFor("File being moved", () -> sJobCluster.getMaster().getJobMaster().listDetailed().stream().anyMatch(x -> x.getName().equals("Move") && x.getStatus().equals(Status.COMPLETED) && x.getAffectedPaths().contains(filePathA.getPath())), sWaitOptions);
    assertTrue(fileSystem.getStatus(filePathA).getFileBlockInfos().get(0).getBlockInfo().getLocations().stream().anyMatch(x -> x.getMediumType().equals(Constants.MEDIUM_SSD)));
    assertEquals(-1, fsShell.run("pin", filePathB.toString(), "NVRAM"));
}

Example 52 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class PinCommandMultipleMediaIntegrationTest method pinToMediumForceEviction.

@Test
public void pinToMediumForceEviction() throws Exception {
    FileSystem fileSystem = sLocalAlluxioClusterResource.get().getClient();
    FileSystemShell fsShell = new FileSystemShell(ServerConfiguration.global());
    AlluxioURI filePathA = new AlluxioURI("/testFileA");
    AlluxioURI dirPath = new AlluxioURI("/testDirA");
    AlluxioURI filePathB = new AlluxioURI(dirPath.getPath() + "/testFileB");
    AlluxioURI filePathC = new AlluxioURI("/testFileC");
    int fileSize = SIZE_BYTES / 2;
    FileSystemTestUtils.createByteFile(fileSystem, filePathA, WritePType.CACHE_THROUGH, fileSize);
    assertTrue(fileExists(fileSystem, filePathA));
    assertEquals(0, fsShell.run("pin", filePathA.toString(), "MEM"));
    URIStatus status = fileSystem.getStatus(filePathA);
    assertTrue(status.isPinned());
    assertTrue(status.getPinnedMediumTypes().contains("MEM"));
    fileSystem.createDirectory(dirPath);
    assertEquals(0, fsShell.run("pin", dirPath.toString(), "MEM"));
    FileSystemTestUtils.createByteFile(fileSystem, filePathB, WritePType.CACHE_THROUGH, fileSize);
    assertTrue(fileExists(fileSystem, filePathB));
    URIStatus statusB = fileSystem.getStatus(filePathB);
    assertTrue(statusB.isPinned());
    assertTrue(statusB.getPinnedMediumTypes().contains("MEM"));
    FileSystemTestUtils.createByteFile(fileSystem, filePathC, WritePType.THROUGH, fileSize);
    assertEquals(100, fileSystem.getStatus(filePathA).getInAlluxioPercentage());
    assertEquals(100, fileSystem.getStatus(filePathB).getInAlluxioPercentage());
    assertEquals(0, fileSystem.getStatus(filePathC).getInAlluxioPercentage());
    assertEquals(0, fsShell.run("pin", filePathC.toString(), "SSD"));
    // Verify files are replicated into the correct tier through job service
    CommonUtils.waitFor("File being loaded", () -> sJobCluster.getMaster().getJobMaster().listDetailed().stream().anyMatch(x -> x.getStatus().equals(Status.COMPLETED) && x.getName().equals("Replicate") && x.getAffectedPaths().contains(filePathC.getPath())), sWaitOptions);
    assertEquals(100, fileSystem.getStatus(filePathC).getInAlluxioPercentage());
    assertEquals(Constants.MEDIUM_MEM, fileSystem.getStatus(filePathA).getFileBlockInfos().get(0).getBlockInfo().getLocations().get(0).getMediumType());
    assertEquals(Constants.MEDIUM_MEM, fileSystem.getStatus(filePathB).getFileBlockInfos().get(0).getBlockInfo().getLocations().get(0).getMediumType());
    assertEquals(Constants.MEDIUM_SSD, fileSystem.getStatus(filePathC).getFileBlockInfos().get(0).getBlockInfo().getLocations().get(0).getMediumType());
    assertEquals(0, fsShell.run("unpin", filePathA.toString()));
    assertEquals(0, fsShell.run("pin", filePathC.toString(), "MEM"));
    status = fileSystem.getStatus(filePathA);
    assertFalse(status.isPinned());
    assertTrue(status.getPinnedMediumTypes().isEmpty());
    // Verify files are migrated from another tier into the correct tier through job service
    // Also verify that eviction works
    CommonUtils.waitFor("File being moved", () -> sJobCluster.getMaster().getJobMaster().listDetailed().stream().anyMatch(x -> x.getStatus().equals(Status.COMPLETED) && x.getName().equals("Move") && x.getAffectedPaths().contains(filePathC.getPath())), sWaitOptions);
    assertEquals(0, fileSystem.getStatus(filePathA).getInAlluxioPercentage());
    assertEquals(Constants.MEDIUM_MEM, fileSystem.getStatus(filePathC).getFileBlockInfos().get(0).getBlockInfo().getLocations().get(0).getMediumType());
}

Example 53 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class SetFaclCommandIntegrationTest method createFiles.

// Helper function to create a set of files in the file system
private URIStatus[] createFiles(String user) throws IOException, AlluxioException {
    FileSystem fs = sFileSystem;
    if (user != null) {
        fs = sLocalAlluxioCluster.getClient(FileSystemContext.create(new TestUserState(user, ServerConfiguration.global()).getSubject(), ServerConfiguration.global()));
    }
    FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileA", WritePType.MUST_CACHE, 10);
    FileSystemTestUtils.createByteFile(fs, "/testRoot/testDir/testFileB", WritePType.MUST_CACHE, 20);
    FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileC", WritePType.THROUGH, 30);
    URIStatus[] files = new URIStatus[4];
    files[0] = fs.getStatus(new AlluxioURI("/testRoot/testFileA"));
    files[1] = fs.getStatus(new AlluxioURI("/testRoot/testDir"));
    files[2] = fs.getStatus(new AlluxioURI("/testRoot/testDir/testFileB"));
    files[3] = fs.getStatus(new AlluxioURI("/testRoot/testFileC"));
    return files;
}

Example 54 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class LsCommandSecurityIntegrationTest method lsWildcard.

/**
 * Tests ls command with wildcard when security is enabled.
 */
@Test
public void lsWildcard() throws Exception {
    FileSystem fs = sLocalAlluxioCluster.getClient(FileSystemContext.create(new TestUserState("test_user_ls", ServerConfiguration.global()).getSubject(), ServerConfiguration.global()));
    String testDir = FileSystemShellUtilsTest.resetFileHierarchy(fs);
    sFsShell.run("ls", testDir + "/*/foo*");
    // CHECKSTYLE.OFF: LineLengthExceed - Improve readability
    checkOutput("-rw-r--r--  test_user_ls   test_user_ls                30   NOT_PERSISTED .+ .+ 100% /testDir/bar/foobar3", "-rw-r--r--  test_user_ls   test_user_ls                10   NOT_PERSISTED .+ .+ 100% /testDir/foo/foobar1", "-rw-r--r--  test_user_ls   test_user_ls                20   NOT_PERSISTED .+ .+ 100% /testDir/foo/foobar2");
    mOutput.reset();
    sFsShell.run("ls", "--sort", "path", testDir + "/*");
    checkOutput("-rw-r--r--  test_user_ls   test_user_ls                30   NOT_PERSISTED .+ .+ 100% /testDir/bar/foobar3", "-rw-r--r--  test_user_ls   test_user_ls                10   NOT_PERSISTED .+ .+ 100% /testDir/foo/foobar1", "-rw-r--r--  test_user_ls   test_user_ls                20   NOT_PERSISTED .+ .+ 100% /testDir/foo/foobar2", "-rw-r--r--  test_user_ls   test_user_ls                40   NOT_PERSISTED .+ .+ 100% /testDir/foobar4");
// CHECKSTYLE.ON: LineLengthExceed
}

Example 55 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class GetBlockInfoCommandIntegrationTest method getBlockInfo.

@Test
public void getBlockInfo() throws IOException, AlluxioException {
    FileSystem fileSystem = mLocalAlluxioCluster.getClient();
    fileSystem.createDirectory(new AlluxioURI("/foo"));
    FileSystemTestUtils.createByteFile(fileSystem, "/foo/foobar1", WritePType.MUST_CACHE, 10);
    long blockId = fileSystem.listStatus(new AlluxioURI("/foo/foobar1")).get(0).getBlockIds().get(0);
    int ret = mFsAdminShell.run("getBlockInfo", String.valueOf(blockId));
    Assert.assertEquals(0, ret);
    String output = mOutput.toString();
    Assert.assertThat(output, containsString("BlockInfo{id=" + blockId + ", length=10, locations=[BlockLocation{workerId="));
    Assert.assertThat(output, containsString("This block belongs to file {id=" + BlockId.getFileId(blockId) + ", path=/foo/foobar1}"));
}