Examples with FileSystemShell alluxio.cli.fs.FileSystemShell used on opensource projects

Example 6 with FileSystemShell

use of alluxio.cli.fs.FileSystemShell in project alluxio by Alluxio.

the class PinCommandMultipleMediaIntegrationTest method pinToMediumForceEviction.

@Test
public void pinToMediumForceEviction() throws Exception {
    FileSystem fileSystem = sLocalAlluxioClusterResource.get().getClient();
    FileSystemShell fsShell = new FileSystemShell(ServerConfiguration.global());
    AlluxioURI filePathA = new AlluxioURI("/testFileA");
    AlluxioURI dirPath = new AlluxioURI("/testDirA");
    AlluxioURI filePathB = new AlluxioURI(dirPath.getPath() + "/testFileB");
    AlluxioURI filePathC = new AlluxioURI("/testFileC");
    int fileSize = SIZE_BYTES / 2;
    FileSystemTestUtils.createByteFile(fileSystem, filePathA, WritePType.CACHE_THROUGH, fileSize);
    assertTrue(fileExists(fileSystem, filePathA));
    assertEquals(0, fsShell.run("pin", filePathA.toString(), "MEM"));
    URIStatus status = fileSystem.getStatus(filePathA);
    assertTrue(status.isPinned());
    assertTrue(status.getPinnedMediumTypes().contains("MEM"));
    fileSystem.createDirectory(dirPath);
    assertEquals(0, fsShell.run("pin", dirPath.toString(), "MEM"));
    FileSystemTestUtils.createByteFile(fileSystem, filePathB, WritePType.CACHE_THROUGH, fileSize);
    assertTrue(fileExists(fileSystem, filePathB));
    URIStatus statusB = fileSystem.getStatus(filePathB);
    assertTrue(statusB.isPinned());
    assertTrue(statusB.getPinnedMediumTypes().contains("MEM"));
    FileSystemTestUtils.createByteFile(fileSystem, filePathC, WritePType.THROUGH, fileSize);
    assertEquals(100, fileSystem.getStatus(filePathA).getInAlluxioPercentage());
    assertEquals(100, fileSystem.getStatus(filePathB).getInAlluxioPercentage());
    assertEquals(0, fileSystem.getStatus(filePathC).getInAlluxioPercentage());
    assertEquals(0, fsShell.run("pin", filePathC.toString(), "SSD"));
    // Verify files are replicated into the correct tier through job service
    CommonUtils.waitFor("File being loaded", () -> sJobCluster.getMaster().getJobMaster().listDetailed().stream().anyMatch(x -> x.getStatus().equals(Status.COMPLETED) && x.getName().equals("Replicate") && x.getAffectedPaths().contains(filePathC.getPath())), sWaitOptions);
    assertEquals(100, fileSystem.getStatus(filePathC).getInAlluxioPercentage());
    assertEquals(Constants.MEDIUM_MEM, fileSystem.getStatus(filePathA).getFileBlockInfos().get(0).getBlockInfo().getLocations().get(0).getMediumType());
    assertEquals(Constants.MEDIUM_MEM, fileSystem.getStatus(filePathB).getFileBlockInfos().get(0).getBlockInfo().getLocations().get(0).getMediumType());
    assertEquals(Constants.MEDIUM_SSD, fileSystem.getStatus(filePathC).getFileBlockInfos().get(0).getBlockInfo().getLocations().get(0).getMediumType());
    assertEquals(0, fsShell.run("unpin", filePathA.toString()));
    assertEquals(0, fsShell.run("pin", filePathC.toString(), "MEM"));
    status = fileSystem.getStatus(filePathA);
    assertFalse(status.isPinned());
    assertTrue(status.getPinnedMediumTypes().isEmpty());
    // Verify files are migrated from another tier into the correct tier through job service
    // Also verify that eviction works
    CommonUtils.waitFor("File being moved", () -> sJobCluster.getMaster().getJobMaster().listDetailed().stream().anyMatch(x -> x.getStatus().equals(Status.COMPLETED) && x.getName().equals("Move") && x.getAffectedPaths().contains(filePathC.getPath())), sWaitOptions);
    assertEquals(0, fileSystem.getStatus(filePathA).getInAlluxioPercentage());
    assertEquals(Constants.MEDIUM_MEM, fileSystem.getStatus(filePathC).getFileBlockInfos().get(0).getBlockInfo().getLocations().get(0).getMediumType());
}

Example 7 with FileSystemShell

use of alluxio.cli.fs.FileSystemShell in project alluxio by Alluxio.

the class AbstractFileSystemShellTest method beforeClass.

@BeforeClass
public static void beforeClass() throws Exception {
    sLocalAlluxioCluster = sLocalAlluxioClusterResource.get();
    sLocalAlluxioJobCluster = new LocalAlluxioJobCluster();
    sLocalAlluxioJobCluster.start();
    sFileSystem = sLocalAlluxioCluster.getClient();
    sJobMaster = sLocalAlluxioJobCluster.getMaster().getJobMaster();
    sJobShell = new alluxio.cli.job.JobShell(ServerConfiguration.global());
    sFsShell = new FileSystemShell(ServerConfiguration.global());
}

Example 8 with FileSystemShell

use of alluxio.cli.fs.FileSystemShell in project alluxio by Alluxio.

the class AddCommandIntegrationTest method immediatelyEffectiveForShellCommands.

@Test
public void immediatelyEffectiveForShellCommands() throws Exception {
    // Tests that after adding some path configuration, it's immediately effective for command
    // line calls afterwards.
    InstancedConfiguration conf = ServerConfiguration.global();
    try (FileSystemShell fsShell = new FileSystemShell(conf);
        FileSystemAdminShell fsAdminShell = new FileSystemAdminShell(conf)) {
        Assert.assertEquals(0, fsAdminShell.run("pathConf", "add", "--property", WRITE_TYPE_THROUGH, PATH1));
        Assert.assertEquals(0, fsAdminShell.run("pathConf", "add", "--property", WRITE_TYPE_CACHE_THROUGH, PATH2));
        FileSystem fs = sLocalAlluxioClusterResource.get().getClient();
        String file = "/file";
        FileSystemTestUtils.createByteFile(fs, file, 100, CreateFilePOptions.getDefaultInstance());
        fs.createDirectory(new AlluxioURI(PATH1), CreateDirectoryPOptions.newBuilder().setRecursive(true).build());
        fs.createDirectory(new AlluxioURI(PATH2), CreateDirectoryPOptions.newBuilder().setRecursive(true).build());
        AlluxioURI target = new AlluxioURI(PATH1 + file);
        Assert.assertEquals(0, fsShell.run("cp", file, target.toString()));
        URIStatus status = fs.getStatus(target);
        Assert.assertEquals(0, status.getInMemoryPercentage());
        Assert.assertEquals(PersistenceState.PERSISTED.toString(), status.getPersistenceState());
        target = new AlluxioURI(PATH2 + file);
        Assert.assertEquals(0, fsShell.run("cp", file, target.toString()));
        status = fs.getStatus(target);
        Assert.assertEquals(100, status.getInMemoryPercentage());
        Assert.assertEquals(PersistenceState.PERSISTED.toString(), status.getPersistenceState());
    }
}

Example 9 with FileSystemShell

use of alluxio.cli.fs.FileSystemShell in project alluxio by Alluxio.

the class PersistCommandTest method persistOnRenameDirectoryBlacklist.

@Test
public void persistOnRenameDirectoryBlacklist() throws Exception {
    InstancedConfiguration conf = new InstancedConfiguration(ServerConfiguration.global());
    conf.set(PropertyKey.USER_FILE_WRITE_TYPE_DEFAULT, "MUST_CACHE");
    conf.set(PropertyKey.USER_FILE_PERSIST_ON_RENAME, true);
    try (FileSystemShell fsShell = new FileSystemShell(conf)) {
        String testDir = FileSystemShellUtilsTest.resetFileHierarchy(sFileSystem);
        String toPersist = testDir + "/foo";
        String persisted = testDir + "/foo_persisted";
        // create the file that is blacklisted, under the directory
        FileSystemTestUtils.createByteFile(sFileSystem, toPersist + "/foobar_blacklist", WritePType.MUST_CACHE, 10);
        assertFalse(sFileSystem.getStatus(new AlluxioURI(testDir)).isPersisted());
        assertFalse(sFileSystem.getStatus(new AlluxioURI(toPersist)).isPersisted());
        assertFalse(sFileSystem.getStatus(new AlluxioURI(toPersist + "/foobar_blacklist")).isPersisted());
        int ret = fsShell.run("mv", toPersist, persisted);
        Assert.assertEquals(0, ret);
        CommonUtils.waitFor("Directory to be persisted", () -> {
            try {
                return sFileSystem.getStatus(new AlluxioURI(persisted)).isPersisted() && sFileSystem.getStatus(new AlluxioURI(persisted + "/foobar1")).isPersisted();
            } catch (Exception e) {
                return false;
            }
        }, WaitForOptions.defaults().setTimeoutMs(10000));
        assertFalse(sFileSystem.getStatus(new AlluxioURI(persisted + "/foobar_blacklist")).isPersisted());
        checkFilePersisted(new AlluxioURI(persisted + "/foobar1"), 10);
    }
}

Example 10 with FileSystemShell

use of alluxio.cli.fs.FileSystemShell in project alluxio by Alluxio.

the class DistributedLoadCommandTest method beforeClass.

@BeforeClass
public static void beforeClass() throws Exception {
    sLocalAlluxioCluster = sResource.get();
    sLocalAlluxioJobCluster = new LocalAlluxioJobCluster();
    sLocalAlluxioJobCluster.start();
    sFileSystem = sLocalAlluxioCluster.getClient();
    sJobMaster = sLocalAlluxioJobCluster.getMaster().getJobMaster();
    sJobShell = new alluxio.cli.job.JobShell(ServerConfiguration.global());
    sFsShell = new FileSystemShell(ServerConfiguration.global());
}