Examples with FileSystem alluxio.client.file.FileSystem used on opensource projects

Example 91 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class LsCommandSecurityIntegrationTest method lsWithExtendedAcl.

@Test
public void lsWithExtendedAcl() throws IOException, AlluxioException {
    int size = 50;
    FileSystem fs = sLocalAlluxioCluster.getClient(FileSystemContext.create(new TestUserState("test_user_ls", ServerConfiguration.global()).getSubject(), ServerConfiguration.global()));
    FileSystemTestUtils.createByteFile(fs, "/testRoot/testDir/testFileB", WritePType.MUST_CACHE, 20);
    FileSystemTestUtils.createByteFile(fs, "/testRoot/testFile", WritePType.MUST_CACHE, size, size);
    sFsShell.run("ls", "--sort", "path", "/testRoot");
    // CHECKSTYLE.OFF: LineLengthExceed - Improve readability
    checkOutput("drwxr-xr-x  test_user_ls   test_user_ls                 1   NOT_PERSISTED .+ .+ DIR /testRoot/testDir", "-rw-r--r--  test_user_ls   test_user_ls                50   NOT_PERSISTED .+ .+ 100% /testRoot/testFile");
    // CHECKSTYLE.ON: LineLengthExceed
    mOutput.reset();
    fs.setAcl(new AlluxioURI("/testRoot/testDir"), SetAclAction.MODIFY, Arrays.asList(AclEntry.fromCliString("default:user:nameduser:rwx")));
    fs.setAcl(new AlluxioURI("/testRoot/testFile"), SetAclAction.MODIFY, Arrays.asList(AclEntry.fromCliString("user:nameduser:rwx")));
    sFsShell.run("ls", "--sort", "path", "/testRoot");
    // CHECKSTYLE.OFF: LineLengthExceed - Improve readability
    checkOutput("drwxr-xr-x\\+ test_user_ls   test_user_ls                 1   NOT_PERSISTED .+ .+  DIR /testRoot/testDir", "-rw-r--r--\\+ test_user_ls   test_user_ls                50   NOT_PERSISTED .+ .+ 100% /testRoot/testFile");
// CHECKSTYLE.ON: LineLengthExceed
}

Example 92 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class LsCommandSecurityIntegrationTest method createFiles.

// Helper function to create a set of files in the file system
private void createFiles() throws Exception {
    FileSystem fs = sLocalAlluxioCluster.getClient(FileSystemContext.create(new TestUserState("test_user_ls", ServerConfiguration.global()).getSubject(), ServerConfiguration.global()));
    FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileA", WritePType.MUST_CACHE, 10);
    FileSystemTestUtils.createByteFile(fs, "/testRoot/testDir/testFileB", WritePType.MUST_CACHE, 20);
    FileSystemTestUtils.createByteFile(fs, "/testRoot/testFileC", WritePType.THROUGH, 30);
}

Example 93 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class ManagerProcessContext method getMountPointList.

/**
 * Get a list of mount points and their information.
 *
 * @return a MountPointList object
 */
public ListMountPointResponse.Payload getMountPointList() {
    List<alluxio.hub.proto.MountPointInfo> mountPoints = new ArrayList<>();
    FileSystem fs = getFileSystem();
    if (fs == null) {
        LOG.debug("Unable to get file system client");
        return ListMountPointResponse.Payload.getDefaultInstance();
    }
    try {
        Map<String, MountPointInfo> mountTable = fs.getMountTable();
        Set<String> syncSet = fs.getSyncPathList().stream().map(syncPoint -> syncPoint.getSyncPointUri().toString()).collect(Collectors.toSet());
        for (Map.Entry<String, MountPointInfo> entry : mountTable.entrySet()) {
            alluxio.hub.proto.MountPointInfo.Builder builder = alluxio.hub.proto.MountPointInfo.newBuilder();
            MountPointInfo mountInfo = entry.getValue();
            // Generic mount point info
            builder.setAlluxioPath(entry.getKey()).setUfsUri(mountInfo.getUfsUri()).setUfsType(mountInfo.getUfsType()).setReadOnly(mountInfo.getReadOnly()).setShared(mountInfo.getShared());
            Map<String, String> prop = new HashMap<>(mountInfo.getProperties());
            // remove properties that are explicitly shown as options
            prop.remove(PropertyKey.UNDERFS_HDFS_CONFIGURATION.getName());
            // HDFS specific mount point info
            if (mountInfo.getUfsType().toLowerCase().equals("hdfs")) {
                prop.remove(PropertyKey.UNDERFS_VERSION.getName());
                HDFSMountPointInfo.Builder hdfsBuilder = HDFSMountPointInfo.newBuilder();
                String[] confFileNames = mountInfo.getProperties().getOrDefault(PropertyKey.UNDERFS_HDFS_CONFIGURATION.getName(), "").split(":");
                String coreSiteFileName = "";
                String hdfsSiteFileName = "";
                for (String confFile : confFileNames) {
                    if (confFile.contains("hdfs-site.xml")) {
                        hdfsSiteFileName = confFile;
                    } else if (confFile.contains("core-site.xml")) {
                        coreSiteFileName = confFile;
                    }
                }
                String hdfsVersion = mountInfo.getProperties().getOrDefault(PropertyKey.UNDERFS_VERSION.getName(), "");
                // Use file name in place of file content for now
                // Need to verify syncing through active sync api
                hdfsBuilder.setEnableSync(syncSet.contains(entry.getKey())).setCoreSiteFilePath(coreSiteFileName).setHdfsSiteFilePath(hdfsSiteFileName).setHdfsVersion(hdfsVersion);
                builder.setHdfsInfo(hdfsBuilder);
            }
            builder.putAllProperties(prop);
            mountPoints.add(builder.build());
        }
    } catch (AlluxioException e) {
        LOG.debug("getMountTable failed due to " + e.getMessage());
        return ListMountPointResponse.Payload.getDefaultInstance();
    } catch (IOException e) {
        LOG.debug("RPC to get mount table failed due to " + e.getMessage());
        return ListMountPointResponse.Payload.getDefaultInstance();
    }
    return ListMountPointResponse.Payload.newBuilder().addAllMountPoint(mountPoints).build();
}

Example 94 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class ManagerProcessContextTest method testSpeedTest.

@Test
public void testSpeedTest() throws Exception {
    UfsIOBench bench = Mockito.mock(UfsIOBench.class);
    FileSystem mockFs = Mockito.mock(FileSystem.class);
    doReturn(mockFs).when(mContext).getFileSystem();
    doReturn(bench).when(mContext).getIOBench();
    Map<String, MountPointInfo> mt = new HashMap<>();
    MountPointInfo info = new MountPointInfo().setMountId(1).setReadOnly(true).setUfsType("hdfs").setUfsUri("hdfs://localhost:9000");
    mt.put("/", info);
    doReturn(mt).when(mockFs).getMountTable();
    IOTaskSummary summ = new IOTaskSummary();
    IOTaskSummary.SpeedStat read = generateSpeedStat();
    IOTaskSummary.SpeedStat write = generateSpeedStat();
    summ.setReadSpeedStat(read);
    summ.setWriteSpeedStat(write);
    doReturn(summ.toJson()).when(bench).run(any());
    SpeedTestResponse.Payload resp = mContext.speedTest(SpeedTestRequest.newBuilder().setPayload(SpeedTestRequest.Payload.newBuilder().setMountPoint("/").setPath("bbb").setClusterParallelism(0).setNodeParallelism(0).setDataSize("1m")).build());
    SpeedStat readResp = resp.getReadSpeedStat();
    SpeedStat writeResp = resp.getWriteSpeedStat();
    List<String> errResp = resp.getErrorList();
    SpeedTestParameter params = resp.getParameters();
    assertEquals(read.mAvgSpeedMbps, readResp.getAvgSpeedMbps(), 1E-10);
    assertEquals(read.mMinSpeedMbps, readResp.getMinSpeedMbps(), 1E-10);
    assertEquals(read.mMaxSpeedMbps, readResp.getMaxSpeedMbps(), 1E-10);
    assertEquals(read.mStdDev, readResp.getStdDev(), 1E-10);
    assertEquals(read.mTotalDurationSeconds, readResp.getTotalDurationSeconds(), 1E-10);
    assertEquals(read.mTotalSizeBytes, readResp.getTotalSizeBytes(), 1E-10);
    assertEquals(write.mAvgSpeedMbps, writeResp.getAvgSpeedMbps(), 1E-10);
    assertEquals(write.mMinSpeedMbps, writeResp.getMinSpeedMbps(), 1E-10);
    assertEquals(write.mMaxSpeedMbps, writeResp.getMaxSpeedMbps(), 1E-10);
    assertEquals(write.mStdDev, writeResp.getStdDev(), 1E-10);
    assertEquals(write.mTotalDurationSeconds, writeResp.getTotalDurationSeconds(), 1E-10);
    assertEquals(write.mTotalSizeBytes, writeResp.getTotalSizeBytes(), 1E-10);
    assertEquals(0, errResp.size());
    summ.setErrors(Collections.singletonList("testErr"));
    doReturn(summ.toJson()).when(bench).run(any());
    resp = mContext.speedTest(SpeedTestRequest.newBuilder().setPayload(SpeedTestRequest.Payload.newBuilder().setMountPoint("/").setPath("bbb").setClusterParallelism(0).setNodeParallelism(0).setDataSize("1m")).build());
    errResp = resp.getErrorList();
    assertEquals(1, errResp.size());
    assertEquals("testErr", errResp.get(0));
}

Example 95 with FileSystem

use of alluxio.client.file.FileSystem in project alluxio by Alluxio.

the class TestDatabase method genTable.

public static void genTable(int numOfTable, int numOfPartitions, boolean generateFiles) {
    DATABASE.mUdbTables.clear();
    FileSystem fs = null;
    if (generateFiles) {
        fs = FileSystem.Factory.create(ServerConfiguration.global());
    }
    for (int i = 0; i < numOfTable; i++) {
        DATABASE.mUdbTables.put(getTableName(i), new TestUdbTable(TEST_UDB_NAME, getTableName(i), numOfPartitions, fs));
    }
}