Search in sources :

Example 1 with UfsIOBench

use of alluxio.stress.cli.UfsIOBench in project alluxio by Alluxio.

the class ManagerProcessContext method speedTest.

/**
 * speed test for ufs.
 *
 * @param request ufs speed test request
 * @return result of the test
 */
public SpeedTestResponse.Payload speedTest(SpeedTestRequest request) {
    SpeedTestRequest.Payload req = request.getPayload();
    String dataSize = req.getDataSize();
    String mountPoint = req.getMountPoint();
    String relativePath = req.getPath();
    int clusterParallelism = req.getClusterParallelism();
    int nodeParallelism = req.getNodeParallelism();
    List<String> errors = new ArrayList<>();
    MountPointInfo mountInfo = null;
    try {
        mountInfo = getFileSystem().getMountTable().get(mountPoint);
    } catch (IOException | AlluxioException e) {
        errors.add("Failed to retrieve mount point information");
        errors.add(ValidationUtils.getErrorInfo(e));
    }
    if (mountInfo == null) {
        errors.add("Failed to retrieve mount point information for " + mountPoint);
        return SpeedTestResponse.Payload.newBuilder().addAllError(errors).build();
    }
    if (clusterParallelism > mAlluxioCluster.size()) {
        errors.add("Can not set cluster parallelism to be " + clusterParallelism + " which is greater than cluster size " + mAlluxioCluster.size());
        return SpeedTestResponse.Payload.newBuilder().addAllError(errors).build();
    }
    if (relativePath.isEmpty()) {
        relativePath = "/";
    }
    String ufsPath = PathUtils.normalizePath(PathUtils.concatPath(mountInfo.getUfsUri(), relativePath), AlluxioURI.SEPARATOR);
    List<String> argList = new ArrayList<>();
    String[] args = new String[] { "--io-size", dataSize, "--threads", String.valueOf(nodeParallelism), "--path", ufsPath, "--cluster-limit", String.valueOf(clusterParallelism), "--cluster", "--use-mount-conf" };
    Collections.addAll(argList, args);
    UfsIOBench bench = getIOBench();
    try {
        String output = bench.run(argList.toArray(new String[0]));
        IOTaskSummary result = (IOTaskSummary) JsonSerializable.fromJson(output);
        SpeedTestResponse.Payload.Builder builder = SpeedTestResponse.Payload.newBuilder();
        if (result.getErrors() != null) {
            builder.addAllError(result.getErrors());
        }
        if (result.getParameters() != null) {
            UfsIOParameters param = result.getParameters();
            builder.setParameters(SpeedTestParameter.newBuilder().setDataSize(param.mDataSize).setPath(param.mPath).setThreads(param.mThreads).putAllProperties(param.mConf).build());
        }
        builder.setReadSpeedStat(convertToProto(result.getReadSpeedStat())).setWriteSpeedStat(convertToProto(result.getWriteSpeedStat()));
        return builder.build();
    } catch (Exception e) {
        errors.add("UfsIOBench failed due to " + ValidationUtils.getErrorInfo(e));
    }
    return SpeedTestResponse.Payload.newBuilder().addAllError(errors).build();
}
Also used : SpeedTestRequest(alluxio.hub.proto.SpeedTestRequest) ArrayList(java.util.ArrayList) ByteString(com.google.protobuf.ByteString) IOException(java.io.IOException) UfsIOParameters(alluxio.stress.worker.UfsIOParameters) AlluxioException(alluxio.exception.AlluxioException) IOException(java.io.IOException) InvalidPathException(alluxio.exception.InvalidPathException) FileDoesNotExistException(alluxio.exception.FileDoesNotExistException) AlluxioStatusException(alluxio.exception.status.AlluxioStatusException) IOTaskSummary(alluxio.stress.worker.IOTaskSummary) MountPointInfo(alluxio.wire.MountPointInfo) HDFSMountPointInfo(alluxio.hub.proto.HDFSMountPointInfo) UfsIOBench(alluxio.stress.cli.UfsIOBench) AlluxioException(alluxio.exception.AlluxioException)

Example 2 with UfsIOBench

use of alluxio.stress.cli.UfsIOBench in project alluxio by Alluxio.

the class ManagerProcessContextTest method testSpeedTest.

@Test
public void testSpeedTest() throws Exception {
    UfsIOBench bench = Mockito.mock(UfsIOBench.class);
    FileSystem mockFs = Mockito.mock(FileSystem.class);
    doReturn(mockFs).when(mContext).getFileSystem();
    doReturn(bench).when(mContext).getIOBench();
    Map<String, MountPointInfo> mt = new HashMap<>();
    MountPointInfo info = new MountPointInfo().setMountId(1).setReadOnly(true).setUfsType("hdfs").setUfsUri("hdfs://localhost:9000");
    mt.put("/", info);
    doReturn(mt).when(mockFs).getMountTable();
    IOTaskSummary summ = new IOTaskSummary();
    IOTaskSummary.SpeedStat read = generateSpeedStat();
    IOTaskSummary.SpeedStat write = generateSpeedStat();
    summ.setReadSpeedStat(read);
    summ.setWriteSpeedStat(write);
    doReturn(summ.toJson()).when(bench).run(any());
    SpeedTestResponse.Payload resp = mContext.speedTest(SpeedTestRequest.newBuilder().setPayload(SpeedTestRequest.Payload.newBuilder().setMountPoint("/").setPath("bbb").setClusterParallelism(0).setNodeParallelism(0).setDataSize("1m")).build());
    SpeedStat readResp = resp.getReadSpeedStat();
    SpeedStat writeResp = resp.getWriteSpeedStat();
    List<String> errResp = resp.getErrorList();
    SpeedTestParameter params = resp.getParameters();
    assertEquals(read.mAvgSpeedMbps, readResp.getAvgSpeedMbps(), 1E-10);
    assertEquals(read.mMinSpeedMbps, readResp.getMinSpeedMbps(), 1E-10);
    assertEquals(read.mMaxSpeedMbps, readResp.getMaxSpeedMbps(), 1E-10);
    assertEquals(read.mStdDev, readResp.getStdDev(), 1E-10);
    assertEquals(read.mTotalDurationSeconds, readResp.getTotalDurationSeconds(), 1E-10);
    assertEquals(read.mTotalSizeBytes, readResp.getTotalSizeBytes(), 1E-10);
    assertEquals(write.mAvgSpeedMbps, writeResp.getAvgSpeedMbps(), 1E-10);
    assertEquals(write.mMinSpeedMbps, writeResp.getMinSpeedMbps(), 1E-10);
    assertEquals(write.mMaxSpeedMbps, writeResp.getMaxSpeedMbps(), 1E-10);
    assertEquals(write.mStdDev, writeResp.getStdDev(), 1E-10);
    assertEquals(write.mTotalDurationSeconds, writeResp.getTotalDurationSeconds(), 1E-10);
    assertEquals(write.mTotalSizeBytes, writeResp.getTotalSizeBytes(), 1E-10);
    assertEquals(0, errResp.size());
    summ.setErrors(Collections.singletonList("testErr"));
    doReturn(summ.toJson()).when(bench).run(any());
    resp = mContext.speedTest(SpeedTestRequest.newBuilder().setPayload(SpeedTestRequest.Payload.newBuilder().setMountPoint("/").setPath("bbb").setClusterParallelism(0).setNodeParallelism(0).setDataSize("1m")).build());
    errResp = resp.getErrorList();
    assertEquals(1, errResp.size());
    assertEquals("testErr", errResp.get(0));
}
Also used : SpeedTestResponse(alluxio.hub.proto.SpeedTestResponse) HashMap(java.util.HashMap) SpeedStat(alluxio.hub.proto.SpeedStat) IOTaskSummary(alluxio.stress.worker.IOTaskSummary) MountPointInfo(alluxio.wire.MountPointInfo) SpeedTestParameter(alluxio.hub.proto.SpeedTestParameter) FileSystem(alluxio.client.file.FileSystem) UfsIOBench(alluxio.stress.cli.UfsIOBench) BaseHubTest(alluxio.hub.test.BaseHubTest) Test(org.junit.Test)

Aggregations

UfsIOBench (alluxio.stress.cli.UfsIOBench)2 IOTaskSummary (alluxio.stress.worker.IOTaskSummary)2 MountPointInfo (alluxio.wire.MountPointInfo)2 FileSystem (alluxio.client.file.FileSystem)1 AlluxioException (alluxio.exception.AlluxioException)1 FileDoesNotExistException (alluxio.exception.FileDoesNotExistException)1 InvalidPathException (alluxio.exception.InvalidPathException)1 AlluxioStatusException (alluxio.exception.status.AlluxioStatusException)1 HDFSMountPointInfo (alluxio.hub.proto.HDFSMountPointInfo)1 SpeedStat (alluxio.hub.proto.SpeedStat)1 SpeedTestParameter (alluxio.hub.proto.SpeedTestParameter)1 SpeedTestRequest (alluxio.hub.proto.SpeedTestRequest)1 SpeedTestResponse (alluxio.hub.proto.SpeedTestResponse)1 BaseHubTest (alluxio.hub.test.BaseHubTest)1 UfsIOParameters (alluxio.stress.worker.UfsIOParameters)1 ByteString (com.google.protobuf.ByteString)1 IOException (java.io.IOException)1 ArrayList (java.util.ArrayList)1 HashMap (java.util.HashMap)1 Test (org.junit.Test)1