Examples with IOTaskSummary alluxio.stress.worker.IOTaskSummary used on opensource projects

Example 1 with IOTaskSummary

use of alluxio.stress.worker.IOTaskSummary in project alluxio by Alluxio.

the class ManagerProcessContext method speedTest.

/**
 * speed test for ufs.
 *
 * @param request ufs speed test request
 * @return result of the test
 */
public SpeedTestResponse.Payload speedTest(SpeedTestRequest request) {
    SpeedTestRequest.Payload req = request.getPayload();
    String dataSize = req.getDataSize();
    String mountPoint = req.getMountPoint();
    String relativePath = req.getPath();
    int clusterParallelism = req.getClusterParallelism();
    int nodeParallelism = req.getNodeParallelism();
    List<String> errors = new ArrayList<>();
    MountPointInfo mountInfo = null;
    try {
        mountInfo = getFileSystem().getMountTable().get(mountPoint);
    } catch (IOException | AlluxioException e) {
        errors.add("Failed to retrieve mount point information");
        errors.add(ValidationUtils.getErrorInfo(e));
    }
    if (mountInfo == null) {
        errors.add("Failed to retrieve mount point information for " + mountPoint);
        return SpeedTestResponse.Payload.newBuilder().addAllError(errors).build();
    }
    if (clusterParallelism > mAlluxioCluster.size()) {
        errors.add("Can not set cluster parallelism to be " + clusterParallelism + " which is greater than cluster size " + mAlluxioCluster.size());
        return SpeedTestResponse.Payload.newBuilder().addAllError(errors).build();
    }
    if (relativePath.isEmpty()) {
        relativePath = "/";
    }
    String ufsPath = PathUtils.normalizePath(PathUtils.concatPath(mountInfo.getUfsUri(), relativePath), AlluxioURI.SEPARATOR);
    List<String> argList = new ArrayList<>();
    String[] args = new String[] { "--io-size", dataSize, "--threads", String.valueOf(nodeParallelism), "--path", ufsPath, "--cluster-limit", String.valueOf(clusterParallelism), "--cluster", "--use-mount-conf" };
    Collections.addAll(argList, args);
    UfsIOBench bench = getIOBench();
    try {
        String output = bench.run(argList.toArray(new String[0]));
        IOTaskSummary result = (IOTaskSummary) JsonSerializable.fromJson(output);
        SpeedTestResponse.Payload.Builder builder = SpeedTestResponse.Payload.newBuilder();
        if (result.getErrors() != null) {
            builder.addAllError(result.getErrors());
        }
        if (result.getParameters() != null) {
            UfsIOParameters param = result.getParameters();
            builder.setParameters(SpeedTestParameter.newBuilder().setDataSize(param.mDataSize).setPath(param.mPath).setThreads(param.mThreads).putAllProperties(param.mConf).build());
        }
        builder.setReadSpeedStat(convertToProto(result.getReadSpeedStat())).setWriteSpeedStat(convertToProto(result.getWriteSpeedStat()));
        return builder.build();
    } catch (Exception e) {
        errors.add("UfsIOBench failed due to " + ValidationUtils.getErrorInfo(e));
    }
    return SpeedTestResponse.Payload.newBuilder().addAllError(errors).build();
}

Example 2 with IOTaskSummary

use of alluxio.stress.worker.IOTaskSummary in project alluxio by Alluxio.

the class ManagerProcessContextTest method testSpeedTest.

@Test
public void testSpeedTest() throws Exception {
    UfsIOBench bench = Mockito.mock(UfsIOBench.class);
    FileSystem mockFs = Mockito.mock(FileSystem.class);
    doReturn(mockFs).when(mContext).getFileSystem();
    doReturn(bench).when(mContext).getIOBench();
    Map<String, MountPointInfo> mt = new HashMap<>();
    MountPointInfo info = new MountPointInfo().setMountId(1).setReadOnly(true).setUfsType("hdfs").setUfsUri("hdfs://localhost:9000");
    mt.put("/", info);
    doReturn(mt).when(mockFs).getMountTable();
    IOTaskSummary summ = new IOTaskSummary();
    IOTaskSummary.SpeedStat read = generateSpeedStat();
    IOTaskSummary.SpeedStat write = generateSpeedStat();
    summ.setReadSpeedStat(read);
    summ.setWriteSpeedStat(write);
    doReturn(summ.toJson()).when(bench).run(any());
    SpeedTestResponse.Payload resp = mContext.speedTest(SpeedTestRequest.newBuilder().setPayload(SpeedTestRequest.Payload.newBuilder().setMountPoint("/").setPath("bbb").setClusterParallelism(0).setNodeParallelism(0).setDataSize("1m")).build());
    SpeedStat readResp = resp.getReadSpeedStat();
    SpeedStat writeResp = resp.getWriteSpeedStat();
    List<String> errResp = resp.getErrorList();
    SpeedTestParameter params = resp.getParameters();
    assertEquals(read.mAvgSpeedMbps, readResp.getAvgSpeedMbps(), 1E-10);
    assertEquals(read.mMinSpeedMbps, readResp.getMinSpeedMbps(), 1E-10);
    assertEquals(read.mMaxSpeedMbps, readResp.getMaxSpeedMbps(), 1E-10);
    assertEquals(read.mStdDev, readResp.getStdDev(), 1E-10);
    assertEquals(read.mTotalDurationSeconds, readResp.getTotalDurationSeconds(), 1E-10);
    assertEquals(read.mTotalSizeBytes, readResp.getTotalSizeBytes(), 1E-10);
    assertEquals(write.mAvgSpeedMbps, writeResp.getAvgSpeedMbps(), 1E-10);
    assertEquals(write.mMinSpeedMbps, writeResp.getMinSpeedMbps(), 1E-10);
    assertEquals(write.mMaxSpeedMbps, writeResp.getMaxSpeedMbps(), 1E-10);
    assertEquals(write.mStdDev, writeResp.getStdDev(), 1E-10);
    assertEquals(write.mTotalDurationSeconds, writeResp.getTotalDurationSeconds(), 1E-10);
    assertEquals(write.mTotalSizeBytes, writeResp.getTotalSizeBytes(), 1E-10);
    assertEquals(0, errResp.size());
    summ.setErrors(Collections.singletonList("testErr"));
    doReturn(summ.toJson()).when(bench).run(any());
    resp = mContext.speedTest(SpeedTestRequest.newBuilder().setPayload(SpeedTestRequest.Payload.newBuilder().setMountPoint("/").setPath("bbb").setClusterParallelism(0).setNodeParallelism(0).setDataSize("1m")).build());
    errResp = resp.getErrorList();
    assertEquals(1, errResp.size());
    assertEquals("testErr", errResp.get(0));
}