Examples with IOTaskResult alluxio.stress.worker.IOTaskResult used on opensource projects

Example 1 with IOTaskResult

use of alluxio.stress.worker.IOTaskResult in project alluxio by Alluxio.

the class UfsIOBench method read.

private IOTaskResult read(ExecutorService pool) throws InterruptedException, ExecutionException {
    UnderFileSystemConfiguration ufsConf;
    UnderFileSystem ufs;
    int numThreads;
    long ioSizeBytes;
    try {
        // Use multiple threads to saturate the bandwidth of this worker
        numThreads = mParameters.mThreads;
        ioSizeBytes = FormatUtils.parseSpaceSize(mParameters.mDataSize);
        ufsConf = UnderFileSystemConfiguration.defaults(mConf).createMountSpecificConf(mParameters.mConf);
        ufs = UnderFileSystem.Factory.create(mDataDir, ufsConf);
        if (!ufs.exists(mDataDir)) {
            // If the directory does not exist, there's no point proceeding
            throw new IOException(String.format("The target directory %s does not exist!", mDataDir));
        }
    } catch (Exception e) {
        LOG.error("Failed to access UFS path {}", mDataDir);
        // If the UFS path is not valid, abort the test
        IOTaskResult result = new IOTaskResult();
        result.setParameters(mParameters);
        result.setBaseParameters(mBaseParameters);
        result.addError(ValidationUtils.getErrorInfo(e));
        return result;
    }
    List<CompletableFuture<IOTaskResult>> futures = new ArrayList<>();
    for (int i = 0; i < numThreads; i++) {
        final int idx = i;
        CompletableFuture<IOTaskResult> future = CompletableFuture.supplyAsync(() -> {
            IOTaskResult result = new IOTaskResult();
            result.setBaseParameters(mBaseParameters);
            result.setParameters(mParameters);
            long startTime = CommonUtils.getCurrentMs();
            String filePath = getFilePath(idx);
            LOG.debug("Reading filePath={}", filePath);
            long readBytes = 0;
            InputStream inStream = null;
            try {
                inStream = ufs.open(filePath);
                byte[] buf = new byte[BUFFER_SIZE];
                int readBufBytes;
                while (readBytes < ioSizeBytes && (readBufBytes = inStream.read(buf)) > 0) {
                    readBytes += readBufBytes;
                }
                long endTime = CommonUtils.getCurrentMs();
                // convert to second
                double duration = (endTime - startTime) / 1000.0;
                IOTaskResult.Point p = new IOTaskResult.Point(IOTaskResult.IOMode.READ, duration, readBytes);
                result.addPoint(p);
                LOG.debug("Read task finished {}", p);
            } catch (Exception e) {
                LOG.error("Failed to read {}", filePath, e);
                result.addError(ValidationUtils.getErrorInfo(e));
            } finally {
                if (inStream != null) {
                    try {
                        inStream.close();
                    } catch (IOException e) {
                        LOG.warn("Failed to close read stream {}", filePath, e);
                        result.addError(e.getMessage());
                    }
                }
            }
            return result;
        }, pool);
        futures.add(future);
    }
    // Collect the result
    CompletableFuture[] cfs = futures.toArray(new CompletableFuture[0]);
    List<IOTaskResult> results = CompletableFuture.allOf(cfs).thenApply(f -> futures.stream().map(CompletableFuture::join).collect(Collectors.toList())).get();
    return IOTaskResult.reduceList(results);
}

Example 2 with IOTaskResult

use of alluxio.stress.worker.IOTaskResult in project alluxio by Alluxio.

the class UfsIOBench method write.

private IOTaskResult write(ExecutorService pool) throws InterruptedException, ExecutionException {
    UnderFileSystemConfiguration ufsConf;
    UnderFileSystem ufs;
    int numThreads;
    long ioSizeBytes;
    try {
        // Use multiple threads to saturate the bandwidth of this worker
        numThreads = mParameters.mThreads;
        ioSizeBytes = FormatUtils.parseSpaceSize(mParameters.mDataSize);
        ufsConf = UnderFileSystemConfiguration.defaults(mConf).createMountSpecificConf(mParameters.mConf);
        // Create a subdir for the IO
        ufs = UnderFileSystem.Factory.create(mDataDir, ufsConf);
        if (!ufs.exists(mDataDir)) {
            LOG.debug("Prepare directory {}", mDataDir);
            ufs.mkdirs(mDataDir);
        }
    } catch (Exception e) {
        LOG.error("Failed to prepare base directory {}", mDataDir);
        // If the UFS path is not valid, abort the test
        IOTaskResult result = new IOTaskResult();
        result.setParameters(mParameters);
        result.setBaseParameters(mBaseParameters);
        result.addError(ValidationUtils.getErrorInfo(e));
        return result;
    }
    List<CompletableFuture<IOTaskResult>> futures = new ArrayList<>();
    final byte[] randomData = CommonUtils.randomBytes(BUFFER_SIZE);
    for (int i = 0; i < numThreads; i++) {
        final int idx = i;
        CompletableFuture<IOTaskResult> future = CompletableFuture.supplyAsync(() -> {
            IOTaskResult result = new IOTaskResult();
            result.setParameters(mParameters);
            result.setBaseParameters(mBaseParameters);
            long startTime = CommonUtils.getCurrentMs();
            String filePath = getFilePath(idx);
            LOG.debug("filePath={}, data to write={}", filePath, mParameters.mDataSize);
            long wroteBytes = 0;
            BufferedOutputStream outStream = null;
            try {
                outStream = new BufferedOutputStream(ufs.create(filePath));
                while (wroteBytes < ioSizeBytes) {
                    long bytesToWrite = Math.min(ioSizeBytes - wroteBytes, BUFFER_SIZE);
                    // bytesToWrite is bounded by BUFFER_SIZE, which is an integer
                    outStream.write(randomData, 0, (int) bytesToWrite);
                    wroteBytes += bytesToWrite;
                }
                outStream.flush();
                long endTime = CommonUtils.getCurrentMs();
                // convert to second
                double duration = (endTime - startTime) / 1000.0;
                IOTaskResult.Point p = new IOTaskResult.Point(IOTaskResult.IOMode.WRITE, duration, wroteBytes);
                result.addPoint(p);
                LOG.debug("Write task finished {}", p);
            } catch (Exception e) {
                LOG.error("Failed to write to UFS: ", e);
                result.addError(e.getMessage());
            } finally {
                if (outStream != null) {
                    try {
                        outStream.close();
                    } catch (IOException e) {
                        LOG.warn("Failed to close stream to UFS: ", e);
                        result.addError(e.getMessage());
                    }
                }
            }
            LOG.debug("Thread {} file={}, IOBench result={}", Thread.currentThread().getName(), filePath, result);
            return result;
        }, pool);
        futures.add(future);
    }
    // Collect the result
    CompletableFuture[] cfs = futures.toArray(new CompletableFuture[0]);
    List<IOTaskResult> results = CompletableFuture.allOf(cfs).thenApply(f -> futures.stream().map(CompletableFuture::join).collect(Collectors.toList())).get();
    return IOTaskResult.reduceList(results);
}

Example 3 with IOTaskResult

use of alluxio.stress.worker.IOTaskResult in project alluxio by Alluxio.

the class UfsIOBench method runIOBench.

private IOTaskResult runIOBench(ExecutorService pool) throws Exception {
    IOTaskResult writeTaskResult = write(pool);
    if (writeTaskResult.getPoints().size() == 0) {
        LOG.error("Failed to write any files. Abort the test.");
        return writeTaskResult;
    }
    IOTaskResult readTaskResult = read(pool);
    cleanUp();
    return writeTaskResult.merge(readTaskResult);
}

Example 4 with IOTaskResult

use of alluxio.stress.worker.IOTaskResult in project alluxio by Alluxio.

the class UfsIOBench method runLocal.

@Override
public IOTaskResult runLocal() throws Exception {
    // UfsIOBench is invoked from the job worker then runs in a standalone process
    // The process type should be set to keep consistent
    boolean switched = CommonUtils.PROCESS_TYPE.compareAndSet(CommonUtils.ProcessType.CLIENT, CommonUtils.ProcessType.JOB_WORKER);
    LOG.debug("Running locally with {} threads", mParameters.mThreads);
    ExecutorService pool = null;
    IOTaskResult result = null;
    try {
        pool = ExecutorServiceFactories.fixedThreadPool("bench-io-thread", mParameters.mThreads).create();
        result = runIOBench(pool);
        LOG.debug("IO benchmark finished with result: {}", result);
        // Aggregate the task results
        return result;
    } catch (Exception e) {
        if (result == null) {
            LOG.error("Failed run UFS IO benchmark on path {}", mParameters.mPath, e);
            result = new IOTaskResult();
            result.setParameters(mParameters);
            result.setBaseParameters(mBaseParameters);
            result.addError(ValidationUtils.getErrorInfo(e));
        }
        return result;
    } finally {
        if (pool != null) {
            pool.shutdownNow();
            pool.awaitTermination(30, TimeUnit.SECONDS);
        }
        if (switched) {
            // restore the process type if it was switched
            CommonUtils.PROCESS_TYPE.set(CommonUtils.ProcessType.CLIENT);
        }
    }
}