Examples with SuppressFBWarnings alluxio.annotation.SuppressFBWarnings used on opensource projects

Example 6 with SuppressFBWarnings

use of alluxio.annotation.SuppressFBWarnings in project alluxio by Alluxio.

the class StressMasterBench method prepare.

@Override
@SuppressFBWarnings("BC_UNCONFIRMED_CAST")
public void prepare() throws Exception {
    if (mParameters.mFixedCount <= 0) {
        throw new IllegalStateException("fixed count must be > 0. fixedCount: " + mParameters.mFixedCount);
    }
    if (!mBaseParameters.mDistributed) {
        // set hdfs conf for preparation client
        Configuration hdfsConf = new Configuration();
        // force delete, create dirs through to UFS
        hdfsConf.set(PropertyKey.Name.USER_FILE_DELETE_UNCHECKED, "true");
        hdfsConf.set(PropertyKey.Name.USER_FILE_WRITE_TYPE_DEFAULT, mParameters.mWriteType);
        // more threads for parallel deletes for cleanup
        hdfsConf.set(PropertyKey.Name.USER_FILE_MASTER_CLIENT_POOL_SIZE_MAX, "256");
        FileSystem prepareFs = FileSystem.get(new URI(mParameters.mBasePath), hdfsConf);
        // initialize the base, for only the non-distributed task (the cluster launching task)
        Path path = new Path(mParameters.mBasePath);
        // the base path depends on the operation
        Path basePath;
        if (mParameters.mOperation == Operation.CREATE_DIR) {
            basePath = new Path(path, "dirs");
        } else {
            basePath = new Path(path, "files");
        }
        if (mParameters.mOperation == Operation.CREATE_FILE || mParameters.mOperation == Operation.CREATE_DIR) {
            LOG.info("Cleaning base path: {}", basePath);
            long start = CommonUtils.getCurrentMs();
            deletePaths(prepareFs, basePath);
            long end = CommonUtils.getCurrentMs();
            LOG.info("Cleanup took: {} s", (end - start) / 1000.0);
            prepareFs.mkdirs(basePath);
        } else {
            // these are read operations. the directory must exist
            if (!prepareFs.exists(basePath)) {
                throw new IllegalStateException(String.format("base path (%s) must exist for operation (%s)", basePath, mParameters.mOperation));
            }
        }
        if (!prepareFs.isDirectory(basePath)) {
            throw new IllegalStateException(String.format("base path (%s) must be a directory for operation (%s)", basePath, mParameters.mOperation));
        }
    }
    // set hdfs conf for all test clients
    Configuration hdfsConf = new Configuration();
    // do not cache these clients
    hdfsConf.set(String.format("fs.%s.impl.disable.cache", (new URI(mParameters.mBasePath)).getScheme()), "true");
    for (Map.Entry<String, String> entry : mParameters.mConf.entrySet()) {
        hdfsConf.set(entry.getKey(), entry.getValue());
    }
    hdfsConf.set(PropertyKey.Name.USER_FILE_WRITE_TYPE_DEFAULT, mParameters.mWriteType);
    if (mParameters.mClientType == FileSystemClientType.ALLUXIO_HDFS) {
        LOG.info("Using ALLUXIO HDFS Compatible API to perform the test.");
        mCachedFs = new FileSystem[mParameters.mClients];
        for (int i = 0; i < mCachedFs.length; i++) {
            mCachedFs[i] = FileSystem.get(new URI(mParameters.mBasePath), hdfsConf);
        }
    } else {
        LOG.info("Using ALLUXIO Native API to perform the test.");
        alluxio.conf.AlluxioProperties alluxioProperties = ConfigurationUtils.defaults();
        alluxioProperties.merge(HadoopConfigurationUtils.getConfigurationFromHadoop(hdfsConf), Source.RUNTIME);
        mCachedNativeFs = new alluxio.client.file.FileSystem[mParameters.mClients];
        for (int i = 0; i < mCachedNativeFs.length; i++) {
            mCachedNativeFs[i] = alluxio.client.file.FileSystem.Factory.create(new InstancedConfiguration(alluxioProperties));
        }
    }
}

Example 7 with SuppressFBWarnings

use of alluxio.annotation.SuppressFBWarnings in project alluxio by Alluxio.

the class StressClientIOBench method prepare.

@Override
@SuppressFBWarnings("BC_UNCONFIRMED_CAST")
public void prepare() throws Exception {
    if (mBaseParameters.mCluster && mBaseParameters.mClusterLimit != 1) {
        throw new IllegalArgumentException(String.format("%s is a single-node client IO stress test, so it cannot be run in cluster mode without" + " flag '%s 1'.", this.getClass().getName(), BaseParameters.CLUSTER_LIMIT_FLAG));
    }
    if (FormatUtils.parseSpaceSize(mParameters.mFileSize) < FormatUtils.parseSpaceSize(mParameters.mBufferSize)) {
        throw new IllegalArgumentException(String.format("File size (%s) must be larger than buffer size (%s)", mParameters.mFileSize, mParameters.mBufferSize));
    }
    if (mParameters.mOperation == ClientIOOperation.WRITE) {
        LOG.warn("Cannot write repeatedly, so warmup is not possible. Setting warmup to 0s.");
        mParameters.mWarmup = "0s";
    }
    if (!mBaseParameters.mDistributed) {
        // set hdfs conf for preparation client
        Configuration hdfsConf = new Configuration();
        hdfsConf.set(PropertyKey.Name.USER_FILE_DELETE_UNCHECKED, "true");
        hdfsConf.set(PropertyKey.Name.USER_FILE_WRITE_TYPE_DEFAULT, mParameters.mWriteType);
        FileSystem prepareFs = FileSystem.get(new URI(mParameters.mBasePath), hdfsConf);
        // initialize the base, for only the non-distributed task (the cluster launching task)
        Path path = new Path(mParameters.mBasePath);
        if (!ClientIOOperation.isRead(mParameters.mOperation)) {
            prepareFs.delete(path, true);
            prepareFs.mkdirs(path);
        }
    }
    ClientIOWritePolicy.setMaxWorkers(mParameters.mWriteNumWorkers);
    // set hdfs conf for all test clients
    Configuration hdfsConf = new Configuration();
    // do not cache these clients
    hdfsConf.set(String.format("fs.%s.impl.disable.cache", (new URI(mParameters.mBasePath)).getScheme()), "true");
    hdfsConf.set(PropertyKey.Name.USER_BLOCK_WRITE_LOCATION_POLICY, ClientIOWritePolicy.class.getName());
    for (Map.Entry<String, String> entry : mParameters.mConf.entrySet()) {
        hdfsConf.set(entry.getKey(), entry.getValue());
    }
    hdfsConf.set(PropertyKey.Name.USER_FILE_WRITE_TYPE_DEFAULT, mParameters.mWriteType);
    if (mParameters.mClientType == FileSystemClientType.ALLUXIO_HDFS) {
        LOG.info("Using ALLUXIO HDFS Compatible API to perform the test.");
        mCachedFs = new FileSystem[mParameters.mClients];
        for (int i = 0; i < mCachedFs.length; i++) {
            mCachedFs[i] = FileSystem.get(new URI(mParameters.mBasePath), hdfsConf);
        }
    } else {
        LOG.info("Using ALLUXIO Native API to perform the test.");
        alluxio.conf.AlluxioProperties alluxioProperties = ConfigurationUtils.defaults();
        alluxioProperties.merge(HadoopConfigurationUtils.getConfigurationFromHadoop(hdfsConf), Source.RUNTIME);
        mCachedNativeFs = new alluxio.client.file.FileSystem[mParameters.mClients];
        for (int i = 0; i < mCachedNativeFs.length; i++) {
            mCachedNativeFs[i] = alluxio.client.file.FileSystem.Factory.create(new InstancedConfiguration(alluxioProperties));
        }
    }
}

Example 8 with SuppressFBWarnings

use of alluxio.annotation.SuppressFBWarnings in project alluxio by Alluxio.

the class StressClientIOBench method runLocal.

@Override
@SuppressFBWarnings("BC_UNCONFIRMED_CAST")
public ClientIOTaskResult runLocal() throws Exception {
    List<Integer> threadCounts = new ArrayList<>(mParameters.mThreads);
    threadCounts.sort(Comparator.comparingInt(i -> i));
    ClientIOTaskResult taskResult = new ClientIOTaskResult();
    taskResult.setBaseParameters(mBaseParameters);
    taskResult.setParameters(mParameters);
    for (Integer numThreads : threadCounts) {
        ClientIOTaskResult.ThreadCountResult threadCountResult = runForThreadCount(numThreads);
        if (!mBaseParameters.mProfileAgent.isEmpty()) {
            taskResult.putTimeToFirstBytePerThread(numThreads, addAdditionalResult(threadCountResult.getRecordStartMs(), threadCountResult.getEndMs()));
        }
        taskResult.addThreadCountResults(numThreads, threadCountResult);
    }
    return taskResult;
}

Example 9 with SuppressFBWarnings

use of alluxio.annotation.SuppressFBWarnings in project alluxio by Alluxio.

the class Benchmark method processMethodProfiles.

/**
 * @param startMs the start time
 * @param endMs the end time
 * @param nameTransformer function which transforms the type and method into a name. If the
 *                        function returns null, then the method is skipped
 * @return a map of names to statistics
 */
@SuppressFBWarnings(value = "DMI_HARDCODED_ABSOLUTE_FILENAME")
protected Map<String, MethodStatistics> processMethodProfiles(long startMs, long endMs, Function<ProfileInput, String> nameTransformer) throws IOException {
    Map<String, MethodStatistics> nameStatistics = new HashMap<>();
    try (final BufferedReader reader = new BufferedReader(new FileReader(BaseParameters.AGENT_OUTPUT_PATH))) {
        String line;
        long bucketSize = (endMs - startMs) / StressConstants.MAX_TIME_COUNT;
        final ObjectMapper objectMapper = new ObjectMapper();
        while ((line = reader.readLine()) != null) {
            final Map<String, Object> lineMap;
            try {
                lineMap = objectMapper.readValue(line, Map.class);
            } catch (JsonParseException | MismatchedInputException e) {
                // skip the last line of a not completed file
                break;
            }
            final String type = (String) lineMap.get("type");
            final String methodName = (String) lineMap.get("methodName");
            final Number timestampNumber = (Number) lineMap.get("timestamp");
            final Number durationNumber = (Number) lineMap.get("duration");
            final Boolean ttfbFlag = (Boolean) lineMap.get("ttfb");
            if (type == null || methodName == null || timestampNumber == null || durationNumber == null || ttfbFlag == null) {
                continue;
            }
            final long timestamp = timestampNumber.longValue();
            final long duration = durationNumber.longValue();
            final boolean ttfb = ttfbFlag.booleanValue();
            if (timestamp <= startMs) {
                continue;
            }
            ProfileInput profileInput = new ProfileInput(type, methodName, ttfb);
            final String name = nameTransformer.apply(profileInput);
            if (name == null) {
                continue;
            }
            if (!nameStatistics.containsKey(name)) {
                nameStatistics.put(name, new MethodStatistics());
            }
            final MethodStatistics statistic = nameStatistics.get(name);
            statistic.mTimeNs.recordValue(duration);
            statistic.mNumSuccess += 1;
            int bucket = Math.min(statistic.mMaxTimeNs.length - 1, (int) ((timestamp - startMs) / bucketSize));
            statistic.mMaxTimeNs[bucket] = Math.max(statistic.mMaxTimeNs[bucket], duration);
        }
    }
    return nameStatistics;
}

Example 10 with SuppressFBWarnings

use of alluxio.annotation.SuppressFBWarnings in project alluxio by Alluxio.

the class StressWorkerBench method runLocal.

@Override
@SuppressFBWarnings("BC_UNCONFIRMED_CAST")
public WorkerBenchTaskResult runLocal() throws Exception {
    ExecutorService service = ExecutorServiceFactories.fixedThreadPool("bench-thread", mParameters.mThreads).create();
    long durationMs = FormatUtils.parseTimeSize(mParameters.mDuration);
    long warmupMs = FormatUtils.parseTimeSize(mParameters.mWarmup);
    long startMs = mBaseParameters.mStartMs;
    if (mBaseParameters.mStartMs == BaseParameters.UNDEFINED_START_MS) {
        startMs = CommonUtils.getCurrentMs() + 5000;
    }
    long endMs = startMs + warmupMs + durationMs;
    BenchContext context = new BenchContext(startMs, endMs);
    List<Callable<Void>> callables = new ArrayList<>(mParameters.mThreads);
    for (int i = 0; i < mParameters.mThreads; i++) {
        callables.add(new BenchThread(context, mCachedFs[i % mCachedFs.length]));
    }
    service.invokeAll(callables, FormatUtils.parseTimeSize(mBaseParameters.mBenchTimeout), TimeUnit.MILLISECONDS);
    service.shutdownNow();
    service.awaitTermination(30, TimeUnit.SECONDS);
    return context.getResult();
}