Examples with DescriptiveStatistics org.apache.commons.math3.stat.descriptive.DescriptiveStatistics used on opensource projects

Example 86 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project hive by apache.

the class ReplStatsTracker method addEntry.

/**
 * Adds an entry for tracking.
 * @param eventType the type of event.
 * @param eventId the event id.
 * @param timeTaken time taken to process the event.
 */
public synchronized void addEntry(String eventType, String eventId, long timeTaken) {
    // Store the last EventId for the JMX.
    lastEventId = eventId;
    // Update the entry in the descriptive statistics.
    DescriptiveStatistics descStatistics = descMap.get(eventType);
    if (descStatistics == null) {
        descStatistics = new DescriptiveStatistics();
        descStatistics.addValue(timeTaken);
        descMap.put(eventType, descStatistics);
    } else {
        descStatistics.addValue(timeTaken);
    }
    // Tracking for top K events, Maintain the list in descending order.
    ListOrderedMap<Long, Long> topKEntries = topKEvents.get(eventType);
    if (topKEntries == null) {
        topKEntries = new ListOrderedMap<>();
        topKEntries.put(Long.parseLong(eventId), timeTaken);
        topKEvents.put(eventType, topKEntries);
    } else {
        // Get the index of insertion, by descending order.
        int index = Collections.binarySearch(new ArrayList(topKEntries.values()), timeTaken, Collections.reverseOrder());
        // If the element comes as top K add it to the topEntries.
        // The index returned from the binary search, is either the index where the element already exist, else
        // (-insertionIndex) -1, so convert it to actual insertion index
        int insertionIndex = index < 0 ? -1 * (index) - 1 : index;
        if (insertionIndex < k && k >= 0) {
            topKEntries.put(insertionIndex, Long.parseLong(eventId), timeTaken);
        }
        // entries, since the list is sorted in descending order.
        if (topKEntries.size() > k) {
            topKEntries.remove(k);
        }
    }
}

Example 87 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project hive by apache.

the class HMSBenchmarks method benchmarkRenameTable.

static DescriptiveStatistics benchmarkRenameTable(@NotNull MicroBenchmark bench, @NotNull BenchData data, int count) {
    final HMSClient client = data.getClient();
    String dbName = data.dbName;
    String tableName = data.tableName;
    BenchmarkUtils.createPartitionedTable(client, dbName, tableName);
    try {
        addManyPartitionsNoException(client, dbName, tableName, null, Collections.singletonList("d"), count);
        Table oldTable = client.getTable(dbName, tableName);
        oldTable.getSd().setLocation("");
        Table newTable = oldTable.deepCopy();
        newTable.setTableName(tableName + "_renamed");
        return bench.measure(() -> {
            // Measuring 2 renames, so the tests are idempotent
            throwingSupplierWrapper(() -> client.alterTable(oldTable.getDbName(), oldTable.getTableName(), newTable));
            throwingSupplierWrapper(() -> client.alterTable(newTable.getDbName(), newTable.getTableName(), oldTable));
        });
    } catch (TException e) {
        e.printStackTrace();
        return new DescriptiveStatistics();
    } finally {
        throwingSupplierWrapper(() -> client.dropTable(dbName, tableName));
    }
}

Example 88 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project hive by apache.

the class HMSBenchmarks method benchmarkListManyPartitions.

static DescriptiveStatistics benchmarkListManyPartitions(@NotNull MicroBenchmark bench, @NotNull BenchData data, int howMany) {
    final HMSClient client = data.getClient();
    String dbName = data.dbName;
    String tableName = data.tableName;
    BenchmarkUtils.createPartitionedTable(client, dbName, tableName);
    try {
        addManyPartitions(client, dbName, tableName, null, Collections.singletonList("d"), howMany);
        LOG.debug("Created {} partitions", howMany);
        LOG.debug("started benchmark... ");
        return bench.measure(() -> throwingSupplierWrapper(() -> client.listPartitions(dbName, tableName)));
    } catch (TException e) {
        e.printStackTrace();
        return new DescriptiveStatistics();
    } finally {
        throwingSupplierWrapper(() -> client.dropTable(dbName, tableName));
    }
}

Example 89 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project hive by apache.

the class HMSBenchmarks method benchmarkListPartition.

static DescriptiveStatistics benchmarkListPartition(@NotNull MicroBenchmark bench, @NotNull BenchData data) {
    final HMSClient client = data.getClient();
    String dbName = data.dbName;
    String tableName = data.tableName;
    BenchmarkUtils.createPartitionedTable(client, dbName, tableName);
    try {
        addManyPartitions(client, dbName, tableName, null, Collections.singletonList("d"), 1);
        return bench.measure(() -> throwingSupplierWrapper(() -> client.listPartitions(dbName, tableName)));
    } catch (TException e) {
        e.printStackTrace();
        return new DescriptiveStatistics();
    } finally {
        throwingSupplierWrapper(() -> client.dropTable(dbName, tableName));
    }
}

Example 90 with DescriptiveStatistics

use of org.apache.commons.math3.stat.descriptive.DescriptiveStatistics in project hive by apache.

the class MicroBenchmark method measure.

/**
 * Run the benchmark and measure run-time statistics in nanoseconds.<p>
 * Before the run the warm-up phase is executed.
 * @param pre Optional pre-test setup
 * @param test Mandatory test
 * @param post Optional post-test cleanup
 * @return Statistics describing the results. All times are in nanoseconds.
 */
public DescriptiveStatistics measure(@Nullable Runnable pre, @NotNull Runnable test, @Nullable Runnable post) {
    // Warmup phase
    for (int i = 0; i < warmup; i++) {
        if (pre != null) {
            pre.run();
        }
        test.run();
        if (post != null) {
            post.run();
        }
    }
    // Run the benchmark
    DescriptiveStatistics stats = new DescriptiveStatistics();
    for (int i = 0; i < iterations; i++) {
        if (pre != null) {
            pre.run();
        }
        long start = System.nanoTime();
        test.run();
        long end = System.nanoTime();
        stats.addValue((double) (end - start) / scaleFactor);
        if (post != null) {
            post.run();
        }
    }
    return stats;
}