Examples with Statistics org.apache.hadoop.fs.FileSystem.Statistics used on opensource projects

Example 1 with Statistics

use of org.apache.hadoop.fs.FileSystem.Statistics in project hadoop by apache.

the class TestLocalFileSystem method testStatistics.

@Test(timeout = 1000)
public void testStatistics() throws Exception {
    int fileSchemeCount = 0;
    for (Statistics stats : FileSystem.getAllStatistics()) {
        if (stats.getScheme().equals("file")) {
            fileSchemeCount++;
        }
    }
    assertEquals(1, fileSchemeCount);
}

Example 2 with Statistics

use of org.apache.hadoop.fs.FileSystem.Statistics in project hadoop by apache.

the class AbstractFileSystem method getAllStatistics.

protected static synchronized Map<URI, Statistics> getAllStatistics() {
    Map<URI, Statistics> statsMap = new HashMap<URI, Statistics>(STATISTICS_TABLE.size());
    for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) {
        URI key = pair.getKey();
        Statistics value = pair.getValue();
        Statistics newStatsObj = new Statistics(value);
        statsMap.put(URI.create(key.toString()), newStatsObj);
    }
    return statsMap;
}

Example 3 with Statistics

use of org.apache.hadoop.fs.FileSystem.Statistics in project hadoop by apache.

the class FCStatisticsBaseTest method testStatisticsThreadLocalDataCleanUp.

@Test(timeout = 70000)
public void testStatisticsThreadLocalDataCleanUp() throws Exception {
    final Statistics stats = new Statistics("test");
    // create a small thread pool to test the statistics
    final int size = 2;
    ExecutorService es = Executors.newFixedThreadPool(size);
    List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>(size);
    for (int i = 0; i < size; i++) {
        tasks.add(new Callable<Boolean>() {

            public Boolean call() {
                // this populates the data set in statistics
                stats.incrementReadOps(1);
                return true;
            }
        });
    }
    // run the threads
    es.invokeAll(tasks);
    // assert that the data size is exactly the number of threads
    final AtomicInteger allDataSize = new AtomicInteger(0);
    allDataSize.set(stats.getAllThreadLocalDataSize());
    Assert.assertEquals(size, allDataSize.get());
    Assert.assertEquals(size, stats.getReadOps());
    // force the GC to collect the threads by shutting down the thread pool
    es.shutdownNow();
    es.awaitTermination(1, TimeUnit.MINUTES);
    es = null;
    // force GC to garbage collect threads
    System.gc();
    // wait for up to 60 seconds
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            int size = stats.getAllThreadLocalDataSize();
            allDataSize.set(size);
            if (size == 0) {
                return true;
            }
            LOG.warn("not all references have been cleaned up; still " + allDataSize.get() + " references left");
            LOG.warn("triggering another GC");
            System.gc();
            return false;
        }
    }, 500, 60 * 1000);
    Assert.assertEquals(0, allDataSize.get());
    Assert.assertEquals(size, stats.getReadOps());
}

Example 4 with Statistics

use of org.apache.hadoop.fs.FileSystem.Statistics in project hadoop by apache.

the class Task method updateCounters.

private synchronized void updateCounters() {
    Map<String, List<FileSystem.Statistics>> map = new HashMap<String, List<FileSystem.Statistics>>();
    for (Statistics stat : FileSystem.getAllStatistics()) {
        String uriScheme = stat.getScheme();
        if (map.containsKey(uriScheme)) {
            List<FileSystem.Statistics> list = map.get(uriScheme);
            list.add(stat);
        } else {
            List<FileSystem.Statistics> list = new ArrayList<FileSystem.Statistics>();
            list.add(stat);
            map.put(uriScheme, list);
        }
    }
    for (Map.Entry<String, List<FileSystem.Statistics>> entry : map.entrySet()) {
        FileSystemStatisticUpdater updater = statisticUpdaters.get(entry.getKey());
        if (updater == null) {
            //new FileSystem has been found in the cache
            updater = new FileSystemStatisticUpdater(entry.getValue(), entry.getKey());
            statisticUpdaters.put(entry.getKey(), updater);
        }
        updater.updateCounters();
    }
    gcUpdater.incrementGcCounter();
    updateResourceCounters();
}

Example 5 with Statistics

use of org.apache.hadoop.fs.FileSystem.Statistics in project hadoop by apache.

the class FCStatisticsBaseTest method testStatisticsOperations.

@Test(timeout = 60000)
public void testStatisticsOperations() throws Exception {
    final Statistics stats = new Statistics("file");
    Assert.assertEquals(0L, stats.getBytesRead());
    Assert.assertEquals(0L, stats.getBytesWritten());
    Assert.assertEquals(0, stats.getWriteOps());
    stats.incrementBytesWritten(1000);
    Assert.assertEquals(1000L, stats.getBytesWritten());
    Assert.assertEquals(0, stats.getWriteOps());
    stats.incrementWriteOps(123);
    Assert.assertEquals(123, stats.getWriteOps());
    Thread thread = new Thread() {

        @Override
        public void run() {
            stats.incrementWriteOps(1);
        }
    };
    thread.start();
    Uninterruptibles.joinUninterruptibly(thread);
    Assert.assertEquals(124, stats.getWriteOps());
    // Test copy constructor and reset function
    Statistics stats2 = new Statistics(stats);
    stats.reset();
    Assert.assertEquals(0, stats.getWriteOps());
    Assert.assertEquals(0L, stats.getBytesWritten());
    Assert.assertEquals(0L, stats.getBytesRead());
    Assert.assertEquals(124, stats2.getWriteOps());
    Assert.assertEquals(1000L, stats2.getBytesWritten());
    Assert.assertEquals(0L, stats2.getBytesRead());
}