Examples with FakeTimer org.apache.hadoop.util.FakeTimer used on opensource projects

Example 6 with FakeTimer

use of org.apache.hadoop.util.FakeTimer in project hadoop by apache.

the class TestStorageLocationChecker method testTimeoutInCheck.

/**
   * Verify that a {@link StorageLocation#check} timeout is correctly detected
   * as a failure.
   *
   * This is hard to test without a {@link Thread#sleep} call.
   *
   * @throws Exception
   */
@Test(timeout = 300000)
public void testTimeoutInCheck() throws Exception {
    final Configuration conf = new HdfsConfiguration();
    conf.setTimeDuration(DFS_DATANODE_DISK_CHECK_TIMEOUT_KEY, 1, TimeUnit.SECONDS);
    conf.setInt(DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, 1);
    final FakeTimer timer = new FakeTimer();
    // Generate a list of storage locations the first of which sleeps
    // for 2 seconds in its check() routine.
    final List<StorageLocation> locations = makeSlowLocations(2000, 1);
    StorageLocationChecker checker = new StorageLocationChecker(conf, timer);
    try {
        // Check the two locations and ensure that only one of them
        // was filtered out.
        List<StorageLocation> filteredList = checker.check(conf, locations);
        assertThat(filteredList.size(), is(1));
    } finally {
        checker.shutdownAndWait(10, TimeUnit.SECONDS);
    }
}

Example 7 with FakeTimer

use of org.apache.hadoop.util.FakeTimer in project hadoop by apache.

the class TestThrottledAsyncChecker method testScheduler.

/**
   * Test various scheduling combinations to ensure scheduling and
   * throttling behave as expected.
   */
@Test(timeout = 60000)
public void testScheduler() throws Exception {
    final NoOpCheckable target1 = new NoOpCheckable();
    final NoOpCheckable target2 = new NoOpCheckable();
    final FakeTimer timer = new FakeTimer();
    ThrottledAsyncChecker<Boolean, Boolean> checker = new ThrottledAsyncChecker<>(timer, MIN_ERROR_CHECK_GAP, 0, getExecutorService());
    // check target1 and ensure we get back the expected result.
    assertTrue(checker.schedule(target1, true).isPresent());
    waitTestCheckableCheckCount(target1, 1L);
    // Check target1 again without advancing the timer. target1 should not
    // be checked again.
    assertFalse(checker.schedule(target1, true).isPresent());
    waitTestCheckableCheckCount(target1, 1L);
    // Schedule target2 scheduled without advancing the timer.
    // target2 should be checked as it has never been checked before.
    assertTrue(checker.schedule(target2, true).isPresent());
    waitTestCheckableCheckCount(target2, 1L);
    // Advance the timer but just short of the min gap.
    // Neither target1 nor target2 should be checked again.
    timer.advance(MIN_ERROR_CHECK_GAP - 1);
    assertFalse(checker.schedule(target1, true).isPresent());
    waitTestCheckableCheckCount(target1, 1L);
    assertFalse(checker.schedule(target2, true).isPresent());
    waitTestCheckableCheckCount(target2, 1L);
    // Advance the timer again.
    // Both targets should be checked now.
    timer.advance(MIN_ERROR_CHECK_GAP);
    assertTrue(checker.schedule(target1, true).isPresent());
    waitTestCheckableCheckCount(target1, 2L);
    assertTrue(checker.schedule(target2, true).isPresent());
    waitTestCheckableCheckCount(target2, 2L);
}

Example 8 with FakeTimer

use of org.apache.hadoop.util.FakeTimer in project hadoop by apache.

the class TestFSNamesystemLock method testDetailedHoldMetrics.

@Test
public void testDetailedHoldMetrics() throws Exception {
    Configuration conf = new Configuration();
    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_LOCK_DETAILED_METRICS_KEY, true);
    FakeTimer timer = new FakeTimer();
    MetricsRegistry registry = new MetricsRegistry("Test");
    MutableRatesWithAggregation rates = registry.newRatesWithAggregation("Test");
    FSNamesystemLock fsLock = new FSNamesystemLock(conf, rates, timer);
    fsLock.readLock();
    timer.advance(1);
    fsLock.readUnlock("foo");
    fsLock.readLock();
    timer.advance(2);
    fsLock.readUnlock("foo");
    fsLock.readLock();
    timer.advance(1);
    fsLock.readLock();
    timer.advance(1);
    fsLock.readUnlock("bar");
    fsLock.readUnlock("bar");
    fsLock.writeLock();
    timer.advance(1);
    fsLock.writeUnlock("baz");
    MetricsRecordBuilder rb = MetricsAsserts.mockMetricsRecordBuilder();
    rates.snapshot(rb, true);
    assertGauge("FSNReadLockFooAvgTime", 1.5, rb);
    assertCounter("FSNReadLockFooNumOps", 2L, rb);
    assertGauge("FSNReadLockBarAvgTime", 2.0, rb);
    assertCounter("FSNReadLockBarNumOps", 1L, rb);
    assertGauge("FSNWriteLockBazAvgTime", 1.0, rb);
    assertCounter("FSNWriteLockBazNumOps", 1L, rb);
}

Example 9 with FakeTimer

use of org.apache.hadoop.util.FakeTimer in project hadoop by apache.

the class TestFSNamesystemLock method testFSWriteLockLongHoldingReport.

/**
   * Test when FSNamesystem write lock is held for a long time,
   * logger will report it.
   */
@Test(timeout = 45000)
public void testFSWriteLockLongHoldingReport() throws Exception {
    final long writeLockReportingThreshold = 100L;
    final long writeLockSuppressWarningInterval = 10000L;
    Configuration conf = new Configuration();
    conf.setLong(DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY, writeLockReportingThreshold);
    conf.setTimeDuration(DFSConfigKeys.DFS_LOCK_SUPPRESS_WARNING_INTERVAL_KEY, writeLockSuppressWarningInterval, TimeUnit.MILLISECONDS);
    final FakeTimer timer = new FakeTimer();
    final FSNamesystemLock fsnLock = new FSNamesystemLock(conf, null, timer);
    timer.advance(writeLockSuppressWarningInterval);
    LogCapturer logs = LogCapturer.captureLogs(FSNamesystem.LOG);
    GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.INFO);
    // Don't report if the write lock is held for a short time
    fsnLock.writeLock();
    fsnLock.writeUnlock();
    assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName()));
    // Report if the write lock is held for a long time
    fsnLock.writeLock();
    timer.advance(writeLockReportingThreshold + 10);
    logs.clearOutput();
    fsnLock.writeUnlock();
    assertTrue(logs.getOutput().contains(GenericTestUtils.getMethodName()));
    // Track but do not report if the write lock is held (interruptibly) for
    // a long time but time since last report does not exceed the suppress
    // warning interval
    fsnLock.writeLockInterruptibly();
    timer.advance(writeLockReportingThreshold + 10);
    logs.clearOutput();
    fsnLock.writeUnlock();
    assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName()));
    // Track but do not report if it's held for a long time when re-entering
    // write lock but time since last report does not exceed the suppress
    // warning interval
    fsnLock.writeLock();
    timer.advance(writeLockReportingThreshold / 2 + 1);
    fsnLock.writeLockInterruptibly();
    timer.advance(writeLockReportingThreshold / 2 + 1);
    fsnLock.writeLock();
    timer.advance(writeLockReportingThreshold / 2);
    logs.clearOutput();
    fsnLock.writeUnlock();
    assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName()));
    logs.clearOutput();
    fsnLock.writeUnlock();
    assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName()));
    logs.clearOutput();
    fsnLock.writeUnlock();
    assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName()));
    // Report if it's held for a long time and time since last report exceeds
    // the supress warning interval
    timer.advance(writeLockSuppressWarningInterval);
    fsnLock.writeLock();
    timer.advance(writeLockReportingThreshold + 100);
    logs.clearOutput();
    fsnLock.writeUnlock();
    assertTrue(logs.getOutput().contains(GenericTestUtils.getMethodName()));
    assertTrue(logs.getOutput().contains("Number of suppressed write-lock reports: 2"));
}

Example 10 with FakeTimer

use of org.apache.hadoop.util.FakeTimer in project hadoop by apache.

the class TestGroupsCaching method testOnlyOneRequestWhenExpiredEntryExists.

@Test
public void testOnlyOneRequestWhenExpiredEntryExists() throws Exception {
    conf.setLong(CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_SECS, 1);
    FakeTimer timer = new FakeTimer();
    final Groups groups = new Groups(conf, timer);
    groups.cacheGroupsAdd(Arrays.asList(myGroups));
    groups.refresh();
    FakeGroupMapping.clearBlackList();
    FakeGroupMapping.setGetGroupsDelayMs(100);
    // We make an initial request to populate the cache
    groups.getGroups("me");
    int startingRequestCount = FakeGroupMapping.getRequestCount();
    // Then expire that entry
    timer.advance(400 * 1000);
    Thread.sleep(100);
    ArrayList<Thread> threads = new ArrayList<Thread>();
    for (int i = 0; i < 10; i++) {
        threads.add(new Thread() {

            public void run() {
                try {
                    assertEquals(2, groups.getGroups("me").size());
                } catch (IOException e) {
                    fail("Should not happen");
                }
            }
        });
    }
    // We start a bunch of threads who all see the cached value
    for (Thread t : threads) {
        t.start();
    }
    for (Thread t : threads) {
        t.join();
    }
    // Only one extra request is made
    assertEquals(startingRequestCount + 1, FakeGroupMapping.getRequestCount());
}