Search in sources :

Example 26 with AtomicReference

use of java.util.concurrent.atomic.AtomicReference in project hadoop by apache.

the class TestEditLogRace method testSaveRightBeforeSync.

/**
   * Most of the FSNamesystem methods have a synchronized section where they
   * update the name system itself and write to the edit log, and then
   * unsynchronized, they call logSync. This test verifies that, if an
   * operation has written to the edit log but not yet synced it,
   * we wait for that sync before entering safe mode.
   */
@Test
public void testSaveRightBeforeSync() throws Exception {
    Configuration conf = getConf();
    NameNode.initMetrics(conf, NamenodeRole.NAMENODE);
    DFSTestUtil.formatNameNode(conf);
    final FSNamesystem namesystem = FSNamesystem.loadFromDisk(conf);
    try {
        FSImage fsimage = namesystem.getFSImage();
        final FSEditLog editLog = fsimage.getEditLog();
        final AtomicReference<Throwable> deferredException = new AtomicReference<Throwable>();
        final CountDownLatch sleepingBeforeSync = new CountDownLatch(1);
        final Thread doAnEditThread = new Thread() {

            @Override
            public void run() {
                try {
                    LOG.info("Starting setOwner");
                    namesystem.writeLock();
                    try {
                        editLog.logSetOwner("/", "test", "test");
                    } finally {
                        namesystem.writeUnlock();
                    }
                    sleepingBeforeSync.countDown();
                    LOG.info("edit thread: sleeping for " + BLOCK_TIME + "secs");
                    Thread.sleep(BLOCK_TIME * 1000);
                    editLog.logSync();
                    LOG.info("edit thread: logSync complete");
                } catch (Throwable ioe) {
                    LOG.fatal("Got exception", ioe);
                    deferredException.set(ioe);
                    sleepingBeforeSync.countDown();
                }
            }
        };
        doAnEditThread.setDaemon(true);
        doAnEditThread.start();
        LOG.info("Main thread: waiting to just before logSync...");
        sleepingBeforeSync.await(200, TimeUnit.MILLISECONDS);
        assertNull(deferredException.get());
        LOG.info("Main thread: detected that logSync about to be called.");
        LOG.info("Trying to enter safe mode.");
        long st = Time.now();
        namesystem.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
        long et = Time.now();
        LOG.info("Entered safe mode after " + (et - st) + "ms");
        // Make sure we didn't wait for the thread that did a logEdit but
        // not logSync.  Going into safemode does a logSyncAll that will flush
        // its edit.
        assertTrue(et - st < (BLOCK_TIME / 2) * 1000);
        // Once we're in safe mode, save namespace.
        namesystem.saveNamespace(0, 0);
        LOG.info("Joining on edit thread...");
        doAnEditThread.join();
        assertNull(deferredException.get());
        // We did 3 edits: begin, txn, and end
        assertEquals(3, verifyEditLogs(namesystem, fsimage, NNStorage.getFinalizedEditsFileName(1, 3), 1));
        // after the save, just the one "begin"
        assertEquals(1, verifyEditLogs(namesystem, fsimage, NNStorage.getInProgressEditsFileName(4), 4));
    } finally {
        LOG.info("Closing nn");
        if (namesystem != null)
            namesystem.close();
    }
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) HdfsConfiguration(org.apache.hadoop.hdfs.HdfsConfiguration) AtomicReference(java.util.concurrent.atomic.AtomicReference) CountDownLatch(java.util.concurrent.CountDownLatch) Test(org.junit.Test)

Example 27 with AtomicReference

use of java.util.concurrent.atomic.AtomicReference in project hbase by apache.

the class HBaseTestingUtility method findLastTableState.

@Nullable
public TableState findLastTableState(final TableName table) throws IOException {
    final AtomicReference<TableState> lastTableState = new AtomicReference<>(null);
    MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {

        @Override
        public boolean visit(Result r) throws IOException {
            if (!Arrays.equals(r.getRow(), table.getName()))
                return false;
            TableState state = MetaTableAccessor.getTableState(r);
            if (state != null)
                lastTableState.set(state);
            return true;
        }
    };
    MetaTableAccessor.scanMeta(connection, null, null, MetaTableAccessor.QueryType.TABLE, Integer.MAX_VALUE, visitor);
    return lastTableState.get();
}
Also used : AtomicReference(java.util.concurrent.atomic.AtomicReference) TableState(org.apache.hadoop.hbase.client.TableState) Result(org.apache.hadoop.hbase.client.Result) Nullable(edu.umd.cs.findbugs.annotations.Nullable)

Example 28 with AtomicReference

use of java.util.concurrent.atomic.AtomicReference in project hadoop by apache.

the class TestAMRMTokens method testAMRMMasterKeysUpdate.

@Test(timeout = 20000)
public void testAMRMMasterKeysUpdate() throws Exception {
    final AtomicReference<AMRMTokenSecretManager> spySecretMgrRef = new AtomicReference<AMRMTokenSecretManager>();
    MockRM rm = new MockRM(conf) {

        @Override
        protected void doSecureLogin() throws IOException {
        // Skip the login.
        }

        @Override
        protected RMSecretManagerService createRMSecretManagerService() {
            return new RMSecretManagerService(conf, rmContext) {

                @Override
                protected AMRMTokenSecretManager createAMRMTokenSecretManager(Configuration conf, RMContext rmContext) {
                    AMRMTokenSecretManager spySecretMgr = spy(super.createAMRMTokenSecretManager(conf, rmContext));
                    spySecretMgrRef.set(spySecretMgr);
                    return spySecretMgr;
                }
            };
        }
    };
    rm.start();
    MockNM nm = rm.registerNode("127.0.0.1:1234", 8000);
    RMApp app = rm.submitApp(200);
    MockAM am = MockRM.launchAndRegisterAM(app, rm, nm);
    AMRMTokenSecretManager spySecretMgr = spySecretMgrRef.get();
    // Do allocate. Should not update AMRMToken
    AllocateResponse response = am.allocate(Records.newRecord(AllocateRequest.class));
    Assert.assertNull(response.getAMRMToken());
    Token<AMRMTokenIdentifier> oldToken = rm.getRMContext().getRMApps().get(app.getApplicationId()).getRMAppAttempt(am.getApplicationAttemptId()).getAMRMToken();
    // roll over the master key
    // Do allocate again. the AM should get the latest AMRMToken
    rm.getRMContext().getAMRMTokenSecretManager().rollMasterKey();
    response = am.allocate(Records.newRecord(AllocateRequest.class));
    Assert.assertNotNull(response.getAMRMToken());
    Token<AMRMTokenIdentifier> amrmToken = ConverterUtils.convertFromYarn(response.getAMRMToken(), new Text(response.getAMRMToken().getService()));
    Assert.assertEquals(amrmToken.decodeIdentifier().getKeyId(), rm.getRMContext().getAMRMTokenSecretManager().getMasterKey().getMasterKey().getKeyId());
    // Do allocate again with the same old token and verify the RM sends
    // back the last generated token instead of generating it again.
    reset(spySecretMgr);
    UserGroupInformation ugi = UserGroupInformation.createUserForTesting(am.getApplicationAttemptId().toString(), new String[0]);
    ugi.addTokenIdentifier(oldToken.decodeIdentifier());
    response = am.doAllocateAs(ugi, Records.newRecord(AllocateRequest.class));
    Assert.assertNotNull(response.getAMRMToken());
    verify(spySecretMgr, never()).createAndGetAMRMToken(isA(ApplicationAttemptId.class));
    // Do allocate again with the updated token and verify we do not
    // receive a new token to use.
    response = am.allocate(Records.newRecord(AllocateRequest.class));
    Assert.assertNull(response.getAMRMToken());
    // Activate the next master key. Since there is new master key generated
    // in AMRMTokenSecretManager. The AMRMToken will not get updated for AM
    rm.getRMContext().getAMRMTokenSecretManager().activateNextMasterKey();
    response = am.allocate(Records.newRecord(AllocateRequest.class));
    Assert.assertNull(response.getAMRMToken());
    rm.stop();
}
Also used : RMContext(org.apache.hadoop.yarn.server.resourcemanager.RMContext) RMApp(org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp) RMSecretManagerService(org.apache.hadoop.yarn.server.resourcemanager.RMSecretManagerService) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) MockNM(org.apache.hadoop.yarn.server.resourcemanager.MockNM) AllocateRequest(org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest) AtomicReference(java.util.concurrent.atomic.AtomicReference) MockRM(org.apache.hadoop.yarn.server.resourcemanager.MockRM) Text(org.apache.hadoop.io.Text) ApplicationAttemptId(org.apache.hadoop.yarn.api.records.ApplicationAttemptId) AllocateResponse(org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse) AMRMTokenIdentifier(org.apache.hadoop.yarn.security.AMRMTokenIdentifier) MockAM(org.apache.hadoop.yarn.server.resourcemanager.MockAM) UserGroupInformation(org.apache.hadoop.security.UserGroupInformation) Test(org.junit.Test)

Example 29 with AtomicReference

use of java.util.concurrent.atomic.AtomicReference in project hbase by apache.

the class TestWeakObjectPool method testCongestion.

@Test(timeout = 1000)
public void testCongestion() throws Exception {
    final int THREAD_COUNT = 100;
    final AtomicBoolean assertionFailed = new AtomicBoolean();
    final AtomicReference<Object> expectedObjRef = new AtomicReference<>();
    final CountDownLatch prepareLatch = new CountDownLatch(THREAD_COUNT);
    final CountDownLatch startLatch = new CountDownLatch(1);
    final CountDownLatch endLatch = new CountDownLatch(THREAD_COUNT);
    for (int i = 0; i < THREAD_COUNT; i++) {
        new Thread() {

            @Override
            public void run() {
                prepareLatch.countDown();
                try {
                    startLatch.await();
                    Object obj = pool.get("a");
                    if (!expectedObjRef.compareAndSet(null, obj)) {
                        if (expectedObjRef.get() != obj) {
                            assertionFailed.set(true);
                        }
                    }
                } catch (Exception e) {
                    assertionFailed.set(true);
                } finally {
                    endLatch.countDown();
                }
            }
        }.start();
    }
    prepareLatch.await();
    startLatch.countDown();
    endLatch.await();
    if (assertionFailed.get()) {
        Assert.fail();
    }
}
Also used : AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) AtomicReference(java.util.concurrent.atomic.AtomicReference) CountDownLatch(java.util.concurrent.CountDownLatch) Test(org.junit.Test)

Example 30 with AtomicReference

use of java.util.concurrent.atomic.AtomicReference in project hbase by apache.

the class HBaseAdmin method getRegion.

/**
   * @param regionName Name of a region.
   * @return a pair of HRegionInfo and ServerName if <code>regionName</code> is
   *  a verified region name (we call {@link
   *  MetaTableAccessor#getRegionLocation(Connection, byte[])}
   *  else null.
   * Throw IllegalArgumentException if <code>regionName</code> is null.
   * @throws IOException
   */
Pair<HRegionInfo, ServerName> getRegion(final byte[] regionName) throws IOException {
    if (regionName == null) {
        throw new IllegalArgumentException("Pass a table name or region name");
    }
    Pair<HRegionInfo, ServerName> pair = MetaTableAccessor.getRegion(connection, regionName);
    if (pair == null) {
        final AtomicReference<Pair<HRegionInfo, ServerName>> result = new AtomicReference<>(null);
        final String encodedName = Bytes.toString(regionName);
        MetaTableAccessor.Visitor visitor = new MetaTableAccessor.Visitor() {

            @Override
            public boolean visit(Result data) throws IOException {
                HRegionInfo info = MetaTableAccessor.getHRegionInfo(data);
                if (info == null) {
                    LOG.warn("No serialized HRegionInfo in " + data);
                    return true;
                }
                RegionLocations rl = MetaTableAccessor.getRegionLocations(data);
                boolean matched = false;
                ServerName sn = null;
                if (rl != null) {
                    for (HRegionLocation h : rl.getRegionLocations()) {
                        if (h != null && encodedName.equals(h.getRegionInfo().getEncodedName())) {
                            sn = h.getServerName();
                            info = h.getRegionInfo();
                            matched = true;
                        }
                    }
                }
                if (!matched)
                    return true;
                result.set(new Pair<>(info, sn));
                // found the region, stop
                return false;
            }
        };
        MetaTableAccessor.fullScanRegions(connection, visitor);
        pair = result.get();
    }
    return pair;
}
Also used : RegionLocations(org.apache.hadoop.hbase.RegionLocations) MetaTableAccessor(org.apache.hadoop.hbase.MetaTableAccessor) AtomicReference(java.util.concurrent.atomic.AtomicReference) HRegionInfo(org.apache.hadoop.hbase.HRegionInfo) HRegionLocation(org.apache.hadoop.hbase.HRegionLocation) ServerName(org.apache.hadoop.hbase.ServerName) Pair(org.apache.hadoop.hbase.util.Pair) NameStringPair(org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameStringPair)

Aggregations

AtomicReference (java.util.concurrent.atomic.AtomicReference)1331 Test (org.junit.Test)668 CountDownLatch (java.util.concurrent.CountDownLatch)437 IOException (java.io.IOException)263 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)205 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)159 ArrayList (java.util.ArrayList)108 HashMap (java.util.HashMap)105 List (java.util.List)95 Map (java.util.Map)77 Test (org.testng.annotations.Test)76 File (java.io.File)64 ExecutionException (java.util.concurrent.ExecutionException)60 HashSet (java.util.HashSet)54 URI (java.net.URI)48 TimeoutException (java.util.concurrent.TimeoutException)48 HttpServletRequest (javax.servlet.http.HttpServletRequest)48 HttpServletResponse (javax.servlet.http.HttpServletResponse)46 MockResponse (okhttp3.mockwebserver.MockResponse)46 ByteBuffer (java.nio.ByteBuffer)44