Examples with NNHAStatusHeartbeat org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat used on opensource projects

Example 1 with NNHAStatusHeartbeat

use of org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat in project hadoop by apache.

the class TestBPOfferService method testTrySendErrorReportWhenNNThrowsIOException.

/**
   * This test case tests whether the {@BPServiceActor#processQueueMessages}
   * adds back the error report back to the queue when 
   * {BPServiceActorAction#reportTo} throws an IOException
   * @throws Exception
   */
@Test
public void testTrySendErrorReportWhenNNThrowsIOException() throws Exception {
    BPOfferService bpos = setupBPOSForNNs(mockNN1, mockNN2);
    bpos.start();
    try {
        waitForInitialization(bpos);
        // Should start with neither NN as active.
        assertNull(bpos.getActiveNN());
        // Have NN1 claim active at txid 1
        mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceState.ACTIVE, 1);
        bpos.triggerHeartbeatForTests();
        // Now mockNN1 is acting like active namenode and mockNN2 as Standby
        assertSame(mockNN1, bpos.getActiveNN());
        Mockito.doAnswer(new Answer<Void>() {

            // Throw an IOException when this function is first called which will
            // in turn add that errorReport back to the bpThreadQueue and let it 
            // process the next time. 
            @Override
            public Void answer(InvocationOnMock invocation) throws Throwable {
                if (firstCallTime == 0) {
                    firstCallTime = Time.now();
                    throw new IOException();
                } else {
                    secondCallTime = Time.now();
                    return null;
                }
            }
        }).when(mockNN1).errorReport(Mockito.any(DatanodeRegistration.class), Mockito.anyInt(), Mockito.anyString());
        String errorString = "Can't send invalid block " + FAKE_BLOCK;
        bpos.trySendErrorReport(DatanodeProtocol.INVALID_BLOCK, errorString);
        Thread.sleep(10000);
        assertTrue("Active namenode didn't add the report back to the queue " + "when errorReport threw IOException", secondCallTime != 0);
    } finally {
        bpos.stop();
        bpos.join();
    }
}

Example 2 with NNHAStatusHeartbeat

use of org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat in project hadoop by apache.

the class TestBPOfferService method testReportBadBlockWhenStandbyNNTimesOut.

/**
   * This test case test the {@link BPOfferService#reportBadBlocks} method
   * such that if call to standby namenode times out then that should not 
   * affect the active namenode heartbeat processing since this function 
   * are in writeLock.
   * @throws Exception
   */
@Test
public void testReportBadBlockWhenStandbyNNTimesOut() throws Exception {
    BPOfferService bpos = setupBPOSForNNs(mockNN1, mockNN2);
    bpos.start();
    try {
        waitForInitialization(bpos);
        // Should start with neither NN as active.
        assertNull(bpos.getActiveNN());
        // Have NN1 claim active at txid 1
        mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceState.ACTIVE, 1);
        bpos.triggerHeartbeatForTests();
        // Now mockNN1 is acting like active namenode and mockNN2 as Standby
        assertSame(mockNN1, bpos.getActiveNN());
        Mockito.doAnswer(new BPOfferServiceSynchronousCallAnswer(0)).when(mockNN1).reportBadBlocks(Mockito.any(LocatedBlock[].class));
        Mockito.doAnswer(new BPOfferServiceSynchronousCallAnswer(1)).when(mockNN2).reportBadBlocks(Mockito.any(LocatedBlock[].class));
        bpos.reportBadBlocks(FAKE_BLOCK, mockFSDataset.getVolume(FAKE_BLOCK).getStorageID(), mockFSDataset.getVolume(FAKE_BLOCK).getStorageType());
        bpos.reportBadBlocks(FAKE_BLOCK, mockFSDataset.getVolume(FAKE_BLOCK).getStorageID(), mockFSDataset.getVolume(FAKE_BLOCK).getStorageType());
        Thread.sleep(10000);
        long difference = secondCallTime - firstCallTime;
        assertTrue("Active namenode reportBadBlock processing should be " + "independent of standby namenode reportBadBlock processing ", difference < 5000);
    } finally {
        bpos.stop();
        bpos.join();
    }
}

Example 3 with NNHAStatusHeartbeat

use of org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat in project hadoop by apache.

the class FSNamesystem method handleHeartbeat.

/**
   * The given node has reported in.  This method should:
   * 1) Record the heartbeat, so the datanode isn't timed out
   * 2) Adjust usage stats for future block allocation
   *
   * If a substantial amount of time passed since the last datanode
   * heartbeat then request an immediate block report.
   *
   * @return an array of datanode commands
   * @throws IOException
   */
HeartbeatResponse handleHeartbeat(DatanodeRegistration nodeReg, StorageReport[] reports, long cacheCapacity, long cacheUsed, int xceiverCount, int xmitsInProgress, int failedVolumes, VolumeFailureSummary volumeFailureSummary, boolean requestFullBlockReportLease, @Nonnull SlowPeerReports slowPeers) throws IOException {
    readLock();
    try {
        //get datanode commands
        final int maxTransfer = blockManager.getMaxReplicationStreams() - xmitsInProgress;
        DatanodeCommand[] cmds = blockManager.getDatanodeManager().handleHeartbeat(nodeReg, reports, getBlockPoolId(), cacheCapacity, cacheUsed, xceiverCount, maxTransfer, failedVolumes, volumeFailureSummary, slowPeers);
        long blockReportLeaseId = 0;
        if (requestFullBlockReportLease) {
            blockReportLeaseId = blockManager.requestBlockReportLeaseId(nodeReg);
        }
        //create ha status
        final NNHAStatusHeartbeat haState = new NNHAStatusHeartbeat(haContext.getState().getServiceState(), getFSImage().getCorrectLastAppliedOrWrittenTxId());
        return new HeartbeatResponse(cmds, haState, rollingUpgradeInfo, blockReportLeaseId);
    } finally {
        readUnlock("handleHeartbeat");
    }
}

Example 4 with NNHAStatusHeartbeat

use of org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat in project hadoop by apache.

the class InternalDataNodeTestUtils method startDNWithMockNN.

/**
   * Starts an instance of DataNode with NN mocked. Called should ensure to
   * shutdown the DN
   *
   * @throws IOException
   */
public static DataNode startDNWithMockNN(Configuration conf, final InetSocketAddress nnSocketAddr, final String dnDataDir) throws IOException {
    FileSystem.setDefaultUri(conf, "hdfs://" + nnSocketAddr.getHostName() + ":" + nnSocketAddr.getPort());
    ArrayList<StorageLocation> locations = new ArrayList<StorageLocation>();
    File dataDir = new File(dnDataDir);
    FileUtil.fullyDelete(dataDir);
    dataDir.mkdirs();
    StorageLocation location = StorageLocation.parse(dataDir.getPath());
    locations.add(location);
    final DatanodeProtocolClientSideTranslatorPB namenode = mock(DatanodeProtocolClientSideTranslatorPB.class);
    Mockito.doAnswer(new Answer<DatanodeRegistration>() {

        @Override
        public DatanodeRegistration answer(InvocationOnMock invocation) throws Throwable {
            return (DatanodeRegistration) invocation.getArguments()[0];
        }
    }).when(namenode).registerDatanode(Mockito.any(DatanodeRegistration.class));
    when(namenode.versionRequest()).thenReturn(new NamespaceInfo(1, TEST_CLUSTER_ID, TEST_POOL_ID, 1L));
    when(namenode.sendHeartbeat(Mockito.any(DatanodeRegistration.class), Mockito.any(StorageReport[].class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt(), Mockito.any(VolumeFailureSummary.class), Mockito.anyBoolean(), Mockito.any(SlowPeerReports.class))).thenReturn(new HeartbeatResponse(new DatanodeCommand[0], new NNHAStatusHeartbeat(HAServiceState.ACTIVE, 1), null, ThreadLocalRandom.current().nextLong() | 1L));
    DataNode dn = new DataNode(conf, locations, null, null) {

        @Override
        DatanodeProtocolClientSideTranslatorPB connectToNN(InetSocketAddress nnAddr) throws IOException {
            Assert.assertEquals(nnSocketAddr, nnAddr);
            return namenode;
        }
    };
    // Trigger a heartbeat so that it acknowledges the NN as active.
    dn.getAllBpOs().get(0).triggerHeartbeatForTests();
    return dn;
}

Example 5 with NNHAStatusHeartbeat

use of org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat in project hadoop by apache.

the class TestBPOfferService method testIBRClearanceForStandbyOnReRegister.

/*
   * HDFS-9917 : Standby IBR accumulation when Standby was down.
   */
@Test
public void testIBRClearanceForStandbyOnReRegister() throws Exception {
    final BPOfferService bpos = setupBPOSForNNs(mockNN1, mockNN2);
    bpos.start();
    try {
        waitForInitialization(bpos);
        // Should start with neither NN as active.
        assertNull(bpos.getActiveNN());
        // Have NN1 claim active at txid 1
        mockHaStatuses[0] = new NNHAStatusHeartbeat(HAServiceState.ACTIVE, 1);
        bpos.triggerHeartbeatForTests();
        // Now mockNN1 is acting like active namenode and mockNN2 as Standby
        assertSame(mockNN1, bpos.getActiveNN());
        // Return nothing when active Active Namenode gets IBRs
        Mockito.doNothing().when(mockNN1).blockReceivedAndDeleted(Mockito.any(DatanodeRegistration.class), Mockito.anyString(), Mockito.any(StorageReceivedDeletedBlocks[].class));
        final IOException re = new IOException("Standby NN is currently not able to process IBR");
        final AtomicBoolean ibrReported = new AtomicBoolean(false);
        // throw exception for standby when first IBR is receieved
        Mockito.doAnswer(new Answer<Void>() {

            @Override
            public Void answer(InvocationOnMock invocation) throws Throwable {
                ibrReported.set(true);
                throw re;
            }
        }).when(mockNN2).blockReceivedAndDeleted(Mockito.any(DatanodeRegistration.class), Mockito.anyString(), Mockito.any(StorageReceivedDeletedBlocks[].class));
        DatanodeStorage storage = Mockito.mock(DatanodeStorage.class);
        Mockito.doReturn(storage).when(mockFSDataset).getStorage("storage0");
        // Add IBRs
        bpos.notifyNamenodeReceivedBlock(FAKE_BLOCK, null, "storage0", false);
        // Send heartbeat so that the BpServiceActor can send IBR to
        // namenode
        bpos.triggerHeartbeatForTests();
        // Wait till first IBR is received at standbyNN. Just for confirmation.
        GenericTestUtils.waitFor(new Supplier<Boolean>() {

            @Override
            public Boolean get() {
                return ibrReported.get();
            }
        }, 100, 1000);
        // Send register command back to Datanode to reRegister().
        // After reRegister IBRs should be cleared.
        datanodeCommands[1] = new DatanodeCommand[] { new RegisterCommand() };
        assertEquals("IBR size before reRegister should be non-0", 1, getStandbyIBRSize(bpos));
        bpos.triggerHeartbeatForTests();
        GenericTestUtils.waitFor(new Supplier<Boolean>() {

            @Override
            public Boolean get() {
                return getStandbyIBRSize(bpos) == 0;
            }
        }, 100, 1000);
    } finally {
        bpos.stop();
        bpos.join();
    }
}