Examples with DatanodeProtocolClientSideTranslatorPB org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB used on opensource projects

Example 16 with DatanodeProtocolClientSideTranslatorPB

use of org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB in project hadoop by apache.

the class InternalDataNodeTestUtils method startDNWithMockNN.

/**
   * Starts an instance of DataNode with NN mocked. Called should ensure to
   * shutdown the DN
   *
   * @throws IOException
   */
public static DataNode startDNWithMockNN(Configuration conf, final InetSocketAddress nnSocketAddr, final String dnDataDir) throws IOException {
    FileSystem.setDefaultUri(conf, "hdfs://" + nnSocketAddr.getHostName() + ":" + nnSocketAddr.getPort());
    ArrayList<StorageLocation> locations = new ArrayList<StorageLocation>();
    File dataDir = new File(dnDataDir);
    FileUtil.fullyDelete(dataDir);
    dataDir.mkdirs();
    StorageLocation location = StorageLocation.parse(dataDir.getPath());
    locations.add(location);
    final DatanodeProtocolClientSideTranslatorPB namenode = mock(DatanodeProtocolClientSideTranslatorPB.class);
    Mockito.doAnswer(new Answer<DatanodeRegistration>() {

        @Override
        public DatanodeRegistration answer(InvocationOnMock invocation) throws Throwable {
            return (DatanodeRegistration) invocation.getArguments()[0];
        }
    }).when(namenode).registerDatanode(Mockito.any(DatanodeRegistration.class));
    when(namenode.versionRequest()).thenReturn(new NamespaceInfo(1, TEST_CLUSTER_ID, TEST_POOL_ID, 1L));
    when(namenode.sendHeartbeat(Mockito.any(DatanodeRegistration.class), Mockito.any(StorageReport[].class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt(), Mockito.any(VolumeFailureSummary.class), Mockito.anyBoolean(), Mockito.any(SlowPeerReports.class))).thenReturn(new HeartbeatResponse(new DatanodeCommand[0], new NNHAStatusHeartbeat(HAServiceState.ACTIVE, 1), null, ThreadLocalRandom.current().nextLong() | 1L));
    DataNode dn = new DataNode(conf, locations, null, null) {

        @Override
        DatanodeProtocolClientSideTranslatorPB connectToNN(InetSocketAddress nnAddr) throws IOException {
            Assert.assertEquals(nnSocketAddr, nnAddr);
            return namenode;
        }
    };
    // Trigger a heartbeat so that it acknowledges the NN as active.
    dn.getAllBpOs().get(0).triggerHeartbeatForTests();
    return dn;
}

Example 17 with DatanodeProtocolClientSideTranslatorPB

use of org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB in project hadoop by apache.

the class TestDataNodeHotSwapVolumes method testFullBlockReportAfterRemovingVolumes.

/** Test that a full block report is sent after hot swapping volumes */
@Test(timeout = 100000)
public void testFullBlockReportAfterRemovingVolumes() throws IOException, ReconfigurationException {
    Configuration conf = new Configuration();
    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
    // Similar to TestTriggerBlockReport, set a really long value for
    // dfs.heartbeat.interval, so that incremental block reports and heartbeats
    // won't be sent during this test unless they're triggered
    // manually.
    conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 10800000L);
    conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1080L);
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
    cluster.waitActive();
    final DataNode dn = cluster.getDataNodes().get(0);
    DatanodeProtocolClientSideTranslatorPB spy = InternalDataNodeTestUtils.spyOnBposToNN(dn, cluster.getNameNode());
    // Remove a data dir from datanode
    File dataDirToKeep = new File(cluster.getDataDirectory(), "data1");
    assertThat("DN did not update its own config", dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, dataDirToKeep.toString()), is(dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY)));
    // We should get 1 full report
    Mockito.verify(spy, timeout(60000).times(1)).blockReport(any(DatanodeRegistration.class), anyString(), any(StorageBlockReport[].class), any(BlockReportContext.class));
}

Example 18 with DatanodeProtocolClientSideTranslatorPB

use of org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB in project hadoop by apache.

the class TestBPOfferService method setupNNMock.

/**
   * Set up a mock NN with the bare minimum for a DN to register to it.
   */
private DatanodeProtocolClientSideTranslatorPB setupNNMock(int nnIdx) throws Exception {
    DatanodeProtocolClientSideTranslatorPB mock = Mockito.mock(DatanodeProtocolClientSideTranslatorPB.class);
    Mockito.doReturn(new NamespaceInfo(1, FAKE_CLUSTERID, FAKE_BPID, 0)).when(mock).versionRequest();
    Mockito.doReturn(DFSTestUtil.getLocalDatanodeRegistration()).when(mock).registerDatanode(Mockito.any(DatanodeRegistration.class));
    Mockito.doAnswer(new HeartbeatAnswer(nnIdx)).when(mock).sendHeartbeat(Mockito.any(DatanodeRegistration.class), Mockito.any(StorageReport[].class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt(), Mockito.any(VolumeFailureSummary.class), Mockito.anyBoolean(), Mockito.any(SlowPeerReports.class));
    mockHaStatuses[nnIdx] = new NNHAStatusHeartbeat(HAServiceState.STANDBY, 0);
    datanodeCommands[nnIdx] = new DatanodeCommand[0];
    return mock;
}

Example 19 with DatanodeProtocolClientSideTranslatorPB

use of org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB in project hadoop by apache.

the class TestBlockRecovery method startUp.

/**
   * Starts an instance of DataNode
   * @throws IOException
   */
@Before
public void startUp() throws IOException, URISyntaxException {
    tearDownDone = false;
    conf = new HdfsConfiguration();
    conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, DATA_DIR);
    conf.set(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, "0.0.0.0:0");
    conf.set(DFSConfigKeys.DFS_DATANODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
    conf.set(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY, "0.0.0.0:0");
    if (currentTestName.getMethodName().contains("DoesNotHoldLock")) {
        // This test requires a very long value for the xceiver stop timeout.
        conf.setLong(DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_KEY, TEST_STOP_WORKER_XCEIVER_STOP_TIMEOUT_MILLIS);
    }
    conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
    FileSystem.setDefaultUri(conf, "hdfs://" + NN_ADDR.getHostName() + ":" + NN_ADDR.getPort());
    ArrayList<StorageLocation> locations = new ArrayList<StorageLocation>();
    File dataDir = new File(DATA_DIR);
    FileUtil.fullyDelete(dataDir);
    dataDir.mkdirs();
    StorageLocation location = StorageLocation.parse(dataDir.getPath());
    locations.add(location);
    final DatanodeProtocolClientSideTranslatorPB namenode = mock(DatanodeProtocolClientSideTranslatorPB.class);
    Mockito.doAnswer(new Answer<DatanodeRegistration>() {

        @Override
        public DatanodeRegistration answer(InvocationOnMock invocation) throws Throwable {
            return (DatanodeRegistration) invocation.getArguments()[0];
        }
    }).when(namenode).registerDatanode(Mockito.any(DatanodeRegistration.class));
    when(namenode.versionRequest()).thenReturn(new NamespaceInfo(1, CLUSTER_ID, POOL_ID, 1L));
    when(namenode.sendHeartbeat(Mockito.any(DatanodeRegistration.class), Mockito.any(StorageReport[].class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt(), Mockito.any(VolumeFailureSummary.class), Mockito.anyBoolean(), Mockito.any(SlowPeerReports.class))).thenReturn(new HeartbeatResponse(new DatanodeCommand[0], new NNHAStatusHeartbeat(HAServiceState.ACTIVE, 1), null, ThreadLocalRandom.current().nextLong() | 1L));
    dn = new DataNode(conf, locations, null, null) {

        @Override
        DatanodeProtocolClientSideTranslatorPB connectToNN(InetSocketAddress nnAddr) throws IOException {
            Assert.assertEquals(NN_ADDR, nnAddr);
            return namenode;
        }
    };
    // Trigger a heartbeat so that it acknowledges the NN as active.
    dn.getAllBpOs().get(0).triggerHeartbeatForTests();
    waitForActiveNN();
    spyDN = spy(dn);
    recoveryWorker = new BlockRecoveryWorker(spyDN);
}

Example 20 with DatanodeProtocolClientSideTranslatorPB

use of org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB in project hadoop by apache.

the class TestDatanodeProtocolRetryPolicy method testDatanodeRegistrationRetry.

/**
   * Verify the following scenario.
   * 1. The initial DatanodeProtocol.registerDatanode succeeds.
   * 2. DN starts heartbeat process.
   * 3. In the first heartbeat, NN asks DN to reregister.
   * 4. DN calls DatanodeProtocol.registerDatanode.
   * 5. DatanodeProtocol.registerDatanode throws EOFException.
   * 6. DN retries.
   * 7. DatanodeProtocol.registerDatanode succeeds.
   */
@Test(timeout = 60000)
public void testDatanodeRegistrationRetry() throws Exception {
    final DatanodeProtocolClientSideTranslatorPB namenode = mock(DatanodeProtocolClientSideTranslatorPB.class);
    Mockito.doAnswer(new Answer<DatanodeRegistration>() {

        int i = 0;

        @Override
        public DatanodeRegistration answer(InvocationOnMock invocation) throws Throwable {
            i++;
            if (i > 1 && i < 5) {
                LOG.info("mockito exception " + i);
                throw new EOFException("TestDatanodeProtocolRetryPolicy");
            } else {
                DatanodeRegistration dr = (DatanodeRegistration) invocation.getArguments()[0];
                datanodeRegistration = new DatanodeRegistration(dr.getDatanodeUuid(), dr);
                LOG.info("mockito succeeded " + datanodeRegistration);
                return datanodeRegistration;
            }
        }
    }).when(namenode).registerDatanode(Mockito.any(DatanodeRegistration.class));
    when(namenode.versionRequest()).thenReturn(new NamespaceInfo(1, CLUSTER_ID, POOL_ID, 1L));
    Mockito.doAnswer(new Answer<HeartbeatResponse>() {

        int i = 0;

        @Override
        public HeartbeatResponse answer(InvocationOnMock invocation) throws Throwable {
            i++;
            HeartbeatResponse heartbeatResponse;
            if (i == 1) {
                LOG.info("mockito heartbeatResponse registration " + i);
                heartbeatResponse = new HeartbeatResponse(new DatanodeCommand[] { RegisterCommand.REGISTER }, new NNHAStatusHeartbeat(HAServiceState.ACTIVE, 1), null, ThreadLocalRandom.current().nextLong() | 1L);
            } else {
                LOG.info("mockito heartbeatResponse " + i);
                heartbeatResponse = new HeartbeatResponse(new DatanodeCommand[0], new NNHAStatusHeartbeat(HAServiceState.ACTIVE, 1), null, ThreadLocalRandom.current().nextLong() | 1L);
            }
            return heartbeatResponse;
        }
    }).when(namenode).sendHeartbeat(Mockito.any(DatanodeRegistration.class), Mockito.any(StorageReport[].class), Mockito.anyLong(), Mockito.anyLong(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt(), Mockito.any(VolumeFailureSummary.class), Mockito.anyBoolean(), Mockito.any(SlowPeerReports.class));
    dn = new DataNode(conf, locations, null, null) {

        @Override
        DatanodeProtocolClientSideTranslatorPB connectToNN(InetSocketAddress nnAddr) throws IOException {
            Assert.assertEquals(NN_ADDR, nnAddr);
            return namenode;
        }
    };
    // Trigger a heartbeat so that it acknowledges the NN as active.
    dn.getAllBpOs().get(0).triggerHeartbeatForTests();
    waitForBlockReport(namenode);
}