Examples with DatanodeProtocolClientSideTranslatorPB org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB used on opensource projects

Example 11 with DatanodeProtocolClientSideTranslatorPB

use of org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB in project hadoop by apache.

the class TestBPOfferService method setupBPOSForNNs.

private BPOfferService setupBPOSForNNs(DataNode mockDn, DatanodeProtocolClientSideTranslatorPB... nns) throws IOException {
    // Set up some fake InetAddresses, then override the connectToNN
    // function to return the corresponding proxies.
    final Map<InetSocketAddress, DatanodeProtocolClientSideTranslatorPB> nnMap = Maps.newLinkedHashMap();
    for (int port = 0; port < nns.length; port++) {
        nnMap.put(new InetSocketAddress(port), nns[port]);
        Mockito.doReturn(nns[port]).when(mockDn).connectToNN(Mockito.eq(new InetSocketAddress(port)));
    }
    return new BPOfferService(Lists.newArrayList(nnMap.keySet()), Collections.<InetSocketAddress>nCopies(nnMap.size(), null), mockDn);
}

Example 12 with DatanodeProtocolClientSideTranslatorPB

use of org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB in project hadoop by apache.

the class TestDatanodeRegister method setUp.

@Before
public void setUp() throws IOException {
    mockDnConf = mock(DNConf.class);
    doReturn(VersionInfo.getVersion()).when(mockDnConf).getMinimumNameNodeVersion();
    DataNode mockDN = mock(DataNode.class);
    doReturn(true).when(mockDN).shouldRun();
    doReturn(mockDnConf).when(mockDN).getDnConf();
    BPOfferService mockBPOS = mock(BPOfferService.class);
    doReturn(mockDN).when(mockBPOS).getDataNode();
    actor = new BPServiceActor(INVALID_ADDR, null, mockBPOS);
    fakeNsInfo = mock(NamespaceInfo.class);
    // Return a a good software version.
    doReturn(VersionInfo.getVersion()).when(fakeNsInfo).getSoftwareVersion();
    // Return a good layout version for now.
    doReturn(HdfsServerConstants.NAMENODE_LAYOUT_VERSION).when(fakeNsInfo).getLayoutVersion();
    DatanodeProtocolClientSideTranslatorPB fakeDnProt = mock(DatanodeProtocolClientSideTranslatorPB.class);
    when(fakeDnProt.versionRequest()).thenReturn(fakeNsInfo);
    actor.setNameNode(fakeDnProt);
}

Example 13 with DatanodeProtocolClientSideTranslatorPB

use of org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB in project hadoop by apache.

the class TestIsMethodSupported method testDatanodeProtocol.

@Test
public void testDatanodeProtocol() throws IOException {
    DatanodeProtocolClientSideTranslatorPB translator = new DatanodeProtocolClientSideTranslatorPB(nnAddress, conf);
    assertTrue(translator.isMethodSupported("sendHeartbeat"));
}

Example 14 with DatanodeProtocolClientSideTranslatorPB

use of org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB in project hadoop by apache.

the class TestBlockListAsLongs method testDatanodeDetect.

@Test
public void testDatanodeDetect() throws ServiceException, IOException {
    final AtomicReference<BlockReportRequestProto> request = new AtomicReference<>();
    // just capture the outgoing PB
    DatanodeProtocolPB mockProxy = mock(DatanodeProtocolPB.class);
    doAnswer(new Answer<BlockReportResponseProto>() {

        public BlockReportResponseProto answer(InvocationOnMock invocation) {
            Object[] args = invocation.getArguments();
            request.set((BlockReportRequestProto) args[1]);
            return BlockReportResponseProto.newBuilder().build();
        }
    }).when(mockProxy).blockReport(any(RpcController.class), any(BlockReportRequestProto.class));
    @SuppressWarnings("resource") DatanodeProtocolClientSideTranslatorPB nn = new DatanodeProtocolClientSideTranslatorPB(mockProxy);
    DatanodeRegistration reg = DFSTestUtil.getLocalDatanodeRegistration();
    NamespaceInfo nsInfo = new NamespaceInfo(1, "cluster", "bp", 1);
    reg.setNamespaceInfo(nsInfo);
    Replica r = new FinalizedReplica(new Block(1, 2, 3), null, null);
    BlockListAsLongs bbl = BlockListAsLongs.encode(Collections.singleton(r));
    DatanodeStorage storage = new DatanodeStorage("s1");
    StorageBlockReport[] sbr = { new StorageBlockReport(storage, bbl) };
    // check DN sends new-style BR
    request.set(null);
    nsInfo.setCapabilities(Capability.STORAGE_BLOCK_REPORT_BUFFERS.getMask());
    nn.blockReport(reg, "pool", sbr, new BlockReportContext(1, 0, System.nanoTime(), 0L, true));
    BlockReportRequestProto proto = request.get();
    assertNotNull(proto);
    assertTrue(proto.getReports(0).getBlocksList().isEmpty());
    assertFalse(proto.getReports(0).getBlocksBuffersList().isEmpty());
    // back up to prior version and check DN sends old-style BR
    request.set(null);
    nsInfo.setCapabilities(Capability.UNKNOWN.getMask());
    BlockListAsLongs blockList = getBlockList(r);
    StorageBlockReport[] obp = new StorageBlockReport[] { new StorageBlockReport(new DatanodeStorage("s1"), blockList) };
    nn.blockReport(reg, "pool", obp, new BlockReportContext(1, 0, System.nanoTime(), 0L, true));
    proto = request.get();
    assertNotNull(proto);
    assertFalse(proto.getReports(0).getBlocksList().isEmpty());
    assertTrue(proto.getReports(0).getBlocksBuffersList().isEmpty());
}

Example 15 with DatanodeProtocolClientSideTranslatorPB

use of org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB in project hadoop by apache.

the class BlockReportTestBase method testOneReplicaRbwReportArrivesAfterBlockCompleted.

/**
   * Test for the case where one of the DNs in the pipeline is in the
   * process of doing a block report exactly when the block is closed.
   * In this case, the block report becomes delayed until after the
   * block is marked completed on the NN, and hence it reports an RBW
   * replica for a COMPLETE block. Such a report should not be marked
   * corrupt.
   * This is a regression test for HDFS-2791.
   */
@Test(timeout = 300000)
public void testOneReplicaRbwReportArrivesAfterBlockCompleted() throws Exception {
    final CountDownLatch brFinished = new CountDownLatch(1);
    DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG) {

        @Override
        protected Object passThrough(InvocationOnMock invocation) throws Throwable {
            try {
                return super.passThrough(invocation);
            } finally {
                // inform the test that our block report went through.
                brFinished.countDown();
            }
        }
    };
    final String METHOD_NAME = GenericTestUtils.getMethodName();
    Path filePath = new Path("/" + METHOD_NAME + ".dat");
    // Start a second DN for this test -- we're checking
    // what happens when one of the DNs is slowed for some reason.
    REPL_FACTOR = 2;
    startDNandWait(null, false);
    NameNode nn = cluster.getNameNode();
    FSDataOutputStream out = fs.create(filePath, REPL_FACTOR);
    try {
        AppendTestUtil.write(out, 0, 10);
        out.hflush();
        // Set up a spy so that we can delay the block report coming
        // from this node.
        DataNode dn = cluster.getDataNodes().get(0);
        DatanodeProtocolClientSideTranslatorPB spy = InternalDataNodeTestUtils.spyOnBposToNN(dn, nn);
        Mockito.doAnswer(delayer).when(spy).blockReport(Mockito.<DatanodeRegistration>anyObject(), Mockito.anyString(), Mockito.<StorageBlockReport[]>anyObject(), Mockito.<BlockReportContext>anyObject());
        // Force a block report to be generated. The block report will have
        // an RBW replica in it. Wait for the RPC to be sent, but block
        // it before it gets to the NN.
        dn.scheduleAllBlockReport(0);
        delayer.waitForCall();
    } finally {
        IOUtils.closeStream(out);
    }
    // Now that the stream is closed, the NN will have the block in COMPLETE
    // state.
    delayer.proceed();
    brFinished.await();
    // Verify that no replicas are marked corrupt, and that the
    // file is still readable.
    BlockManagerTestUtil.updateState(nn.getNamesystem().getBlockManager());
    assertEquals(0, nn.getNamesystem().getCorruptReplicaBlocks());
    DFSTestUtil.readFile(fs, filePath);
    // Ensure that the file is readable even from the DN that we futzed with.
    cluster.stopDataNode(1);
    DFSTestUtil.readFile(fs, filePath);
}