Examples with DatanodeInfo org.apache.hadoop.hdfs.protocol.DatanodeInfo used on opensource projects

Example 91 with DatanodeInfo

use of org.apache.hadoop.hdfs.protocol.DatanodeInfo in project hadoop by apache.

the class TestDFSClientRetries method testIdempotentAllocateBlockAndClose.

/**
   * Test that getAdditionalBlock() and close() are idempotent. This allows
   * a client to safely retry a call and still produce a correct
   * file. See HDFS-3031.
   */
@Test
public void testIdempotentAllocateBlockAndClose() throws Exception {
    final String src = "/testIdempotentAllocateBlock";
    Path file = new Path(src);
    conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 4096);
    final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
    try {
        cluster.waitActive();
        FileSystem fs = cluster.getFileSystem();
        NamenodeProtocols preSpyNN = cluster.getNameNodeRpc();
        NamenodeProtocols spyNN = spy(preSpyNN);
        DFSClient client = new DFSClient(null, spyNN, conf, null);
        // Make the call to addBlock() get called twice, as if it were retried
        // due to an IPC issue.
        doAnswer(new Answer<LocatedBlock>() {

            private int getBlockCount(LocatedBlock ret) throws IOException {
                LocatedBlocks lb = cluster.getNameNodeRpc().getBlockLocations(src, 0, Long.MAX_VALUE);
                assertEquals(lb.getLastLocatedBlock().getBlock(), ret.getBlock());
                return lb.getLocatedBlocks().size();
            }

            @Override
            public LocatedBlock answer(InvocationOnMock invocation) throws Throwable {
                LOG.info("Called addBlock: " + Arrays.toString(invocation.getArguments()));
                // call first time
                // warp NotReplicatedYetException with RemoteException as rpc does.
                final LocatedBlock ret;
                try {
                    ret = (LocatedBlock) invocation.callRealMethod();
                } catch (NotReplicatedYetException e) {
                    throw new RemoteException(e.getClass().getName(), e.getMessage());
                }
                final int blockCount = getBlockCount(ret);
                // Retrying should result in a new block at the end of the file.
                // (abandoning the old one)
                // It should not have NotReplicatedYetException.
                final LocatedBlock ret2;
                try {
                    ret2 = (LocatedBlock) invocation.callRealMethod();
                } catch (NotReplicatedYetException e) {
                    throw new AssertionError("Unexpected exception", e);
                }
                final int blockCount2 = getBlockCount(ret2);
                // We shouldn't have gained an extra block by the RPC.
                assertEquals(blockCount, blockCount2);
                return ret2;
            }
        }).when(spyNN).addBlock(Mockito.anyString(), Mockito.anyString(), Mockito.<ExtendedBlock>any(), Mockito.<DatanodeInfo[]>any(), Mockito.anyLong(), Mockito.<String[]>any(), Mockito.<EnumSet<AddBlockFlag>>any());
        doAnswer(new Answer<Boolean>() {

            @Override
            public Boolean answer(InvocationOnMock invocation) throws Throwable {
                // complete() may return false a few times before it returns
                // true. We want to wait until it returns true, and then
                // make it retry one more time after that.
                LOG.info("Called complete:");
                if (!(Boolean) invocation.callRealMethod()) {
                    LOG.info("Complete call returned false, not faking a retry RPC");
                    return false;
                }
                // We got a successful close. Call it again to check idempotence.
                try {
                    boolean ret = (Boolean) invocation.callRealMethod();
                    LOG.info("Complete call returned true, faked second RPC. " + "Returned: " + ret);
                    return ret;
                } catch (Throwable t) {
                    LOG.error("Idempotent retry threw exception", t);
                    throw t;
                }
            }
        }).when(spyNN).complete(Mockito.anyString(), Mockito.anyString(), Mockito.<ExtendedBlock>any(), anyLong());
        OutputStream stm = client.create(file.toString(), true);
        try {
            AppendTestUtil.write(stm, 0, 10000);
            stm.close();
            stm = null;
        } finally {
            IOUtils.cleanup(LOG, stm);
        }
        // Make sure the mock was actually properly injected.
        Mockito.verify(spyNN, Mockito.atLeastOnce()).addBlock(Mockito.anyString(), Mockito.anyString(), Mockito.<ExtendedBlock>any(), Mockito.<DatanodeInfo[]>any(), Mockito.anyLong(), Mockito.<String[]>any(), Mockito.<EnumSet<AddBlockFlag>>any());
        Mockito.verify(spyNN, Mockito.atLeastOnce()).complete(Mockito.anyString(), Mockito.anyString(), Mockito.<ExtendedBlock>any(), anyLong());
        AppendTestUtil.check(fs, file, 10000);
    } finally {
        cluster.shutdown();
    }
}

Example 92 with DatanodeInfo

use of org.apache.hadoop.hdfs.protocol.DatanodeInfo in project hadoop by apache.

the class TestDFSOutputStream method testCongestionBackoff.

@Test
public void testCongestionBackoff() throws IOException {
    DfsClientConf dfsClientConf = mock(DfsClientConf.class);
    DFSClient client = mock(DFSClient.class);
    when(client.getConf()).thenReturn(dfsClientConf);
    when(client.getTracer()).thenReturn(FsTracer.get(new Configuration()));
    client.clientRunning = true;
    DataStreamer stream = new DataStreamer(mock(HdfsFileStatus.class), mock(ExtendedBlock.class), client, "foo", null, null, null, null, null, null);
    DataOutputStream blockStream = mock(DataOutputStream.class);
    doThrow(new IOException()).when(blockStream).flush();
    Whitebox.setInternalState(stream, "blockStream", blockStream);
    Whitebox.setInternalState(stream, "stage", BlockConstructionStage.PIPELINE_CLOSE);
    @SuppressWarnings("unchecked") LinkedList<DFSPacket> dataQueue = (LinkedList<DFSPacket>) Whitebox.getInternalState(stream, "dataQueue");
    @SuppressWarnings("unchecked") ArrayList<DatanodeInfo> congestedNodes = (ArrayList<DatanodeInfo>) Whitebox.getInternalState(stream, "congestedNodes");
    congestedNodes.add(mock(DatanodeInfo.class));
    DFSPacket packet = mock(DFSPacket.class);
    when(packet.getTraceParents()).thenReturn(new SpanId[] {});
    dataQueue.add(packet);
    stream.run();
    Assert.assertTrue(congestedNodes.isEmpty());
}

Example 93 with DatanodeInfo

use of org.apache.hadoop.hdfs.protocol.DatanodeInfo in project hadoop by apache.

the class TestDFSStripedOutputStreamWithFailure method runTest.

/**
   * runTest implementation.
   * @param length file length
   * @param killPos killing positions in ascending order
   * @param dnIndex DN index to kill when meets killing positions
   * @param tokenExpire wait token to expire when kill a DN
   * @throws Exception
   */
private void runTest(final int length, final int[] killPos, final int[] dnIndex, final boolean tokenExpire) throws Exception {
    if (killPos[0] <= FLUSH_POS) {
        LOG.warn("killPos=" + Arrays.toString(killPos) + " <= FLUSH_POS=" + FLUSH_POS + ", length=" + length + ", dnIndex=" + Arrays.toString(dnIndex));
        //skip test
        return;
    }
    Preconditions.checkArgument(length > killPos[0], "length=%s <= killPos=%s", length, killPos);
    Preconditions.checkArgument(killPos.length == dnIndex.length);
    final Path p = new Path(dir, "dn" + Arrays.toString(dnIndex) + "len" + length + "kill" + Arrays.toString(killPos));
    final String fullPath = p.toString();
    LOG.info("fullPath=" + fullPath);
    if (tokenExpire) {
        final NameNode nn = cluster.getNameNode();
        final BlockManager bm = nn.getNamesystem().getBlockManager();
        final BlockTokenSecretManager sm = bm.getBlockTokenSecretManager();
        // set a short token lifetime (1 second)
        SecurityTestUtil.setBlockTokenLifetime(sm, 1000L);
    }
    final AtomicInteger pos = new AtomicInteger();
    final FSDataOutputStream out = dfs.create(p);
    final DFSStripedOutputStream stripedOut = (DFSStripedOutputStream) out.getWrappedStream();
    // first GS of this block group which never proceeds blockRecovery
    long firstGS = -1;
    // the old GS before bumping
    long oldGS = -1;
    List<Long> gsList = new ArrayList<>();
    final List<DatanodeInfo> killedDN = new ArrayList<>();
    int numKilled = 0;
    for (; pos.get() < length; ) {
        final int i = pos.getAndIncrement();
        if (numKilled < killPos.length && i == killPos[numKilled]) {
            assertTrue(firstGS != -1);
            final long gs = getGenerationStamp(stripedOut);
            if (numKilled == 0) {
                assertEquals(firstGS, gs);
            } else {
                //TODO: implement hflush/hsync and verify gs strict greater than oldGS
                assertTrue(gs >= oldGS);
            }
            oldGS = gs;
            if (tokenExpire) {
                DFSTestUtil.flushInternal(stripedOut);
                waitTokenExpires(out);
            }
            killedDN.add(killDatanode(cluster, stripedOut, dnIndex[numKilled], pos));
            numKilled++;
        }
        write(out, i);
        if (i % blockGroupSize == FLUSH_POS) {
            firstGS = getGenerationStamp(stripedOut);
            oldGS = firstGS;
        }
        if (i > 0 && (i + 1) % blockGroupSize == 0) {
            gsList.add(oldGS);
        }
    }
    gsList.add(oldGS);
    out.close();
    assertEquals(dnIndex.length, numKilled);
    StripedFileTestUtil.waitBlockGroupsReported(dfs, fullPath, numKilled);
    cluster.triggerBlockReports();
    StripedFileTestUtil.checkData(dfs, p, length, killedDN, gsList, blockGroupSize);
}

Example 94 with DatanodeInfo

use of org.apache.hadoop.hdfs.protocol.DatanodeInfo in project hadoop by apache.

the class TestDFSStripedOutputStreamWithFailure method getDatanodes.

static DatanodeInfo getDatanodes(StripedDataStreamer streamer) {
    for (; ; ) {
        DatanodeInfo[] datanodes = streamer.getNodes();
        if (datanodes == null) {
            // try peeking following block.
            final LocatedBlock lb = streamer.peekFollowingBlock();
            if (lb != null) {
                datanodes = lb.getLocations();
            }
        }
        if (datanodes != null) {
            Assert.assertEquals(1, datanodes.length);
            Assert.assertNotNull(datanodes[0]);
            return datanodes[0];
        }
        try {
            Thread.sleep(100);
        } catch (InterruptedException ie) {
            Assert.fail(StringUtils.stringifyException(ie));
            return null;
        }
    }
}

Example 95 with DatanodeInfo

use of org.apache.hadoop.hdfs.protocol.DatanodeInfo in project hadoop by apache.

the class TestDataNodeMetrics method testRoundTripAckMetric.

/**
   * Tests that round-trip acks in a datanode write pipeline are correctly 
   * measured. 
   */
@Test
public void testRoundTripAckMetric() throws Exception {
    final int datanodeCount = 2;
    final int interval = 1;
    Configuration conf = new HdfsConfiguration();
    conf.set(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY, "" + interval);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(datanodeCount).build();
    try {
        cluster.waitActive();
        FileSystem fs = cluster.getFileSystem();
        // Open a file and get the head of the pipeline
        Path testFile = new Path("/testRoundTripAckMetric.txt");
        FSDataOutputStream fsout = fs.create(testFile, (short) datanodeCount);
        DFSOutputStream dout = (DFSOutputStream) fsout.getWrappedStream();
        // Slow down the writes to catch the write pipeline
        dout.setChunksPerPacket(5);
        dout.setArtificialSlowdown(3000);
        fsout.write(new byte[10000]);
        DatanodeInfo[] pipeline = null;
        int count = 0;
        while (pipeline == null && count < 5) {
            pipeline = dout.getPipeline();
            System.out.println("Waiting for pipeline to be created.");
            Thread.sleep(1000);
            count++;
        }
        // Get the head node that should be receiving downstream acks
        DatanodeInfo headInfo = pipeline[0];
        DataNode headNode = null;
        for (DataNode datanode : cluster.getDataNodes()) {
            if (datanode.getDatanodeId().equals(headInfo)) {
                headNode = datanode;
                break;
            }
        }
        assertNotNull("Could not find the head of the datanode write pipeline", headNode);
        // Close the file and wait for the metrics to rollover
        Thread.sleep((interval + 1) * 1000);
        // Check the ack was received
        MetricsRecordBuilder dnMetrics = getMetrics(headNode.getMetrics().name());
        assertTrue("Expected non-zero number of acks", getLongCounter("PacketAckRoundTripTimeNanosNumOps", dnMetrics) > 0);
        assertQuantileGauges("PacketAckRoundTripTimeNanos" + interval + "s", dnMetrics);
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}