Examples with StorageBlockReport org.apache.hadoop.hdfs.server.protocol.StorageBlockReport used on opensource projects

Example 16 with StorageBlockReport

use of org.apache.hadoop.hdfs.server.protocol.StorageBlockReport in project hadoop by apache.

the class BlockReportTestBase method blockReport_07.

/**
   * Similar to BlockReport_03() but works with two DNs
   * Test writes a file and closes it.
   * The second datanode is started in the cluster.
   * As soon as the replication process is completed test finds a block from
   * the second DN and sets its GS to be < of original one.
   * this is the markBlockAsCorrupt case 3 so we expect one pending deletion
   * Block report is forced and the check for # of currupted blocks is performed.
   * Another block is chosen and its length is set to a lesser than original.
   * A check for another corrupted block is performed after yet another
   * BlockReport
   *
   * @throws IOException in case of an error
   */
@Test(timeout = 300000)
public void blockReport_07() throws Exception {
    final String METHOD_NAME = GenericTestUtils.getMethodName();
    Path filePath = new Path("/" + METHOD_NAME + ".dat");
    final int DN_N1 = DN_N0 + 1;
    // write file and start second node to be "older" than the original
    writeFile(METHOD_NAME, FILE_SIZE, filePath);
    startDNandWait(filePath, true);
    // all blocks belong to the same file, hence same BP
    DataNode dn = cluster.getDataNodes().get(DN_N1);
    String poolId = cluster.getNamesystem().getBlockPoolId();
    DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
    StorageBlockReport[] reports = getBlockReports(dn, poolId, true, false);
    sendBlockReports(dnR, poolId, reports);
    printStats();
    assertThat("Wrong number of corrupt blocks", cluster.getNamesystem().getCorruptReplicaBlocks(), is(0L));
    assertThat("Wrong number of PendingDeletion blocks", cluster.getNamesystem().getPendingDeletionBlocks(), is(1L));
    assertThat("Wrong number of PendingReplication blocks", cluster.getNamesystem().getPendingReplicationBlocks(), is(0L));
    reports = getBlockReports(dn, poolId, false, true);
    sendBlockReports(dnR, poolId, reports);
    printStats();
    assertThat("Wrong number of corrupt blocks", cluster.getNamesystem().getCorruptReplicaBlocks(), is(1L));
    assertThat("Wrong number of PendingDeletion blocks", cluster.getNamesystem().getPendingDeletionBlocks(), is(1L));
    assertThat("Wrong number of PendingReplication blocks", cluster.getNamesystem().getPendingReplicationBlocks(), is(0L));
    printStats();
}

Example 17 with StorageBlockReport

use of org.apache.hadoop.hdfs.server.protocol.StorageBlockReport in project hadoop by apache.

the class BlockReportTestBase method testOneReplicaRbwReportArrivesAfterBlockCompleted.

/**
   * Test for the case where one of the DNs in the pipeline is in the
   * process of doing a block report exactly when the block is closed.
   * In this case, the block report becomes delayed until after the
   * block is marked completed on the NN, and hence it reports an RBW
   * replica for a COMPLETE block. Such a report should not be marked
   * corrupt.
   * This is a regression test for HDFS-2791.
   */
@Test(timeout = 300000)
public void testOneReplicaRbwReportArrivesAfterBlockCompleted() throws Exception {
    final CountDownLatch brFinished = new CountDownLatch(1);
    DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG) {

        @Override
        protected Object passThrough(InvocationOnMock invocation) throws Throwable {
            try {
                return super.passThrough(invocation);
            } finally {
                // inform the test that our block report went through.
                brFinished.countDown();
            }
        }
    };
    final String METHOD_NAME = GenericTestUtils.getMethodName();
    Path filePath = new Path("/" + METHOD_NAME + ".dat");
    // Start a second DN for this test -- we're checking
    // what happens when one of the DNs is slowed for some reason.
    REPL_FACTOR = 2;
    startDNandWait(null, false);
    NameNode nn = cluster.getNameNode();
    FSDataOutputStream out = fs.create(filePath, REPL_FACTOR);
    try {
        AppendTestUtil.write(out, 0, 10);
        out.hflush();
        // Set up a spy so that we can delay the block report coming
        // from this node.
        DataNode dn = cluster.getDataNodes().get(0);
        DatanodeProtocolClientSideTranslatorPB spy = InternalDataNodeTestUtils.spyOnBposToNN(dn, nn);
        Mockito.doAnswer(delayer).when(spy).blockReport(Mockito.<DatanodeRegistration>anyObject(), Mockito.anyString(), Mockito.<StorageBlockReport[]>anyObject(), Mockito.<BlockReportContext>anyObject());
        // Force a block report to be generated. The block report will have
        // an RBW replica in it. Wait for the RPC to be sent, but block
        // it before it gets to the NN.
        dn.scheduleAllBlockReport(0);
        delayer.waitForCall();
    } finally {
        IOUtils.closeStream(out);
    }
    // Now that the stream is closed, the NN will have the block in COMPLETE
    // state.
    delayer.proceed();
    brFinished.await();
    // Verify that no replicas are marked corrupt, and that the
    // file is still readable.
    BlockManagerTestUtil.updateState(nn.getNamesystem().getBlockManager());
    assertEquals(0, nn.getNamesystem().getCorruptReplicaBlocks());
    DFSTestUtil.readFile(fs, filePath);
    // Ensure that the file is readable even from the DN that we futzed with.
    cluster.stopDataNode(1);
    DFSTestUtil.readFile(fs, filePath);
}

Example 18 with StorageBlockReport

use of org.apache.hadoop.hdfs.server.protocol.StorageBlockReport in project hadoop by apache.

the class BlockReportTestBase method blockReport_08.

/**
   * The test set the configuration parameters for a large block size and
   * restarts initiated single-node cluster.
   * Then it writes a file > block_size and closes it.
   * The second datanode is started in the cluster.
   * As soon as the replication process is started and at least one TEMPORARY
   * replica is found test forces BlockReport process and checks
   * if the TEMPORARY replica isn't reported on it.
   * Eventually, the configuration is being restored into the original state.
   *
   * @throws IOException in case of an error
   */
@Test(timeout = 300000)
public void blockReport_08() throws IOException {
    final String METHOD_NAME = GenericTestUtils.getMethodName();
    Path filePath = new Path("/" + METHOD_NAME + ".dat");
    final int DN_N1 = DN_N0 + 1;
    final int bytesChkSum = 1024 * 1000;
    conf.setInt(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, bytesChkSum);
    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 6 * bytesChkSum);
    shutDownCluster();
    startUpCluster();
    try {
        ArrayList<Block> blocks = writeFile(METHOD_NAME, 12 * bytesChkSum, filePath);
        Block bl = findBlock(filePath, 12 * bytesChkSum);
        BlockChecker bc = new BlockChecker(filePath);
        bc.start();
        waitForTempReplica(bl, DN_N1);
        // all blocks belong to the same file, hence same BP
        DataNode dn = cluster.getDataNodes().get(DN_N1);
        String poolId = cluster.getNamesystem().getBlockPoolId();
        DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
        StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false);
        sendBlockReports(dnR, poolId, reports);
        printStats();
        assertEquals("Wrong number of PendingReplication blocks", blocks.size(), cluster.getNamesystem().getPendingReplicationBlocks());
        try {
            bc.join();
        } catch (InterruptedException e) {
        }
    } finally {
        // return the initial state of the configuration
        resetConfiguration();
    }
}

Example 19 with StorageBlockReport

use of org.apache.hadoop.hdfs.server.protocol.StorageBlockReport in project hadoop by apache.

the class BlockReportTestBase method blockReport_01.

/**
   * Test write a file, verifies and closes it. Then the length of the blocks
   * are messed up and BlockReport is forced.
   * The modification of blocks' length has to be ignored
   *
   * @throws java.io.IOException on an error
   */
@Test(timeout = 300000)
public void blockReport_01() throws IOException {
    final String METHOD_NAME = GenericTestUtils.getMethodName();
    Path filePath = new Path("/" + METHOD_NAME + ".dat");
    ArrayList<Block> blocks = prepareForRide(filePath, METHOD_NAME, FILE_SIZE);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Number of blocks allocated " + blocks.size());
    }
    long[] oldLengths = new long[blocks.size()];
    int tempLen;
    for (int i = 0; i < blocks.size(); i++) {
        Block b = blocks.get(i);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Block " + b.getBlockName() + " before\t" + "Size " + b.getNumBytes());
        }
        oldLengths[i] = b.getNumBytes();
        if (LOG.isDebugEnabled()) {
            LOG.debug("Setting new length");
        }
        tempLen = rand.nextInt(BLOCK_SIZE);
        b.set(b.getBlockId(), tempLen, b.getGenerationStamp());
        if (LOG.isDebugEnabled()) {
            LOG.debug("Block " + b.getBlockName() + " after\t " + "Size " + b.getNumBytes());
        }
    }
    // all blocks belong to the same file, hence same BP
    DataNode dn = cluster.getDataNodes().get(DN_N0);
    String poolId = cluster.getNamesystem().getBlockPoolId();
    DatanodeRegistration dnR = dn.getDNRegistrationForBP(poolId);
    StorageBlockReport[] reports = getBlockReports(dn, poolId, false, false);
    sendBlockReports(dnR, poolId, reports);
    List<LocatedBlock> blocksAfterReport = DFSTestUtil.getAllBlocks(fs.open(filePath));
    if (LOG.isDebugEnabled()) {
        LOG.debug("After mods: Number of blocks allocated " + blocksAfterReport.size());
    }
    for (int i = 0; i < blocksAfterReport.size(); i++) {
        ExtendedBlock b = blocksAfterReport.get(i).getBlock();
        assertEquals("Length of " + i + "th block is incorrect", oldLengths[i], b.getNumBytes());
    }
}

Example 20 with StorageBlockReport

use of org.apache.hadoop.hdfs.server.protocol.StorageBlockReport in project hadoop by apache.

the class BlockReportTestBase method blockReport_02.

/**
   * Test write a file, verifies and closes it. Then a couple of random blocks
   * is removed and BlockReport is forced; the FSNamesystem is pushed to
   * recalculate required DN's activities such as replications and so on.
   * The number of missing and under-replicated blocks should be the same in
   * case of a single-DN cluster.
   *
   * @throws IOException in case of errors
   */
@Test(timeout = 300000)
public void blockReport_02() throws IOException {
    final String METHOD_NAME = GenericTestUtils.getMethodName();
    LOG.info("Running test " + METHOD_NAME);
    Path filePath = new Path("/" + METHOD_NAME + ".dat");
    DFSTestUtil.createFile(fs, filePath, FILE_SIZE, REPL_FACTOR, rand.nextLong());
    // mock around with newly created blocks and delete some
    File dataDir = new File(cluster.getDataDirectory());
    assertTrue(dataDir.isDirectory());
    List<ExtendedBlock> blocks2Remove = new ArrayList<ExtendedBlock>();
    List<Integer> removedIndex = new ArrayList<Integer>();
    List<LocatedBlock> lBlocks = cluster.getNameNodeRpc().getBlockLocations(filePath.toString(), FILE_START, FILE_SIZE).getLocatedBlocks();
    while (removedIndex.size() != 2) {
        int newRemoveIndex = rand.nextInt(lBlocks.size());
        if (!removedIndex.contains(newRemoveIndex))
            removedIndex.add(newRemoveIndex);
    }
    for (Integer aRemovedIndex : removedIndex) {
        blocks2Remove.add(lBlocks.get(aRemovedIndex).getBlock());
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Number of blocks allocated " + lBlocks.size());
    }
    final DataNode dn0 = cluster.getDataNodes().get(DN_N0);
    for (ExtendedBlock b : blocks2Remove) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Removing the block " + b.getBlockName());
        }
        for (File f : findAllFiles(dataDir, new MyFileFilter(b.getBlockName(), true))) {
            DataNodeTestUtils.getFSDataset(dn0).unfinalizeBlock(b);
            if (!f.delete()) {
                LOG.warn("Couldn't delete " + b.getBlockName());
            } else {
                LOG.debug("Deleted file " + f.toString());
            }
        }
    }
    waitTil(TimeUnit.SECONDS.toMillis(DN_RESCAN_EXTRA_WAIT));
    // all blocks belong to the same file, hence same BP
    String poolId = cluster.getNamesystem().getBlockPoolId();
    DatanodeRegistration dnR = dn0.getDNRegistrationForBP(poolId);
    StorageBlockReport[] reports = getBlockReports(dn0, poolId, false, false);
    sendBlockReports(dnR, poolId, reports);
    BlockManagerTestUtil.getComputedDatanodeWork(cluster.getNamesystem().getBlockManager());
    printStats();
    assertEquals("Wrong number of MissingBlocks is found", blocks2Remove.size(), cluster.getNamesystem().getMissingBlocksCount());
    assertEquals("Wrong number of UnderReplicatedBlocks is found", blocks2Remove.size(), cluster.getNamesystem().getUnderReplicatedBlocks());
}