Examples with RecoveringBlock org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock used on opensource projects

Example 1 with RecoveringBlock

use of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock in project hadoop by apache.

the class TestInterDatanodeProtocol method testUpdateReplicaUnderRecovery.

/** 
   * Test  for
   * {@link FsDatasetImpl#updateReplicaUnderRecovery(ExtendedBlock, long, long)} 
   * */
@Test
public void testUpdateReplicaUnderRecovery() throws IOException {
    MiniDFSCluster cluster = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
        cluster.waitActive();
        //create a file
        DistributedFileSystem dfs = cluster.getFileSystem();
        String filestr = "/foo";
        Path filepath = new Path(filestr);
        DFSTestUtil.createFile(dfs, filepath, 1024L, (short) 3, 0L);
        //get block info
        final LocatedBlock locatedblock = getLastLocatedBlock(DFSClientAdapter.getDFSClient(dfs).getNamenode(), filestr);
        final DatanodeInfo[] datanodeinfo = locatedblock.getLocations();
        Assert.assertTrue(datanodeinfo.length > 0);
        //get DataNode and FSDataset objects
        final DataNode datanode = cluster.getDataNode(datanodeinfo[0].getIpcPort());
        Assert.assertTrue(datanode != null);
        //initReplicaRecovery
        final ExtendedBlock b = locatedblock.getBlock();
        final long recoveryid = b.getGenerationStamp() + 1;
        final long newlength = b.getNumBytes() - 1;
        final FsDatasetSpi<?> fsdataset = DataNodeTestUtils.getFSDataset(datanode);
        final ReplicaRecoveryInfo rri = fsdataset.initReplicaRecovery(new RecoveringBlock(b, null, recoveryid));
        //check replica
        final Replica replica = cluster.getFsDatasetTestUtils(datanode).fetchReplica(b);
        Assert.assertEquals(ReplicaState.RUR, replica.getState());
        //check meta data before update
        cluster.getFsDatasetTestUtils(datanode).checkStoredReplica(replica);
        //case "THIS IS NOT SUPPOSED TO HAPPEN"
        //with (block length) != (stored replica's on disk length). 
        {
            //create a block with same id and gs but different length.
            final ExtendedBlock tmp = new ExtendedBlock(b.getBlockPoolId(), rri.getBlockId(), rri.getNumBytes() - 1, rri.getGenerationStamp());
            try {
                //update should fail
                fsdataset.updateReplicaUnderRecovery(tmp, recoveryid, tmp.getBlockId(), newlength);
                Assert.fail();
            } catch (IOException ioe) {
                System.out.println("GOOD: getting " + ioe);
            }
        }
        //update
        final Replica r = fsdataset.updateReplicaUnderRecovery(new ExtendedBlock(b.getBlockPoolId(), rri), recoveryid, rri.getBlockId(), newlength);
        assertTrue(r != null);
        assertTrue(r.getStorageUuid() != null);
    } finally {
        if (cluster != null)
            cluster.shutdown();
    }
}

Example 2 with RecoveringBlock

use of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock in project hadoop by apache.

the class InterDatanodeProtocolServerSideTranslatorPB method initReplicaRecovery.

@Override
public InitReplicaRecoveryResponseProto initReplicaRecovery(RpcController unused, InitReplicaRecoveryRequestProto request) throws ServiceException {
    RecoveringBlock b = PBHelper.convert(request.getBlock());
    ReplicaRecoveryInfo r;
    try {
        r = impl.initReplicaRecovery(b);
    } catch (IOException e) {
        throw new ServiceException(e);
    }
    if (r == null) {
        return InitReplicaRecoveryResponseProto.newBuilder().setReplicaFound(false).build();
    } else {
        return InitReplicaRecoveryResponseProto.newBuilder().setReplicaFound(true).setBlock(PBHelperClient.convert(r)).setState(PBHelper.convert(r.getOriginalReplicaState())).build();
    }
}

Example 3 with RecoveringBlock

use of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock in project hadoop by apache.

the class PBHelper method convert.

public static BlockRecoveryCommand convert(BlockRecoveryCommandProto recoveryCmd) {
    List<RecoveringBlockProto> list = recoveryCmd.getBlocksList();
    List<RecoveringBlock> recoveringBlocks = new ArrayList<RecoveringBlock>(list.size());
    for (RecoveringBlockProto rbp : list) {
        recoveringBlocks.add(PBHelper.convert(rbp));
    }
    return new BlockRecoveryCommand(recoveringBlocks);
}

Example 4 with RecoveringBlock

use of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock in project hadoop by apache.

the class TestBlockRecovery method testRaceBetweenReplicaRecoveryAndFinalizeBlock.

/**
   * Test to verify the race between finalizeBlock and Lease recovery
   * 
   * @throws Exception
   */
@Test(timeout = 20000)
public void testRaceBetweenReplicaRecoveryAndFinalizeBlock() throws Exception {
    // Stop the Mocked DN started in startup()
    tearDown();
    Configuration conf = new HdfsConfiguration();
    conf.set(DFSConfigKeys.DFS_DATANODE_XCEIVER_STOP_TIMEOUT_MILLIS_KEY, "1000");
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    try {
        cluster.waitClusterUp();
        DistributedFileSystem fs = cluster.getFileSystem();
        Path path = new Path("/test");
        FSDataOutputStream out = fs.create(path);
        out.writeBytes("data");
        out.hsync();
        List<LocatedBlock> blocks = DFSTestUtil.getAllBlocks(fs.open(path));
        final LocatedBlock block = blocks.get(0);
        final DataNode dataNode = cluster.getDataNodes().get(0);
        final AtomicBoolean recoveryInitResult = new AtomicBoolean(true);
        Thread recoveryThread = new Thread() {

            @Override
            public void run() {
                try {
                    DatanodeInfo[] locations = block.getLocations();
                    final RecoveringBlock recoveringBlock = new RecoveringBlock(block.getBlock(), locations, block.getBlock().getGenerationStamp() + 1);
                    try (AutoCloseableLock lock = dataNode.data.acquireDatasetLock()) {
                        Thread.sleep(2000);
                        dataNode.initReplicaRecovery(recoveringBlock);
                    }
                } catch (Exception e) {
                    recoveryInitResult.set(false);
                }
            }
        };
        recoveryThread.start();
        try {
            out.close();
        } catch (IOException e) {
            Assert.assertTrue("Writing should fail", e.getMessage().contains("are bad. Aborting..."));
        } finally {
            recoveryThread.join();
        }
        Assert.assertTrue("Recovery should be initiated successfully", recoveryInitResult.get());
        dataNode.updateReplicaUnderRecovery(block.getBlock(), block.getBlock().getGenerationStamp() + 1, block.getBlock().getBlockId(), block.getBlockSize());
    } finally {
        if (null != cluster) {
            cluster.shutdown();
            cluster = null;
        }
    }
}

Example 5 with RecoveringBlock

use of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock in project hadoop by apache.

the class TestBlockRecovery method testErrorReplicas.

/**
   * BlockRecoveryFI_06. all datanodes throws an exception.
   *
   * @throws IOException
   *           in case of an error
   */
@Test(timeout = 60000)
public void testErrorReplicas() throws IOException, InterruptedException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Running " + GenericTestUtils.getMethodName());
    }
    doThrow(new IOException()).when(spyDN).initReplicaRecovery(any(RecoveringBlock.class));
    for (RecoveringBlock rBlock : initRecoveringBlocks()) {
        BlockRecoveryWorker.RecoveryTaskContiguous RecoveryTaskContiguous = recoveryWorker.new RecoveryTaskContiguous(rBlock);
        BlockRecoveryWorker.RecoveryTaskContiguous spyTask = spy(RecoveryTaskContiguous);
        try {
            spyTask.recover();
            fail();
        } catch (IOException e) {
            GenericTestUtils.assertExceptionContains("All datanodes failed", e);
        }
        verify(spyTask, never()).syncBlock(anyListOf(BlockRecord.class));
    }
}