Examples with ReplicaHandler org.apache.hadoop.hdfs.server.datanode.ReplicaHandler used on opensource projects

Example 6 with ReplicaHandler

use of org.apache.hadoop.hdfs.server.datanode.ReplicaHandler in project hadoop by apache.

the class TestFsDatasetImpl method testRemoveVolumes.

@Test(timeout = 30000)
public void testRemoveVolumes() throws IOException {
    // Feed FsDataset with block metadata.
    final int NUM_BLOCKS = 100;
    for (int i = 0; i < NUM_BLOCKS; i++) {
        String bpid = BLOCK_POOL_IDS[NUM_BLOCKS % BLOCK_POOL_IDS.length];
        ExtendedBlock eb = new ExtendedBlock(bpid, i);
        try (ReplicaHandler replica = dataset.createRbw(StorageType.DEFAULT, eb, false)) {
        }
    }
    final String[] dataDirs = conf.get(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY).split(",");
    final String volumePathToRemove = dataDirs[0];
    Set<StorageLocation> volumesToRemove = new HashSet<>();
    volumesToRemove.add(StorageLocation.parse(volumePathToRemove));
    FsVolumeReferences volReferences = dataset.getFsVolumeReferences();
    FsVolumeImpl volumeToRemove = null;
    for (FsVolumeSpi vol : volReferences) {
        if (vol.getStorageLocation().equals(volumesToRemove.iterator().next())) {
            volumeToRemove = (FsVolumeImpl) vol;
        }
    }
    assertTrue(volumeToRemove != null);
    volReferences.close();
    dataset.removeVolumes(volumesToRemove, true);
    int expectedNumVolumes = dataDirs.length - 1;
    assertEquals("The volume has been removed from the volumeList.", expectedNumVolumes, getNumVolumes());
    assertEquals("The volume has been removed from the storageMap.", expectedNumVolumes, dataset.storageMap.size());
    try {
        dataset.asyncDiskService.execute(volumeToRemove, new Runnable() {

            @Override
            public void run() {
            }
        });
        fail("Expect RuntimeException: the volume has been removed from the " + "AsyncDiskService.");
    } catch (RuntimeException e) {
        GenericTestUtils.assertExceptionContains("Cannot find volume", e);
    }
    int totalNumReplicas = 0;
    for (String bpid : dataset.volumeMap.getBlockPoolList()) {
        totalNumReplicas += dataset.volumeMap.size(bpid);
    }
    assertEquals("The replica infos on this volume has been removed from the " + "volumeMap.", NUM_BLOCKS / NUM_INIT_VOLUMES, totalNumReplicas);
}

Example 7 with ReplicaHandler

use of org.apache.hadoop.hdfs.server.datanode.ReplicaHandler in project hadoop by apache.

the class FsDatasetImpl method recoverRbwImpl.

private ReplicaHandler recoverRbwImpl(ReplicaInPipeline rbw, ExtendedBlock b, long newGS, long minBytesRcvd, long maxBytesRcvd) throws IOException {
    try (AutoCloseableLock lock = datasetLock.acquire()) {
        // check generation stamp
        long replicaGenerationStamp = rbw.getGenerationStamp();
        if (replicaGenerationStamp < b.getGenerationStamp() || replicaGenerationStamp > newGS) {
            throw new ReplicaNotFoundException(ReplicaNotFoundException.UNEXPECTED_GS_REPLICA + b + ". Expected GS range is [" + b.getGenerationStamp() + ", " + newGS + "].");
        }
        // check replica length
        long bytesAcked = rbw.getBytesAcked();
        long numBytes = rbw.getNumBytes();
        if (bytesAcked < minBytesRcvd || numBytes > maxBytesRcvd) {
            throw new ReplicaNotFoundException("Unmatched length replica " + rbw + ": BytesAcked = " + bytesAcked + " BytesRcvd = " + numBytes + " are not in the range of [" + minBytesRcvd + ", " + maxBytesRcvd + "].");
        }
        FsVolumeReference ref = rbw.getReplicaInfo().getVolume().obtainReference();
        try {
            // any corrupt data written after the acked length can go unnoticed.
            if (numBytes > bytesAcked) {
                rbw.getReplicaInfo().truncateBlock(bytesAcked);
                rbw.setNumBytes(bytesAcked);
                rbw.setLastChecksumAndDataLen(bytesAcked, null);
            }
            // bump the replica's generation stamp to newGS
            rbw.getReplicaInfo().bumpReplicaGS(newGS);
        } catch (IOException e) {
            IOUtils.cleanup(null, ref);
            throw e;
        }
        return new ReplicaHandler(rbw, ref);
    }
}

Example 8 with ReplicaHandler

use of org.apache.hadoop.hdfs.server.datanode.ReplicaHandler in project hadoop by apache.

the class TestFsDatasetImpl method testRemoveVolumeBeingWritten.

@Test(timeout = 60000)
public void testRemoveVolumeBeingWritten() throws Exception {
    // Will write and remove on dn0.
    final ExtendedBlock eb = new ExtendedBlock(BLOCK_POOL_IDS[0], 0);
    final CountDownLatch startFinalizeLatch = new CountDownLatch(1);
    final CountDownLatch blockReportReceivedLatch = new CountDownLatch(1);
    final CountDownLatch volRemoveStartedLatch = new CountDownLatch(1);
    final CountDownLatch volRemoveCompletedLatch = new CountDownLatch(1);
    class BlockReportThread extends Thread {

        public void run() {
            // Lets wait for the volume remove process to start
            try {
                volRemoveStartedLatch.await();
            } catch (Exception e) {
                LOG.info("Unexpected exception when waiting for vol removal:", e);
            }
            LOG.info("Getting block report");
            dataset.getBlockReports(eb.getBlockPoolId());
            LOG.info("Successfully received block report");
            blockReportReceivedLatch.countDown();
        }
    }
    class ResponderThread extends Thread {

        public void run() {
            try (ReplicaHandler replica = dataset.createRbw(StorageType.DEFAULT, eb, false)) {
                LOG.info("CreateRbw finished");
                startFinalizeLatch.countDown();
                // Ignore any interrupts coming out of volume shutdown.
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException ie) {
                    LOG.info("Ignoring ", ie);
                }
                // Lets wait for the other thread finish getting block report
                blockReportReceivedLatch.await();
                dataset.finalizeBlock(eb);
                LOG.info("FinalizeBlock finished");
            } catch (Exception e) {
                LOG.warn("Exception caught. This should not affect the test", e);
            }
        }
    }
    class VolRemoveThread extends Thread {

        public void run() {
            Set<StorageLocation> volumesToRemove = new HashSet<>();
            try {
                volumesToRemove.add(dataset.getVolume(eb).getStorageLocation());
            } catch (Exception e) {
                LOG.info("Problem preparing volumes to remove: ", e);
                Assert.fail("Exception in remove volume thread, check log for " + "details.");
            }
            LOG.info("Removing volume " + volumesToRemove);
            dataset.removeVolumes(volumesToRemove, true);
            volRemoveCompletedLatch.countDown();
            LOG.info("Removed volume " + volumesToRemove);
        }
    }
    // Start the volume write operation
    ResponderThread responderThread = new ResponderThread();
    responderThread.start();
    startFinalizeLatch.await();
    // Start the block report get operation
    final BlockReportThread blockReportThread = new BlockReportThread();
    blockReportThread.start();
    // Start the volume remove operation
    VolRemoveThread volRemoveThread = new VolRemoveThread();
    volRemoveThread.start();
    // Let volume write and remove operation be
    // blocked for few seconds
    Thread.sleep(2000);
    // Signal block report receiver and volume writer
    // thread to complete their operations so that vol
    // remove can proceed
    volRemoveStartedLatch.countDown();
    // Verify if block report can be received
    // when volume is in use and also being removed
    blockReportReceivedLatch.await();
    // Verify if volume can be removed safely when there
    // are read/write operation in-progress
    volRemoveCompletedLatch.await();
}