Examples with RecoveringBlock org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock used on opensource projects

Example 6 with RecoveringBlock

use of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock in project hadoop by apache.

the class TestBlockRecovery method initRecoveringBlocks.

private Collection<RecoveringBlock> initRecoveringBlocks() throws IOException {
    Collection<RecoveringBlock> blocks = new ArrayList<RecoveringBlock>(1);
    DatanodeInfo mockOtherDN = DFSTestUtil.getLocalDatanodeInfo();
    DatanodeInfo[] locs = new DatanodeInfo[] { new DatanodeInfoBuilder().setNodeID(dn.getDNRegistrationForBP(block.getBlockPoolId())).build(), mockOtherDN };
    RecoveringBlock rBlock = new RecoveringBlock(block, locs, RECOVERY_ID);
    blocks.add(rBlock);
    return blocks;
}

Example 7 with RecoveringBlock

use of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock in project hadoop by apache.

the class TestBlockRecovery method testRURReplicas.

/**
   * DNs report RUR instead of RBW, RWR or FINALIZED. Primary DN expected to
   * throw an exception.
   * @throws Exception
   */
@Test(timeout = 60000)
public void testRURReplicas() throws Exception {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Running " + GenericTestUtils.getMethodName());
    }
    doReturn(new ReplicaRecoveryInfo(block.getBlockId(), block.getNumBytes(), block.getGenerationStamp(), ReplicaState.RUR)).when(spyDN).initReplicaRecovery(any(RecoveringBlock.class));
    boolean exceptionThrown = false;
    try {
        for (RecoveringBlock rBlock : initRecoveringBlocks()) {
            BlockRecoveryWorker.RecoveryTaskContiguous RecoveryTaskContiguous = recoveryWorker.new RecoveryTaskContiguous(rBlock);
            BlockRecoveryWorker.RecoveryTaskContiguous spyTask = spy(RecoveryTaskContiguous);
            spyTask.recover();
        }
    } catch (IOException e) {
        // expect IOException to be thrown here
        e.printStackTrace();
        assertTrue("Wrong exception was thrown: " + e.getMessage(), e.getMessage().contains("Found 1 replica(s) for block " + block + " but none is in RWR or better state"));
        exceptionThrown = true;
    } finally {
        assertTrue(exceptionThrown);
    }
}

Example 8 with RecoveringBlock

use of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock in project hadoop by apache.

the class TestBlockRecovery method testStopWorker.

/**
   * Test that an FsDatasetImpl operation does not hold the lock for an
   * unreasonable amount of time if a writer is taking a long time to stop.
   */
private void testStopWorker(final TestStopWorkerRunnable tswr) throws Exception {
    LOG.debug("Running " + currentTestName.getMethodName());
    // We need a long value for the data xceiver stop timeout.
    // Otherwise the timeout will trigger, and we will not have tested that
    // thread join was done locklessly.
    Assert.assertEquals(TEST_STOP_WORKER_XCEIVER_STOP_TIMEOUT_MILLIS, dn.getDnConf().getXceiverStopTimeout());
    final TestStopWorkerSemaphore progressParent = new TestStopWorkerSemaphore();
    final TestStopWorkerSemaphore terminateSlowWriter = new TestStopWorkerSemaphore();
    final AtomicReference<String> failure = new AtomicReference<String>(null);
    Collection<RecoveringBlock> recoveringBlocks = initRecoveringBlocks();
    final RecoveringBlock recoveringBlock = Iterators.get(recoveringBlocks.iterator(), 0);
    final ExtendedBlock block = recoveringBlock.getBlock();
    Thread slowWriterThread = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                // Register this thread as the writer for the recoveringBlock.
                LOG.debug("slowWriter creating rbw");
                ReplicaHandler replicaHandler = spyDN.data.createRbw(StorageType.DISK, block, false);
                replicaHandler.close();
                LOG.debug("slowWriter created rbw");
                // Tell the parent thread to start progressing.
                progressParent.sem.release();
                terminateSlowWriter.uninterruptiblyAcquire(60000);
                LOG.debug("slowWriter exiting");
            } catch (Throwable t) {
                LOG.error("slowWriter got exception", t);
                failure.compareAndSet(null, "slowWriter got exception " + t.getMessage());
            }
        }
    });
    // Start the slow worker thread and wait for it to take ownership of the
    // ReplicaInPipeline
    slowWriterThread.start();
    progressParent.uninterruptiblyAcquire(60000);
    // Start a worker thread which will attempt to stop the writer.
    Thread stopWriterThread = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                LOG.debug("initiating " + tswr.opName());
                tswr.run(recoveringBlock);
                LOG.debug("finished " + tswr.opName());
            } catch (Throwable t) {
                LOG.error("stopWriterThread got unexpected exception for " + tswr.opName(), t);
                failure.compareAndSet(null, "stopWriterThread got unexpected " + "exception for " + tswr.opName() + ": " + t.getMessage());
            }
        }
    });
    stopWriterThread.start();
    while (!terminateSlowWriter.gotInterruption.get()) {
        // Wait until stopWriterThread attempts to stop our slow writer by sending
        // it an InterruptedException.
        Thread.sleep(1);
    }
    // We know that stopWriterThread is in the process of joining our slow
    // writer.  It must not hold the lock during this operation.
    // In order to test that it does not, we attempt to do an operation that
    // requires the lock-- getReplicaString.
    spyDN.getFSDataset().getReplicaString(recoveringBlock.getBlock().getBlockPoolId(), recoveringBlock.getBlock().getBlockId());
    // Tell the slow writer to exit, and then wait for all threads to join.
    terminateSlowWriter.sem.release();
    slowWriterThread.join();
    stopWriterThread.join();
    // Check that our worker threads exited cleanly.  This is not checked by the
    // unit test framework, so we have to do it manually here.
    String failureReason = failure.get();
    if (failureReason != null) {
        Assert.fail("Thread failure: " + failureReason);
    }
}

Example 9 with RecoveringBlock

use of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock in project hadoop by apache.

the class TestBlockRecovery method testRecoveryInProgressException.

/**
   * BlockRecoveryFI_05. One DN throws RecoveryInProgressException.
   *
   * @throws IOException
   *           in case of an error
   */
@Test(timeout = 60000)
public void testRecoveryInProgressException() throws IOException, InterruptedException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Running " + GenericTestUtils.getMethodName());
    }
    doThrow(new RecoveryInProgressException("Replica recovery is in progress")).when(spyDN).initReplicaRecovery(any(RecoveringBlock.class));
    for (RecoveringBlock rBlock : initRecoveringBlocks()) {
        BlockRecoveryWorker.RecoveryTaskContiguous RecoveryTaskContiguous = recoveryWorker.new RecoveryTaskContiguous(rBlock);
        BlockRecoveryWorker.RecoveryTaskContiguous spyTask = spy(RecoveryTaskContiguous);
        spyTask.recover();
        verify(spyTask, never()).syncBlock(anyListOf(BlockRecord.class));
    }
}

Example 10 with RecoveringBlock

use of org.apache.hadoop.hdfs.server.protocol.BlockRecoveryCommand.RecoveringBlock in project hadoop by apache.

the class TestBlockRecovery method testSyncReplicas.

/** Sync two replicas */
private void testSyncReplicas(ReplicaRecoveryInfo replica1, ReplicaRecoveryInfo replica2, InterDatanodeProtocol dn1, InterDatanodeProtocol dn2, long expectLen) throws IOException {
    DatanodeInfo[] locs = new DatanodeInfo[] { mock(DatanodeInfo.class), mock(DatanodeInfo.class) };
    RecoveringBlock rBlock = new RecoveringBlock(block, locs, RECOVERY_ID);
    ArrayList<BlockRecord> syncList = new ArrayList<BlockRecord>(2);
    BlockRecord record1 = new BlockRecord(DFSTestUtil.getDatanodeInfo("1.2.3.4", "bogus", 1234), dn1, replica1);
    BlockRecord record2 = new BlockRecord(DFSTestUtil.getDatanodeInfo("1.2.3.4", "bogus", 1234), dn2, replica2);
    syncList.add(record1);
    syncList.add(record2);
    when(dn1.updateReplicaUnderRecovery((ExtendedBlock) anyObject(), anyLong(), anyLong(), anyLong())).thenReturn("storage1");
    when(dn2.updateReplicaUnderRecovery((ExtendedBlock) anyObject(), anyLong(), anyLong(), anyLong())).thenReturn("storage2");
    BlockRecoveryWorker.RecoveryTaskContiguous RecoveryTaskContiguous = recoveryWorker.new RecoveryTaskContiguous(rBlock);
    RecoveryTaskContiguous.syncBlock(syncList);
}