Examples with DelayAnswer org.apache.hadoop.test.GenericTestUtils.DelayAnswer used on opensource projects

Example 6 with DelayAnswer

use of org.apache.hadoop.test.GenericTestUtils.DelayAnswer in project hadoop by apache.

the class TestIPCLoggerChannel method testQueueLimiting.

/**
   * Test that, once the queue eclipses the configure size limit,
   * calls to journal more data are rejected.
   */
@Test
public void testQueueLimiting() throws Exception {
    // Block the underlying fake proxy from actually completing any calls.
    DelayAnswer delayer = new DelayAnswer(LOG);
    Mockito.doAnswer(delayer).when(mockProxy).journal(Mockito.<RequestInfo>any(), Mockito.eq(1L), Mockito.eq(1L), Mockito.eq(1), Mockito.same(FAKE_DATA));
    // Queue up the maximum number of calls.
    int numToQueue = LIMIT_QUEUE_SIZE_BYTES / FAKE_DATA.length;
    for (int i = 1; i <= numToQueue; i++) {
        ch.sendEdits(1L, (long) i, 1, FAKE_DATA);
    }
    // The accounting should show the correct total number queued.
    assertEquals(LIMIT_QUEUE_SIZE_BYTES, ch.getQueuedEditsSize());
    // Trying to queue any more should fail.
    try {
        ch.sendEdits(1L, numToQueue + 1, 1, FAKE_DATA).get(1, TimeUnit.SECONDS);
        fail("Did not fail to queue more calls after queue was full");
    } catch (ExecutionException ee) {
        if (!(ee.getCause() instanceof LoggerTooFarBehindException)) {
            throw ee;
        }
    }
    delayer.proceed();
    // After we allow it to proceeed, it should chug through the original queue
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        @Override
        public Boolean get() {
            return ch.getQueuedEditsSize() == 0;
        }
    }, 10, 1000);
}

Example 7 with DelayAnswer

use of org.apache.hadoop.test.GenericTestUtils.DelayAnswer in project hadoop by apache.

the class BlockReportTestBase method testOneReplicaRbwReportArrivesAfterBlockCompleted.

/**
   * Test for the case where one of the DNs in the pipeline is in the
   * process of doing a block report exactly when the block is closed.
   * In this case, the block report becomes delayed until after the
   * block is marked completed on the NN, and hence it reports an RBW
   * replica for a COMPLETE block. Such a report should not be marked
   * corrupt.
   * This is a regression test for HDFS-2791.
   */
@Test(timeout = 300000)
public void testOneReplicaRbwReportArrivesAfterBlockCompleted() throws Exception {
    final CountDownLatch brFinished = new CountDownLatch(1);
    DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG) {

        @Override
        protected Object passThrough(InvocationOnMock invocation) throws Throwable {
            try {
                return super.passThrough(invocation);
            } finally {
                // inform the test that our block report went through.
                brFinished.countDown();
            }
        }
    };
    final String METHOD_NAME = GenericTestUtils.getMethodName();
    Path filePath = new Path("/" + METHOD_NAME + ".dat");
    // Start a second DN for this test -- we're checking
    // what happens when one of the DNs is slowed for some reason.
    REPL_FACTOR = 2;
    startDNandWait(null, false);
    NameNode nn = cluster.getNameNode();
    FSDataOutputStream out = fs.create(filePath, REPL_FACTOR);
    try {
        AppendTestUtil.write(out, 0, 10);
        out.hflush();
        // Set up a spy so that we can delay the block report coming
        // from this node.
        DataNode dn = cluster.getDataNodes().get(0);
        DatanodeProtocolClientSideTranslatorPB spy = InternalDataNodeTestUtils.spyOnBposToNN(dn, nn);
        Mockito.doAnswer(delayer).when(spy).blockReport(Mockito.<DatanodeRegistration>anyObject(), Mockito.anyString(), Mockito.<StorageBlockReport[]>anyObject(), Mockito.<BlockReportContext>anyObject());
        // Force a block report to be generated. The block report will have
        // an RBW replica in it. Wait for the RPC to be sent, but block
        // it before it gets to the NN.
        dn.scheduleAllBlockReport(0);
        delayer.waitForCall();
    } finally {
        IOUtils.closeStream(out);
    }
    // Now that the stream is closed, the NN will have the block in COMPLETE
    // state.
    delayer.proceed();
    brFinished.await();
    // Verify that no replicas are marked corrupt, and that the
    // file is still readable.
    BlockManagerTestUtil.updateState(nn.getNamesystem().getBlockManager());
    assertEquals(0, nn.getNamesystem().getCorruptReplicaBlocks());
    DFSTestUtil.readFile(fs, filePath);
    // Ensure that the file is readable even from the DN that we futzed with.
    cluster.stopDataNode(1);
    DFSTestUtil.readFile(fs, filePath);
}

Example 8 with DelayAnswer

use of org.apache.hadoop.test.GenericTestUtils.DelayAnswer in project hadoop by apache.

the class TestCheckpoint method testMultipleSecondaryNNsAgainstSameNN2.

/**
   * Test case where two secondary namenodes are checkpointing the same
   * NameNode. This differs from {@link #testMultipleSecondaryNamenodes()}
   * since that test runs against two distinct NNs.
   * 
   * This case tests the following interleaving:
   * - 2NN A) calls rollEdits()
   * - 2NN B) calls rollEdits()
   * - 2NN A) paused at getRemoteEditLogManifest()
   * - 2NN B) calls getRemoteEditLogManifest() (returns up to txid 4)
   * - 2NN B) uploads checkpoint fsimage_4
   * - 2NN A) allowed to proceed, also returns up to txid 4
   * - 2NN A) uploads checkpoint fsimage_4 as well, should fail gracefully
   * 
   * It verifies that one of the two gets an error that it's uploading a
   * duplicate checkpoint, and the other one succeeds.
   */
@Test
public void testMultipleSecondaryNNsAgainstSameNN2() throws Exception {
    Configuration conf = new HdfsConfiguration();
    MiniDFSCluster cluster = null;
    SecondaryNameNode secondary1 = null, secondary2 = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).format(true).build();
        // Start 2NNs
        secondary1 = startSecondaryNameNode(conf, 1);
        secondary2 = startSecondaryNameNode(conf, 2);
        // Make the first 2NN's checkpoint process delayable - we can pause it
        // right before it calls getRemoteEditLogManifest.
        // The method to set up a spy on an RPC protocol is a little bit involved
        // since we can't spy directly on a proxy object. This sets up a mock
        // which delegates all its calls to the original object, instead.
        final NamenodeProtocol origNN = secondary1.getNameNode();
        final Answer<Object> delegator = new GenericTestUtils.DelegateAnswer(origNN);
        NamenodeProtocol spyNN = Mockito.mock(NamenodeProtocol.class, delegator);
        DelayAnswer delayer = new DelayAnswer(LOG) {

            @Override
            protected Object passThrough(InvocationOnMock invocation) throws Throwable {
                return delegator.answer(invocation);
            }
        };
        secondary1.setNameNode(spyNN);
        Mockito.doAnswer(delayer).when(spyNN).getEditLogManifest(Mockito.anyLong());
        // Set up a thread to do a checkpoint from the first 2NN
        DoCheckpointThread checkpointThread = new DoCheckpointThread(secondary1);
        checkpointThread.start();
        // Wait for the first checkpointer to be about to call getEditLogManifest
        delayer.waitForCall();
        // Now make the second checkpointer run an entire checkpoint
        secondary2.doCheckpoint();
        // NN should have now received fsimage_4
        NNStorage storage = cluster.getNameNode().getFSImage().getStorage();
        assertEquals(4, storage.getMostRecentCheckpointTxId());
        // Let the first one finish
        delayer.proceed();
        // Letting the first node continue, it should try to upload the
        // same image, and gracefully ignore it, while logging an
        // error message.
        checkpointThread.join();
        checkpointThread.propagateExceptions();
        // primary should still consider fsimage_4 the latest
        assertEquals(4, storage.getMostRecentCheckpointTxId());
        // Now have second one checkpoint one more time just to make sure that
        // the NN isn't left in a broken state
        secondary2.doCheckpoint();
        assertEquals(6, storage.getMostRecentCheckpointTxId());
        // Should have accepted both checkpoints
        assertNNHasCheckpoints(cluster, ImmutableList.of(4, 6));
        // Let the first one also go again on its own to make sure it can
        // continue at next checkpoint
        secondary1.setNameNode(origNN);
        secondary1.doCheckpoint();
        // NN should have received new checkpoint
        assertEquals(8, storage.getMostRecentCheckpointTxId());
        // Validate invariant that files named the same are the same.
        assertParallelFilesInvariant(cluster, ImmutableList.of(secondary1, secondary2));
        // Validate that the NN received checkpoints at expected txids
        // (i.e that both checkpoints went through)
        assertNNHasCheckpoints(cluster, ImmutableList.of(6, 8));
    } finally {
        cleanup(secondary1);
        secondary1 = null;
        cleanup(secondary2);
        secondary2 = null;
        cleanup(cluster);
        cluster = null;
    }
}

Example 9 with DelayAnswer

use of org.apache.hadoop.test.GenericTestUtils.DelayAnswer in project hadoop by apache.

the class TestDeleteRace method testDeleteAndCommitBlockSynchronizationRace.

/**
   * Test race between delete operation and commitBlockSynchronization method.
   * See HDFS-6825.
   * @param hasSnapshot
   * @throws Exception
   */
private void testDeleteAndCommitBlockSynchronizationRace(boolean hasSnapshot) throws Exception {
    LOG.info("Start testing, hasSnapshot: " + hasSnapshot);
    ArrayList<AbstractMap.SimpleImmutableEntry<String, Boolean>> testList = new ArrayList<AbstractMap.SimpleImmutableEntry<String, Boolean>>();
    testList.add(new AbstractMap.SimpleImmutableEntry<String, Boolean>("/test-file", false));
    testList.add(new AbstractMap.SimpleImmutableEntry<String, Boolean>("/test-file1", true));
    testList.add(new AbstractMap.SimpleImmutableEntry<String, Boolean>("/testdir/testdir1/test-file", false));
    testList.add(new AbstractMap.SimpleImmutableEntry<String, Boolean>("/testdir/testdir1/test-file1", true));
    final Path rootPath = new Path("/");
    final Configuration conf = new Configuration();
    // Disable permissions so that another user can recover the lease.
    conf.setBoolean(DFSConfigKeys.DFS_PERMISSIONS_ENABLED_KEY, false);
    conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
    FSDataOutputStream stm = null;
    Map<DataNode, DatanodeProtocolClientSideTranslatorPB> dnMap = new HashMap<DataNode, DatanodeProtocolClientSideTranslatorPB>();
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
        cluster.waitActive();
        DistributedFileSystem fs = cluster.getFileSystem();
        int stId = 0;
        for (AbstractMap.SimpleImmutableEntry<String, Boolean> stest : testList) {
            String testPath = stest.getKey();
            Boolean mkSameDir = stest.getValue();
            LOG.info("test on " + testPath + " mkSameDir: " + mkSameDir + " snapshot: " + hasSnapshot);
            Path fPath = new Path(testPath);
            //find grandest non-root parent
            Path grandestNonRootParent = fPath;
            while (!grandestNonRootParent.getParent().equals(rootPath)) {
                grandestNonRootParent = grandestNonRootParent.getParent();
            }
            stm = fs.create(fPath);
            LOG.info("test on " + testPath + " created " + fPath);
            // write a half block
            AppendTestUtil.write(stm, 0, BLOCK_SIZE / 2);
            stm.hflush();
            if (hasSnapshot) {
                SnapshotTestHelper.createSnapshot(fs, rootPath, "st" + String.valueOf(stId));
                ++stId;
            }
            // Look into the block manager on the active node for the block
            // under construction.
            NameNode nn = cluster.getNameNode();
            ExtendedBlock blk = DFSTestUtil.getFirstBlock(fs, fPath);
            DatanodeDescriptor expectedPrimary = DFSTestUtil.getExpectedPrimaryNode(nn, blk);
            LOG.info("Expecting block recovery to be triggered on DN " + expectedPrimary);
            // Find the corresponding DN daemon, and spy on its connection to the
            // active.
            DataNode primaryDN = cluster.getDataNode(expectedPrimary.getIpcPort());
            DatanodeProtocolClientSideTranslatorPB nnSpy = dnMap.get(primaryDN);
            if (nnSpy == null) {
                nnSpy = InternalDataNodeTestUtils.spyOnBposToNN(primaryDN, nn);
                dnMap.put(primaryDN, nnSpy);
            }
            // Delay the commitBlockSynchronization call
            DelayAnswer delayer = new DelayAnswer(LOG);
            Mockito.doAnswer(delayer).when(nnSpy).commitBlockSynchronization(Mockito.eq(blk), // new genstamp
            Mockito.anyInt(), // new length
            Mockito.anyLong(), // close file
            Mockito.eq(true), // delete block
            Mockito.eq(false), // new targets
            (DatanodeID[]) Mockito.anyObject(), // new target storages
            (String[]) Mockito.anyObject());
            fs.recoverLease(fPath);
            LOG.info("Waiting for commitBlockSynchronization call from primary");
            delayer.waitForCall();
            LOG.info("Deleting recursively " + grandestNonRootParent);
            fs.delete(grandestNonRootParent, true);
            if (mkSameDir && !grandestNonRootParent.toString().equals(testPath)) {
                LOG.info("Recreate dir " + grandestNonRootParent + " testpath: " + testPath);
                fs.mkdirs(grandestNonRootParent);
            }
            delayer.proceed();
            LOG.info("Now wait for result");
            delayer.waitForResult();
            Throwable t = delayer.getThrown();
            if (t != null) {
                LOG.info("Result exception (snapshot: " + hasSnapshot + "): " + t);
            }
        }
        // end of loop each fPath
        LOG.info("Now check we can restart");
        cluster.restartNameNodes();
        LOG.info("Restart finished");
    } finally {
        if (stm != null) {
            IOUtils.closeStream(stm);
        }
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 10 with DelayAnswer

use of org.apache.hadoop.test.GenericTestUtils.DelayAnswer in project hadoop by apache.

the class TestSaveNamespace method testCancelSaveNamespace.

@Test(timeout = 20000)
public void testCancelSaveNamespace() throws Exception {
    Configuration conf = getConf();
    NameNode.initMetrics(conf, NamenodeRole.NAMENODE);
    DFSTestUtil.formatNameNode(conf);
    FSNamesystem fsn = FSNamesystem.loadFromDisk(conf);
    // Replace the FSImage with a spy
    final FSImage image = fsn.getFSImage();
    NNStorage storage = image.getStorage();
    // unlock any directories that
    // FSNamesystem's initialization may have locked
    storage.close();
    storage.setStorageDirectories(FSNamesystem.getNamespaceDirs(conf), FSNamesystem.getNamespaceEditsDirs(conf));
    FSNamesystem spyFsn = spy(fsn);
    final FSNamesystem finalFsn = spyFsn;
    DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG);
    BlockIdManager bid = spy(spyFsn.getBlockManager().getBlockIdManager());
    Whitebox.setInternalState(finalFsn.getBlockManager(), "blockIdManager", bid);
    doAnswer(delayer).when(bid).getGenerationStamp();
    ExecutorService pool = Executors.newFixedThreadPool(2);
    try {
        doAnEdit(fsn, 1);
        final Canceler canceler = new Canceler();
        // Save namespace
        fsn.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
        try {
            Future<Void> saverFuture = pool.submit(new Callable<Void>() {

                @Override
                public Void call() throws Exception {
                    image.saveNamespace(finalFsn, NameNodeFile.IMAGE, canceler);
                    return null;
                }
            });
            // Wait until saveNamespace calls getGenerationStamp
            delayer.waitForCall();
            // then cancel the saveNamespace
            Future<Void> cancelFuture = pool.submit(new Callable<Void>() {

                @Override
                public Void call() throws Exception {
                    canceler.cancel("cancelled");
                    return null;
                }
            });
            // give the cancel call time to run
            Thread.sleep(500);
            // allow saveNamespace to proceed - it should check the cancel flag
            // after this point and throw an exception
            delayer.proceed();
            cancelFuture.get();
            saverFuture.get();
            fail("saveNamespace did not fail even though cancelled!");
        } catch (Throwable t) {
            GenericTestUtils.assertExceptionContains("SaveNamespaceCancelledException", t);
        }
        LOG.info("Successfully cancelled a saveNamespace");
        // Check that we have only the original image and not any
        // cruft left over from half-finished images
        FSImageTestUtil.logStorageContents(LOG, storage);
        for (StorageDirectory sd : storage.dirIterable(null)) {
            File curDir = sd.getCurrentDir();
            GenericTestUtils.assertGlobEquals(curDir, "fsimage_.*", NNStorage.getImageFileName(0), NNStorage.getImageFileName(0) + MD5FileUtils.MD5_SUFFIX);
        }
    } finally {
        fsn.close();
    }
}