Examples with NamenodeProtocols org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols used on opensource projects

Example 46 with NamenodeProtocols

use of org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols in project hadoop by apache.

the class TestCheckpoint method testCheckpointWithSeparateDirsAfterNameFails.

/**
   * Test case where the NN is configured with a name-only and an edits-only
   * dir, with storage-restore turned on. In this case, if the name-only dir
   * disappears and comes back, a new checkpoint after it has been restored
   * should function correctly.
   * @throws Exception
   */
@Test
public void testCheckpointWithSeparateDirsAfterNameFails() throws Exception {
    MiniDFSCluster cluster = null;
    SecondaryNameNode secondary = null;
    File currentDir = null;
    Configuration conf = new HdfsConfiguration();
    File base_dir = new File(MiniDFSCluster.getBaseDirectory());
    conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_RESTORE_KEY, true);
    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, MiniDFSCluster.getBaseDirectory() + "/name-only");
    conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, MiniDFSCluster.getBaseDirectory() + "/edits-only");
    conf.set(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_DIR_KEY, fileAsURI(new File(base_dir, "namesecondary1")).toString());
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).format(true).manageNameDfsDirs(false).build();
        secondary = startSecondaryNameNode(conf);
        // Checkpoint once
        secondary.doCheckpoint();
        // Now primary NN experiences failure of its only name dir -- fake by
        // setting its current dir to a-x permissions
        NamenodeProtocols nn = cluster.getNameNodeRpc();
        NNStorage storage = cluster.getNameNode().getFSImage().getStorage();
        StorageDirectory sd0 = storage.getStorageDir(0);
        assertEquals(NameNodeDirType.IMAGE, sd0.getStorageDirType());
        currentDir = sd0.getCurrentDir();
        assertEquals(0, FileUtil.chmod(currentDir.getAbsolutePath(), "000"));
        // valid storage dirs
        try {
            secondary.doCheckpoint();
            fail("Did not fail to checkpoint when there are no valid storage dirs");
        } catch (IOException ioe) {
            GenericTestUtils.assertExceptionContains("No targets in destination storage", ioe);
        }
        // Restore the good dir
        assertEquals(0, FileUtil.chmod(currentDir.getAbsolutePath(), "755"));
        nn.restoreFailedStorage("true");
        nn.rollEditLog();
        // Checkpoint again -- this should upload to the restored name dir
        secondary.doCheckpoint();
        assertNNHasCheckpoints(cluster, ImmutableList.of(8));
        assertParallelFilesInvariant(cluster, ImmutableList.of(secondary));
    } finally {
        if (currentDir != null) {
            FileUtil.chmod(currentDir.getAbsolutePath(), "755");
        }
        cleanup(secondary);
        secondary = null;
        cleanup(cluster);
        cluster = null;
    }
}

Example 47 with NamenodeProtocols

use of org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols in project hadoop by apache.

the class TestBackupNode method testBackupNodeTailsEdits.

/**
   * Ensure that the backupnode will tail edits from the NN
   * and keep in sync, even while the NN rolls, checkpoints
   * occur, etc.
   */
@Test
public void testBackupNodeTailsEdits() throws Exception {
    Configuration conf = new HdfsConfiguration();
    HAUtil.setAllowStandbyReads(conf, true);
    MiniDFSCluster cluster = null;
    FileSystem fileSys = null;
    BackupNode backup = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
        fileSys = cluster.getFileSystem();
        backup = startBackupNode(conf, StartupOption.BACKUP, 1);
        BackupImage bnImage = (BackupImage) backup.getFSImage();
        testBNInSync(cluster, backup, 1);
        // Force a roll -- BN should roll with NN.
        NameNode nn = cluster.getNameNode();
        NamenodeProtocols nnRpc = nn.getRpcServer();
        nnRpc.rollEditLog();
        assertEquals(bnImage.getEditLog().getCurSegmentTxId(), nn.getFSImage().getEditLog().getCurSegmentTxId());
        // BN should stay in sync after roll
        testBNInSync(cluster, backup, 2);
        long nnImageBefore = nn.getFSImage().getStorage().getMostRecentCheckpointTxId();
        // BN checkpoint
        backup.doCheckpoint();
        // NN should have received a new image
        long nnImageAfter = nn.getFSImage().getStorage().getMostRecentCheckpointTxId();
        assertTrue("nn should have received new checkpoint. before: " + nnImageBefore + " after: " + nnImageAfter, nnImageAfter > nnImageBefore);
        // BN should stay in sync after checkpoint
        testBNInSync(cluster, backup, 3);
        // Stop BN
        StorageDirectory sd = bnImage.getStorage().getStorageDir(0);
        backup.stop();
        backup = null;
        // When shutting down the BN, it shouldn't finalize logs that are
        // still open on the NN
        EditLogFile editsLog = FSImageTestUtil.findLatestEditsLog(sd);
        assertEquals(editsLog.getFirstTxId(), nn.getFSImage().getEditLog().getCurSegmentTxId());
        assertTrue("Should not have finalized " + editsLog, editsLog.isInProgress());
        // do some edits
        assertTrue(fileSys.mkdirs(new Path("/edit-while-bn-down")));
        // start a new backup node
        backup = startBackupNode(conf, StartupOption.BACKUP, 1);
        testBNInSync(cluster, backup, 4);
        assertNotNull(backup.getNamesystem().getFileInfo("/edit-while-bn-down", false));
        // Trigger an unclean shutdown of the backup node. Backup node will not
        // unregister from the active when this is done simulating a node crash.
        backup.stop(false);
        // do some edits on the active. This should go through without failing.
        // This will verify that active is still up and can add entries to
        // master editlog.
        assertTrue(fileSys.mkdirs(new Path("/edit-while-bn-down-2")));
    } finally {
        LOG.info("Shutting down...");
        if (backup != null)
            backup.stop();
        if (fileSys != null)
            fileSys.close();
        if (cluster != null)
            cluster.shutdown();
    }
    assertStorageDirsMatch(cluster.getNameNode(), backup);
}

Example 48 with NamenodeProtocols

use of org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols in project hadoop by apache.

the class TestAddBlockRetry method testRetryAddBlockWhileInChooseTarget.

/**
   * Retry addBlock() while another thread is in chooseTarget().
   * See HDFS-4452.
   */
@Test
public void testRetryAddBlockWhileInChooseTarget() throws Exception {
    final String src = "/testRetryAddBlockWhileInChooseTarget";
    final FSNamesystem ns = cluster.getNamesystem();
    final NamenodeProtocols nn = cluster.getNameNodeRpc();
    // create file
    nn.create(src, FsPermission.getFileDefault(), "clientName", new EnumSetWritable<CreateFlag>(EnumSet.of(CreateFlag.CREATE)), true, (short) 3, 1024, null);
    // start first addBlock()
    LOG.info("Starting first addBlock for " + src);
    LocatedBlock[] onRetryBlock = new LocatedBlock[1];
    ns.readLock();
    FSDirWriteFileOp.ValidateAddBlockResult r;
    FSPermissionChecker pc = Mockito.mock(FSPermissionChecker.class);
    try {
        r = FSDirWriteFileOp.validateAddBlock(ns, pc, src, HdfsConstants.GRANDFATHER_INODE_ID, "clientName", null, onRetryBlock);
    } finally {
        ns.readUnlock();
        ;
    }
    DatanodeStorageInfo[] targets = FSDirWriteFileOp.chooseTargetForNewBlock(ns.getBlockManager(), src, null, null, null, r);
    assertNotNull("Targets must be generated", targets);
    // run second addBlock()
    LOG.info("Starting second addBlock for " + src);
    nn.addBlock(src, "clientName", null, null, HdfsConstants.GRANDFATHER_INODE_ID, null, null);
    assertTrue("Penultimate block must be complete", checkFileProgress(src, false));
    LocatedBlocks lbs = nn.getBlockLocations(src, 0, Long.MAX_VALUE);
    assertEquals("Must be one block", 1, lbs.getLocatedBlocks().size());
    LocatedBlock lb2 = lbs.get(0);
    assertEquals("Wrong replication", REPLICATION, lb2.getLocations().length);
    // continue first addBlock()
    ns.writeLock();
    LocatedBlock newBlock;
    try {
        newBlock = FSDirWriteFileOp.storeAllocatedBlock(ns, src, HdfsConstants.GRANDFATHER_INODE_ID, "clientName", null, targets);
    } finally {
        ns.writeUnlock();
    }
    assertEquals("Blocks are not equal", lb2.getBlock(), newBlock.getBlock());
    // check locations
    lbs = nn.getBlockLocations(src, 0, Long.MAX_VALUE);
    assertEquals("Must be one block", 1, lbs.getLocatedBlocks().size());
    LocatedBlock lb1 = lbs.get(0);
    assertEquals("Wrong replication", REPLICATION, lb1.getLocations().length);
    assertEquals("Blocks are not equal", lb1.getBlock(), lb2.getBlock());
}

Example 49 with NamenodeProtocols

use of org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols in project hadoop by apache.

the class TestDFSClientRetries method testNotYetReplicatedErrors.

// more tests related to different failure cases can be added here.
/**
   * Verify that client will correctly give up after the specified number
   * of times trying to add a block
   */
@SuppressWarnings({ "serial", "unchecked" })
@Test
public void testNotYetReplicatedErrors() throws IOException {
    final String exceptionMsg = "Nope, not replicated yet...";
    // Allow one retry (total of two calls)
    final int maxRetries = 1;
    conf.setInt(HdfsClientConfigKeys.BlockWrite.LOCATEFOLLOWINGBLOCK_RETRIES_KEY, maxRetries);
    NamenodeProtocols mockNN = mock(NamenodeProtocols.class);
    Answer<Object> answer = new ThrowsException(new IOException()) {

        int retryCount = 0;

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            retryCount++;
            System.out.println("addBlock has been called " + retryCount + " times");
            if (// First call was not a retry
            retryCount > maxRetries + 1)
                throw new IOException("Retried too many times: " + retryCount);
            else
                throw new RemoteException(NotReplicatedYetException.class.getName(), exceptionMsg);
        }
    };
    when(mockNN.addBlock(anyString(), anyString(), any(ExtendedBlock.class), any(DatanodeInfo[].class), anyLong(), any(String[].class), Matchers.<EnumSet<AddBlockFlag>>any())).thenAnswer(answer);
    Mockito.doReturn(new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission((short) 777), "owner", "group", new byte[0], new byte[0], 1010, 0, null, (byte) 0, null)).when(mockNN).getFileInfo(anyString());
    Mockito.doReturn(new HdfsFileStatus(0, false, 1, 1024, 0, 0, new FsPermission((short) 777), "owner", "group", new byte[0], new byte[0], 1010, 0, null, (byte) 0, null)).when(mockNN).create(anyString(), (FsPermission) anyObject(), anyString(), (EnumSetWritable<CreateFlag>) anyObject(), anyBoolean(), anyShort(), anyLong(), (CryptoProtocolVersion[]) anyObject());
    final DFSClient client = new DFSClient(null, mockNN, conf, null);
    OutputStream os = client.create("testfile", true);
    // write one random byte
    os.write(20);
    try {
        os.close();
    } catch (Exception e) {
        assertTrue("Retries are not being stopped correctly: " + e.getMessage(), e.getMessage().equals(exceptionMsg));
    }
}

Example 50 with NamenodeProtocols

use of org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols in project hadoop by apache.

the class TestClientProtocolForPipelineRecovery method testGetNewStamp.

@Test
public void testGetNewStamp() throws IOException {
    int numDataNodes = 1;
    Configuration conf = new HdfsConfiguration();
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
    try {
        cluster.waitActive();
        FileSystem fileSys = cluster.getFileSystem();
        NamenodeProtocols namenode = cluster.getNameNodeRpc();
        /* Test writing to finalized replicas */
        Path file = new Path("dataprotocol.dat");
        DFSTestUtil.createFile(fileSys, file, 1L, (short) numDataNodes, 0L);
        // get the first blockid for the file
        ExtendedBlock firstBlock = DFSTestUtil.getFirstBlock(fileSys, file);
        // test getNewStampAndToken on a finalized block
        try {
            namenode.updateBlockForPipeline(firstBlock, "");
            Assert.fail("Can not get a new GS from a finalized block");
        } catch (IOException e) {
            Assert.assertTrue(e.getMessage().contains("not " + BlockUCState.UNDER_CONSTRUCTION));
        }
        // test getNewStampAndToken on a non-existent block
        try {
            long newBlockId = firstBlock.getBlockId() + 1;
            ExtendedBlock newBlock = new ExtendedBlock(firstBlock.getBlockPoolId(), newBlockId, 0, firstBlock.getGenerationStamp());
            namenode.updateBlockForPipeline(newBlock, "");
            Assert.fail("Cannot get a new GS from a non-existent block");
        } catch (IOException e) {
            Assert.assertTrue(e.getMessage().contains("does not exist"));
        }
        /* Test RBW replicas */
        // change first block to a RBW
        DFSOutputStream out = null;
        try {
            out = (DFSOutputStream) (fileSys.append(file).getWrappedStream());
            out.write(1);
            out.hflush();
            FSDataInputStream in = null;
            try {
                in = fileSys.open(file);
                firstBlock = DFSTestUtil.getAllBlocks(in).get(0).getBlock();
            } finally {
                IOUtils.closeStream(in);
            }
            // test non-lease holder
            DFSClient dfs = ((DistributedFileSystem) fileSys).dfs;
            try {
                namenode.updateBlockForPipeline(firstBlock, "test" + dfs.clientName);
                Assert.fail("Cannot get a new GS for a non lease holder");
            } catch (LeaseExpiredException e) {
                Assert.assertTrue(e.getMessage().startsWith("Lease mismatch"));
            }
            // test null lease holder
            try {
                namenode.updateBlockForPipeline(firstBlock, null);
                Assert.fail("Cannot get a new GS for a null lease holder");
            } catch (LeaseExpiredException e) {
                Assert.assertTrue(e.getMessage().startsWith("Lease mismatch"));
            }
            // test getNewStampAndToken on a rbw block
            namenode.updateBlockForPipeline(firstBlock, dfs.clientName);
        } finally {
            IOUtils.closeStream(out);
        }
    } finally {
        cluster.shutdown();
    }
}