Examples with StorageDirectory org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory used on opensource projects

Example 56 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class TestFileJournalManager method testNormalOperation.

/** 
   * Test the normal operation of loading transactions from
   * file journal manager. 3 edits directories are setup without any
   * failures. Test that we read in the expected number of transactions.
   */
@Test
public void testNormalOperation() throws IOException {
    File f1 = new File(TestEditLog.TEST_DIR + "/normtest0");
    File f2 = new File(TestEditLog.TEST_DIR + "/normtest1");
    File f3 = new File(TestEditLog.TEST_DIR + "/normtest2");
    List<URI> editUris = ImmutableList.of(f1.toURI(), f2.toURI(), f3.toURI());
    NNStorage storage = setupEdits(editUris, 5);
    long numJournals = 0;
    for (StorageDirectory sd : storage.dirIterable(NameNodeDirType.EDITS)) {
        FileJournalManager jm = new FileJournalManager(conf, sd, storage);
        assertEquals(6 * TXNS_PER_ROLL, getNumberOfTransactions(jm, 1, true, false));
        numJournals++;
    }
    assertEquals(3, numJournals);
}

Example 57 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class TestFileJournalManager method testAskForTransactionsMidfile.

/**
   * Make requests with starting transaction ids which don't match the beginning
   * txid of some log segments.
   * 
   * This should succeed.
   */
@Test
public void testAskForTransactionsMidfile() throws IOException {
    File f = new File(TestEditLog.TEST_DIR + "/askfortransactionsmidfile");
    NNStorage storage = setupEdits(Collections.<URI>singletonList(f.toURI()), 10);
    StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
    FileJournalManager jm = new FileJournalManager(conf, sd, storage);
    // 10 rolls, so 11 rolled files, 110 txids total.
    final int TOTAL_TXIDS = 10 * 11;
    for (int txid = 1; txid <= TOTAL_TXIDS; txid++) {
        assertEquals((TOTAL_TXIDS - txid) + 1, getNumberOfTransactions(jm, txid, true, false));
    }
}

Example 58 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class TestFileJournalManager method testDoPreUpgradeIOError.

/**
   * Tests that internal renames are done using native code on platforms that
   * have it.  The native rename includes more detailed information about the
   * failure, which can be useful for troubleshooting.
   */
@Test
public void testDoPreUpgradeIOError() throws IOException {
    File storageDir = new File(TestEditLog.TEST_DIR, "preupgradeioerror");
    List<URI> editUris = Collections.singletonList(storageDir.toURI());
    NNStorage storage = setupEdits(editUris, 5);
    StorageDirectory sd = storage.dirIterator(NameNodeDirType.EDITS).next();
    assertNotNull(sd);
    // Change storage directory so that renaming current to previous.tmp fails.
    FileUtil.setWritable(storageDir, false);
    FileJournalManager jm = null;
    try {
        jm = new FileJournalManager(conf, sd, storage);
        exception.expect(IOException.class);
        if (NativeCodeLoader.isNativeCodeLoaded()) {
            exception.expectMessage("failure in native rename");
        }
        jm.doPreUpgrade();
    } finally {
        IOUtils.cleanup(LOG, jm);
        // Restore permissions on storage directory and make sure we can delete.
        FileUtil.setWritable(storageDir, true);
        FileUtil.fullyDelete(storageDir);
    }
}

Example 59 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class TestFileJournalManager method testInprogressRecoveryAll.

/** 
   * Test that FileJournalManager behaves correctly despite inprogress
   * files in all its edit log directories. Set up 3 directories and fail
   * all on the last roll. Verify that the correct number of transaction 
   * are then loaded.
   */
@Test
public void testInprogressRecoveryAll() throws IOException {
    File f1 = new File(TestEditLog.TEST_DIR + "/failalltest0");
    File f2 = new File(TestEditLog.TEST_DIR + "/failalltest1");
    File f3 = new File(TestEditLog.TEST_DIR + "/failalltest2");
    List<URI> editUris = ImmutableList.of(f1.toURI(), f2.toURI(), f3.toURI());
    // abort after the 5th roll 
    NNStorage storage = setupEdits(editUris, 5, new AbortSpec(5, 0), new AbortSpec(5, 1), new AbortSpec(5, 2));
    Iterator<StorageDirectory> dirs = storage.dirIterator(NameNodeDirType.EDITS);
    StorageDirectory sd = dirs.next();
    FileJournalManager jm = new FileJournalManager(conf, sd, storage);
    assertEquals(5 * TXNS_PER_ROLL + TXNS_PER_FAIL, getNumberOfTransactions(jm, 1, true, false));
    sd = dirs.next();
    jm = new FileJournalManager(conf, sd, storage);
    assertEquals(5 * TXNS_PER_ROLL + TXNS_PER_FAIL, getNumberOfTransactions(jm, 1, true, false));
    sd = dirs.next();
    jm = new FileJournalManager(conf, sd, storage);
    assertEquals(5 * TXNS_PER_ROLL + TXNS_PER_FAIL, getNumberOfTransactions(jm, 1, true, false));
}

Example 60 with StorageDirectory

use of org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory in project hadoop by apache.

the class TestNameNodeRecovery method testNameNodeRecoveryImpl.

static void testNameNodeRecoveryImpl(Corruptor corruptor, boolean finalize) throws IOException {
    final String TEST_PATH = "/test/path/dir";
    final String TEST_PATH2 = "/second/dir";
    final boolean needRecovery = corruptor.needRecovery(finalize);
    // start a cluster
    Configuration conf = getConf();
    setupRecoveryTestConf(conf);
    MiniDFSCluster cluster = null;
    FileSystem fileSys = null;
    StorageDirectory sd = null;
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).manageNameDfsDirs(false).build();
        cluster.waitActive();
        if (!finalize) {
            // Normally, the in-progress edit log would be finalized by
            // FSEditLog#endCurrentLogSegment.  For testing purposes, we
            // disable that here.
            FSEditLog spyLog = spy(cluster.getNameNode().getFSImage().getEditLog());
            doNothing().when(spyLog).endCurrentLogSegment(true);
            DFSTestUtil.setEditLogForTesting(cluster.getNamesystem(), spyLog);
        }
        fileSys = cluster.getFileSystem();
        final FSNamesystem namesystem = cluster.getNamesystem();
        FSImage fsimage = namesystem.getFSImage();
        fileSys.mkdirs(new Path(TEST_PATH));
        fileSys.mkdirs(new Path(TEST_PATH2));
        sd = fsimage.getStorage().dirIterator(NameNodeDirType.EDITS).next();
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
    File editFile = FSImageTestUtil.findLatestEditsLog(sd).getFile();
    assertTrue("Should exist: " + editFile, editFile.exists());
    // Corrupt the edit log
    LOG.info("corrupting edit log file '" + editFile + "'");
    corruptor.corrupt(editFile);
    // If needRecovery == true, make sure that we can't start the
    // cluster normally before recovery
    cluster = null;
    try {
        LOG.debug("trying to start normally (this should fail)...");
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).enableManagedDfsDirsRedundancy(false).format(false).build();
        cluster.waitActive();
        cluster.shutdown();
        if (needRecovery) {
            fail("expected the corrupted edit log to prevent normal startup");
        }
    } catch (IOException e) {
        if (!needRecovery) {
            LOG.error("Got unexpected failure with " + corruptor.getName() + corruptor, e);
            fail("got unexpected exception " + e.getMessage());
        }
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
    // Perform NameNode recovery.
    // Even if there was nothing wrong previously (needRecovery == false),
    // this should still work fine.
    cluster = null;
    try {
        LOG.debug("running recovery...");
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).enableManagedDfsDirsRedundancy(false).format(false).startupOption(recoverStartOpt).build();
    } catch (IOException e) {
        fail("caught IOException while trying to recover. " + "message was " + e.getMessage() + "\nstack trace\n" + StringUtils.stringifyException(e));
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
    // Make sure that we can start the cluster normally after recovery
    cluster = null;
    try {
        LOG.debug("starting cluster normally after recovery...");
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).enableManagedDfsDirsRedundancy(false).format(false).build();
        LOG.debug("successfully recovered the " + corruptor.getName() + " corrupted edit log");
        cluster.waitActive();
        assertTrue(cluster.getFileSystem().exists(new Path(TEST_PATH)));
    } catch (IOException e) {
        fail("failed to recover.  Error message: " + e.getMessage());
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}