Examples with EditLogInputStream org.apache.hadoop.hdfs.server.namenode.EditLogInputStream used on opensource projects

Example 1 with EditLogInputStream

use of org.apache.hadoop.hdfs.server.namenode.EditLogInputStream in project hadoop by apache.

the class QuorumJournalManager method selectInputStreams.

@Override
public void selectInputStreams(Collection<EditLogInputStream> streams, long fromTxnId, boolean inProgressOk, boolean onlyDurableTxns) throws IOException {
    QuorumCall<AsyncLogger, RemoteEditLogManifest> q = loggers.getEditLogManifest(fromTxnId, inProgressOk);
    Map<AsyncLogger, RemoteEditLogManifest> resps = loggers.waitForWriteQuorum(q, selectInputStreamsTimeoutMs, "selectInputStreams");
    LOG.debug("selectInputStream manifests:\n" + Joiner.on("\n").withKeyValueSeparator(": ").join(resps));
    final PriorityQueue<EditLogInputStream> allStreams = new PriorityQueue<EditLogInputStream>(64, JournalSet.EDIT_LOG_INPUT_STREAM_COMPARATOR);
    for (Map.Entry<AsyncLogger, RemoteEditLogManifest> e : resps.entrySet()) {
        AsyncLogger logger = e.getKey();
        RemoteEditLogManifest manifest = e.getValue();
        long committedTxnId = manifest.getCommittedTxnId();
        for (RemoteEditLog remoteLog : manifest.getLogs()) {
            URL url = logger.buildURLToFetchLogs(remoteLog.getStartTxId());
            long endTxId = remoteLog.getEndTxId();
            // than committedTxnId. This ensures the consistency.
            if (onlyDurableTxns && inProgressOk) {
                endTxId = Math.min(endTxId, committedTxnId);
            }
            EditLogInputStream elis = EditLogFileInputStream.fromUrl(connectionFactory, url, remoteLog.getStartTxId(), endTxId, remoteLog.isInProgress());
            allStreams.add(elis);
        }
    }
    JournalSet.chainAndMakeRedundantStreams(streams, allStreams, fromTxnId);
}

Example 2 with EditLogInputStream

use of org.apache.hadoop.hdfs.server.namenode.EditLogInputStream in project hadoop by apache.

the class QJMTestUtil method verifyEdits.

/**
   * Verify that the given list of streams contains exactly the range of
   * transactions specified, inclusive.
   */
public static void verifyEdits(List<EditLogInputStream> streams, int firstTxnId, int lastTxnId) throws IOException {
    Iterator<EditLogInputStream> iter = streams.iterator();
    assertTrue(iter.hasNext());
    EditLogInputStream stream = iter.next();
    for (int expected = firstTxnId; expected <= lastTxnId; expected++) {
        FSEditLogOp op = stream.readOp();
        while (op == null) {
            assertTrue("Expected to find txid " + expected + ", " + "but no more streams available to read from", iter.hasNext());
            stream = iter.next();
            op = stream.readOp();
        }
        assertEquals(FSEditLogOpCodes.OP_MKDIR, op.opCode);
        assertEquals(expected, op.getTransactionId());
    }
    assertNull(stream.readOp());
    assertFalse("Expected no more txns after " + lastTxnId + " but more streams are available", iter.hasNext());
}

Example 3 with EditLogInputStream

use of org.apache.hadoop.hdfs.server.namenode.EditLogInputStream in project hadoop by apache.

the class TestQuorumJournalManager method testReaderWhileAnotherWrites.

@Test
public void testReaderWhileAnotherWrites() throws Exception {
    QuorumJournalManager readerQjm = closeLater(createSpyingQJM());
    List<EditLogInputStream> streams = Lists.newArrayList();
    readerQjm.selectInputStreams(streams, 0, false);
    assertEquals(0, streams.size());
    writeSegment(cluster, qjm, 1, 3, true);
    readerQjm.selectInputStreams(streams, 0, false);
    try {
        assertEquals(1, streams.size());
        // Validate the actual stream contents.
        EditLogInputStream stream = streams.get(0);
        assertEquals(1, stream.getFirstTxId());
        assertEquals(3, stream.getLastTxId());
        verifyEdits(streams, 1, 3);
        assertNull(stream.readOp());
    } finally {
        IOUtils.cleanup(LOG, streams.toArray(new Closeable[0]));
        streams.clear();
    }
    // Ensure correct results when there is a stream in-progress, but we don't
    // ask for in-progress.
    writeSegment(cluster, qjm, 4, 3, false);
    readerQjm.selectInputStreams(streams, 0, false);
    try {
        assertEquals(1, streams.size());
        EditLogInputStream stream = streams.get(0);
        assertEquals(1, stream.getFirstTxId());
        assertEquals(3, stream.getLastTxId());
        verifyEdits(streams, 1, 3);
    } finally {
        IOUtils.cleanup(LOG, streams.toArray(new Closeable[0]));
        streams.clear();
    }
    // TODO: check results for selectInputStreams with inProgressOK = true.
    // This doesn't currently work, due to a bug where RedundantEditInputStream
    // throws an exception if there are any unvalidated in-progress edits in the list!
    // But, it shouldn't be necessary for current use cases.
    qjm.finalizeLogSegment(4, 6);
    readerQjm.selectInputStreams(streams, 0, false);
    try {
        assertEquals(2, streams.size());
        assertEquals(4, streams.get(1).getFirstTxId());
        assertEquals(6, streams.get(1).getLastTxId());
        verifyEdits(streams, 1, 6);
    } finally {
        IOUtils.cleanup(LOG, streams.toArray(new Closeable[0]));
        streams.clear();
    }
}

Example 4 with EditLogInputStream

use of org.apache.hadoop.hdfs.server.namenode.EditLogInputStream in project hadoop by apache.

the class EditLogTailer method doTailEdits.

@VisibleForTesting
void doTailEdits() throws IOException, InterruptedException {
    // Write lock needs to be interruptible here because the 
    // transitionToActive RPC takes the write lock before calling
    // tailer.stop() -- so if we're not interruptible, it will
    // deadlock.
    namesystem.writeLockInterruptibly();
    try {
        FSImage image = namesystem.getFSImage();
        long lastTxnId = image.getLastAppliedTxId();
        if (LOG.isDebugEnabled()) {
            LOG.debug("lastTxnId: " + lastTxnId);
        }
        Collection<EditLogInputStream> streams;
        try {
            streams = editLog.selectInputStreams(lastTxnId + 1, 0, null, inProgressOk, true);
        } catch (IOException ioe) {
            // This is acceptable. If we try to tail edits in the middle of an edits
            // log roll, i.e. the last one has been finalized but the new inprogress
            // edits file hasn't been started yet.
            LOG.warn("Edits tailer failed to find any streams. Will try again " + "later.", ioe);
            return;
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("edit streams to load from: " + streams.size());
        }
        // Once we have streams to load, errors encountered are legitimate cause
        // for concern, so we don't catch them here. Simple errors reading from
        // disk are ignored.
        long editsLoaded = 0;
        try {
            editsLoaded = image.loadEdits(streams, namesystem);
        } catch (EditLogInputException elie) {
            editsLoaded = elie.getNumEditsLoaded();
            throw elie;
        } finally {
            if (editsLoaded > 0 || LOG.isDebugEnabled()) {
                LOG.debug(String.format("Loaded %d edits starting from txid %d ", editsLoaded, lastTxnId));
            }
        }
        if (editsLoaded > 0) {
            lastLoadTimeMs = monotonicNow();
        }
        lastLoadedTxnId = image.getLastAppliedTxId();
    } finally {
        namesystem.writeUnlock();
    }
}

Example 5 with EditLogInputStream

use of org.apache.hadoop.hdfs.server.namenode.EditLogInputStream in project hadoop by apache.

the class QJMTestUtil method recoverAndReturnLastTxn.

public static long recoverAndReturnLastTxn(QuorumJournalManager qjm) throws IOException {
    qjm.recoverUnfinalizedSegments();
    long lastRecoveredTxn = 0;
    List<EditLogInputStream> streams = Lists.newArrayList();
    try {
        qjm.selectInputStreams(streams, 0, false);
        for (EditLogInputStream elis : streams) {
            assertTrue(elis.getFirstTxId() > lastRecoveredTxn);
            lastRecoveredTxn = elis.getLastTxId();
        }
    } finally {
        IOUtils.cleanup(null, streams.toArray(new Closeable[0]));
    }
    return lastRecoveredTxn;
}