Examples with Reader org.apache.hadoop.hbase.wal.WAL.Reader used on opensource projects

Example 1 with Reader

use of org.apache.hadoop.hbase.wal.WAL.Reader in project hbase by apache.

the class WALSplitter method getReader.

/**
   * Create a new {@link Reader} for reading logs to split.
   *
   * @param file
   * @return A new Reader instance, caller should close
   * @throws IOException
   * @throws CorruptedLogFileException
   */
protected Reader getReader(FileStatus file, boolean skipErrors, CancelableProgressable reporter) throws IOException, CorruptedLogFileException {
    Path path = file.getPath();
    long length = file.getLen();
    Reader in;
    // HDFS-878 is committed.
    if (length <= 0) {
        LOG.warn("File " + path + " might be still open, length is 0");
    }
    try {
        FSUtils.getInstance(fs, conf).recoverFileLease(fs, path, conf, reporter);
        try {
            in = getReader(path, reporter);
        } catch (EOFException e) {
            if (length <= 0) {
                // TODO should we ignore an empty, not-last log file if skip.errors
                // is false? Either way, the caller should decide what to do. E.g.
                // ignore if this is the last log in sequence.
                // TODO is this scenario still possible if the log has been
                // recovered (i.e. closed)
                LOG.warn("Could not open " + path + " for reading. File is empty", e);
                return null;
            } else {
                // EOFException being ignored
                return null;
            }
        }
    } catch (IOException e) {
        if (e instanceof FileNotFoundException) {
            // A wal file may not exist anymore. Nothing can be recovered so move on
            LOG.warn("File " + path + " doesn't exist anymore.", e);
            return null;
        }
        if (!skipErrors || e instanceof InterruptedIOException) {
            // Don't mark the file corrupted if interrupted, or not skipErrors
            throw e;
        }
        CorruptedLogFileException t = new CorruptedLogFileException("skipErrors=true Could not open wal " + path + " ignoring");
        t.initCause(e);
        throw t;
    }
    return in;
}

Example 2 with Reader

use of org.apache.hadoop.hbase.wal.WAL.Reader in project hbase by apache.

the class TestWALSplit method logsAreEqual.

private boolean logsAreEqual(Path p1, Path p2) throws IOException {
    Reader in1, in2;
    in1 = wals.createReader(fs, p1);
    in2 = wals.createReader(fs, p2);
    Entry entry1;
    Entry entry2;
    while ((entry1 = in1.next()) != null) {
        entry2 = in2.next();
        if ((entry1.getKey().compareTo(entry2.getKey()) != 0) || (!entry1.getEdit().toString().equals(entry2.getEdit().toString()))) {
            return false;
        }
    }
    in1.close();
    in2.close();
    return true;
}

Example 3 with Reader

use of org.apache.hadoop.hbase.wal.WAL.Reader in project hbase by apache.

the class TestWALSplit method ignoreCorruption.

private void ignoreCorruption(final Corruptions corruption, final int entryCount, final int expectedCount) throws IOException {
    conf.setBoolean(HBASE_SKIP_ERRORS, false);
    final String REGION = "region__1";
    REGIONS.clear();
    REGIONS.add(REGION);
    Path c1 = new Path(WALDIR, WAL_FILE_PREFIX + "0");
    generateWALs(1, entryCount, -1, 0);
    corruptWAL(c1, corruption, true);
    useDifferentDFSClient();
    WALSplitter.split(HBASEDIR, WALDIR, OLDLOGDIR, fs, conf, wals);
    Path[] splitLog = getLogForRegion(HBASEDIR, TABLE_NAME, REGION);
    assertEquals(1, splitLog.length);
    int actualCount = 0;
    Reader in = wals.createReader(fs, splitLog[0]);
    @SuppressWarnings("unused") Entry entry;
    while ((entry = in.next()) != null) ++actualCount;
    assertEquals(expectedCount, actualCount);
    in.close();
    // should not have stored the EOF files as corrupt
    FileStatus[] archivedLogs = fs.listStatus(CORRUPTDIR);
    assertEquals(archivedLogs.length, 0);
}

Example 4 with Reader

use of org.apache.hadoop.hbase.wal.WAL.Reader in project hbase by apache.

the class TestWALSplit method doTestThreading.

/**
   * Sets up a log splitter with a mock reader and writer. The mock reader generates
   * a specified number of edits spread across 5 regions. The mock writer optionally
   * sleeps for each edit it is fed.
   * *
   * After the split is complete, verifies that the statistics show the correct number
   * of edits output into each region.
   *
   * @param numFakeEdits number of fake edits to push through pipeline
   * @param bufferSize size of in-memory buffer
   * @param writerSlowness writer threads will sleep this many ms per edit
   */
private void doTestThreading(final int numFakeEdits, final int bufferSize, final int writerSlowness) throws Exception {
    Configuration localConf = new Configuration(conf);
    localConf.setInt("hbase.regionserver.hlog.splitlog.buffersize", bufferSize);
    // Create a fake log file (we'll override the reader to produce a stream of edits)
    Path logPath = new Path(WALDIR, WAL_FILE_PREFIX + ".fake");
    FSDataOutputStream out = fs.create(logPath);
    out.close();
    // Make region dirs for our destination regions so the output doesn't get skipped
    final List<String> regions = ImmutableList.of("r0", "r1", "r2", "r3", "r4");
    makeRegionDirs(regions);
    // Create a splitter that reads and writes the data without touching disk
    WALSplitter logSplitter = new WALSplitter(wals, localConf, HBASEDIR, fs, null, null, this.mode) {

        /* Produce a mock writer that doesn't write anywhere */
        @Override
        protected Writer createWriter(Path logfile) throws IOException {
            Writer mockWriter = Mockito.mock(Writer.class);
            Mockito.doAnswer(new Answer<Void>() {

                int expectedIndex = 0;

                @Override
                public Void answer(InvocationOnMock invocation) {
                    if (writerSlowness > 0) {
                        try {
                            Thread.sleep(writerSlowness);
                        } catch (InterruptedException ie) {
                            Thread.currentThread().interrupt();
                        }
                    }
                    Entry entry = (Entry) invocation.getArguments()[0];
                    WALEdit edit = entry.getEdit();
                    List<Cell> cells = edit.getCells();
                    assertEquals(1, cells.size());
                    Cell cell = cells.get(0);
                    // Check that the edits come in the right order.
                    assertEquals(expectedIndex, Bytes.toInt(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
                    expectedIndex++;
                    return null;
                }
            }).when(mockWriter).append(Mockito.<Entry>any());
            return mockWriter;
        }

        /* Produce a mock reader that generates fake entries */
        @Override
        protected Reader getReader(Path curLogFile, CancelableProgressable reporter) throws IOException {
            Reader mockReader = Mockito.mock(Reader.class);
            Mockito.doAnswer(new Answer<Entry>() {

                int index = 0;

                @Override
                public Entry answer(InvocationOnMock invocation) throws Throwable {
                    if (index >= numFakeEdits)
                        return null;
                    // Generate r0 through r4 in round robin fashion
                    int regionIdx = index % regions.size();
                    byte[] region = new byte[] { (byte) 'r', (byte) (0x30 + regionIdx) };
                    Entry ret = createTestEntry(TABLE_NAME, region, Bytes.toBytes((int) (index / regions.size())), FAMILY, QUALIFIER, VALUE, index);
                    index++;
                    return ret;
                }
            }).when(mockReader).next();
            return mockReader;
        }
    };
    logSplitter.splitLogFile(fs.getFileStatus(logPath), null);
    // Verify number of written edits per region
    Map<byte[], Long> outputCounts = logSplitter.outputSink.getOutputCounts();
    for (Map.Entry<byte[], Long> entry : outputCounts.entrySet()) {
        LOG.info("Got " + entry.getValue() + " output edits for region " + Bytes.toString(entry.getKey()));
        assertEquals((long) entry.getValue(), numFakeEdits / regions.size());
    }
    assertEquals("Should have as many outputs as regions", regions.size(), outputCounts.size());
}

Example 5 with Reader

use of org.apache.hadoop.hbase.wal.WAL.Reader in project hbase by apache.

the class TestWALSplit method countWAL.

private int countWAL(Path log) throws IOException {
    int count = 0;
    Reader in = wals.createReader(fs, log);
    while (in.next() != null) {
        count++;
    }
    in.close();
    return count;
}