Examples with Entry org.apache.hadoop.hbase.wal.WAL.Entry used on opensource projects

Example 21 with Entry

use of org.apache.hadoop.hbase.wal.WAL.Entry in project hbase by apache.

the class TestWALSplit method ignoreCorruption.

private void ignoreCorruption(final Corruptions corruption, final int entryCount, final int expectedCount) throws IOException {
    conf.setBoolean(HBASE_SKIP_ERRORS, false);
    final String REGION = "region__1";
    REGIONS.clear();
    REGIONS.add(REGION);
    Path c1 = new Path(WALDIR, WAL_FILE_PREFIX + "0");
    generateWALs(1, entryCount, -1, 0);
    corruptWAL(c1, corruption, true);
    useDifferentDFSClient();
    WALSplitter.split(HBASEDIR, WALDIR, OLDLOGDIR, fs, conf, wals);
    Path[] splitLog = getLogForRegion(HBASEDIR, TABLE_NAME, REGION);
    assertEquals(1, splitLog.length);
    int actualCount = 0;
    Reader in = wals.createReader(fs, splitLog[0]);
    @SuppressWarnings("unused") Entry entry;
    while ((entry = in.next()) != null) ++actualCount;
    assertEquals(expectedCount, actualCount);
    in.close();
    // should not have stored the EOF files as corrupt
    FileStatus[] archivedLogs = fs.listStatus(CORRUPTDIR);
    assertEquals(archivedLogs.length, 0);
}

Example 22 with Entry

use of org.apache.hadoop.hbase.wal.WAL.Entry in project hbase by apache.

the class TestWALSplit method doTestThreading.

/**
   * Sets up a log splitter with a mock reader and writer. The mock reader generates
   * a specified number of edits spread across 5 regions. The mock writer optionally
   * sleeps for each edit it is fed.
   * *
   * After the split is complete, verifies that the statistics show the correct number
   * of edits output into each region.
   *
   * @param numFakeEdits number of fake edits to push through pipeline
   * @param bufferSize size of in-memory buffer
   * @param writerSlowness writer threads will sleep this many ms per edit
   */
private void doTestThreading(final int numFakeEdits, final int bufferSize, final int writerSlowness) throws Exception {
    Configuration localConf = new Configuration(conf);
    localConf.setInt("hbase.regionserver.hlog.splitlog.buffersize", bufferSize);
    // Create a fake log file (we'll override the reader to produce a stream of edits)
    Path logPath = new Path(WALDIR, WAL_FILE_PREFIX + ".fake");
    FSDataOutputStream out = fs.create(logPath);
    out.close();
    // Make region dirs for our destination regions so the output doesn't get skipped
    final List<String> regions = ImmutableList.of("r0", "r1", "r2", "r3", "r4");
    makeRegionDirs(regions);
    // Create a splitter that reads and writes the data without touching disk
    WALSplitter logSplitter = new WALSplitter(wals, localConf, HBASEDIR, fs, null, null, this.mode) {

        /* Produce a mock writer that doesn't write anywhere */
        @Override
        protected Writer createWriter(Path logfile) throws IOException {
            Writer mockWriter = Mockito.mock(Writer.class);
            Mockito.doAnswer(new Answer<Void>() {

                int expectedIndex = 0;

                @Override
                public Void answer(InvocationOnMock invocation) {
                    if (writerSlowness > 0) {
                        try {
                            Thread.sleep(writerSlowness);
                        } catch (InterruptedException ie) {
                            Thread.currentThread().interrupt();
                        }
                    }
                    Entry entry = (Entry) invocation.getArguments()[0];
                    WALEdit edit = entry.getEdit();
                    List<Cell> cells = edit.getCells();
                    assertEquals(1, cells.size());
                    Cell cell = cells.get(0);
                    // Check that the edits come in the right order.
                    assertEquals(expectedIndex, Bytes.toInt(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
                    expectedIndex++;
                    return null;
                }
            }).when(mockWriter).append(Mockito.<Entry>any());
            return mockWriter;
        }

        /* Produce a mock reader that generates fake entries */
        @Override
        protected Reader getReader(Path curLogFile, CancelableProgressable reporter) throws IOException {
            Reader mockReader = Mockito.mock(Reader.class);
            Mockito.doAnswer(new Answer<Entry>() {

                int index = 0;

                @Override
                public Entry answer(InvocationOnMock invocation) throws Throwable {
                    if (index >= numFakeEdits)
                        return null;
                    // Generate r0 through r4 in round robin fashion
                    int regionIdx = index % regions.size();
                    byte[] region = new byte[] { (byte) 'r', (byte) (0x30 + regionIdx) };
                    Entry ret = createTestEntry(TABLE_NAME, region, Bytes.toBytes((int) (index / regions.size())), FAMILY, QUALIFIER, VALUE, index);
                    index++;
                    return ret;
                }
            }).when(mockReader).next();
            return mockReader;
        }
    };
    logSplitter.splitLogFile(fs.getFileStatus(logPath), null);
    // Verify number of written edits per region
    Map<byte[], Long> outputCounts = logSplitter.outputSink.getOutputCounts();
    for (Map.Entry<byte[], Long> entry : outputCounts.entrySet()) {
        LOG.info("Got " + entry.getValue() + " output edits for region " + Bytes.toString(entry.getKey()));
        assertEquals((long) entry.getValue(), numFakeEdits / regions.size());
    }
    assertEquals("Should have as many outputs as regions", regions.size(), outputCounts.size());
}

Example 23 with Entry

use of org.apache.hadoop.hbase.wal.WAL.Entry in project hbase by apache.

the class TestWALSplit method createTestEntry.

private static Entry createTestEntry(TableName table, byte[] region, byte[] row, byte[] family, byte[] qualifier, byte[] value, long seq) {
    long time = System.nanoTime();
    seq++;
    final KeyValue cell = new KeyValue(row, family, qualifier, time, KeyValue.Type.Put, value);
    WALEdit edit = new WALEdit();
    edit.add(cell);
    return new Entry(new WALKey(region, table, seq, time, HConstants.DEFAULT_CLUSTER_ID), edit);
}

Example 24 with Entry

use of org.apache.hadoop.hbase.wal.WAL.Entry in project hbase by apache.

the class WALSplitter method splitLogFile.

/**
   * log splitting implementation, splits one log file.
   * @param logfile should be an actual log file.
   */
@VisibleForTesting
boolean splitLogFile(FileStatus logfile, CancelableProgressable reporter) throws IOException {
    Preconditions.checkState(status == null);
    Preconditions.checkArgument(logfile.isFile(), "passed in file status is for something other than a regular file.");
    boolean isCorrupted = false;
    boolean skipErrors = conf.getBoolean("hbase.hlog.split.skip.errors", SPLIT_SKIP_ERRORS_DEFAULT);
    int interval = conf.getInt("hbase.splitlog.report.interval.loglines", 1024);
    Path logPath = logfile.getPath();
    boolean outputSinkStarted = false;
    boolean progress_failed = false;
    int editsCount = 0;
    int editsSkipped = 0;
    status = TaskMonitor.get().createStatus("Splitting log file " + logfile.getPath() + "into a temporary staging area.");
    Reader in = null;
    this.fileBeingSplit = logfile;
    try {
        long logLength = logfile.getLen();
        LOG.info("Splitting wal: " + logPath + ", length=" + logLength);
        LOG.info("DistributedLogReplay = " + this.distributedLogReplay);
        status.setStatus("Opening log file");
        if (reporter != null && !reporter.progress()) {
            progress_failed = true;
            return false;
        }
        in = getReader(logfile, skipErrors, reporter);
        if (in == null) {
            LOG.warn("Nothing to split in log file " + logPath);
            return true;
        }
        int numOpenedFilesBeforeReporting = conf.getInt("hbase.splitlog.report.openedfiles", 3);
        int numOpenedFilesLastCheck = 0;
        outputSink.setReporter(reporter);
        outputSink.startWriterThreads();
        outputSinkStarted = true;
        Entry entry;
        Long lastFlushedSequenceId = -1L;
        // THIS IS BROKEN!!!! GETTING SERVERNAME FROM PATH IS NOT GOING TO WORK IF LAYOUT CHANGES!!!
        // TODO: Fix.
        ServerName serverName = AbstractFSWALProvider.getServerNameFromWALDirectoryName(logPath);
        failedServerName = (serverName == null) ? "" : serverName.getServerName();
        while ((entry = getNextLogLine(in, logPath, skipErrors)) != null) {
            byte[] region = entry.getKey().getEncodedRegionName();
            String encodedRegionNameAsStr = Bytes.toString(region);
            lastFlushedSequenceId = lastFlushedSequenceIds.get(encodedRegionNameAsStr);
            if (lastFlushedSequenceId == null) {
                if (this.distributedLogReplay) {
                    RegionStoreSequenceIds ids = csm.getSplitLogWorkerCoordination().getRegionFlushedSequenceId(failedServerName, encodedRegionNameAsStr);
                    if (ids != null) {
                        lastFlushedSequenceId = ids.getLastFlushedSequenceId();
                        if (LOG.isDebugEnabled()) {
                            LOG.debug("DLR Last flushed sequenceid for " + encodedRegionNameAsStr + ": " + TextFormat.shortDebugString(ids));
                        }
                    }
                } else if (sequenceIdChecker != null) {
                    RegionStoreSequenceIds ids = sequenceIdChecker.getLastSequenceId(region);
                    Map<byte[], Long> maxSeqIdInStores = new TreeMap<>(Bytes.BYTES_COMPARATOR);
                    for (StoreSequenceId storeSeqId : ids.getStoreSequenceIdList()) {
                        maxSeqIdInStores.put(storeSeqId.getFamilyName().toByteArray(), storeSeqId.getSequenceId());
                    }
                    regionMaxSeqIdInStores.put(encodedRegionNameAsStr, maxSeqIdInStores);
                    lastFlushedSequenceId = ids.getLastFlushedSequenceId();
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("DLS Last flushed sequenceid for " + encodedRegionNameAsStr + ": " + TextFormat.shortDebugString(ids));
                    }
                }
                if (lastFlushedSequenceId == null) {
                    lastFlushedSequenceId = -1L;
                }
                lastFlushedSequenceIds.put(encodedRegionNameAsStr, lastFlushedSequenceId);
            }
            if (lastFlushedSequenceId >= entry.getKey().getSequenceId()) {
                editsSkipped++;
                continue;
            }
            // Don't send Compaction/Close/Open region events to recovered edit type sinks.
            if (entry.getEdit().isMetaEdit() && !outputSink.keepRegionEvent(entry)) {
                editsSkipped++;
                continue;
            }
            entryBuffers.appendEntry(entry);
            editsCount++;
            int moreWritersFromLastCheck = this.getNumOpenWriters() - numOpenedFilesLastCheck;
            // If sufficient edits have passed, check if we should report progress.
            if (editsCount % interval == 0 || moreWritersFromLastCheck > numOpenedFilesBeforeReporting) {
                numOpenedFilesLastCheck = this.getNumOpenWriters();
                String countsStr = (editsCount - (editsSkipped + outputSink.getSkippedEdits())) + " edits, skipped " + editsSkipped + " edits.";
                status.setStatus("Split " + countsStr);
                if (reporter != null && !reporter.progress()) {
                    progress_failed = true;
                    return false;
                }
            }
        }
    } catch (InterruptedException ie) {
        IOException iie = new InterruptedIOException();
        iie.initCause(ie);
        throw iie;
    } catch (CorruptedLogFileException e) {
        LOG.warn("Could not parse, corrupted log file " + logPath, e);
        if (this.csm != null) {
            // Some tests pass in a csm of null.
            this.csm.getSplitLogWorkerCoordination().markCorrupted(rootDir, logfile.getPath().getName(), fs);
        } else {
            // for tests only
            ZKSplitLog.markCorrupted(rootDir, logfile.getPath().getName(), fs);
        }
        isCorrupted = true;
    } catch (IOException e) {
        e = e instanceof RemoteException ? ((RemoteException) e).unwrapRemoteException() : e;
        throw e;
    } finally {
        LOG.debug("Finishing writing output logs and closing down.");
        try {
            if (null != in) {
                in.close();
            }
        } catch (IOException exception) {
            LOG.warn("Could not close wal reader: " + exception.getMessage());
            LOG.debug("exception details", exception);
        }
        try {
            if (outputSinkStarted) {
                // Set progress_failed to true as the immediate following statement will reset its value
                // when finishWritingAndClose() throws exception, progress_failed has the right value
                progress_failed = true;
                progress_failed = outputSink.finishWritingAndClose() == null;
            }
        } finally {
            String msg = "Processed " + editsCount + " edits across " + outputSink.getNumberOfRecoveredRegions() + " regions; edits skipped=" + editsSkipped + "; log file=" + logPath + ", length=" + // See if length got updated post lease recovery
            logfile.getLen() + ", corrupted=" + isCorrupted + ", progress failed=" + progress_failed;
            LOG.info(msg);
            status.markComplete(msg);
        }
    }
    return !progress_failed;
}

Example 25 with Entry

use of org.apache.hadoop.hbase.wal.WAL.Entry in project hbase by apache.

the class ReplicationProtbufUtil method buildReplicateWALEntryRequest.

/**
   * Create a new ReplicateWALEntryRequest from a list of WAL entries
   * @param entries the WAL entries to be replicated
   * @param encodedRegionName alternative region name to use if not null
   * @param replicationClusterId Id which will uniquely identify source cluster FS client
   *          configurations in the replication configuration directory
   * @param sourceBaseNamespaceDir Path to source cluster base namespace directory
   * @param sourceHFileArchiveDir Path to the source cluster hfile archive directory
   * @return a pair of ReplicateWALEntryRequest and a CellScanner over all the WALEdit values found.
   */
public static Pair<AdminProtos.ReplicateWALEntryRequest, CellScanner> buildReplicateWALEntryRequest(final Entry[] entries, byte[] encodedRegionName, String replicationClusterId, Path sourceBaseNamespaceDir, Path sourceHFileArchiveDir) {
    // Accumulate all the Cells seen in here.
    List<List<? extends Cell>> allCells = new ArrayList<>(entries.length);
    int size = 0;
    WALProtos.FamilyScope.Builder scopeBuilder = WALProtos.FamilyScope.newBuilder();
    AdminProtos.WALEntry.Builder entryBuilder = AdminProtos.WALEntry.newBuilder();
    AdminProtos.ReplicateWALEntryRequest.Builder builder = AdminProtos.ReplicateWALEntryRequest.newBuilder();
    HBaseProtos.UUID.Builder uuidBuilder = HBaseProtos.UUID.newBuilder();
    for (Entry entry : entries) {
        entryBuilder.clear();
        // TODO: this duplicates a lot in WALKey#getBuilder
        WALProtos.WALKey.Builder keyBuilder = entryBuilder.getKeyBuilder();
        WALKey key = entry.getKey();
        keyBuilder.setEncodedRegionName(UnsafeByteOperations.unsafeWrap(encodedRegionName == null ? key.getEncodedRegionName() : encodedRegionName));
        keyBuilder.setTableName(UnsafeByteOperations.unsafeWrap(key.getTablename().getName()));
        keyBuilder.setLogSequenceNumber(key.getLogSeqNum());
        keyBuilder.setWriteTime(key.getWriteTime());
        if (key.getNonce() != HConstants.NO_NONCE) {
            keyBuilder.setNonce(key.getNonce());
        }
        if (key.getNonceGroup() != HConstants.NO_NONCE) {
            keyBuilder.setNonceGroup(key.getNonceGroup());
        }
        for (UUID clusterId : key.getClusterIds()) {
            uuidBuilder.setLeastSigBits(clusterId.getLeastSignificantBits());
            uuidBuilder.setMostSigBits(clusterId.getMostSignificantBits());
            keyBuilder.addClusterIds(uuidBuilder.build());
        }
        if (key.getOrigLogSeqNum() > 0) {
            keyBuilder.setOrigSequenceNumber(key.getOrigLogSeqNum());
        }
        WALEdit edit = entry.getEdit();
        NavigableMap<byte[], Integer> scopes = key.getReplicationScopes();
        if (scopes != null && !scopes.isEmpty()) {
            for (Map.Entry<byte[], Integer> scope : scopes.entrySet()) {
                scopeBuilder.setFamily(UnsafeByteOperations.unsafeWrap(scope.getKey()));
                WALProtos.ScopeType scopeType = WALProtos.ScopeType.valueOf(scope.getValue().intValue());
                scopeBuilder.setScopeType(scopeType);
                keyBuilder.addScopes(scopeBuilder.build());
            }
        }
        List<Cell> cells = edit.getCells();
        // Add up the size.  It is used later serializing out the kvs.
        for (Cell cell : cells) {
            size += CellUtil.estimatedSerializedSizeOf(cell);
        }
        // Collect up the cells
        allCells.add(cells);
        // Write out how many cells associated with this entry.
        entryBuilder.setAssociatedCellCount(cells.size());
        builder.addEntry(entryBuilder.build());
    }
    if (replicationClusterId != null) {
        builder.setReplicationClusterId(replicationClusterId);
    }
    if (sourceBaseNamespaceDir != null) {
        builder.setSourceBaseNamespaceDirPath(sourceBaseNamespaceDir.toString());
    }
    if (sourceHFileArchiveDir != null) {
        builder.setSourceHFileArchiveDirPath(sourceHFileArchiveDir.toString());
    }
    return new Pair<>(builder.build(), getCellScanner(allCells, size));
}