Examples with Entry org.apache.hadoop.hbase.wal.WAL.Entry used on opensource projects

Example 1 with Entry

use of org.apache.hadoop.hbase.wal.WAL.Entry in project hbase by apache.

the class HBaseInterClusterReplicationEndpoint method replicate.

/**
   * Do the shipping logic
   */
@Override
public boolean replicate(ReplicateContext replicateContext) {
    CompletionService<Integer> pool = new ExecutorCompletionService<>(this.exec);
    List<Entry> entries = replicateContext.getEntries();
    String walGroupId = replicateContext.getWalGroupId();
    int sleepMultiplier = 1;
    int numReplicated = 0;
    if (!peersSelected && this.isRunning()) {
        connectToPeers();
        peersSelected = true;
    }
    int numSinks = replicationSinkMgr.getNumSinks();
    if (numSinks == 0) {
        LOG.warn("No replication sinks found, returning without replicating. The source should retry" + " with the same set of edits.");
        return false;
    }
    // minimum of: configured threads, number of 100-waledit batches,
    //  and number of current sinks
    int n = Math.min(Math.min(this.maxThreads, entries.size() / 100 + 1), numSinks);
    List<List<Entry>> entryLists = new ArrayList<>(n);
    if (n == 1) {
        entryLists.add(entries);
    } else {
        for (int i = 0; i < n; i++) {
            entryLists.add(new ArrayList<>(entries.size() / n + 1));
        }
        // now group by region
        for (Entry e : entries) {
            entryLists.get(Math.abs(Bytes.hashCode(e.getKey().getEncodedRegionName()) % n)).add(e);
        }
    }
    while (this.isRunning() && !exec.isShutdown()) {
        if (!isPeerEnabled()) {
            if (sleepForRetries("Replication is disabled", sleepMultiplier)) {
                sleepMultiplier++;
            }
            continue;
        }
        try {
            if (LOG.isTraceEnabled()) {
                LOG.trace("Replicating " + entries.size() + " entries of total size " + replicateContext.getSize());
            }
            int futures = 0;
            for (int i = 0; i < entryLists.size(); i++) {
                if (!entryLists.get(i).isEmpty()) {
                    if (LOG.isTraceEnabled()) {
                        LOG.trace("Submitting " + entryLists.get(i).size() + " entries of total size " + replicateContext.getSize());
                    }
                    // RuntimeExceptions encountered here bubble up and are handled in ReplicationSource
                    pool.submit(createReplicator(entryLists.get(i), i));
                    futures++;
                }
            }
            IOException iox = null;
            for (int i = 0; i < futures; i++) {
                try {
                    // wait for all futures, remove successful parts
                    // (only the remaining parts will be retried)
                    Future<Integer> f = pool.take();
                    int index = f.get().intValue();
                    int batchSize = entryLists.get(index).size();
                    entryLists.set(index, Collections.<Entry>emptyList());
                    // Now, we have marked the batch as done replicating, record its size
                    numReplicated += batchSize;
                } catch (InterruptedException ie) {
                    iox = new IOException(ie);
                } catch (ExecutionException ee) {
                    // cause must be an IOException
                    iox = (IOException) ee.getCause();
                }
            }
            if (iox != null) {
                // if we had any exceptions, try again
                throw iox;
            }
            if (numReplicated != entries.size()) {
                // Something went wrong here and we don't know what, let's just fail and retry.
                LOG.warn("The number of edits replicated is different from the number received," + " failing for now.");
                return false;
            }
            // update metrics
            this.metrics.setAgeOfLastShippedOp(entries.get(entries.size() - 1).getKey().getWriteTime(), walGroupId);
            return true;
        } catch (IOException ioe) {
            // Didn't ship anything, but must still age the last time we did
            this.metrics.refreshAgeOfLastShippedOp(walGroupId);
            if (ioe instanceof RemoteException) {
                ioe = ((RemoteException) ioe).unwrapRemoteException();
                LOG.warn("Can't replicate because of an error on the remote cluster: ", ioe);
                if (ioe instanceof TableNotFoundException) {
                    if (sleepForRetries("A table is missing in the peer cluster. " + "Replication cannot proceed without losing data.", sleepMultiplier)) {
                        sleepMultiplier++;
                    }
                } else if (ioe instanceof SaslException) {
                    LOG.warn("Peer encountered SaslException, rechecking all sinks: ", ioe);
                    replicationSinkMgr.chooseSinks();
                }
            } else {
                if (ioe instanceof SocketTimeoutException) {
                    // This exception means we waited for more than 60s and nothing
                    // happened, the cluster is alive and calling it right away
                    // even for a test just makes things worse.
                    sleepForRetries("Encountered a SocketTimeoutException. Since the " + "call to the remote cluster timed out, which is usually " + "caused by a machine failure or a massive slowdown", this.socketTimeoutMultiplier);
                } else if (ioe instanceof ConnectException) {
                    LOG.warn("Peer is unavailable, rechecking all sinks: ", ioe);
                    replicationSinkMgr.chooseSinks();
                } else {
                    LOG.warn("Can't replicate because of a local or network error: ", ioe);
                }
            }
            if (sleepForRetries("Since we are unable to replicate", sleepMultiplier)) {
                sleepMultiplier++;
            }
        }
    }
    // in case we exited before replicating
    return false;
}

Example 2 with Entry

use of org.apache.hadoop.hbase.wal.WAL.Entry in project hbase by apache.

the class ReplicationSourceWALReaderThread method run.

@Override
public void run() {
    int sleepMultiplier = 1;
    while (isReaderRunning()) {
        // we only loop back here if something fatal happened to our stream
        try (WALEntryStream entryStream = new WALEntryStream(logQueue, fs, conf, currentPosition, metrics)) {
            while (isReaderRunning()) {
                // loop here to keep reusing stream while we can
                if (!checkQuota()) {
                    continue;
                }
                WALEntryBatch batch = null;
                while (entryStream.hasNext()) {
                    if (batch == null) {
                        batch = new WALEntryBatch(replicationBatchCountCapacity, entryStream.getCurrentPath());
                    }
                    Entry entry = entryStream.next();
                    if (updateSerialReplPos(batch, entry)) {
                        batch.lastWalPosition = entryStream.getPosition();
                        break;
                    }
                    entry = filterEntry(entry);
                    if (entry != null) {
                        WALEdit edit = entry.getEdit();
                        if (edit != null && !edit.isEmpty()) {
                            long entrySize = getEntrySize(entry);
                            batch.addEntry(entry);
                            updateBatchStats(batch, entry, entryStream.getPosition(), entrySize);
                            boolean totalBufferTooLarge = acquireBufferQuota(entrySize);
                            // Stop if too many entries or too big
                            if (totalBufferTooLarge || batch.getHeapSize() >= replicationBatchSizeCapacity || batch.getNbEntries() >= replicationBatchCountCapacity) {
                                break;
                            }
                        }
                    }
                }
                if (batch != null && (!batch.getLastSeqIds().isEmpty() || batch.getNbEntries() > 0)) {
                    if (LOG.isTraceEnabled()) {
                        LOG.trace(String.format("Read %s WAL entries eligible for replication", batch.getNbEntries()));
                    }
                    entryBatchQueue.put(batch);
                    sleepMultiplier = 1;
                } else {
                    // got no entries and didn't advance position in WAL
                    LOG.trace("Didn't read any new entries from WAL");
                    if (replicationQueueInfo.isQueueRecovered()) {
                        // we're done with queue recovery, shut ourself down
                        setReaderRunning(false);
                        // shuts down shipper thread immediately
                        entryBatchQueue.put(batch != null ? batch : new WALEntryBatch(replicationBatchCountCapacity, entryStream.getCurrentPath()));
                    } else {
                        Thread.sleep(sleepForRetries);
                    }
                }
                currentPosition = entryStream.getPosition();
                // reuse stream
                entryStream.reset();
            }
        } catch (IOException | WALEntryStreamRuntimeException e) {
            // stream related
            if (sleepMultiplier < maxRetriesMultiplier) {
                LOG.debug("Failed to read stream of replication entries: " + e);
                sleepMultiplier++;
            } else {
                LOG.error("Failed to read stream of replication entries", e);
            }
            Threads.sleep(sleepForRetries * sleepMultiplier);
        } catch (InterruptedException e) {
            LOG.trace("Interrupted while sleeping between WAL reads");
            Thread.currentThread().interrupt();
        }
    }
}

Example 3 with Entry

use of org.apache.hadoop.hbase.wal.WAL.Entry in project hbase by apache.

the class WALEntryStream method next.

/**
   * @return the next WAL entry in this stream
   * @throws WALEntryStreamRuntimeException if there was an IOException
   * @throws NoSuchElementException if no more entries in the stream.
   */
@Override
public Entry next() {
    if (!hasNext())
        throw new NoSuchElementException();
    Entry save = currentEntry;
    // gets reloaded by hasNext()
    currentEntry = null;
    return save;
}

Example 4 with Entry

use of org.apache.hadoop.hbase.wal.WAL.Entry in project hbase by apache.

the class WALEditsReplaySink method replayEntries.

/**
   * Replay an array of actions of the same region directly into the newly assigned Region Server
   * @param entries to replay
   * @throws IOException on IO failure
   */
public void replayEntries(List<Pair<HRegionLocation, Entry>> entries) throws IOException {
    if (entries.isEmpty()) {
        return;
    }
    int batchSize = entries.size();
    Map<HRegionInfo, List<Entry>> entriesByRegion = new HashMap<>();
    HRegionLocation loc = null;
    Entry entry = null;
    List<Entry> regionEntries = null;
    // Build the action list.
    for (int i = 0; i < batchSize; i++) {
        loc = entries.get(i).getFirst();
        entry = entries.get(i).getSecond();
        if (entriesByRegion.containsKey(loc.getRegionInfo())) {
            regionEntries = entriesByRegion.get(loc.getRegionInfo());
        } else {
            regionEntries = new ArrayList<>();
            entriesByRegion.put(loc.getRegionInfo(), regionEntries);
        }
        regionEntries.add(entry);
    }
    long startTime = EnvironmentEdgeManager.currentTime();
    // replaying edits by region
    for (Map.Entry<HRegionInfo, List<Entry>> _entry : entriesByRegion.entrySet()) {
        HRegionInfo curRegion = _entry.getKey();
        List<Entry> allActions = _entry.getValue();
        // send edits in chunks
        int totalActions = allActions.size();
        int replayedActions = 0;
        int curBatchSize = 0;
        for (; replayedActions < totalActions; ) {
            curBatchSize = (totalActions > (MAX_BATCH_SIZE + replayedActions)) ? MAX_BATCH_SIZE : (totalActions - replayedActions);
            replayEdits(loc, curRegion, allActions.subList(replayedActions, replayedActions + curBatchSize));
            replayedActions += curBatchSize;
        }
    }
    long endTime = EnvironmentEdgeManager.currentTime() - startTime;
    LOG.debug("number of rows:" + entries.size() + " are sent by batch! spent " + endTime + "(ms)!");
    metrics.updateReplayTime(endTime);
    metrics.updateReplayBatchSize(batchSize);
    this.totalReplayedEdits.addAndGet(batchSize);
}

Example 5 with Entry

use of org.apache.hadoop.hbase.wal.WAL.Entry in project hbase by apache.

the class ReaderBase method next.

@Override
public Entry next(Entry reuse) throws IOException {
    Entry e = reuse;
    if (e == null) {
        e = new Entry(new WALKey(), new WALEdit());
    }
    if (compressionContext != null) {
        e.setCompressionContext(compressionContext);
    }
    boolean hasEntry = false;
    try {
        hasEntry = readNext(e);
    } catch (IllegalArgumentException iae) {
        TableName tableName = e.getKey().getTablename();
        if (tableName != null && tableName.equals(TableName.OLD_ROOT_TABLE_NAME)) {
            // It is old ROOT table edit, ignore it
            LOG.info("Got an old ROOT edit, ignoring ");
            return next(e);
        } else
            throw iae;
    }
    edit++;
    if (compressionContext != null && emptyCompressionContext) {
        emptyCompressionContext = false;
    }
    return hasEntry ? e : null;
}