Examples with WALEdit org.apache.hadoop.hbase.wal.WALEdit used on opensource projects

Example 31 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class HRegion method doMiniBatchMutate.

/**
 * Called to do a piece of the batch that came in to {@link #batchMutate(Mutation[])}
 * In here we also handle replay of edits on region recover. Also gets change in size brought
 * about by applying {@code batchOp}.
 */
private void doMiniBatchMutate(BatchOperation<?> batchOp) throws IOException {
    boolean success = false;
    WALEdit walEdit = null;
    WriteEntry writeEntry = null;
    boolean locked = false;
    // We try to set up a batch in the range [batchOp.nextIndexToProcess,lastIndexExclusive)
    MiniBatchOperationInProgress<Mutation> miniBatchOp = null;
    /**
     * Keep track of the locks we hold so we can release them in finally clause
     */
    List<RowLock> acquiredRowLocks = Lists.newArrayListWithCapacity(batchOp.size());
    // Check for thread interrupt status in case we have been signaled from
    // #interruptRegionOperation.
    checkInterrupt();
    try {
        // STEP 1. Try to acquire as many locks as we can and build mini-batch of operations with
        // locked rows
        miniBatchOp = batchOp.lockRowsAndBuildMiniBatch(acquiredRowLocks);
        // Ensure we acquire at least one.
        if (miniBatchOp.getReadyToWriteCount() <= 0) {
            // NoSuchColumnFamily?
            return;
        }
        // Check for thread interrupt status in case we have been signaled from
        // #interruptRegionOperation. Do it before we take the lock and disable interrupts for
        // the WAL append.
        checkInterrupt();
        lock(this.updatesLock.readLock(), miniBatchOp.getReadyToWriteCount());
        locked = true;
        // From this point until memstore update this operation should not be interrupted.
        disableInterrupts();
        // STEP 2. Update mini batch of all operations in progress with LATEST_TIMESTAMP timestamp
        // We should record the timestamp only after we have acquired the rowLock,
        // otherwise, newer puts/deletes/increment/append are not guaranteed to have a newer
        // timestamp
        long now = EnvironmentEdgeManager.currentTime();
        batchOp.prepareMiniBatchOperations(miniBatchOp, now, acquiredRowLocks);
        // STEP 3. Build WAL edit
        List<Pair<NonceKey, WALEdit>> walEdits = batchOp.buildWALEdits(miniBatchOp);
        for (Iterator<Pair<NonceKey, WALEdit>> it = walEdits.iterator(); it.hasNext(); ) {
            Pair<NonceKey, WALEdit> nonceKeyWALEditPair = it.next();
            walEdit = nonceKeyWALEditPair.getSecond();
            NonceKey nonceKey = nonceKeyWALEditPair.getFirst();
            if (walEdit != null && !walEdit.isEmpty()) {
                writeEntry = doWALAppend(walEdit, batchOp.durability, batchOp.getClusterIds(), now, nonceKey.getNonceGroup(), nonceKey.getNonce(), batchOp.getOrigLogSeqNum());
            }
            // Complete mvcc for all but last writeEntry (for replay case)
            if (it.hasNext() && writeEntry != null) {
                mvcc.complete(writeEntry);
                writeEntry = null;
            }
        }
        // STEP 5. Write back to memStore
        // NOTE: writeEntry can be null here
        writeEntry = batchOp.writeMiniBatchOperationsToMemStore(miniBatchOp, writeEntry);
        // STEP 6. Complete MiniBatchOperations: If required calls postBatchMutate() CP hook and
        // complete mvcc for last writeEntry
        batchOp.completeMiniBatchOperations(miniBatchOp, writeEntry);
        writeEntry = null;
        success = true;
    } finally {
        // Call complete rather than completeAndWait because we probably had error if walKey != null
        if (writeEntry != null)
            mvcc.complete(writeEntry);
        if (locked) {
            this.updatesLock.readLock().unlock();
        }
        releaseRowLocks(acquiredRowLocks);
        enableInterrupts();
        final int finalLastIndexExclusive = miniBatchOp != null ? miniBatchOp.getLastIndexExclusive() : batchOp.size();
        final boolean finalSuccess = success;
        batchOp.visitBatchOperations(true, finalLastIndexExclusive, (int i) -> {
            Mutation mutation = batchOp.getMutation(i);
            if (mutation instanceof Increment || mutation instanceof Append) {
                if (finalSuccess) {
                    batchOp.retCodeDetails[i] = new OperationStatus(OperationStatusCode.SUCCESS, batchOp.results[i]);
                } else {
                    batchOp.retCodeDetails[i] = OperationStatus.FAILURE;
                }
            } else {
                batchOp.retCodeDetails[i] = finalSuccess ? OperationStatus.SUCCESS : OperationStatus.FAILURE;
            }
            return true;
        });
        batchOp.doPostOpCleanupForMiniBatch(miniBatchOp, walEdit, finalSuccess);
        batchOp.nextIndexToProcess = finalLastIndexExclusive;
    }
}

Example 32 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class ReplaySyncReplicationWALCallable method filter.

// return whether we should include this entry.
private boolean filter(Entry entry) {
    WALEdit edit = entry.getEdit();
    WALUtil.filterCells(edit, c -> CellUtil.matchingFamily(c, WALEdit.METAFAMILY) ? null : c);
    return !edit.isEmpty();
}

Example 33 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class HBaseInterClusterReplicationEndpoint method filterNotExistColumnFamilyEdits.

List<List<Entry>> filterNotExistColumnFamilyEdits(final List<List<Entry>> oldEntryList) {
    List<List<Entry>> entryList = new ArrayList<>();
    Map<TableName, Set<String>> existColumnFamilyMap = new HashMap<>();
    try (Connection localConn = ConnectionFactory.createConnection(ctx.getLocalConfiguration());
        Admin localAdmin = localConn.getAdmin()) {
        for (List<Entry> oldEntries : oldEntryList) {
            List<Entry> entries = new ArrayList<>();
            for (Entry e : oldEntries) {
                TableName tableName = e.getKey().getTableName();
                if (!existColumnFamilyMap.containsKey(tableName)) {
                    try {
                        Set<String> cfs = localAdmin.getDescriptor(tableName).getColumnFamilyNames().stream().map(Bytes::toString).collect(Collectors.toSet());
                        existColumnFamilyMap.put(tableName, cfs);
                    } catch (Exception ex) {
                        LOG.warn("Exception getting cf names for local table {}", tableName, ex);
                        // if catch any exception, we are not sure about table's description,
                        // so replicate raw entry
                        entries.add(e);
                        continue;
                    }
                }
                Set<String> existColumnFamilies = existColumnFamilyMap.get(tableName);
                Set<String> missingCFs = new HashSet<>();
                WALEdit walEdit = new WALEdit();
                walEdit.getCells().addAll(e.getEdit().getCells());
                WALUtil.filterCells(walEdit, cell -> {
                    String cf = Bytes.toString(CellUtil.cloneFamily(cell));
                    if (existColumnFamilies.contains(cf)) {
                        return cell;
                    } else {
                        missingCFs.add(cf);
                        return null;
                    }
                });
                if (!walEdit.isEmpty()) {
                    Entry newEntry = new Entry(e.getKey(), walEdit);
                    entries.add(newEntry);
                }
                if (!missingCFs.isEmpty()) {
                    // Would potentially be better to retry in one of the outer loops
                    // and add a table filter there; but that would break the encapsulation,
                    // so we're doing the filtering here.
                    LOG.warn("Missing column family detected at sink, local column family also does not exist," + " filtering edits for table '{}',column family '{}'", tableName, missingCFs);
                }
            }
            if (!entries.isEmpty()) {
                entryList.add(entries);
            }
        }
    } catch (IOException iox) {
        LOG.warn("Exception when creating connection to check local table", iox);
        return oldEntryList;
    }
    return entryList;
}

Example 34 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class ReplicationSourceWALReader method getEntrySizeExcludeBulkLoad.

public static long getEntrySizeExcludeBulkLoad(Entry entry) {
    WALEdit edit = entry.getEdit();
    WALKey key = entry.getKey();
    return edit.heapSize() + key.estimatedSerializedSizeOf();
}

Example 35 with WALEdit

use of org.apache.hadoop.hbase.wal.WALEdit in project hbase by apache.

the class ReplicationSourceWALReader method addEntryToBatch.

// returns true if we reach the size limit for batch, i.e, we need to finish the batch and return.
protected final boolean addEntryToBatch(WALEntryBatch batch, Entry entry) {
    WALEdit edit = entry.getEdit();
    if (edit == null || edit.isEmpty()) {
        LOG.debug("Edit null or empty for entry {} ", entry);
        return false;
    }
    LOG.debug("updating TimeStampOfLastAttempted to {}, from entry {}, for source queue: {}", entry.getKey().getWriteTime(), entry.getKey(), this.source.getQueueId());
    long entrySize = getEntrySizeIncludeBulkLoad(entry);
    long entrySizeExcludeBulkLoad = getEntrySizeExcludeBulkLoad(entry);
    batch.addEntry(entry, entrySize);
    updateBatchStats(batch, entry, entrySize);
    boolean totalBufferTooLarge = acquireBufferQuota(entrySizeExcludeBulkLoad);
    // Stop if too many entries or too big
    return totalBufferTooLarge || batch.getHeapSize() >= replicationBatchSizeCapacity || batch.getNbEntries() >= replicationBatchCountCapacity;
}