Examples with NavigableMap java.util.NavigableMap used on opensource projects

Example 66 with NavigableMap

use of java.util.NavigableMap in project incubator-atlas by apache.

the class HBaseStoreManager method normalizeKeyBounds.

/**
     * Given a map produced by {@link HTable#getRegionLocations()}, transform
     * each key from an {@link HRegionInfo} to a {@link KeyRange} expressing the
     * region's start and end key bounds using Titan-partitioning-friendly
     * conventions (start inclusive, end exclusive, zero bytes appended where
     * necessary to make all keys at least 4 bytes long).
     * <p/>
     * This method iterates over the entries in its map parameter and performs
     * the following conditional conversions on its keys. "Require" below means
     * either a {@link Preconditions} invocation or an assertion. HRegionInfo
     * sometimes returns start and end keys of zero length; this method replaces
     * zero length keys with null before doing any of the checks described
     * below. The parameter map and the values it contains are only read and
     * never modified.
     *
     * <ul>
     * <li>If an entry's HRegionInfo has null start and end keys, then first
     * require that the parameter map is a singleton, and then return a
     * single-entry map whose {@code KeyRange} has start and end buffers that
     * are both four bytes of zeros.</li>
     * <li>If the entry has a null end key (but non-null start key), put an
     * equivalent entry in the result map with a start key identical to the
     * input, except that zeros are appended to values less than 4 bytes long,
     * and an end key that is four bytes of zeros.
     * <li>If the entry has a null start key (but non-null end key), put an
     * equivalent entry in the result map where the start key is four bytes of
     * zeros, and the end key has zeros appended, if necessary, to make it at
     * least 4 bytes long, after which one is added to the padded value in
     * unsigned 32-bit arithmetic with overflow allowed.</li>
     * <li>Any entry which matches none of the above criteria results in an
     * equivalent entry in the returned map, except that zeros are appended to
     * both keys to make each at least 4 bytes long, and the end key is then
     * incremented as described in the last bullet point.</li>
     * </ul>
     *
     * After iterating over the parameter map, this method checks that it either
     * saw no entries with null keys, one entry with a null start key and a
     * different entry with a null end key, or one entry with both start and end
     * keys null. If any null keys are observed besides these three cases, the
     * method will die with a precondition failure.
     *
     * @param raw
     *            A map of HRegionInfo and ServerName from HBase
     * @return Titan-friendly expression of each region's rowkey boundaries
     */
private Map<KeyRange, ServerName> normalizeKeyBounds(NavigableMap<HRegionInfo, ServerName> raw) {
    Map.Entry<HRegionInfo, ServerName> nullStart = null;
    Map.Entry<HRegionInfo, ServerName> nullEnd = null;
    ImmutableMap.Builder<KeyRange, ServerName> b = ImmutableMap.builder();
    for (Map.Entry<HRegionInfo, ServerName> e : raw.entrySet()) {
        HRegionInfo regionInfo = e.getKey();
        byte[] startKey = regionInfo.getStartKey();
        byte[] endKey = regionInfo.getEndKey();
        if (0 == startKey.length) {
            startKey = null;
            logger.trace("Converted zero-length HBase startKey byte array to null");
        }
        if (0 == endKey.length) {
            endKey = null;
            logger.trace("Converted zero-length HBase endKey byte array to null");
        }
        if (null == startKey && null == endKey) {
            Preconditions.checkState(1 == raw.size());
            logger.debug("HBase table {} has a single region {}", tableName, regionInfo);
            // Choose arbitrary shared value = startKey = endKey
            return b.put(new KeyRange(FOUR_ZERO_BYTES, FOUR_ZERO_BYTES), e.getValue()).build();
        } else if (null == startKey) {
            logger.debug("Found HRegionInfo with null startKey on server {}: {}", e.getValue(), regionInfo);
            Preconditions.checkState(null == nullStart);
            nullStart = e;
            // I thought endBuf would be inclusive from the HBase javadoc, but in practice it is exclusive
            StaticBuffer endBuf = StaticArrayBuffer.of(zeroExtend(endKey));
            // Replace null start key with zeroes
            b.put(new KeyRange(FOUR_ZERO_BYTES, endBuf), e.getValue());
        } else if (null == endKey) {
            logger.debug("Found HRegionInfo with null endKey on server {}: {}", e.getValue(), regionInfo);
            Preconditions.checkState(null == nullEnd);
            nullEnd = e;
            // Replace null end key with zeroes
            b.put(new KeyRange(StaticArrayBuffer.of(zeroExtend(startKey)), FOUR_ZERO_BYTES), e.getValue());
        } else {
            Preconditions.checkState(null != startKey);
            Preconditions.checkState(null != endKey);
            // Convert HBase's inclusive end keys into exclusive Titan end keys
            StaticBuffer startBuf = StaticArrayBuffer.of(zeroExtend(startKey));
            StaticBuffer endBuf = StaticArrayBuffer.of(zeroExtend(endKey));
            KeyRange kr = new KeyRange(startBuf, endBuf);
            b.put(kr, e.getValue());
            logger.debug("Found HRegionInfo with non-null end and start keys on server {}: {}", e.getValue(), regionInfo);
        }
    }
    // Require either no null key bounds or a pair of them
    Preconditions.checkState((null == nullStart) == (null == nullEnd));
    // Check that every key in the result is at least 4 bytes long
    Map<KeyRange, ServerName> result = b.build();
    for (KeyRange kr : result.keySet()) {
        Preconditions.checkState(4 <= kr.getStart().length());
        Preconditions.checkState(4 <= kr.getEnd().length());
    }
    return result;
}

Example 67 with NavigableMap

use of java.util.NavigableMap in project cdap by caskdata.

the class IncrementHandler method preIncrementAfterRowLock.

@Override
public Result preIncrementAfterRowLock(ObserverContext<RegionCoprocessorEnvironment> e, Increment increment) throws IOException {
    // this should only trigger for a transactional readless increment
    boolean isIncrement = increment.getAttribute(HBaseTable.DELTA_WRITE) != null;
    boolean transactional = state.containsTransactionalFamily(increment.getFamilyCellMap().keySet());
    if (!isIncrement || !transactional) {
        return null;
    }
    byte[] txBytes = increment.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY);
    if (txBytes == null) {
        throw new IllegalArgumentException("Attribute " + TxConstants.TX_OPERATION_ATTRIBUTE_KEY + " must be set for transactional readless increments");
    }
    byte[] wpBytes = increment.getAttribute(HBaseTable.WRITE_POINTER);
    if (wpBytes == null) {
        throw new IllegalArgumentException("Attribute " + HBaseTable.WRITE_POINTER + " must be set for transactional readless increments");
    }
    long writeVersion = Bytes.toLong(wpBytes);
    Get get = new Get(increment.getRow());
    get.setAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY, txBytes);
    for (Map.Entry<byte[], NavigableMap<byte[], Long>> entry : increment.getFamilyMapOfLongs().entrySet()) {
        byte[] family = entry.getKey();
        for (byte[] column : entry.getValue().keySet()) {
            get.addColumn(family, column);
        }
    }
    Result result = region.get(get);
    Put put = new Put(increment.getRow());
    for (Map.Entry<byte[], NavigableMap<byte[], Long>> entry : increment.getFamilyMapOfLongs().entrySet()) {
        byte[] family = entry.getKey();
        for (Map.Entry<byte[], Long> colEntry : entry.getValue().entrySet()) {
            byte[] column = colEntry.getKey();
            long value = colEntry.getValue();
            byte[] existingValue = result.getValue(family, column);
            if (existingValue != null) {
                long delta = Bytes.toLong(existingValue);
                value += delta;
            }
            put.add(new KeyValue(increment.getRow(), family, column, writeVersion, Bytes.toBytes(value)));
        }
    }
    if (!put.isEmpty()) {
        region.put(put);
    }
    e.bypass();
    return new Result();
}

Example 68 with NavigableMap

use of java.util.NavigableMap in project cdap by caskdata.

the class IncrementHandler method preDelete.

@Override
public void preDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete, WALEdit edit, Durability durability) throws IOException {
    boolean transactional = state.containsTransactionalFamily(delete.getFamilyCellMap().keySet());
    if (!transactional) {
        long tsToAssign = state.getUniqueTimestamp();
        delete.setTimestamp(tsToAssign);
        // new key values
        NavigableMap<byte[], List<Cell>> newFamilyMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
        for (Map.Entry<byte[], List<Cell>> entry : delete.getFamilyCellMap().entrySet()) {
            List<Cell> newCells = new ArrayList<>(entry.getValue().size());
            for (Cell kv : entry.getValue()) {
                // replace the timestamp
                newCells.add(CellUtil.createCell(CellUtil.cloneRow(kv), CellUtil.cloneFamily(kv), CellUtil.cloneQualifier(kv), tsToAssign, kv.getTypeByte(), CellUtil.cloneValue(kv)));
            }
            newFamilyMap.put(entry.getKey(), newCells);
        }
        delete.setFamilyCellMap(newFamilyMap);
    }
}

Example 69 with NavigableMap

use of java.util.NavigableMap in project cdap by caskdata.

the class IncrementHandler method preIncrementAfterRowLock.

@Override
public Result preIncrementAfterRowLock(ObserverContext<RegionCoprocessorEnvironment> e, Increment increment) throws IOException {
    // this should only trigger for a transactional readless increment
    boolean isIncrement = increment.getAttribute(HBaseTable.DELTA_WRITE) != null;
    boolean transactional = state.containsTransactionalFamily(increment.getFamilyCellMap().keySet());
    if (!isIncrement || !transactional) {
        return null;
    }
    byte[] txBytes = increment.getAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY);
    if (txBytes == null) {
        throw new IllegalArgumentException("Attribute " + TxConstants.TX_OPERATION_ATTRIBUTE_KEY + " must be set for transactional readless increments");
    }
    byte[] wpBytes = increment.getAttribute(HBaseTable.WRITE_POINTER);
    if (wpBytes == null) {
        throw new IllegalArgumentException("Attribute " + HBaseTable.WRITE_POINTER + " must be set for transactional readless increments");
    }
    long writeVersion = Bytes.toLong(wpBytes);
    Get get = new Get(increment.getRow());
    get.setAttribute(TxConstants.TX_OPERATION_ATTRIBUTE_KEY, txBytes);
    for (Map.Entry<byte[], NavigableMap<byte[], Long>> entry : increment.getFamilyMapOfLongs().entrySet()) {
        byte[] family = entry.getKey();
        for (byte[] column : entry.getValue().keySet()) {
            get.addColumn(family, column);
        }
    }
    Result result = region.get(get);
    Put put = new Put(increment.getRow());
    for (Map.Entry<byte[], NavigableMap<byte[], Long>> entry : increment.getFamilyMapOfLongs().entrySet()) {
        byte[] family = entry.getKey();
        for (Map.Entry<byte[], Long> colEntry : entry.getValue().entrySet()) {
            byte[] column = colEntry.getKey();
            long value = colEntry.getValue();
            byte[] existingValue = result.getValue(family, column);
            if (existingValue != null) {
                long delta = Bytes.toLong(existingValue);
                value += delta;
            }
            put.add(new KeyValue(increment.getRow(), family, column, writeVersion, Bytes.toBytes(value)));
        }
    }
    if (!put.isEmpty()) {
        region.put(put);
    }
    e.bypass();
    return new Result();
}

Example 70 with NavigableMap

use of java.util.NavigableMap in project cdap by caskdata.

the class IncrementHandler method prePut.

@Override
public void prePut(ObserverContext<RegionCoprocessorEnvironment> ctx, Put put, WALEdit edit, Durability durability) throws IOException {
    // we assume that if any of the column families written to are transactional, the entire write is transactional
    boolean transactional = state.containsTransactionalFamily(put.getFamilyCellMap().keySet());
    boolean isIncrement = put.getAttribute(HBaseTable.DELTA_WRITE) != null;
    if (isIncrement || !transactional) {
        // incremental write
        NavigableMap<byte[], List<Cell>> newFamilyMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
        long tsToAssign = 0;
        if (!transactional) {
            tsToAssign = state.getUniqueTimestamp();
        }
        for (Map.Entry<byte[], List<Cell>> entry : put.getFamilyCellMap().entrySet()) {
            List<Cell> newCells = new ArrayList<>(entry.getValue().size());
            for (Cell cell : entry.getValue()) {
                // rewrite the cell value with a special prefix to identify it as a delta
                // for 0.98 we can update this to use cell tags
                byte[] newValue = isIncrement ? Bytes.add(IncrementHandlerState.DELTA_MAGIC_PREFIX, CellUtil.cloneValue(cell)) : CellUtil.cloneValue(cell);
                newCells.add(CellUtil.createCell(CellUtil.cloneRow(cell), CellUtil.cloneFamily(cell), CellUtil.cloneQualifier(cell), transactional ? cell.getTimestamp() : tsToAssign, cell.getTypeByte(), newValue));
            }
            newFamilyMap.put(entry.getKey(), newCells);
        }
        put.setFamilyCellMap(newFamilyMap);
    }
// put completes normally with value prefix marker
}