Examples with KeyRange com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyRange used on opensource projects

Example 1 with KeyRange

use of com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyRange in project titan by thinkaurelius.

the class HBaseStoreManager method getLocalKeyPartition.

@Override
public List<KeyRange> getLocalKeyPartition() throws BackendException {
    List<KeyRange> result = new LinkedList<KeyRange>();
    HTable table = null;
    try {
        ensureTableExists(tableName, getCfNameForStoreName(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME), 0);
        table = new HTable(hconf, tableName);
        Map<KeyRange, ServerName> normed = normalizeKeyBounds(table.getRegionLocations());
        for (Map.Entry<KeyRange, ServerName> e : normed.entrySet()) {
            if (NetworkUtil.isLocalConnection(e.getValue().getHostname())) {
                result.add(e.getKey());
                logger.debug("Found local key/row partition {} on host {}", e.getKey(), e.getValue());
            } else {
                logger.debug("Discarding remote {}", e.getValue());
            }
        }
    } catch (MasterNotRunningException e) {
        logger.warn("Unexpected MasterNotRunningException", e);
    } catch (ZooKeeperConnectionException e) {
        logger.warn("Unexpected ZooKeeperConnectionException", e);
    } catch (IOException e) {
        logger.warn("Unexpected IOException", e);
    } finally {
        IOUtils.closeQuietly(table);
    }
    return result;
}

Example 2 with KeyRange

use of com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyRange in project titan by thinkaurelius.

the class HBaseStoreManager method normalizeKeyBounds.

/**
     * Given a map produced by {@link HTable#getRegionLocations()}, transform
     * each key from an {@link HRegionInfo} to a {@link KeyRange} expressing the
     * region's start and end key bounds using Titan-partitioning-friendly
     * conventions (start inclusive, end exclusive, zero bytes appended where
     * necessary to make all keys at least 4 bytes long).
     * <p/>
     * This method iterates over the entries in its map parameter and performs
     * the following conditional conversions on its keys. "Require" below means
     * either a {@link Preconditions} invocation or an assertion. HRegionInfo
     * sometimes returns start and end keys of zero length; this method replaces
     * zero length keys with null before doing any of the checks described
     * below. The parameter map and the values it contains are only read and
     * never modified.
     *
     * <ul>
     * <li>If an entry's HRegionInfo has null start and end keys, then first
     * require that the parameter map is a singleton, and then return a
     * single-entry map whose {@code KeyRange} has start and end buffers that
     * are both four bytes of zeros.</li>
     * <li>If the entry has a null end key (but non-null start key), put an
     * equivalent entry in the result map with a start key identical to the
     * input, except that zeros are appended to values less than 4 bytes long,
     * and an end key that is four bytes of zeros.
     * <li>If the entry has a null start key (but non-null end key), put an
     * equivalent entry in the result map where the start key is four bytes of
     * zeros, and the end key has zeros appended, if necessary, to make it at
     * least 4 bytes long, after which one is added to the padded value in
     * unsigned 32-bit arithmetic with overflow allowed.</li>
     * <li>Any entry which matches none of the above criteria results in an
     * equivalent entry in the returned map, except that zeros are appended to
     * both keys to make each at least 4 bytes long, and the end key is then
     * incremented as described in the last bullet point.</li>
     * </ul>
     *
     * After iterating over the parameter map, this method checks that it either
     * saw no entries with null keys, one entry with a null start key and a
     * different entry with a null end key, or one entry with both start and end
     * keys null. If any null keys are observed besides these three cases, the
     * method will die with a precondition failure.
     *
     * @param raw
     *            A map of HRegionInfo and ServerName from HBase
     * @return Titan-friendly expression of each region's rowkey boundaries
     */
private Map<KeyRange, ServerName> normalizeKeyBounds(NavigableMap<HRegionInfo, ServerName> raw) {
    Map.Entry<HRegionInfo, ServerName> nullStart = null;
    Map.Entry<HRegionInfo, ServerName> nullEnd = null;
    ImmutableMap.Builder<KeyRange, ServerName> b = ImmutableMap.builder();
    for (Map.Entry<HRegionInfo, ServerName> e : raw.entrySet()) {
        HRegionInfo regionInfo = e.getKey();
        byte[] startKey = regionInfo.getStartKey();
        byte[] endKey = regionInfo.getEndKey();
        if (0 == startKey.length) {
            startKey = null;
            logger.trace("Converted zero-length HBase startKey byte array to null");
        }
        if (0 == endKey.length) {
            endKey = null;
            logger.trace("Converted zero-length HBase endKey byte array to null");
        }
        if (null == startKey && null == endKey) {
            Preconditions.checkState(1 == raw.size());
            logger.debug("HBase table {} has a single region {}", tableName, regionInfo);
            // Choose arbitrary shared value = startKey = endKey
            return b.put(new KeyRange(FOUR_ZERO_BYTES, FOUR_ZERO_BYTES), e.getValue()).build();
        } else if (null == startKey) {
            logger.debug("Found HRegionInfo with null startKey on server {}: {}", e.getValue(), regionInfo);
            Preconditions.checkState(null == nullStart);
            nullStart = e;
            // I thought endBuf would be inclusive from the HBase javadoc, but in practice it is exclusive
            StaticBuffer endBuf = StaticArrayBuffer.of(zeroExtend(endKey));
            // Replace null start key with zeroes
            b.put(new KeyRange(FOUR_ZERO_BYTES, endBuf), e.getValue());
        } else if (null == endKey) {
            logger.debug("Found HRegionInfo with null endKey on server {}: {}", e.getValue(), regionInfo);
            Preconditions.checkState(null == nullEnd);
            nullEnd = e;
            // Replace null end key with zeroes
            b.put(new KeyRange(StaticArrayBuffer.of(zeroExtend(startKey)), FOUR_ZERO_BYTES), e.getValue());
        } else {
            Preconditions.checkState(null != startKey);
            Preconditions.checkState(null != endKey);
            // Convert HBase's inclusive end keys into exclusive Titan end keys
            StaticBuffer startBuf = StaticArrayBuffer.of(zeroExtend(startKey));
            StaticBuffer endBuf = StaticArrayBuffer.of(zeroExtend(endKey));
            KeyRange kr = new KeyRange(startBuf, endBuf);
            b.put(kr, e.getValue());
            logger.debug("Found HRegionInfo with non-null end and start keys on server {}: {}", e.getValue(), regionInfo);
        }
    }
    // Require either no null key bounds or a pair of them
    Preconditions.checkState(!(null == nullStart ^ null == nullEnd));
    // Check that every key in the result is at least 4 bytes long
    Map<KeyRange, ServerName> result = b.build();
    for (KeyRange kr : result.keySet()) {
        Preconditions.checkState(4 <= kr.getStart().length());
        Preconditions.checkState(4 <= kr.getEnd().length());
    }
    return result;
}

Example 3 with KeyRange

use of com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyRange in project incubator-atlas by apache.

the class HBaseStoreManager method normalizeKeyBounds.

/**
     * Given a map produced by {@link HTable#getRegionLocations()}, transform
     * each key from an {@link HRegionInfo} to a {@link KeyRange} expressing the
     * region's start and end key bounds using Titan-partitioning-friendly
     * conventions (start inclusive, end exclusive, zero bytes appended where
     * necessary to make all keys at least 4 bytes long).
     * <p/>
     * This method iterates over the entries in its map parameter and performs
     * the following conditional conversions on its keys. "Require" below means
     * either a {@link Preconditions} invocation or an assertion. HRegionInfo
     * sometimes returns start and end keys of zero length; this method replaces
     * zero length keys with null before doing any of the checks described
     * below. The parameter map and the values it contains are only read and
     * never modified.
     *
     * <ul>
     * <li>If an entry's HRegionInfo has null start and end keys, then first
     * require that the parameter map is a singleton, and then return a
     * single-entry map whose {@code KeyRange} has start and end buffers that
     * are both four bytes of zeros.</li>
     * <li>If the entry has a null end key (but non-null start key), put an
     * equivalent entry in the result map with a start key identical to the
     * input, except that zeros are appended to values less than 4 bytes long,
     * and an end key that is four bytes of zeros.
     * <li>If the entry has a null start key (but non-null end key), put an
     * equivalent entry in the result map where the start key is four bytes of
     * zeros, and the end key has zeros appended, if necessary, to make it at
     * least 4 bytes long, after which one is added to the padded value in
     * unsigned 32-bit arithmetic with overflow allowed.</li>
     * <li>Any entry which matches none of the above criteria results in an
     * equivalent entry in the returned map, except that zeros are appended to
     * both keys to make each at least 4 bytes long, and the end key is then
     * incremented as described in the last bullet point.</li>
     * </ul>
     *
     * After iterating over the parameter map, this method checks that it either
     * saw no entries with null keys, one entry with a null start key and a
     * different entry with a null end key, or one entry with both start and end
     * keys null. If any null keys are observed besides these three cases, the
     * method will die with a precondition failure.
     *
     * @param raw
     *            A map of HRegionInfo and ServerName from HBase
     * @return Titan-friendly expression of each region's rowkey boundaries
     */
private Map<KeyRange, ServerName> normalizeKeyBounds(NavigableMap<HRegionInfo, ServerName> raw) {
    Map.Entry<HRegionInfo, ServerName> nullStart = null;
    Map.Entry<HRegionInfo, ServerName> nullEnd = null;
    ImmutableMap.Builder<KeyRange, ServerName> b = ImmutableMap.builder();
    for (Map.Entry<HRegionInfo, ServerName> e : raw.entrySet()) {
        HRegionInfo regionInfo = e.getKey();
        byte[] startKey = regionInfo.getStartKey();
        byte[] endKey = regionInfo.getEndKey();
        if (0 == startKey.length) {
            startKey = null;
            logger.trace("Converted zero-length HBase startKey byte array to null");
        }
        if (0 == endKey.length) {
            endKey = null;
            logger.trace("Converted zero-length HBase endKey byte array to null");
        }
        if (null == startKey && null == endKey) {
            Preconditions.checkState(1 == raw.size());
            logger.debug("HBase table {} has a single region {}", tableName, regionInfo);
            // Choose arbitrary shared value = startKey = endKey
            return b.put(new KeyRange(FOUR_ZERO_BYTES, FOUR_ZERO_BYTES), e.getValue()).build();
        } else if (null == startKey) {
            logger.debug("Found HRegionInfo with null startKey on server {}: {}", e.getValue(), regionInfo);
            Preconditions.checkState(null == nullStart);
            nullStart = e;
            // I thought endBuf would be inclusive from the HBase javadoc, but in practice it is exclusive
            StaticBuffer endBuf = StaticArrayBuffer.of(zeroExtend(endKey));
            // Replace null start key with zeroes
            b.put(new KeyRange(FOUR_ZERO_BYTES, endBuf), e.getValue());
        } else if (null == endKey) {
            logger.debug("Found HRegionInfo with null endKey on server {}: {}", e.getValue(), regionInfo);
            Preconditions.checkState(null == nullEnd);
            nullEnd = e;
            // Replace null end key with zeroes
            b.put(new KeyRange(StaticArrayBuffer.of(zeroExtend(startKey)), FOUR_ZERO_BYTES), e.getValue());
        } else {
            Preconditions.checkState(null != startKey);
            Preconditions.checkState(null != endKey);
            // Convert HBase's inclusive end keys into exclusive Titan end keys
            StaticBuffer startBuf = StaticArrayBuffer.of(zeroExtend(startKey));
            StaticBuffer endBuf = StaticArrayBuffer.of(zeroExtend(endKey));
            KeyRange kr = new KeyRange(startBuf, endBuf);
            b.put(kr, e.getValue());
            logger.debug("Found HRegionInfo with non-null end and start keys on server {}: {}", e.getValue(), regionInfo);
        }
    }
    // Require either no null key bounds or a pair of them
    Preconditions.checkState((null == nullStart) == (null == nullEnd));
    // Check that every key in the result is at least 4 bytes long
    Map<KeyRange, ServerName> result = b.build();
    for (KeyRange kr : result.keySet()) {
        Preconditions.checkState(4 <= kr.getStart().length());
        Preconditions.checkState(4 <= kr.getEnd().length());
    }
    return result;
}

Example 4 with KeyRange

use of com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyRange in project incubator-atlas by apache.

the class HBaseStoreManager method getLocalKeyPartition.

@Override
public List<KeyRange> getLocalKeyPartition() throws BackendException {
    List<KeyRange> result = new LinkedList<>();
    TableMask table = null;
    try {
        ensureTableExists(tableName, getCfNameForStoreName(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME), 0);
        table = cnx.getTable(tableName);
        HTable hTable = (HTable) table.getTableObject();
        Map<KeyRange, ServerName> normed = normalizeKeyBounds(hTable.getRegionLocations());
        for (Map.Entry<KeyRange, ServerName> e : normed.entrySet()) {
            if (NetworkUtil.isLocalConnection(e.getValue().getHostname())) {
                result.add(e.getKey());
                logger.debug("Found local key/row partition {} on host {}", e.getKey(), e.getValue());
            } else {
                logger.debug("Discarding remote {}", e.getValue());
            }
        }
    } catch (MasterNotRunningException e) {
        logger.warn("Unexpected MasterNotRunningException", e);
    } catch (ZooKeeperConnectionException e) {
        logger.warn("Unexpected ZooKeeperConnectionException", e);
    } catch (IOException e) {
        logger.warn("Unexpected IOException", e);
    } finally {
        IOUtils.closeQuietly(table);
    }
    return result;
}

Example 5 with KeyRange

use of com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyRange in project titan by thinkaurelius.

the class PartitionIDRange method getIDRanges.

public static List<PartitionIDRange> getIDRanges(final int partitionBits, final List<KeyRange> locals) {
    Preconditions.checkArgument(partitionBits > 0 && partitionBits < (Integer.SIZE - 1));
    Preconditions.checkArgument(locals != null && !locals.isEmpty(), "KeyRanges are empty");
    final int partitionIdBound = (1 << (partitionBits));
    final int backShift = Integer.SIZE - partitionBits;
    List<PartitionIDRange> partitionRanges = Lists.newArrayList();
    for (KeyRange local : locals) {
        Preconditions.checkArgument(local.getStart().length() >= 4);
        Preconditions.checkArgument(local.getEnd().length() >= 4);
        if (local.getStart().equals(local.getEnd())) {
            //Start=End => Partition spans entire range
            partitionRanges.add(new PartitionIDRange(0, partitionIdBound, partitionIdBound));
            continue;
        }
        int startInt = local.getStart().getInt(0);
        int lowerID = startInt >>> backShift;
        assert lowerID >= 0 && lowerID < partitionIdBound;
        //Lower id must be inclusive, so check that we did not truncate anything!
        boolean truncatedBits = (lowerID << backShift) != startInt;
        StaticBuffer start = local.getAt(0);
        for (int i = 4; i < start.length() && !truncatedBits; i++) {
            if (start.getByte(i) != 0)
                truncatedBits = true;
        }
        //adjust to make sure we are inclusive
        if (truncatedBits)
            lowerID += 1;
        //upper id is exclusive
        int upperID = local.getEnd().getInt(0) >>> backShift;
        //Check that we haven't jumped order indicating that the interval was too small
        if ((local.getStart().compareTo(local.getEnd()) < 0 && lowerID >= upperID)) {
            discardRange(local);
            continue;
        }
        //ensure that lowerID remains within range
        lowerID = lowerID % partitionIdBound;
        if (lowerID == upperID) {
            //After re-normalizing, check for interval colision
            discardRange(local);
            continue;
        }
        partitionRanges.add(new PartitionIDRange(lowerID, upperID, partitionIdBound));
    }
    return partitionRanges;
}