Examples with KeyRange org.janusgraph.diskstorage.keycolumnvalue.KeyRange used on opensource projects

Example 1 with KeyRange

use of org.janusgraph.diskstorage.keycolumnvalue.KeyRange in project janusgraph by JanusGraph.

the class CassandraHelper method transformRange.

public static KeyRange transformRange(Token leftKeyExclusive, Token rightKeyInclusive) {
    if (!(leftKeyExclusive instanceof BytesToken))
        throw new UnsupportedOperationException();
    // if left part is BytesToken, right part should be too, otherwise there is no sense in the ring
    assert rightKeyInclusive instanceof BytesToken;
    // l is exclusive, r is inclusive
    BytesToken l = (BytesToken) leftKeyExclusive;
    BytesToken r = (BytesToken) rightKeyInclusive;
    byte[] leftTokenValue = l.getTokenValue();
    byte[] rightTokenValue = r.getTokenValue();
    Preconditions.checkArgument(leftTokenValue.length == rightTokenValue.length, "Tokens have unequal length");
    int tokenLength = leftTokenValue.length;
    byte[][] tokens = new byte[][] { leftTokenValue, rightTokenValue };
    byte[][] plusOne = new byte[2][tokenLength];
    for (int j = 0; j < 2; j++) {
        boolean carry = true;
        for (int i = tokenLength - 1; i >= 0; i--) {
            byte b = tokens[j][i];
            if (carry) {
                b++;
                carry = b == 0;
            }
            plusOne[j][i] = b;
        }
    }
    StaticBuffer lb = StaticArrayBuffer.of(plusOne[0]);
    StaticBuffer rb = StaticArrayBuffer.of(plusOne[1]);
    Preconditions.checkArgument(lb.length() == tokenLength, lb.length());
    Preconditions.checkArgument(rb.length() == tokenLength, rb.length());
    return new KeyRange(lb, rb);
}

Example 2 with KeyRange

use of org.janusgraph.diskstorage.keycolumnvalue.KeyRange in project janusgraph by JanusGraph.

the class CassandraThriftStoreManager method getLocalKeyPartition.

@Override
public List<KeyRange> getLocalKeyPartition() throws BackendException {
    CTConnection conn = null;
    IPartitioner partitioner = getCassandraPartitioner();
    if (!(partitioner instanceof AbstractByteOrderedPartitioner))
        throw new UnsupportedOperationException("getLocalKeyPartition() only supported by byte ordered partitioner.");
    Token.TokenFactory tokenFactory = partitioner.getTokenFactory();
    try {
        // Resist the temptation to describe SYSTEM_KS.  It has no ring.
        // Instead, we'll create our own keyspace (or check that it exists), then describe it.
        ensureKeyspaceExists(keySpaceName);
        conn = pool.borrowObject(keySpaceName);
        final List<TokenRange> ranges = conn.getClient().describe_ring(keySpaceName);
        final List<KeyRange> keyRanges = new ArrayList<>(ranges.size());
        for (TokenRange range : ranges) {
            if (!NetworkUtil.hasLocalAddress(range.endpoints))
                continue;
            keyRanges.add(CassandraHelper.transformRange(tokenFactory.fromString(range.start_token), tokenFactory.fromString(range.end_token)));
        }
        return keyRanges;
    } catch (Exception e) {
        throw CassandraThriftKeyColumnValueStore.convertException(e);
    } finally {
        pool.returnObjectUnsafe(keySpaceName, conn);
    }
}

Example 3 with KeyRange

use of org.janusgraph.diskstorage.keycolumnvalue.KeyRange in project janusgraph by JanusGraph.

the class HBaseStoreManager method normalizeKeyBounds.

/**
 * Given a map produced by {@link Connection#getRegionLocator(TableName)}, transform
 * each key from an {@link RegionInfo} to a {@link KeyRange} expressing the
 * region's start and end key bounds using JanusGraph-partitioning-friendly
 * conventions (start inclusive, end exclusive, zero bytes appended where
 * necessary to make all keys at least 4 bytes long).
 * <p/>
 * This method iterates over the entries in its map parameter and performs
 * the following conditional conversions on its keys. "Require" below means
 * either a {@link Preconditions} invocation or an assertion. HRegionInfo
 * sometimes returns start and end keys of zero length; this method replaces
 * zero length keys with null before doing any of the checks described
 * below. The parameter map and the values it contains are only read and
 * never modified.
 *
 * <ul>
 * <li>If an entry's HRegionInfo has null start and end keys, then first
 * require that the parameter map is a singleton, and then return a
 * single-entry map whose {@code KeyRange} has start and end buffers that
 * are both four bytes of zeros.</li>
 * <li>If the entry has a null end key (but non-null start key), put an
 * equivalent entry in the result map with a start key identical to the
 * input, except that zeros are appended to values less than 4 bytes long,
 * and an end key that is four bytes of zeros.
 * <li>If the entry has a null start key (but non-null end key), put an
 * equivalent entry in the result map where the start key is four bytes of
 * zeros, and the end key has zeros appended, if necessary, to make it at
 * least 4 bytes long, after which one is added to the padded value in
 * unsigned 32-bit arithmetic with overflow allowed.</li>
 * <li>Any entry which matches none of the above criteria results in an
 * equivalent entry in the returned map, except that zeros are appended to
 * both keys to make each at least 4 bytes long, and the end key is then
 * incremented as described in the last bullet point.</li>
 * </ul>
 *
 * After iterating over the parameter map, this method checks that it either
 * saw no entries with null keys, one entry with a null start key and a
 * different entry with a null end key, or one entry with both start and end
 * keys null. If any null keys are observed besides these three cases, the
 * method will die with a precondition failure.
 *
 * @param locations A list of HRegionInfo
 * @return JanusGraph-friendly expression of each region's rowkey boundaries
 */
private Map<KeyRange, ServerName> normalizeKeyBounds(List<HRegionLocation> locations) {
    HRegionLocation nullStart = null;
    HRegionLocation nullEnd = null;
    ImmutableMap.Builder<KeyRange, ServerName> b = ImmutableMap.builder();
    for (HRegionLocation location : locations) {
        RegionInfo regionInfo = location.getRegion();
        ServerName serverName = location.getServerName();
        byte[] startKey = regionInfo.getStartKey();
        byte[] endKey = regionInfo.getEndKey();
        if (0 == startKey.length) {
            startKey = null;
            logger.trace("Converted zero-length HBase startKey byte array to null");
        }
        if (0 == endKey.length) {
            endKey = null;
            logger.trace("Converted zero-length HBase endKey byte array to null");
        }
        if (null == startKey && null == endKey) {
            Preconditions.checkState(1 == locations.size());
            logger.debug("HBase table {} has a single region {}", tableName, regionInfo);
            // Choose arbitrary shared value = startKey = endKey
            return b.put(new KeyRange(FOUR_ZERO_BYTES, FOUR_ZERO_BYTES), serverName).build();
        } else if (null == startKey) {
            logger.debug("Found HRegionInfo with null startKey on server {}: {}", serverName, regionInfo);
            Preconditions.checkState(null == nullStart);
            nullStart = location;
            // I thought endBuf would be inclusive from the HBase javadoc, but in practice it is exclusive
            StaticBuffer endBuf = StaticArrayBuffer.of(zeroExtend(endKey));
            // Replace null start key with zeroes
            b.put(new KeyRange(FOUR_ZERO_BYTES, endBuf), serverName);
        } else if (null == endKey) {
            logger.debug("Found HRegionInfo with null endKey on server {}: {}", serverName, regionInfo);
            Preconditions.checkState(null == nullEnd);
            nullEnd = location;
            // Replace null end key with zeroes
            b.put(new KeyRange(StaticArrayBuffer.of(zeroExtend(startKey)), FOUR_ZERO_BYTES), serverName);
        } else {
            // Convert HBase's inclusive end keys into exclusive JanusGraph end keys
            StaticBuffer startBuf = StaticArrayBuffer.of(zeroExtend(startKey));
            StaticBuffer endBuf = StaticArrayBuffer.of(zeroExtend(endKey));
            KeyRange kr = new KeyRange(startBuf, endBuf);
            b.put(kr, serverName);
            logger.debug("Found HRegionInfo with non-null end and start keys on server {}: {}", serverName, regionInfo);
        }
    }
    // Require either no null key bounds or a pair of them
    Preconditions.checkState((null == nullStart) == (null == nullEnd));
    // Check that every key in the result is at least 4 bytes long
    Map<KeyRange, ServerName> result = b.build();
    for (KeyRange kr : result.keySet()) {
        Preconditions.checkState(4 <= kr.getStart().length());
        Preconditions.checkState(4 <= kr.getEnd().length());
    }
    return result;
}

Example 4 with KeyRange

use of org.janusgraph.diskstorage.keycolumnvalue.KeyRange in project janusgraph by JanusGraph.

the class HBaseStoreManager method getLocalKeyPartition.

@Override
public List<KeyRange> getLocalKeyPartition() throws BackendException {
    List<KeyRange> result = new LinkedList<>();
    try {
        if (skipSchemaCheck) {
            logger.debug("Skipping schema check");
            if (!exists())
                throw new PermanentBackendException("Table " + tableName + " doesn't exist in HBase!");
        } else {
            logger.debug("Performing schema check");
            ensureTableExists(tableName, getCfNameForStoreName(GraphDatabaseConfiguration.SYSTEM_PROPERTIES_STORE_NAME), 0);
        }
        Map<KeyRange, ServerName> normed = normalizeKeyBounds(getRegionLocations(tableName));
        for (Map.Entry<KeyRange, ServerName> e : normed.entrySet()) {
            if (NetworkUtil.isLocalConnection(e.getValue().getHostname())) {
                result.add(e.getKey());
                logger.debug("Found local key/row partition {} on host {}", e.getKey(), e.getValue());
            } else {
                logger.debug("Discarding remote {}", e.getValue());
            }
        }
    } catch (MasterNotRunningException e) {
        logger.warn("Unexpected MasterNotRunningException", e);
    } catch (ZooKeeperConnectionException e) {
        logger.warn("Unexpected ZooKeeperConnectionException", e);
    } catch (IOException e) {
        logger.warn("Unexpected IOException", e);
    }
    return result;
}

Example 5 with KeyRange

use of org.janusgraph.diskstorage.keycolumnvalue.KeyRange in project janusgraph by JanusGraph.

the class PartitionIDRange method getIDRanges.

public static List<PartitionIDRange> getIDRanges(final int partitionBits, final List<KeyRange> locals) {
    Preconditions.checkArgument(partitionBits > 0 && partitionBits < (Integer.SIZE - 1));
    Preconditions.checkArgument(locals != null && !locals.isEmpty(), "KeyRanges are empty");
    final int partitionIdBound = (1 << (partitionBits));
    final int backShift = Integer.SIZE - partitionBits;
    List<PartitionIDRange> partitionRanges = new ArrayList<>();
    for (KeyRange local : locals) {
        Preconditions.checkArgument(local.getStart().length() >= 4);
        Preconditions.checkArgument(local.getEnd().length() >= 4);
        if (local.getStart().equals(local.getEnd())) {
            // Start=End => Partition spans entire range
            partitionRanges.add(new PartitionIDRange(0, partitionIdBound, partitionIdBound));
            continue;
        }
        int startInt = local.getStart().getInt(0);
        int lowerID = startInt >>> backShift;
        assert lowerID >= 0 && lowerID < partitionIdBound;
        // Lower id must be inclusive, so check that we did not truncate anything!
        boolean truncatedBits = (lowerID << backShift) != startInt;
        StaticBuffer start = local.getAt(0);
        for (int i = 4; i < start.length() && !truncatedBits; i++) {
            if (start.getByte(i) != 0)
                truncatedBits = true;
        }
        // adjust to make sure we are inclusive
        if (truncatedBits)
            lowerID += 1;
        // upper id is exclusive
        int upperID = local.getEnd().getInt(0) >>> backShift;
        // Check that we haven't jumped order indicating that the interval was too small
        if ((local.getStart().compareTo(local.getEnd()) < 0 && lowerID >= upperID)) {
            discardRange(local);
            continue;
        }
        // ensure that lowerID remains within range
        lowerID = lowerID % partitionIdBound;
        if (lowerID == upperID) {
            // After re-normalizing, check for interval collision
            discardRange(local);
            continue;
        }
        partitionRanges.add(new PartitionIDRange(lowerID, upperID, partitionIdBound));
    }
    return partitionRanges;
}