Examples with LongSet com.carrotsearch.hppc.LongSet used on opensource projects

Example 1 with LongSet

use of com.carrotsearch.hppc.LongSet in project cassandra by apache.

the class OnDiskIndexTest method testCombiningOfThePartitionedSA.

@Test
public void testCombiningOfThePartitionedSA() throws Exception {
    OnDiskIndexBuilder builderA = new OnDiskIndexBuilder(UTF8Type.instance, LongType.instance, OnDiskIndexBuilder.Mode.PREFIX);
    OnDiskIndexBuilder builderB = new OnDiskIndexBuilder(UTF8Type.instance, LongType.instance, OnDiskIndexBuilder.Mode.PREFIX);
    TreeMap<Long, TreeMap<Long, LongSet>> expected = new TreeMap<>();
    for (long i = 0; i <= 100; i++) {
        TreeMap<Long, LongSet> offsets = expected.get(i);
        if (offsets == null)
            expected.put(i, (offsets = new TreeMap<>()));
        builderA.add(LongType.instance.decompose(i), keyAt(i), i);
        putAll(offsets, keyBuilder(i));
    }
    for (long i = 50; i < 100; i++) {
        TreeMap<Long, LongSet> offsets = expected.get(i);
        if (offsets == null)
            expected.put(i, (offsets = new TreeMap<>()));
        long position = 100L + i;
        builderB.add(LongType.instance.decompose(i), keyAt(position), position);
        putAll(offsets, keyBuilder(100L + i));
    }
    File indexA = File.createTempFile("on-disk-sa-partition-a", ".db");
    indexA.deleteOnExit();
    File indexB = File.createTempFile("on-disk-sa-partition-b", ".db");
    indexB.deleteOnExit();
    builderA.finish(indexA);
    builderB.finish(indexB);
    OnDiskIndex a = new OnDiskIndex(indexA, LongType.instance, new KeyConverter());
    OnDiskIndex b = new OnDiskIndex(indexB, LongType.instance, new KeyConverter());
    RangeIterator<OnDiskIndex.DataTerm, CombinedTerm> union = OnDiskIndexIterator.union(a, b);
    TreeMap<Long, TreeMap<Long, LongSet>> actual = new TreeMap<>();
    while (union.hasNext()) {
        CombinedTerm term = union.next();
        Long composedTerm = LongType.instance.compose(term.getTerm());
        TreeMap<Long, LongSet> offsets = actual.get(composedTerm);
        if (offsets == null)
            actual.put(composedTerm, (offsets = new TreeMap<>()));
        putAll(offsets, term.getTokenTreeBuilder());
    }
    Assert.assertEquals(actual, expected);
    File indexC = File.createTempFile("on-disk-sa-partition-final", ".db");
    indexC.deleteOnExit();
    OnDiskIndexBuilder combined = new OnDiskIndexBuilder(UTF8Type.instance, LongType.instance, OnDiskIndexBuilder.Mode.PREFIX);
    combined.finish(Pair.create(keyAt(0).getKey(), keyAt(100).getKey()), indexC, new CombinedTermIterator(a, b));
    OnDiskIndex c = new OnDiskIndex(indexC, LongType.instance, new KeyConverter());
    union = OnDiskIndexIterator.union(c);
    actual.clear();
    while (union.hasNext()) {
        CombinedTerm term = union.next();
        Long composedTerm = LongType.instance.compose(term.getTerm());
        TreeMap<Long, LongSet> offsets = actual.get(composedTerm);
        if (offsets == null)
            actual.put(composedTerm, (offsets = new TreeMap<>()));
        putAll(offsets, term.getTokenTreeBuilder());
    }
    Assert.assertEquals(actual, expected);
    a.close();
    b.close();
}

Example 2 with LongSet

use of com.carrotsearch.hppc.LongSet in project cassandra by apache.

the class OnDiskIndexTest method convert.

private static Set<DecoratedKey> convert(TokenTreeBuilder offsets) {
    Set<DecoratedKey> result = new HashSet<>();
    Iterator<Pair<Long, LongSet>> offsetIter = offsets.iterator();
    while (offsetIter.hasNext()) {
        LongSet v = offsetIter.next().right;
        for (LongCursor offset : v) result.add(keyAt(offset.value));
    }
    return result;
}

Example 3 with LongSet

use of com.carrotsearch.hppc.LongSet in project cassandra by apache.

the class TokenTreeTest method buildSerializeAndIterate.

public void buildSerializeAndIterate(TokenTreeBuilder builder, SortedMap<Long, LongSet> tokenMap) throws Exception {
    builder.finish();
    final File treeFile = File.createTempFile("token-tree-iterate-test1", "tt");
    treeFile.deleteOnExit();
    try (SequentialWriter writer = new SequentialWriter(treeFile, DEFAULT_OPT)) {
        builder.write(writer);
        writer.sync();
    }
    final RandomAccessReader reader = RandomAccessReader.open(treeFile);
    final TokenTree tokenTree = new TokenTree(new MappedBuffer(reader));
    final Iterator<Token> tokenIterator = tokenTree.iterator(KEY_CONVERTER);
    final Iterator<Map.Entry<Long, LongSet>> listIterator = tokenMap.entrySet().iterator();
    while (tokenIterator.hasNext() && listIterator.hasNext()) {
        Token treeNext = tokenIterator.next();
        Map.Entry<Long, LongSet> listNext = listIterator.next();
        Assert.assertEquals(listNext.getKey(), treeNext.get());
        Assert.assertEquals(convert(listNext.getValue()), convert(treeNext));
    }
    Assert.assertFalse("token iterator not finished", tokenIterator.hasNext());
    Assert.assertFalse("list iterator not finished", listIterator.hasNext());
    reader.close();
}

Example 4 with LongSet

use of com.carrotsearch.hppc.LongSet in project cassandra by apache.

the class RangeIntersectionIteratorTest method testIntersectionOfRandomRanges.

private void testIntersectionOfRandomRanges(Strategy strategy) {
    for (int attempt = 0; attempt < 16; attempt++) {
        final ThreadLocalRandom random = ThreadLocalRandom.current();
        final int maxRanges = random.nextInt(2, 16);
        // generate randomize ranges
        long[][] ranges = new long[maxRanges][];
        for (int i = 0; i < ranges.length; i++) {
            int rangeSize = random.nextInt(16, 512);
            LongSet range = new LongOpenHashSet(rangeSize);
            for (int j = 0; j < rangeSize; j++) range.add(random.nextLong(0, 100));
            ranges[i] = range.toArray();
            Arrays.sort(ranges[i]);
        }
        List<Long> expected = new ArrayList<>();
        // determine unique tokens which intersect every range
        for (long token : ranges[0]) {
            boolean intersectsAll = true;
            for (int i = 1; i < ranges.length; i++) {
                if (Arrays.binarySearch(ranges[i], token) < 0) {
                    intersectsAll = false;
                    break;
                }
            }
            if (intersectsAll)
                expected.add(token);
        }
        RangeIterator.Builder<Long, Token> builder = RangeIntersectionIterator.builder(strategy);
        for (long[] range : ranges) builder.add(new LongIterator(range));
        Assert.assertEquals(expected, convert(builder.build()));
    }
}

Example 5 with LongSet

use of com.carrotsearch.hppc.LongSet in project titan by thinkaurelius.

the class EdgeSerializer method writeRelation.

public StaticArrayEntry writeRelation(InternalRelation relation, InternalRelationType type, int position, TypeInspector tx) {
    assert type == relation.getType() || type.getBaseType().equals(relation.getType());
    Direction dir = EdgeDirection.fromPosition(position);
    Preconditions.checkArgument(type.isUnidirected(Direction.BOTH) || type.isUnidirected(dir));
    long typeid = type.longId();
    DirectionID dirID = getDirID(dir, relation.isProperty() ? RelationCategory.PROPERTY : RelationCategory.EDGE);
    DataOutput out = serializer.getDataOutput(DEFAULT_CAPACITY);
    int valuePosition;
    IDHandler.writeRelationType(out, typeid, dirID, type.isInvisibleType());
    Multiplicity multiplicity = type.multiplicity();
    long[] sortKey = type.getSortKey();
    assert !multiplicity.isConstrained() || sortKey.length == 0 : type.name();
    int keyStartPos = out.getPosition();
    if (!multiplicity.isConstrained()) {
        writeInlineTypes(sortKey, relation, out, tx, InlineType.KEY);
    }
    int keyEndPos = out.getPosition();
    long relationId = relation.longId();
    //How multiplicity is handled for edges and properties is slightly different
    if (relation.isEdge()) {
        long otherVertexId = relation.getVertex((position + 1) % 2).longId();
        if (multiplicity.isConstrained()) {
            if (multiplicity.isUnique(dir)) {
                valuePosition = out.getPosition();
                VariableLong.writePositive(out, otherVertexId);
            } else {
                VariableLong.writePositiveBackward(out, otherVertexId);
                valuePosition = out.getPosition();
            }
            VariableLong.writePositive(out, relationId);
        } else {
            VariableLong.writePositiveBackward(out, otherVertexId);
            VariableLong.writePositiveBackward(out, relationId);
            valuePosition = out.getPosition();
        }
    } else {
        assert relation.isProperty();
        Preconditions.checkArgument(relation.isProperty());
        Object value = ((TitanVertexProperty) relation).value();
        Preconditions.checkNotNull(value);
        PropertyKey key = (PropertyKey) type;
        assert key.dataType().isInstance(value);
        if (multiplicity.isConstrained()) {
            if (multiplicity.isUnique(dir)) {
                //Cardinality=SINGLE
                valuePosition = out.getPosition();
                writePropertyValue(out, key, value);
            } else {
                //Cardinality=SET
                writePropertyValue(out, key, value);
                valuePosition = out.getPosition();
            }
            VariableLong.writePositive(out, relationId);
        } else {
            assert multiplicity.getCardinality() == Cardinality.LIST;
            VariableLong.writePositiveBackward(out, relationId);
            valuePosition = out.getPosition();
            writePropertyValue(out, key, value);
        }
    }
    //Write signature
    long[] signature = type.getSignature();
    writeInlineTypes(signature, relation, out, tx, InlineType.SIGNATURE);
    //Write remaining properties
    LongSet writtenTypes = new LongHashSet(sortKey.length + signature.length);
    if (sortKey.length > 0 || signature.length > 0) {
        for (long id : sortKey) writtenTypes.add(id);
        for (long id : signature) writtenTypes.add(id);
    }
    LongArrayList remainingTypes = new LongArrayList(8);
    for (PropertyKey t : relation.getPropertyKeysDirect()) {
        if (!(t instanceof ImplicitKey) && !writtenTypes.contains(t.longId())) {
            remainingTypes.add(t.longId());
        }
    }
    //Sort types before writing to ensure that value is always written the same way
    long[] remaining = remainingTypes.toArray();
    Arrays.sort(remaining);
    for (long tid : remaining) {
        PropertyKey t = tx.getExistingPropertyKey(tid);
        writeInline(out, t, relation.getValueDirect(t), InlineType.NORMAL);
    }
    assert valuePosition > 0;
    StaticArrayEntry entry = new StaticArrayEntry(type.getSortOrder() == Order.DESC ? out.getStaticBufferFlipBytes(keyStartPos, keyEndPos) : out.getStaticBuffer(), valuePosition);
    return entry;
}