Examples with LongSet com.carrotsearch.hppc.LongSet used on opensource projects

Example 16 with LongSet

use of com.carrotsearch.hppc.LongSet in project cassandra by apache.

the class TokenTreeTest method generateTree.

private static TokenTree generateTree(final long minToken, final long maxToken, boolean isStatic) throws IOException {
    final SortedMap<Long, LongSet> toks = new TreeMap<Long, LongSet>() {

        {
            for (long i = minToken; i <= maxToken; i++) {
                LongSet offsetSet = new LongOpenHashSet();
                offsetSet.add(i);
                put(i, offsetSet);
            }
        }
    };
    final TokenTreeBuilder builder = isStatic ? new StaticTokenTreeBuilder(new FakeCombinedTerm(toks)) : new DynamicTokenTreeBuilder(toks);
    builder.finish();
    final File treeFile = File.createTempFile("token-tree-get-test", "tt");
    treeFile.deleteOnExit();
    try (SequentialWriter writer = new SequentialWriter(treeFile, DEFAULT_OPT)) {
        builder.write(writer);
        writer.sync();
    }
    RandomAccessReader reader = null;
    try {
        reader = RandomAccessReader.open(treeFile);
        return new TokenTree(new MappedBuffer(reader));
    } finally {
        FileUtils.closeQuietly(reader);
    }
}

Example 17 with LongSet

use of com.carrotsearch.hppc.LongSet in project cassandra by apache.

the class TokenTreeTest method testMergingOfEqualTokenTrees.

public void testMergingOfEqualTokenTrees(SortedMap<Long, LongSet> tokensMap) throws Exception {
    TokenTreeBuilder tokensA = new DynamicTokenTreeBuilder(tokensMap);
    TokenTreeBuilder tokensB = new DynamicTokenTreeBuilder(tokensMap);
    TokenTree a = buildTree(tokensA);
    TokenTree b = buildTree(tokensB);
    TokenTreeBuilder tokensC = new StaticTokenTreeBuilder(new CombinedTerm(null, null) {

        public RangeIterator<Long, Token> getTokenIterator() {
            RangeIterator.Builder<Long, Token> union = RangeUnionIterator.builder();
            union.add(a.iterator(new KeyConverter()));
            union.add(b.iterator(new KeyConverter()));
            return union.build();
        }
    });
    TokenTree c = buildTree(tokensC);
    Assert.assertEquals(tokensMap.size(), c.getCount());
    Iterator<Token> tokenIterator = c.iterator(KEY_CONVERTER);
    Iterator<Map.Entry<Long, LongSet>> listIterator = tokensMap.entrySet().iterator();
    while (tokenIterator.hasNext() && listIterator.hasNext()) {
        Token treeNext = tokenIterator.next();
        Map.Entry<Long, LongSet> listNext = listIterator.next();
        Assert.assertEquals(listNext.getKey(), treeNext.get());
        Assert.assertEquals(convert(listNext.getValue()), convert(treeNext));
    }
    for (Map.Entry<Long, LongSet> entry : tokensMap.entrySet()) {
        TokenTree.OnDiskToken result = c.get(entry.getKey(), KEY_CONVERTER);
        Assert.assertNotNull("failed to find object for token " + entry.getKey(), result);
        LongSet found = result.getOffsets();
        Assert.assertEquals(entry.getValue(), found);
    }
}

Example 18 with LongSet

use of com.carrotsearch.hppc.LongSet in project titan by thinkaurelius.

the class VertexIDAssignerTest method testIDAssignment.

@Test
public void testIDAssignment() {
    LongSet vertexIds = new LongHashSet();
    LongSet relationIds = new LongHashSet();
    int totalRelations = 0;
    int totalVertices = 0;
    for (int trial = 0; trial < 10; trial++) {
        for (boolean flush : new boolean[] { true, false }) {
            TitanGraph graph = getInMemoryGraph();
            int numVertices = 1000;
            List<TitanVertex> vertices = new ArrayList<TitanVertex>(numVertices);
            List<InternalRelation> relations = new ArrayList<InternalRelation>();
            TitanVertex old = null;
            totalRelations += 2 * numVertices;
            totalVertices += numVertices;
            try {
                for (int i = 0; i < numVertices; i++) {
                    TitanVertex next = graph.addVertex();
                    InternalRelation edge = null;
                    if (old != null) {
                        edge = (InternalRelation) old.addEdge("knows", next);
                    }
                    InternalRelation property = (InternalRelation) next.property("age", 25);
                    if (flush) {
                        idAssigner.assignID((InternalVertex) next, next.vertexLabel());
                        idAssigner.assignID(property);
                        if (edge != null)
                            idAssigner.assignID(edge);
                    }
                    relations.add(property);
                    if (edge != null)
                        relations.add(edge);
                    vertices.add(next);
                    old = next;
                }
                if (!flush)
                    idAssigner.assignIDs(relations);
                //Check if we should have exhausted the id pools
                if (totalRelations > maxIDAssignments || totalVertices > maxIDAssignments)
                    fail();
                //Verify that ids are set and unique
                for (TitanVertex v : vertices) {
                    assertTrue(v.hasId());
                    long id = v.longId();
                    assertTrue(id > 0 && id < Long.MAX_VALUE);
                    assertTrue(vertexIds.add(id));
                }
                for (InternalRelation r : relations) {
                    assertTrue(r.hasId());
                    long id = r.longId();
                    assertTrue(id > 0 && id < Long.MAX_VALUE);
                    assertTrue(relationIds.add(id));
                }
            } catch (IDPoolExhaustedException e) {
                //Since the id assignment process is randomized, we divide by 3/2 to account for minor variations
                assertTrue("Max Avail: " + maxIDAssignments + " vs. [" + totalVertices + "," + totalRelations + "]", totalRelations >= maxIDAssignments / 3 * 2 || totalVertices >= maxIDAssignments / 3 * 2);
            } finally {
                graph.tx().rollback();
                graph.close();
            }
        }
    }
}

Example 19 with LongSet

use of com.carrotsearch.hppc.LongSet in project janusgraph by JanusGraph.

the class IDAuthorityTest method testManyThreadsOneIDAuthority.

@Test
public void testManyThreadsOneIDAuthority() throws InterruptedException, ExecutionException {
    ExecutorService es = Executors.newFixedThreadPool(CONCURRENCY);
    final IDAuthority targetAuthority = idAuthorities[0];
    targetAuthority.setIDBlockSizer(new InnerIDBlockSizer());
    final int targetPartition = 0;
    final int targetNamespace = 2;
    final ConcurrentLinkedQueue<IDBlock> blocks = new ConcurrentLinkedQueue<>();
    final int blocksPerThread = 40;
    final List<Future<Void>> futures = new ArrayList<>(CONCURRENCY);
    // Start some concurrent threads getting blocks the same ID authority and same partition in that authority
    for (int c = 0; c < CONCURRENCY; c++) {
        futures.add(es.submit(new Callable<Void>() {

            @Override
            public Void call() {
                try {
                    getBlock();
                } catch (BackendException e) {
                    throw new RuntimeException(e);
                }
                return null;
            }

            private void getBlock() throws BackendException {
                for (int i = 0; i < blocksPerThread; i++) {
                    IDBlock block = targetAuthority.getIDBlock(targetPartition, targetNamespace, GET_ID_BLOCK_TIMEOUT);
                    Assert.assertNotNull(block);
                    blocks.add(block);
                }
            }
        }));
    }
    for (Future<Void> f : futures) {
        f.get();
    }
    es.shutdownNow();
    assertEquals(blocksPerThread * CONCURRENCY, blocks.size());
    LongSet ids = new LongHashSet((int) blockSize * blocksPerThread * CONCURRENCY);
    for (IDBlock block : blocks) checkBlock(block, ids);
}

Example 20 with LongSet

use of com.carrotsearch.hppc.LongSet in project janusgraph by JanusGraph.

the class IDAuthorityTest method testMultiIDAcquisition.

@Test
public void testMultiIDAcquisition() throws Throwable {
    final int numPartitions = MAX_NUM_PARTITIONS;
    final int numAcquisitionsPerThreadPartition = 100;
    final IDBlockSizer blockSizer = new InnerIDBlockSizer();
    for (int i = 0; i < CONCURRENCY; i++) idAuthorities[i].setIDBlockSizer(blockSizer);
    final List<ConcurrentLinkedQueue<IDBlock>> ids = new ArrayList<>(numPartitions);
    for (int i = 0; i < numPartitions; i++) {
        ids.add(new ConcurrentLinkedQueue<>());
    }
    final int maxIterations = numAcquisitionsPerThreadPartition * numPartitions * 2;
    final Collection<Future<?>> futures = new ArrayList<>(CONCURRENCY);
    ExecutorService es = Executors.newFixedThreadPool(CONCURRENCY);
    final Set<String> uniqueIds = new HashSet<>(CONCURRENCY);
    for (int i = 0; i < CONCURRENCY; i++) {
        final IDAuthority idAuthority = idAuthorities[i];
        final IDStressor stressRunnable = new IDStressor(numAcquisitionsPerThreadPartition, numPartitions, maxIterations, idAuthority, ids);
        uniqueIds.add(idAuthority.getUniqueID());
        futures.add(es.submit(stressRunnable));
    }
    // If this fails, it's likely to be a bug in the test rather than the
    // IDAuthority (the latter is technically possible, just less likely)
    assertEquals(CONCURRENCY, uniqueIds.size());
    for (Future<?> f : futures) {
        try {
            f.get();
        } catch (ExecutionException e) {
            throw e.getCause();
        }
    }
    for (int i = 0; i < numPartitions; i++) {
        ConcurrentLinkedQueue<IDBlock> list = ids.get(i);
        assertEquals(numAcquisitionsPerThreadPartition * CONCURRENCY, list.size());
        LongSet idSet = new LongHashSet((int) blockSize * list.size());
        for (IDBlock block : list) checkBlock(block, idSet);
    }
    es.shutdownNow();
}