Examples with Token org.apache.cassandra.index.sasi.disk.Token used on opensource projects

Example 1 with Token

use of org.apache.cassandra.index.sasi.disk.Token in project cassandra by apache.

the class ColumnIndex method searchMemtable.

public RangeIterator<Long, Token> searchMemtable(Expression e) {
    RangeIterator.Builder<Long, Token> builder = new RangeUnionIterator.Builder<>();
    builder.add(getCurrentMemtable().search(e));
    for (IndexMemtable memtable : getPendingMemtables()) builder.add(memtable.search(e));
    return builder.build();
}

Example 2 with Token

use of org.apache.cassandra.index.sasi.disk.Token in project cassandra by apache.

the class SkipListMemIndex method search.

@SuppressWarnings("resource")
public RangeIterator<Long, Token> search(Expression expression) {
    ByteBuffer min = expression.lower == null ? null : expression.lower.value;
    ByteBuffer max = expression.upper == null ? null : expression.upper.value;
    SortedMap<ByteBuffer, ConcurrentSkipListSet<DecoratedKey>> search;
    if (min == null && max == null) {
        throw new IllegalArgumentException();
    }
    if (min != null && max != null) {
        search = index.subMap(min, expression.lower.inclusive, max, expression.upper.inclusive);
    } else if (min == null) {
        search = index.headMap(max, expression.upper.inclusive);
    } else {
        search = index.tailMap(min, expression.lower.inclusive);
    }
    RangeUnionIterator.Builder<Long, Token> builder = RangeUnionIterator.builder();
    for (ConcurrentSkipListSet<DecoratedKey> keys : search.values()) {
        int size;
        if ((size = keys.size()) > 0)
            builder.add(new KeyRangeIterator(keys, size));
    }
    return builder.build();
}

Example 3 with Token

use of org.apache.cassandra.index.sasi.disk.Token in project cassandra by apache.

the class TermIterator method build.

@SuppressWarnings("resource")
public static TermIterator build(final Expression e, Set<SSTableIndex> perSSTableIndexes) {
    final List<RangeIterator<Long, Token>> tokens = new CopyOnWriteArrayList<>();
    final AtomicLong tokenCount = new AtomicLong(0);
    RangeIterator<Long, Token> memtableIterator = e.index.searchMemtable(e);
    if (memtableIterator != null) {
        tokens.add(memtableIterator);
        tokenCount.addAndGet(memtableIterator.getCount());
    }
    final Set<SSTableIndex> referencedIndexes = new CopyOnWriteArraySet<>();
    try {
        final CountDownLatch latch = newCountDownLatch(perSSTableIndexes.size());
        final ExecutorService searchExecutor = SEARCH_EXECUTOR.get();
        for (final SSTableIndex index : perSSTableIndexes) {
            if (e.getOp() == PREFIX && index.mode() == CONTAINS && !index.hasMarkedPartials())
                throw new UnsupportedOperationException(format("The index %s has not yet been upgraded " + "to support prefix queries in CONTAINS mode. " + "Wait for compaction or rebuild the index.", index.getPath()));
            if (!index.reference()) {
                latch.decrement();
                continue;
            }
            // add to referenced right after the reference was acquired,
            // that helps to release index if something goes bad inside of the search
            referencedIndexes.add(index);
            searchExecutor.submit((Runnable) () -> {
                try {
                    e.checkpoint();
                    RangeIterator<Long, Token> keyIterator = index.search(e);
                    if (keyIterator == null) {
                        releaseIndex(referencedIndexes, index);
                        return;
                    }
                    tokens.add(keyIterator);
                    tokenCount.getAndAdd(keyIterator.getCount());
                } catch (Throwable e1) {
                    releaseIndex(referencedIndexes, index);
                    if (logger.isDebugEnabled())
                        logger.debug(format("Failed search an index %s, skipping.", index.getPath()), e1);
                } finally {
                    latch.decrement();
                }
            });
        }
        latch.awaitUninterruptibly();
        // checkpoint right away after all indexes complete search because we might have crossed the quota
        e.checkpoint();
        RangeIterator<Long, Token> ranges = RangeUnionIterator.build(tokens);
        return new TermIterator(e, ranges, referencedIndexes);
    } catch (Throwable ex) {
        // if execution quota was exceeded while opening indexes or something else happened
        // local (yet to be tracked) indexes should be released first before re-throwing exception
        referencedIndexes.forEach(TermIterator::releaseQuietly);
        throw ex;
    }
}

Example 4 with Token

use of org.apache.cassandra.index.sasi.disk.Token in project cassandra by apache.

the class RangeUnionIteratorTest method testRandomSequences.

@Test
public void testRandomSequences() {
    ThreadLocalRandom random = ThreadLocalRandom.current();
    long[][] values = new long[random.nextInt(1, 20)][];
    int numTests = random.nextInt(10, 20);
    for (int tests = 0; tests < numTests; tests++) {
        RangeUnionIterator.Builder<Long, Token> builder = RangeUnionIterator.builder();
        int totalCount = 0;
        for (int i = 0; i < values.length; i++) {
            long[] part = new long[random.nextInt(1, 500)];
            for (int j = 0; j < part.length; j++) part[j] = random.nextLong();
            // all of the parts have to be sorted to mimic SSTable
            Arrays.sort(part);
            values[i] = part;
            builder.add(new LongIterator(part));
            totalCount += part.length;
        }
        long[] totalOrdering = new long[totalCount];
        int index = 0;
        for (long[] part : values) {
            for (long value : part) totalOrdering[index++] = value;
        }
        Arrays.sort(totalOrdering);
        int count = 0;
        RangeIterator<Long, Token> tokens = builder.build();
        Assert.assertNotNull(tokens);
        while (tokens.hasNext()) Assert.assertEquals(totalOrdering[count++], (long) tokens.next().get());
        Assert.assertEquals(totalCount, count);
    }
}

Example 5 with Token

use of org.apache.cassandra.index.sasi.disk.Token in project cassandra by apache.

the class RangeIntersectionIteratorTest method testIntersectionOfRandomRanges.

private void testIntersectionOfRandomRanges(Strategy strategy) {
    for (int attempt = 0; attempt < 16; attempt++) {
        final ThreadLocalRandom random = ThreadLocalRandom.current();
        final int maxRanges = random.nextInt(2, 16);
        // generate randomize ranges
        long[][] ranges = new long[maxRanges][];
        for (int i = 0; i < ranges.length; i++) {
            int rangeSize = random.nextInt(16, 512);
            LongSet range = new LongHashSet(rangeSize);
            for (int j = 0; j < rangeSize; j++) range.add(random.nextLong(0, 100));
            ranges[i] = range.toArray();
            Arrays.sort(ranges[i]);
        }
        List<Long> expected = new ArrayList<>();
        // determine unique tokens which intersect every range
        for (long token : ranges[0]) {
            boolean intersectsAll = true;
            for (int i = 1; i < ranges.length; i++) {
                if (Arrays.binarySearch(ranges[i], token) < 0) {
                    intersectsAll = false;
                    break;
                }
            }
            if (intersectsAll)
                expected.add(token);
        }
        RangeIterator.Builder<Long, Token> builder = RangeIntersectionIterator.builder(strategy);
        for (long[] range : ranges) builder.add(new LongIterator(range));
        Assert.assertEquals(expected, convert(builder.build()));
    }
}