Examples with PartitionPosition org.apache.cassandra.db.PartitionPosition used on opensource projects

Example 1 with PartitionPosition

use of org.apache.cassandra.db.PartitionPosition in project cassandra by apache.

the class StreamSession method getSSTableSectionsForRanges.

@VisibleForTesting
public static List<SSTableStreamingSections> getSSTableSectionsForRanges(Collection<Range<Token>> ranges, Collection<ColumnFamilyStore> stores, long overriddenRepairedAt, UUID pendingRepair) {
    Refs<SSTableReader> refs = new Refs<>();
    try {
        for (ColumnFamilyStore cfStore : stores) {
            final List<Range<PartitionPosition>> keyRanges = new ArrayList<>(ranges.size());
            for (Range<Token> range : ranges) keyRanges.add(Range.makeRowRange(range));
            refs.addAll(cfStore.selectAndReference(view -> {
                Set<SSTableReader> sstables = Sets.newHashSet();
                SSTableIntervalTree intervalTree = SSTableIntervalTree.build(view.select(SSTableSet.CANONICAL));
                Predicate<SSTableReader> predicate;
                if (pendingRepair == ActiveRepairService.NO_PENDING_REPAIR) {
                    predicate = Predicates.alwaysTrue();
                } else {
                    predicate = s -> s.isPendingRepair() && s.getSSTableMetadata().pendingRepair.equals(pendingRepair);
                }
                for (Range<PartitionPosition> keyRange : keyRanges) {
                    for (SSTableReader sstable : Iterables.filter(View.sstablesInBounds(keyRange.left, keyRange.right, intervalTree), predicate)) {
                        sstables.add(sstable);
                    }
                }
                if (logger.isDebugEnabled())
                    logger.debug("ViewFilter for {}/{} sstables", sstables.size(), Iterables.size(view.select(SSTableSet.CANONICAL)));
                return sstables;
            }).refs);
        }
        List<SSTableStreamingSections> sections = new ArrayList<>(refs.size());
        for (SSTableReader sstable : refs) {
            long repairedAt = overriddenRepairedAt;
            if (overriddenRepairedAt == ActiveRepairService.UNREPAIRED_SSTABLE)
                repairedAt = sstable.getSSTableMetadata().repairedAt;
            sections.add(new SSTableStreamingSections(refs.get(sstable), sstable.getPositionsForRanges(ranges), sstable.estimatedKeysForRanges(ranges), repairedAt));
        }
        return sections;
    } catch (Throwable t) {
        refs.release();
        throw t;
    }
}

Example 2 with PartitionPosition

use of org.apache.cassandra.db.PartitionPosition in project cassandra by apache.

the class ViewTest method testSSTablesInBounds.

@Test
public void testSSTablesInBounds() {
    ColumnFamilyStore cfs = MockSchema.newCFS();
    View initialView = fakeView(0, 5, cfs);
    for (int i = 0; i < 5; i++) {
        for (int j = i; j < 5; j++) {
            PartitionPosition min = MockSchema.readerBounds(i);
            PartitionPosition max = MockSchema.readerBounds(j);
            for (//, false} )
            boolean minInc : //, false} )
            new boolean[] { true }) {
                for (//, false} )
                boolean maxInc : //, false} )
                new boolean[] { true }) {
                    if (i == j && !(minInc && maxInc))
                        continue;
                    AbstractBounds<PartitionPosition> bounds = AbstractBounds.bounds(min, minInc, max, maxInc);
                    List<SSTableReader> r = ImmutableList.copyOf(initialView.liveSSTablesInBounds(bounds.left, bounds.right));
                    Assert.assertEquals(String.format("%d(%s) %d(%s)", i, minInc, j, maxInc), j - i + (minInc ? 0 : -1) + (maxInc ? 1 : 0), r.size());
                }
            }
        }
    }
}

Example 3 with PartitionPosition

use of org.apache.cassandra.db.PartitionPosition in project cassandra by apache.

the class SSTableScannerTest method dataRanges.

private static Iterable<DataRange> dataRanges(TableMetadata metadata, int start, int end, boolean inclusiveStart, boolean inclusiveEnd) {
    List<DataRange> ranges = new ArrayList<>();
    if (start == end + 1) {
        assert !inclusiveStart && inclusiveEnd;
        ranges.add(dataRange(metadata, min(start), false, max(end), true));
        ranges.add(dataRange(metadata, min(start), false, min(end + 1), true));
        ranges.add(dataRange(metadata, max(start - 1), false, max(end), true));
        ranges.add(dataRange(metadata, dk(start - 1), false, dk(start - 1), true));
    } else {
        for (PartitionPosition s : starts(start, inclusiveStart)) {
            for (PartitionPosition e : ends(end, inclusiveEnd)) {
                if (end < start && e.compareTo(s) > 0)
                    continue;
                if (!isEmpty(new AbstractBounds.Boundary<>(s, inclusiveStart), new AbstractBounds.Boundary<>(e, inclusiveEnd)))
                    continue;
                ranges.add(dataRange(metadata, s, inclusiveStart, e, inclusiveEnd));
            }
        }
    }
    return ranges;
}

Example 4 with PartitionPosition

use of org.apache.cassandra.db.PartitionPosition in project cassandra by apache.

the class LeveledManifest method getOverlappingStarvedSSTables.

/**
     * If we do something that makes many levels contain too little data (cleanup, change sstable size) we will "never"
     * compact the high levels.
     *
     * This method finds if we have gone many compaction rounds without doing any high-level compaction, if so
     * we start bringing in one sstable from the highest level until that level is either empty or is doing compaction.
     *
     * @param targetLevel the level the candidates will be compacted into
     * @param candidates the original sstables to compact
     * @return
     */
private Collection<SSTableReader> getOverlappingStarvedSSTables(int targetLevel, Collection<SSTableReader> candidates) {
    Set<SSTableReader> withStarvedCandidate = new HashSet<>(candidates);
    for (int i = generations.length - 1; i > 0; i--) compactionCounter[i]++;
    compactionCounter[targetLevel] = 0;
    if (logger.isTraceEnabled()) {
        for (int j = 0; j < compactionCounter.length; j++) logger.trace("CompactionCounter: {}: {}", j, compactionCounter[j]);
    }
    for (int i = generations.length - 1; i > 0; i--) {
        if (getLevelSize(i) > 0) {
            if (compactionCounter[i] > NO_COMPACTION_LIMIT) {
                // we try to find an sstable that is fully contained within  the boundaries we are compacting;
                // say we are compacting 3 sstables: 0->30 in L1 and 0->12, 12->33 in L2
                // this means that we will not create overlap in L2 if we add an sstable
                // contained within 0 -> 33 to the compaction
                PartitionPosition max = null;
                PartitionPosition min = null;
                for (SSTableReader candidate : candidates) {
                    if (min == null || candidate.first.compareTo(min) < 0)
                        min = candidate.first;
                    if (max == null || candidate.last.compareTo(max) > 0)
                        max = candidate.last;
                }
                if (// single partition sstables - we cannot include a high level sstable.
                min == null || max == null || min.equals(max))
                    return candidates;
                Set<SSTableReader> compacting = cfs.getTracker().getCompacting();
                Range<PartitionPosition> boundaries = new Range<>(min, max);
                for (SSTableReader sstable : getLevel(i)) {
                    Range<PartitionPosition> r = new Range<PartitionPosition>(sstable.first, sstable.last);
                    if (boundaries.contains(r) && !compacting.contains(sstable)) {
                        logger.info("Adding high-level (L{}) {} to candidates", sstable.getSSTableLevel(), sstable);
                        withStarvedCandidate.add(sstable);
                        return withStarvedCandidate;
                    }
                }
            }
            return candidates;
        }
    }
    return candidates;
}

Example 5 with PartitionPosition

use of org.apache.cassandra.db.PartitionPosition in project cassandra by apache.

the class LeveledManifest method getCandidatesFor.

/**
     * @return highest-priority sstables to compact for the given level.
     * If no compactions are possible (because of concurrent compactions or because some sstables are blacklisted
     * for prior failure), will return an empty list.  Never returns null.
     */
private Collection<SSTableReader> getCandidatesFor(int level) {
    assert !getLevel(level).isEmpty();
    logger.trace("Choosing candidates for L{}", level);
    final Set<SSTableReader> compacting = cfs.getTracker().getCompacting();
    if (level == 0) {
        Set<SSTableReader> compactingL0 = getCompacting(0);
        PartitionPosition lastCompactingKey = null;
        PartitionPosition firstCompactingKey = null;
        for (SSTableReader candidate : compactingL0) {
            if (firstCompactingKey == null || candidate.first.compareTo(firstCompactingKey) < 0)
                firstCompactingKey = candidate.first;
            if (lastCompactingKey == null || candidate.last.compareTo(lastCompactingKey) > 0)
                lastCompactingKey = candidate.last;
        }
        // L0 is the dumping ground for new sstables which thus may overlap each other.
        //
        // We treat L0 compactions specially:
        // 1a. add sstables to the candidate set until we have at least maxSSTableSizeInMB
        // 1b. prefer choosing older sstables as candidates, to newer ones
        // 1c. any L0 sstables that overlap a candidate, will also become candidates
        // 2. At most MAX_COMPACTING_L0 sstables from L0 will be compacted at once
        // 3. If total candidate size is less than maxSSTableSizeInMB, we won't bother compacting with L1,
        //    and the result of the compaction will stay in L0 instead of being promoted (see promote())
        //
        // Note that we ignore suspect-ness of L1 sstables here, since if an L1 sstable is suspect we're
        // basically screwed, since we expect all or most L0 sstables to overlap with each L1 sstable.
        // So if an L1 sstable is suspect we can't do much besides try anyway and hope for the best.
        Set<SSTableReader> candidates = new HashSet<>();
        Map<SSTableReader, Bounds<Token>> remaining = genBounds(Iterables.filter(getLevel(0), Predicates.not(suspectP)));
        for (SSTableReader sstable : ageSortedSSTables(remaining.keySet())) {
            if (candidates.contains(sstable))
                continue;
            Sets.SetView<SSTableReader> overlappedL0 = Sets.union(Collections.singleton(sstable), overlappingWithBounds(sstable, remaining));
            if (!Sets.intersection(overlappedL0, compactingL0).isEmpty())
                continue;
            for (SSTableReader newCandidate : overlappedL0) {
                if (firstCompactingKey == null || lastCompactingKey == null || overlapping(firstCompactingKey.getToken(), lastCompactingKey.getToken(), Arrays.asList(newCandidate)).size() == 0)
                    candidates.add(newCandidate);
                remaining.remove(newCandidate);
            }
            if (candidates.size() > MAX_COMPACTING_L0) {
                // limit to only the MAX_COMPACTING_L0 oldest candidates
                candidates = new HashSet<>(ageSortedSSTables(candidates).subList(0, MAX_COMPACTING_L0));
                break;
            }
        }
        // leave everything in L0 if we didn't end up with a full sstable's worth of data
        if (SSTableReader.getTotalBytes(candidates) > maxSSTableSizeInBytes) {
            // add sstables from L1 that overlap candidates
            // if the overlapping ones are already busy in a compaction, leave it out.
            // TODO try to find a set of L0 sstables that only overlaps with non-busy L1 sstables
            Set<SSTableReader> l1overlapping = overlapping(candidates, getLevel(1));
            if (Sets.intersection(l1overlapping, compacting).size() > 0)
                return Collections.emptyList();
            if (!overlapping(candidates, compactingL0).isEmpty())
                return Collections.emptyList();
            candidates = Sets.union(candidates, l1overlapping);
        }
        if (candidates.size() < 2)
            return Collections.emptyList();
        else
            return candidates;
    }
    // for non-L0 compactions, pick up where we left off last time
    Collections.sort(getLevel(level), SSTableReader.sstableComparator);
    // handles case where the prior compaction touched the very last range
    int start = 0;
    for (int i = 0; i < getLevel(level).size(); i++) {
        SSTableReader sstable = getLevel(level).get(i);
        if (sstable.first.compareTo(lastCompactedKeys[level]) > 0) {
            start = i;
            break;
        }
    }
    // look for a non-suspect keyspace to compact with, starting with where we left off last time,
    // and wrapping back to the beginning of the generation if necessary
    Map<SSTableReader, Bounds<Token>> sstablesNextLevel = genBounds(getLevel(level + 1));
    for (int i = 0; i < getLevel(level).size(); i++) {
        SSTableReader sstable = getLevel(level).get((start + i) % getLevel(level).size());
        Set<SSTableReader> candidates = Sets.union(Collections.singleton(sstable), overlappingWithBounds(sstable, sstablesNextLevel));
        if (Iterables.any(candidates, suspectP))
            continue;
        if (Sets.intersection(candidates, compacting).isEmpty())
            return candidates;
    }
    // all the sstables were suspect or overlapped with something suspect
    return Collections.emptyList();
}