Examples with OutgoingStream org.apache.cassandra.streaming.OutgoingStream used on opensource projects

Example 1 with OutgoingStream

use of org.apache.cassandra.streaming.OutgoingStream in project cassandra by apache.

the class LegacySSTableTest method streamLegacyTable.

private void streamLegacyTable(String tablePattern, String legacyVersion) throws Exception {
    String table = String.format(tablePattern, legacyVersion);
    SSTableReader sstable = SSTableReader.open(getDescriptor(legacyVersion, table));
    IPartitioner p = sstable.getPartitioner();
    List<Range<Token>> ranges = new ArrayList<>();
    ranges.add(new Range<>(p.getMinimumToken(), p.getToken(ByteBufferUtil.bytes("100"))));
    ranges.add(new Range<>(p.getToken(ByteBufferUtil.bytes("100")), p.getMinimumToken()));
    List<OutgoingStream> streams = Lists.newArrayList(new CassandraOutgoingFile(StreamOperation.OTHER, sstable.ref(), sstable.getPositionsForRanges(ranges), ranges, sstable.estimatedKeysForRanges(ranges)));
    new StreamPlan(StreamOperation.OTHER).transferStreams(FBUtilities.getBroadcastAddressAndPort(), streams).execute().get();
}

Example 2 with OutgoingStream

use of org.apache.cassandra.streaming.OutgoingStream in project cassandra by apache.

the class CassandraStreamManager method createOutgoingStreams.

@Override
public Collection<OutgoingStream> createOutgoingStreams(StreamSession session, RangesAtEndpoint replicas, UUID pendingRepair, PreviewKind previewKind) {
    Refs<SSTableReader> refs = new Refs<>();
    try {
        final List<Range<PartitionPosition>> keyRanges = new ArrayList<>(replicas.size());
        for (Replica replica : replicas) keyRanges.add(Range.makeRowRange(replica.range()));
        refs.addAll(cfs.selectAndReference(view -> {
            Set<SSTableReader> sstables = Sets.newHashSet();
            SSTableIntervalTree intervalTree = SSTableIntervalTree.build(view.select(SSTableSet.CANONICAL));
            Predicate<SSTableReader> predicate;
            if (previewKind.isPreview()) {
                predicate = previewKind.predicate();
            } else if (pendingRepair == ActiveRepairService.NO_PENDING_REPAIR) {
                predicate = Predicates.alwaysTrue();
            } else {
                predicate = s -> s.isPendingRepair() && s.getSSTableMetadata().pendingRepair.equals(pendingRepair);
            }
            for (Range<PartitionPosition> keyRange : keyRanges) {
                // still actually selecting what we wanted.
                for (SSTableReader sstable : Iterables.filter(View.sstablesInBounds(keyRange.left, keyRange.right, intervalTree), predicate)) {
                    sstables.add(sstable);
                }
            }
            if (logger.isDebugEnabled())
                logger.debug("ViewFilter for {}/{} sstables", sstables.size(), Iterables.size(view.select(SSTableSet.CANONICAL)));
            return sstables;
        }).refs);
        List<Range<Token>> normalizedFullRanges = Range.normalize(replicas.onlyFull().ranges());
        List<Range<Token>> normalizedAllRanges = Range.normalize(replicas.ranges());
        // Create outgoing file streams for ranges possibly skipping repaired ranges in sstables
        List<OutgoingStream> streams = new ArrayList<>(refs.size());
        for (SSTableReader sstable : refs) {
            List<Range<Token>> ranges = sstable.isRepaired() ? normalizedFullRanges : normalizedAllRanges;
            List<SSTableReader.PartitionPositionBounds> sections = sstable.getPositionsForRanges(ranges);
            Ref<SSTableReader> ref = refs.get(sstable);
            if (sections.isEmpty()) {
                ref.release();
                continue;
            }
            streams.add(new CassandraOutgoingFile(session.getStreamOperation(), ref, sections, ranges, sstable.estimatedKeysForRanges(ranges)));
        }
        return streams;
    } catch (Throwable t) {
        refs.release();
        throw t;
    }
}

Example 3 with OutgoingStream

use of org.apache.cassandra.streaming.OutgoingStream in project cassandra by apache.

the class CassandraStreamManagerTest method sstablesFromStreams.

private static Set<SSTableReader> sstablesFromStreams(Collection<OutgoingStream> streams) {
    Set<SSTableReader> sstables = new HashSet<>();
    for (OutgoingStream stream : streams) {
        Ref<SSTableReader> ref = CassandraOutgoingFile.fromStream(stream).getRef();
        sstables.add(ref.get());
        ref.release();
    }
    return sstables;
}

Example 4 with OutgoingStream

use of org.apache.cassandra.streaming.OutgoingStream in project cassandra by apache.

the class CassandraStreamManagerTest method selectReaders.

private Set<SSTableReader> selectReaders(UUID pendingRepair) {
    IPartitioner partitioner = DatabaseDescriptor.getPartitioner();
    Collection<Range<Token>> ranges = Lists.newArrayList(new Range<Token>(partitioner.getMinimumToken(), partitioner.getMinimumToken()));
    Collection<OutgoingStream> streams = cfs.getStreamManager().createOutgoingStreams(session(pendingRepair), RangesAtEndpoint.toDummyList(ranges), pendingRepair, PreviewKind.NONE);
    return sstablesFromStreams(streams);
}

Example 5 with OutgoingStream

use of org.apache.cassandra.streaming.OutgoingStream in project cassandra by apache.

the class EntireSSTableStreamConcurrentComponentMutationTest method testStreamWithConcurrentComponentMutation.

private void testStreamWithConcurrentComponentMutation(Callable<?> runBeforeStreaming, Callable<?> runConcurrentWithStreaming) throws Throwable {
    ByteBuf serializedFile = Unpooled.buffer(8192);
    InetAddressAndPort peer = FBUtilities.getBroadcastAddressAndPort();
    StreamSession session = setupStreamingSessionForTest();
    Collection<OutgoingStream> outgoingStreams = store.getStreamManager().createOutgoingStreams(session, rangesAtEndpoint, NO_PENDING_REPAIR, PreviewKind.NONE);
    CassandraOutgoingFile outgoingFile = (CassandraOutgoingFile) Iterables.getOnlyElement(outgoingStreams);
    Future<?> streaming = executeAsync(() -> {
        runBeforeStreaming.call();
        try (AsyncStreamingOutputPlus out = new AsyncStreamingOutputPlus(createMockNettyChannel(serializedFile))) {
            outgoingFile.write(session, out, MessagingService.current_version);
            assertTrue(sstable.descriptor.getTemporaryFiles().isEmpty());
        }
        return null;
    });
    Future<?> concurrentMutations = executeAsync(runConcurrentWithStreaming);
    streaming.get(3, TimeUnit.MINUTES);
    concurrentMutations.get(3, TimeUnit.MINUTES);
    session.prepareReceiving(new StreamSummary(sstable.metadata().id, 1, 5104));
    StreamMessageHeader messageHeader = new StreamMessageHeader(sstable.metadata().id, peer, session.planId(), false, 0, 0, 0, null);
    try (DataInputBuffer in = new DataInputBuffer(serializedFile.nioBuffer(), false)) {
        CassandraStreamHeader header = CassandraStreamHeader.serializer.deserialize(in, MessagingService.current_version);
        CassandraEntireSSTableStreamReader reader = new CassandraEntireSSTableStreamReader(messageHeader, header, session);
        SSTableReader streamedSSTable = Iterables.getOnlyElement(reader.read(in).finished());
        SSTableUtils.assertContentEquals(sstable, streamedSSTable);
    }
}