Examples with CassandraOutgoingFile org.apache.cassandra.db.streaming.CassandraOutgoingFile used on opensource projects

Example 1 with CassandraOutgoingFile

use of org.apache.cassandra.db.streaming.CassandraOutgoingFile in project cassandra by apache.

the class LegacySSTableTest method streamLegacyTable.

private void streamLegacyTable(String tablePattern, String legacyVersion) throws Exception {
    String table = String.format(tablePattern, legacyVersion);
    SSTableReader sstable = SSTableReader.open(getDescriptor(legacyVersion, table));
    IPartitioner p = sstable.getPartitioner();
    List<Range<Token>> ranges = new ArrayList<>();
    ranges.add(new Range<>(p.getMinimumToken(), p.getToken(ByteBufferUtil.bytes("100"))));
    ranges.add(new Range<>(p.getToken(ByteBufferUtil.bytes("100")), p.getMinimumToken()));
    List<OutgoingStream> streams = Lists.newArrayList(new CassandraOutgoingFile(StreamOperation.OTHER, sstable.ref(), sstable.getPositionsForRanges(ranges), ranges, sstable.estimatedKeysForRanges(ranges)));
    new StreamPlan(StreamOperation.OTHER).transferStreams(FBUtilities.getBroadcastAddressAndPort(), streams).execute().get();
}

Example 2 with CassandraOutgoingFile

use of org.apache.cassandra.db.streaming.CassandraOutgoingFile in project cassandra by apache.

the class StreamTransferTaskTest method testScheduleTimeout.

@Test
public void testScheduleTimeout() throws Exception {
    InetAddressAndPort peer = FBUtilities.getBroadcastAddressAndPort();
    StreamSession session = new StreamSession(StreamOperation.BOOTSTRAP, peer, FACTORY, null, current_version, false, 0, UUID.randomUUID(), PreviewKind.ALL);
    session.init(new StreamResultFuture(UUID.randomUUID(), StreamOperation.OTHER, UUID.randomUUID(), PreviewKind.NONE));
    ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD);
    // create two sstables
    for (int i = 0; i < 2; i++) {
        SchemaLoader.insertData(KEYSPACE1, CF_STANDARD, i, 1);
        cfs.forceBlockingFlush();
    }
    // create streaming task that streams those two sstables
    session.state(StreamSession.State.PREPARING);
    StreamTransferTask task = new StreamTransferTask(session, cfs.metadata.id);
    for (SSTableReader sstable : cfs.getLiveSSTables()) {
        List<Range<Token>> ranges = new ArrayList<>();
        ranges.add(new Range<>(sstable.first.getToken(), sstable.last.getToken()));
        task.addTransferStream(new CassandraOutgoingFile(StreamOperation.BOOTSTRAP, sstable.selfRef(), sstable.getPositionsForRanges(ranges), ranges, 1));
    }
    assertEquals(14, task.getTotalNumberOfFiles());
    // if file sending completes before timeout then the task should be canceled.
    session.state(StreamSession.State.STREAMING);
    Future f = task.scheduleTimeout(0, 0, TimeUnit.NANOSECONDS);
    f.get();
    // when timeout runs on second file, task should be completed
    f = task.scheduleTimeout(1, 10, TimeUnit.MILLISECONDS);
    task.complete(1);
    try {
        f.get();
        Assert.assertTrue(false);
    } catch (CancellationException ex) {
    }
    assertEquals(StreamSession.State.WAIT_COMPLETE, session.state());
    // when all streaming are done, time out task should not be scheduled.
    assertNull(task.scheduleTimeout(1, 1, TimeUnit.SECONDS));
}

Example 3 with CassandraOutgoingFile

use of org.apache.cassandra.db.streaming.CassandraOutgoingFile in project cassandra by apache.

the class SSTableLoader method openSSTables.

@SuppressWarnings("resource")
protected Collection<SSTableReader> openSSTables(final Map<InetAddressAndPort, Collection<Range<Token>>> ranges) {
    outputHandler.output("Opening sstables and calculating sections to stream");
    LifecycleTransaction.getFiles(directory.toPath(), (file, type) -> {
        File dir = file.parent();
        String name = file.name();
        if (type != Directories.FileType.FINAL) {
            outputHandler.output(String.format("Skipping temporary file %s", name));
            return false;
        }
        Pair<Descriptor, Component> p = SSTable.tryComponentFromFilename(file);
        Descriptor desc = p == null ? null : p.left;
        if (p == null || !p.right.equals(Component.DATA))
            return false;
        if (!new File(desc.filenameFor(Component.PRIMARY_INDEX)).exists()) {
            outputHandler.output(String.format("Skipping file %s because index is missing", name));
            return false;
        }
        TableMetadataRef metadata = client.getTableMetadata(desc.cfname);
        if (// we did not find metadata
        metadata == null && // and it's likely we hit CASSANDRA-16235
        directory.name().equals(Directories.BACKUPS_SUBDIR)) {
            File parentDirectory = directory.parent();
            File parentParentDirectory = parentDirectory != null ? parentDirectory.parent() : null;
            // check that descriptor's cfname and ksname are 1 directory closer to root than they should be
            if (parentDirectory != null && parentParentDirectory != null && desc.cfname.equals(parentDirectory.name()) && desc.ksname.equals(parentParentDirectory.name())) {
                Descriptor newDesc = new Descriptor(desc.directory, desc.ksname, Directories.BACKUPS_SUBDIR, desc.generation, desc.formatType);
                metadata = client.getTableMetadata(newDesc.cfname);
                if (metadata != null)
                    desc = newDesc;
            }
        }
        if (metadata == null) {
            outputHandler.output(String.format("Skipping file %s: table %s.%s doesn't exist", name, keyspace, desc.cfname));
            return false;
        }
        Set<Component> components = new HashSet<>();
        components.add(Component.DATA);
        components.add(Component.PRIMARY_INDEX);
        if (new File(desc.filenameFor(Component.SUMMARY)).exists())
            components.add(Component.SUMMARY);
        if (new File(desc.filenameFor(Component.COMPRESSION_INFO)).exists())
            components.add(Component.COMPRESSION_INFO);
        if (new File(desc.filenameFor(Component.STATS)).exists())
            components.add(Component.STATS);
        try {
            // To conserve memory, open SSTableReaders without bloom filters and discard
            // the index summary after calculating the file sections to stream and the estimated
            // number of keys for each endpoint. See CASSANDRA-5555 for details.
            SSTableReader sstable = SSTableReader.openForBatch(desc, components, metadata);
            sstables.add(sstable);
            // keys per host
            for (Map.Entry<InetAddressAndPort, Collection<Range<Token>>> entry : ranges.entrySet()) {
                InetAddressAndPort endpoint = entry.getKey();
                List<Range<Token>> tokenRanges = Range.normalize(entry.getValue());
                List<SSTableReader.PartitionPositionBounds> sstableSections = sstable.getPositionsForRanges(tokenRanges);
                // will generate an error on the server. See CASSANDRA-16349 for details.
                if (sstableSections.isEmpty())
                    continue;
                long estimatedKeys = sstable.estimatedKeysForRanges(tokenRanges);
                Ref<SSTableReader> ref = sstable.ref();
                OutgoingStream stream = new CassandraOutgoingFile(StreamOperation.BULK_LOAD, ref, sstableSections, tokenRanges, estimatedKeys);
                streamingDetails.put(endpoint, stream);
            }
            // to conserve heap space when bulk loading
            sstable.releaseSummary();
        } catch (FSError e) {
            // todo: should we really continue if we can't open all sstables?
            outputHandler.output(String.format("Skipping file %s, error opening it: %s", name, e.getMessage()));
        }
        return false;
    }, Directories.OnTxnErr.IGNORE);
    return sstables;
}

Example 4 with CassandraOutgoingFile

use of org.apache.cassandra.db.streaming.CassandraOutgoingFile in project cassandra by apache.

the class EntireSSTableStreamingCorrectFilesCountTest method test.

@Test
public void test() throws Exception {
    FileCountingStreamEventHandler streamEventHandler = new FileCountingStreamEventHandler();
    StreamSession session = setupStreamingSessionForTest(streamEventHandler);
    Collection<OutgoingStream> outgoingStreams = store.getStreamManager().createOutgoingStreams(session, rangesAtEndpoint, NO_PENDING_REPAIR, PreviewKind.NONE);
    session.addTransferStreams(outgoingStreams);
    AsyncStreamingOutputPlus out = constructDataOutputStream();
    for (OutgoingStream outgoingStream : outgoingStreams) {
        outgoingStream.write(session, out, MessagingService.VERSION_40);
        // verify hardlinks are removed after streaming
        Descriptor descriptor = ((CassandraOutgoingFile) outgoingStream).getRef().get().descriptor;
        assertTrue(descriptor.getTemporaryFiles().isEmpty());
    }
    int totalNumberOfFiles = session.transfers.get(store.metadata.id).getTotalNumberOfFiles();
    assertEquals(ComponentManifest.create(sstable.descriptor).components().size(), totalNumberOfFiles);
    assertEquals(streamEventHandler.fileNames.size(), totalNumberOfFiles);
}

Example 5 with CassandraOutgoingFile

use of org.apache.cassandra.db.streaming.CassandraOutgoingFile in project cassandra by apache.

the class StreamTransferTaskTest method testFailSessionDuringTransferShouldNotReleaseReferences.

@Test
public void testFailSessionDuringTransferShouldNotReleaseReferences() throws Exception {
    InetAddressAndPort peer = FBUtilities.getBroadcastAddressAndPort();
    StreamCoordinator streamCoordinator = new StreamCoordinator(StreamOperation.BOOTSTRAP, 1, new NettyStreamingConnectionFactory(), false, false, null, PreviewKind.NONE);
    StreamResultFuture future = StreamResultFuture.createInitiator(UUID.randomUUID(), StreamOperation.OTHER, Collections.<StreamEventHandler>emptyList(), streamCoordinator);
    StreamSession session = new StreamSession(StreamOperation.BOOTSTRAP, peer, FACTORY, null, current_version, false, 0, null, PreviewKind.NONE);
    session.init(future);
    ColumnFamilyStore cfs = Keyspace.open(KEYSPACE1).getColumnFamilyStore(CF_STANDARD);
    // create two sstables
    for (int i = 0; i < 2; i++) {
        SchemaLoader.insertData(KEYSPACE1, CF_STANDARD, i, 1);
        cfs.forceBlockingFlush();
    }
    // create streaming task that streams those two sstables
    StreamTransferTask task = new StreamTransferTask(session, cfs.metadata.id);
    List<Ref<SSTableReader>> refs = new ArrayList<>(cfs.getLiveSSTables().size());
    for (SSTableReader sstable : cfs.getLiveSSTables()) {
        List<Range<Token>> ranges = new ArrayList<>();
        ranges.add(new Range<>(sstable.first.getToken(), sstable.last.getToken()));
        Ref<SSTableReader> ref = sstable.selfRef();
        refs.add(ref);
        task.addTransferStream(new CassandraOutgoingFile(StreamOperation.BOOTSTRAP, ref, sstable.getPositionsForRanges(ranges), ranges, 1));
    }
    assertEquals(14, task.getTotalNumberOfFiles());
    // add task to stream session, so it is aborted when stream session fails
    session.transfers.put(TableId.generate(), task);
    // make a copy of outgoing file messages, since task is cleared when it's aborted
    Collection<OutgoingStreamMessage> files = new LinkedList<>(task.streams.values());
    // simulate start transfer
    for (OutgoingStreamMessage file : files) {
        file.startTransfer();
    }
    // fail stream session mid-transfer
    session.onError(new Exception("Fake exception")).get(5, TimeUnit.SECONDS);
    // make sure reference was not released
    for (Ref<SSTableReader> ref : refs) {
        assertEquals(1, ref.globalCount());
    }
    // wait for stream to abort asynchronously
    int tries = 10;
    while (ScheduledExecutors.nonPeriodicTasks.getActiveTaskCount() > 0) {
        if (tries < 1)
            throw new RuntimeException("test did not complete in time");
        Thread.sleep(10);
        tries--;
    }
    // simulate finish transfer
    for (OutgoingStreamMessage file : files) {
        file.finishTransfer();
    }
    // now reference should be released
    for (Ref<SSTableReader> ref : refs) {
        assertEquals(0, ref.globalCount());
    }
}