Examples with DataInputStreamPlus org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus used on opensource projects

Example 1 with DataInputStreamPlus

use of org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus in project cassandra by apache.

the class AutoSavingCache method loadSaved.

public int loadSaved() {
    int count = 0;
    long start = System.nanoTime();
    // modern format, allows both key and value (so key cache load can be purely sequential)
    File dataPath = getCacheDataPath(CURRENT_VERSION);
    File crcPath = getCacheCrcPath(CURRENT_VERSION);
    if (dataPath.exists() && crcPath.exists()) {
        DataInputStreamPlus in = null;
        try {
            logger.info("reading saved cache {}", dataPath);
            in = new DataInputStreamPlus(new LengthAvailableInputStream(new BufferedInputStream(streamFactory.getInputStream(dataPath, crcPath)), dataPath.length()));
            //Check the schema has not changed since CFs are looked up by name which is ambiguous
            UUID schemaVersion = new UUID(in.readLong(), in.readLong());
            if (!schemaVersion.equals(Schema.instance.getVersion()))
                throw new RuntimeException("Cache schema version " + schemaVersion + " does not match current schema version " + Schema.instance.getVersion());
            ArrayDeque<Future<Pair<K, V>>> futures = new ArrayDeque<Future<Pair<K, V>>>();
            while (in.available() > 0) {
                //tableId and indexName are serialized by the serializers in CacheService
                //That is delegated there because there are serializer specific conditions
                //where a cache key is skipped and not written
                TableId tableId = TableId.deserialize(in);
                String indexName = in.readUTF();
                if (indexName.isEmpty())
                    indexName = null;
                ColumnFamilyStore cfs = Schema.instance.getColumnFamilyStoreInstance(tableId);
                if (indexName != null && cfs != null)
                    cfs = cfs.indexManager.getIndexByName(indexName).getBackingTable().orElse(null);
                Future<Pair<K, V>> entryFuture = cacheLoader.deserialize(in, cfs);
                // Key cache entry can return null, if the SSTable doesn't exist.
                if (entryFuture == null)
                    continue;
                futures.offer(entryFuture);
                count++;
                /*
                     * Kind of unwise to accrue an unbounded number of pending futures
                     * So now there is this loop to keep a bounded number pending.
                     */
                do {
                    while (futures.peek() != null && futures.peek().isDone()) {
                        Future<Pair<K, V>> future = futures.poll();
                        Pair<K, V> entry = future.get();
                        if (entry != null && entry.right != null)
                            put(entry.left, entry.right);
                    }
                    if (futures.size() > 1000)
                        Thread.yield();
                } while (futures.size() > 1000);
            }
            Future<Pair<K, V>> future = null;
            while ((future = futures.poll()) != null) {
                Pair<K, V> entry = future.get();
                if (entry != null && entry.right != null)
                    put(entry.left, entry.right);
            }
        } catch (CorruptFileException e) {
            JVMStabilityInspector.inspectThrowable(e);
            logger.warn(String.format("Non-fatal checksum error reading saved cache %s", dataPath.getAbsolutePath()), e);
        } catch (Throwable t) {
            JVMStabilityInspector.inspectThrowable(t);
            logger.info(String.format("Harmless error reading saved cache %s", dataPath.getAbsolutePath()), t);
        } finally {
            FileUtils.closeQuietly(in);
        }
    }
    if (logger.isTraceEnabled())
        logger.trace("completed reading ({} ms; {} keys) saved cache {}", TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start), count, dataPath);
    return count;
}

Example 2 with DataInputStreamPlus

use of org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus in project cassandra by apache.

the class IncomingTcpConnection method receiveMessages.

// Not closing constructed DataInputPlus's as the stream needs to remain open.
@SuppressWarnings("resource")
private void receiveMessages() throws IOException {
    // handshake (true) endpoint versions
    DataOutputStream out = new DataOutputStream(socket.getOutputStream());
    // if this version is < the MS version the other node is trying
    // to connect with, the other node will disconnect
    out.writeInt(MessagingService.current_version);
    out.flush();
    DataInputPlus in = new DataInputStreamPlus(socket.getInputStream());
    int maxVersion = in.readInt();
    // outbound side will reconnect if necessary to upgrade version
    assert version <= MessagingService.current_version;
    from = CompactEndpointSerializationHelper.deserialize(in);
    // record the (true) version of the endpoint
    MessagingService.instance().setVersion(from, maxVersion);
    logger.trace("Set version for {} to {} (will use {})", from, maxVersion, MessagingService.instance().getVersion(from));
    if (compressed) {
        logger.trace("Upgrading incoming connection to be compressed");
        LZ4FastDecompressor decompressor = LZ4Factory.fastestInstance().fastDecompressor();
        Checksum checksum = XXHashFactory.fastestInstance().newStreamingHash32(OutboundTcpConnection.LZ4_HASH_SEED).asChecksum();
        in = new DataInputStreamPlus(new LZ4BlockInputStream(socket.getInputStream(), decompressor, checksum));
    } else {
        ReadableByteChannel channel = socket.getChannel();
        in = new NIODataInputStream(channel != null ? channel : Channels.newChannel(socket.getInputStream()), BUFFER_SIZE);
    }
    while (true) {
        MessagingService.validateMagic(in.readInt());
        receiveMessage(in, version);
    }
}

Example 3 with DataInputStreamPlus

use of org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus in project cassandra by apache.

the class MessagingServiceTest method addDCLatency.

private static void addDCLatency(long sentAt, long nowTime) throws IOException {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    try (DataOutputStreamPlus out = new WrappedDataOutputStreamPlus(baos)) {
        out.writeInt((int) sentAt);
    }
    DataInputStreamPlus in = new DataInputStreamPlus(new ByteArrayInputStream(baos.toByteArray()));
    MessageIn.readConstructionTime(InetAddress.getLocalHost(), in, nowTime);
}

Example 4 with DataInputStreamPlus

use of org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus in project cassandra by apache.

the class SerializationsTest method testSyncCompleteRead.

@Test
public void testSyncCompleteRead() throws IOException {
    if (EXECUTE_WRITES)
        testSyncCompleteWrite();
    InetAddress src = InetAddress.getByAddress(new byte[] { 127, 0, 0, 2 });
    InetAddress dest = InetAddress.getByAddress(new byte[] { 127, 0, 0, 3 });
    NodePair nodes = new NodePair(src, dest);
    try (DataInputStreamPlus in = getInput("service.SyncComplete.bin")) {
        // success
        RepairMessage message = RepairMessage.serializer.deserialize(in, getVersion());
        assert message.messageType == RepairMessage.Type.SYNC_COMPLETE;
        assert DESC.equals(message.desc);
        assert nodes.equals(((SyncComplete) message).nodes);
        assert ((SyncComplete) message).success;
        // fail
        message = RepairMessage.serializer.deserialize(in, getVersion());
        assert message.messageType == RepairMessage.Type.SYNC_COMPLETE;
        assert DESC.equals(message.desc);
        assert nodes.equals(((SyncComplete) message).nodes);
        assert !((SyncComplete) message).success;
        // MessageOuts
        for (int i = 0; i < 2; i++) assert MessageIn.read(in, getVersion(), -1) != null;
    }
}

Example 5 with DataInputStreamPlus

use of org.apache.cassandra.io.util.DataInputPlus.DataInputStreamPlus in project cassandra by apache.

the class SerializationsTest method testEndpointStateRead.

@Test
public void testEndpointStateRead() throws IOException {
    if (EXECUTE_WRITES)
        testEndpointStateWrite();
    DataInputStreamPlus in = getInput("gms.EndpointState.bin");
    assert HeartBeatState.serializer.deserialize(in, getVersion()) != null;
    assert EndpointState.serializer.deserialize(in, getVersion()) != null;
    assert VersionedValue.serializer.deserialize(in, getVersion()) != null;
    assert VersionedValue.serializer.deserialize(in, getVersion()) != null;
    in.close();
}