Examples with CompressionCodec org.apache.pulsar.common.compression.CompressionCodec used on opensource projects

Example 1 with CompressionCodec

use of org.apache.pulsar.common.compression.CompressionCodec in project incubator-pulsar by apache.

the class PersistentTopicsBase method internalPeekNthMessage.

protected Response internalPeekNthMessage(String subName, int messagePosition, boolean authoritative) {
    if (topicName.isGlobal()) {
        validateGlobalNamespaceOwnership(namespaceName);
    }
    PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(topicName, authoritative);
    if (partitionMetadata.partitions > 0) {
        throw new RestException(Status.METHOD_NOT_ALLOWED, "Peek messages on a partitioned topic is not allowed");
    }
    validateAdminOperationOnTopic(authoritative);
    if (!(getTopicReference(topicName) instanceof PersistentTopic)) {
        log.error("[{}] Not supported operation of non-persistent topic {} {}", clientAppId(), topicName, subName);
        throw new RestException(Status.METHOD_NOT_ALLOWED, "Skip messages on a non-persistent topic is not allowed");
    }
    PersistentTopic topic = (PersistentTopic) getTopicReference(topicName);
    PersistentReplicator repl = null;
    PersistentSubscription sub = null;
    Entry entry = null;
    if (subName.startsWith(topic.replicatorPrefix)) {
        repl = getReplicatorReference(subName, topic);
    } else {
        sub = (PersistentSubscription) getSubscriptionReference(subName, topic);
    }
    try {
        if (subName.startsWith(topic.replicatorPrefix)) {
            entry = repl.peekNthMessage(messagePosition).get();
        } else {
            entry = sub.peekNthMessage(messagePosition).get();
        }
        checkNotNull(entry);
        PositionImpl pos = (PositionImpl) entry.getPosition();
        ByteBuf metadataAndPayload = entry.getDataBuffer();
        // moves the readerIndex to the payload
        MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload);
        ResponseBuilder responseBuilder = Response.ok();
        responseBuilder.header("X-Pulsar-Message-ID", pos.toString());
        for (KeyValue keyValue : metadata.getPropertiesList()) {
            responseBuilder.header("X-Pulsar-PROPERTY-" + keyValue.getKey(), keyValue.getValue());
        }
        if (metadata.hasPublishTime()) {
            responseBuilder.header("X-Pulsar-publish-time", DateFormatter.format(metadata.getPublishTime()));
        }
        if (metadata.hasEventTime()) {
            responseBuilder.header("X-Pulsar-event-time", DateFormatter.format(metadata.getEventTime()));
        }
        if (metadata.hasNumMessagesInBatch()) {
            responseBuilder.header("X-Pulsar-num-batch-message", metadata.getNumMessagesInBatch());
        }
        // Decode if needed
        CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(metadata.getCompression());
        ByteBuf uncompressedPayload = codec.decode(metadataAndPayload, metadata.getUncompressedSize());
        // Copy into a heap buffer for output stream compatibility
        ByteBuf data = PooledByteBufAllocator.DEFAULT.heapBuffer(uncompressedPayload.readableBytes(), uncompressedPayload.readableBytes());
        data.writeBytes(uncompressedPayload);
        uncompressedPayload.release();
        StreamingOutput stream = new StreamingOutput() {

            @Override
            public void write(OutputStream output) throws IOException, WebApplicationException {
                output.write(data.array(), data.arrayOffset(), data.readableBytes());
                data.release();
            }
        };
        return responseBuilder.entity(stream).build();
    } catch (NullPointerException npe) {
        throw new RestException(Status.NOT_FOUND, "Message not found");
    } catch (Exception exception) {
        log.error("[{}] Failed to get message at position {} from {} {}", clientAppId(), messagePosition, topicName, subName, exception);
        throw new RestException(exception);
    } finally {
        if (entry != null) {
            entry.release();
        }
    }
}

Example 2 with CompressionCodec

use of org.apache.pulsar.common.compression.CompressionCodec in project incubator-pulsar by apache.

the class ConsumerImpl method uncompressPayloadIfNeeded.

private ByteBuf uncompressPayloadIfNeeded(MessageIdData messageId, MessageMetadata msgMetadata, ByteBuf payload, ClientCnx currentCnx) {
    CompressionType compressionType = msgMetadata.getCompression();
    CompressionCodec codec = codecProvider.getCodec(compressionType);
    int uncompressedSize = msgMetadata.getUncompressedSize();
    int payloadSize = payload.readableBytes();
    if (payloadSize > PulsarDecoder.MaxMessageSize) {
        // payload size is itself corrupted since it cannot be bigger than the MaxMessageSize
        log.error("[{}][{}] Got corrupted payload message size {} at {}", topic, subscription, payloadSize, messageId);
        discardCorruptedMessage(messageId, currentCnx, ValidationError.UncompressedSizeCorruption);
        return null;
    }
    try {
        ByteBuf uncompressedPayload = codec.decode(payload, uncompressedSize);
        return uncompressedPayload;
    } catch (IOException e) {
        log.error("[{}][{}] Failed to decompress message with {} at {}: {}", topic, subscription, compressionType, messageId, e.getMessage(), e);
        discardCorruptedMessage(messageId, currentCnx, ValidationError.DecompressionError);
        return null;
    }
}

Example 3 with CompressionCodec

use of org.apache.pulsar.common.compression.CompressionCodec in project incubator-pulsar by apache.

the class CompressorCodecTest method testCodecProvider.

@Test(dataProvider = "codec")
void testCodecProvider(CompressionType type) throws IOException {
    CompressionCodecProvider provider = new CompressionCodecProvider();
    CompressionCodec codec1 = provider.getCodec(type);
    CompressionCodec codec2 = provider.getCodec(type);
    // A single provider instance must return the same codec instance every time
    assertTrue(codec1 == codec2);
}

Example 4 with CompressionCodec

use of org.apache.pulsar.common.compression.CompressionCodec in project incubator-pulsar by apache.

the class CompressorCodecTest method testMultpileUsages.

@Test(dataProvider = "codec")
void testMultpileUsages(CompressionType type) throws IOException {
    CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(type);
    byte[] data = text.getBytes();
    for (int i = 0; i < 5; i++) {
        ByteBuf raw = PooledByteBufAllocator.DEFAULT.buffer();
        raw.writeBytes(data);
        ByteBuf compressed = codec.encode(raw);
        assertEquals(raw.readableBytes(), data.length);
        int compressedSize = compressed.readableBytes();
        ByteBuf uncompressed = codec.decode(compressed, data.length);
        assertEquals(compressed.readableBytes(), compressedSize);
        assertEquals(uncompressed.readableBytes(), data.length);
        assertEquals(uncompressed, raw);
        raw.release();
        compressed.release();
        uncompressed.release();
    }
}

Example 5 with CompressionCodec

use of org.apache.pulsar.common.compression.CompressionCodec in project incubator-pulsar by apache.

the class CompressorCodecTest method testEmptyInput.

@Test(dataProvider = "codec")
void testEmptyInput(CompressionType type) throws IOException {
    CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(type);
    ByteBuf compressed = codec.encode(Unpooled.EMPTY_BUFFER);
    ByteBuf uncompressed = codec.decode(compressed, 0);
    assertEquals(uncompressed, Unpooled.EMPTY_BUFFER);
}