Examples with Compressor org.opensearch.common.compress.Compressor used on opensource projects

Example 1 with Compressor

use of org.opensearch.common.compress.Compressor in project OpenSearch by opensearch-project.

the class XContentHelper method writeRawField.

/**
 * Writes a "raw" (bytes) field, handling cases where the bytes are compressed, and tries to optimize writing using
 * {@link XContentBuilder#rawField(String, InputStream, XContentType)}.
 */
public static void writeRawField(String field, BytesReference source, XContentType xContentType, XContentBuilder builder, Params params) throws IOException {
    Objects.requireNonNull(xContentType);
    Compressor compressor = CompressorFactory.compressor(source);
    if (compressor != null) {
        try (InputStream compressedStreamInput = compressor.threadLocalInputStream(source.streamInput())) {
            builder.rawField(field, compressedStreamInput, xContentType);
        }
    } else {
        try (InputStream stream = source.streamInput()) {
            builder.rawField(field, stream, xContentType);
        }
    }
}

Example 2 with Compressor

use of org.opensearch.common.compress.Compressor in project OpenSearch by opensearch-project.

the class XContentHelper method createParser.

/**
 * Creates a parser based on the bytes provided
 * @deprecated use {@link #createParser(NamedXContentRegistry, DeprecationHandler, BytesReference, XContentType)}
 * to avoid content type auto-detection
 */
@Deprecated
public static XContentParser createParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, BytesReference bytes) throws IOException {
    Compressor compressor = CompressorFactory.compressor(bytes);
    if (compressor != null) {
        InputStream compressedInput = null;
        try {
            compressedInput = compressor.threadLocalInputStream(bytes.streamInput());
            if (compressedInput.markSupported() == false) {
                compressedInput = new BufferedInputStream(compressedInput);
            }
            final XContentType contentType = XContentFactory.xContentType(compressedInput);
            return XContentFactory.xContent(contentType).createParser(xContentRegistry, deprecationHandler, compressedInput);
        } catch (Exception e) {
            if (compressedInput != null)
                compressedInput.close();
            throw e;
        }
    } else {
        return XContentFactory.xContent(xContentType(bytes)).createParser(xContentRegistry, deprecationHandler, bytes.streamInput());
    }
}

Example 3 with Compressor

use of org.opensearch.common.compress.Compressor in project OpenSearch by opensearch-project.

the class PublicationTransportHandler method handleIncomingPublishRequest.

private PublishWithJoinResponse handleIncomingPublishRequest(BytesTransportRequest request) throws IOException {
    final Compressor compressor = CompressorFactory.compressor(request.bytes());
    StreamInput in = request.bytes().streamInput();
    try {
        if (compressor != null) {
            in = new InputStreamStreamInput(compressor.threadLocalInputStream(in));
        }
        in = new NamedWriteableAwareStreamInput(in, namedWriteableRegistry);
        in.setVersion(request.version());
        // If true we received full cluster state - otherwise diffs
        if (in.readBoolean()) {
            final ClusterState incomingState;
            // Close early to release resources used by the de-compression as early as possible
            try (StreamInput input = in) {
                incomingState = ClusterState.readFrom(input, transportService.getLocalNode());
            } catch (Exception e) {
                logger.warn("unexpected error while deserializing an incoming cluster state", e);
                throw e;
            }
            fullClusterStateReceivedCount.incrementAndGet();
            logger.debug("received full cluster state version [{}] with size [{}]", incomingState.version(), request.bytes().length());
            final PublishWithJoinResponse response = acceptState(incomingState);
            lastSeenClusterState.set(incomingState);
            return response;
        } else {
            final ClusterState lastSeen = lastSeenClusterState.get();
            if (lastSeen == null) {
                logger.debug("received diff for but don't have any local cluster state - requesting full state");
                incompatibleClusterStateDiffReceivedCount.incrementAndGet();
                throw new IncompatibleClusterStateVersionException("have no local cluster state");
            } else {
                ClusterState incomingState;
                try {
                    final Diff<ClusterState> diff;
                    // Close stream early to release resources used by the de-compression as early as possible
                    try (StreamInput input = in) {
                        diff = ClusterState.readDiffFrom(input, lastSeen.nodes().getLocalNode());
                    }
                    // might throw IncompatibleClusterStateVersionException
                    incomingState = diff.apply(lastSeen);
                } catch (IncompatibleClusterStateVersionException e) {
                    incompatibleClusterStateDiffReceivedCount.incrementAndGet();
                    throw e;
                } catch (Exception e) {
                    logger.warn("unexpected error while deserializing an incoming cluster state", e);
                    throw e;
                }
                compatibleClusterStateDiffReceivedCount.incrementAndGet();
                logger.debug("received diff cluster state version [{}] with uuid [{}], diff size [{}]", incomingState.version(), incomingState.stateUUID(), request.bytes().length());
                final PublishWithJoinResponse response = acceptState(incomingState);
                lastSeenClusterState.compareAndSet(lastSeen, incomingState);
                return response;
            }
        }
    } finally {
        IOUtils.close(in);
    }
}

Example 4 with Compressor

use of org.opensearch.common.compress.Compressor in project OpenSearch by opensearch-project.

the class XContentHelper method createParser.

/**
 * Creates a parser for the bytes using the supplied content-type
 */
public static XContentParser createParser(NamedXContentRegistry xContentRegistry, DeprecationHandler deprecationHandler, BytesReference bytes, XContentType xContentType) throws IOException {
    Objects.requireNonNull(xContentType);
    Compressor compressor = CompressorFactory.compressor(bytes);
    if (compressor != null) {
        InputStream compressedInput = null;
        try {
            compressedInput = compressor.threadLocalInputStream(bytes.streamInput());
            if (compressedInput.markSupported() == false) {
                compressedInput = new BufferedInputStream(compressedInput);
            }
            return XContentFactory.xContent(xContentType).createParser(xContentRegistry, deprecationHandler, compressedInput);
        } catch (Exception e) {
            if (compressedInput != null)
                compressedInput.close();
            throw e;
        }
    } else {
        if (bytes instanceof BytesArray) {
            final BytesArray array = (BytesArray) bytes;
            return xContentType.xContent().createParser(xContentRegistry, deprecationHandler, array.array(), array.offset(), array.length());
        }
        return xContentType.xContent().createParser(xContentRegistry, deprecationHandler, bytes.streamInput());
    }
}

Example 5 with Compressor

use of org.opensearch.common.compress.Compressor in project OpenSearch by opensearch-project.

the class XContentHelper method convertToMap.

/**
 * Converts the given bytes into a map that is optionally ordered. The provided {@link XContentType} must be non-null.
 */
public static Tuple<XContentType, Map<String, Object>> convertToMap(BytesReference bytes, boolean ordered, XContentType xContentType) throws OpenSearchParseException {
    try {
        final XContentType contentType;
        InputStream input;
        Compressor compressor = CompressorFactory.compressor(bytes);
        if (compressor != null) {
            InputStream compressedStreamInput = compressor.threadLocalInputStream(bytes.streamInput());
            if (compressedStreamInput.markSupported() == false) {
                compressedStreamInput = new BufferedInputStream(compressedStreamInput);
            }
            input = compressedStreamInput;
        } else if (bytes instanceof BytesArray) {
            final BytesArray arr = (BytesArray) bytes;
            final byte[] raw = arr.array();
            final int offset = arr.offset();
            final int length = arr.length();
            contentType = xContentType != null ? xContentType : XContentFactory.xContentType(raw, offset, length);
            return new Tuple<>(Objects.requireNonNull(contentType), convertToMap(XContentFactory.xContent(contentType), raw, offset, length, ordered));
        } else {
            input = bytes.streamInput();
        }
        try (InputStream stream = input) {
            contentType = xContentType != null ? xContentType : XContentFactory.xContentType(stream);
            return new Tuple<>(Objects.requireNonNull(contentType), convertToMap(XContentFactory.xContent(contentType), stream, ordered));
        }
    } catch (IOException e) {
        throw new OpenSearchParseException("Failed to parse content to map", e);
    }
}