Examples with CompressionType com.yahoo.compress.CompressionType used on opensource projects

Example 1 with CompressionType

use of com.yahoo.compress.CompressionType in project vespa by vespa-engine.

the class FastSearcher method doPartialFill.

/**
 * Perform a partial docsum fill for a temporary result
 * representing a partition of the complete fill request.
 *
 * @param result result containing a partition of the unfilled hits
 * @param summaryClass the summary class we want to fill with
 */
protected void doPartialFill(Result result, String summaryClass) {
    if (result.isFilled(summaryClass))
        return;
    Query query = result.getQuery();
    traceQuery(getName(), "fill", query, query.getOffset(), query.getHits(), 2, quotedSummaryClass(summaryClass));
    if (wantsRPCSummaryFill(query)) {
        CompressionType compression = CompressionType.valueOf(query.properties().getString(dispatchCompression, "LZ4").toUpperCase());
        fillSDDocName(result);
        dispatcher.fill(result, summaryClass, compression);
        return;
    }
    CacheKey cacheKey = null;
    PacketWrapper packetWrapper = null;
    if (getCacheControl().useCache(query)) {
        cacheKey = fetchCacheKeyFromHits(result.hits(), summaryClass);
        if (cacheKey == null) {
            QueryPacket queryPacket = QueryPacket.create(query);
            cacheKey = new CacheKey(queryPacket);
        }
        packetWrapper = cacheLookupTwoPhase(cacheKey, result, summaryClass);
    }
    FS4Channel channel = chooseBackend(query).openChannel();
    channel.setQuery(query);
    Packet[] receivedPackets;
    try {
        DocsumPacketKey[] packetKeys;
        if (countFastHits(result) > 0) {
            packetKeys = getPacketKeys(result, summaryClass, false);
            if (packetKeys.length == 0) {
                receivedPackets = new Packet[0];
            } else {
                try {
                    receivedPackets = fetchSummaries(channel, result, summaryClass);
                } catch (InvalidChannelException e) {
                    result.hits().addError(ErrorMessage.createBackendCommunicationError("Invalid channel " + getName() + " (summary fetch)"));
                    return;
                } catch (ChannelTimeoutException e) {
                    result.hits().addError(ErrorMessage.createTimeout("timeout waiting for summaries from " + getName()));
                    return;
                } catch (IOException e) {
                    result.hits().addError(ErrorMessage.createBackendCommunicationError("IO error while talking on channel " + getName() + " (summary fetch): " + e.getMessage()));
                    return;
                }
                if (receivedPackets.length == 0) {
                    result.hits().addError(ErrorMessage.createBackendCommunicationError(getName() + " got no packets back (summary fetch)"));
                    return;
                }
            }
        } else {
            packetKeys = new DocsumPacketKey[0];
            receivedPackets = new Packet[0];
        }
        int skippedHits;
        try {
            FillHitsResult fillHitsResult = fillHits(result, receivedPackets, summaryClass);
            skippedHits = fillHitsResult.skippedHits;
            if (fillHitsResult.error != null) {
                result.hits().addError(ErrorMessage.createTimeout(fillHitsResult.error));
                return;
            }
        } catch (TimeoutException e) {
            result.hits().addError(ErrorMessage.createTimeout(e.getMessage()));
            return;
        } catch (IOException e) {
            result.hits().addError(ErrorMessage.createBackendCommunicationError("Error filling hits with summary fields, source: " + getName() + " Exception thrown: " + e.getMessage()));
            return;
        }
        if (skippedHits == 0 && packetWrapper != null) {
            cacheControl.updateCacheEntry(cacheKey, query, packetKeys, receivedPackets);
        }
        if (skippedHits > 0)
            result.hits().addError(com.yahoo.search.result.ErrorMessage.createEmptyDocsums("Missing hit data for summary '" + summaryClass + "' for " + skippedHits + " hits"));
        result.analyzeHits();
        if (query.getTraceLevel() >= 3) {
            int hitNumber = 0;
            for (Iterator<com.yahoo.search.result.Hit> i = hitIterator(result); i.hasNext(); ) {
                com.yahoo.search.result.Hit hit = i.next();
                if (!(hit instanceof FastHit))
                    continue;
                FastHit fastHit = (FastHit) hit;
                String traceMsg = "Hit: " + (hitNumber++) + " from " + (fastHit.isCached() ? "cache" : "backend");
                if (!fastHit.isFilled(summaryClass))
                    traceMsg += ". Error, hit, not filled";
                query.trace(traceMsg, false, 3);
            }
        }
    } finally {
        channel.close();
    }
}

Example 2 with CompressionType

use of com.yahoo.compress.CompressionType in project vespa by vespa-engine.

the class BaseStructDataType method setCompressionConfig.

/**
 * Set the config to the compressor used to compress data of this type
 */
public void setCompressionConfig(CompressionConfig config) {
    CompressionType type = config.type;
    compressor = new Compressor(type, config.compressionLevel, config.thresholdFactor(), (int) config.minsize);
}

Example 3 with CompressionType

use of com.yahoo.compress.CompressionType in project vespa by vespa-engine.

the class RPCUtil method decodeStateBundleFromSetDistributionStatesRequest.

public static ClusterStateBundle decodeStateBundleFromSetDistributionStatesRequest(Request req) {
    final CompressionType type = CompressionType.valueOf(req.parameters().get(0).asInt8());
    final int uncompressedSize = req.parameters().get(1).asInt32();
    final byte[] compressedPayload = req.parameters().get(2).asData();
    SlimeClusterStateBundleCodec codec = new SlimeClusterStateBundleCodec();
    Compressor.Compression compression = new Compressor.Compression(type, uncompressedSize, compressedPayload);
    return codec.decode(EncodedClusterStateBundle.fromCompressionBuffer(compression));
}

Example 4 with CompressionType

use of com.yahoo.compress.CompressionType in project vespa by vespa-engine.

the class MockClient method getDocsums.

@Override
public void getDocsums(List<FastHit> hitsContext, NodeConnection node, CompressionType compression, int uncompressedSize, byte[] compressedSlime, Dispatcher.GetDocsumsResponseReceiver responseReceiver, double timeoutSeconds) {
    if (malfunctioning) {
        responseReceiver.receive(GetDocsumsResponseOrError.fromError("Malfunctioning"));
        return;
    }
    Inspector request = BinaryFormat.decode(compressor.decompress(compressedSlime, compression, uncompressedSize)).get();
    String docsumClass = request.field("class").asString();
    List<Map<String, Object>> docsumsToReturn = new ArrayList<>();
    request.field("gids").traverse((ArrayTraverser) (index, gid) -> {
        GlobalId docId = new GlobalId(gid.asData());
        docsumsToReturn.add(docsums.get(new DocsumKey(node.toString(), docId, docsumClass)));
    });
    Slime responseSlime = new Slime();
    Cursor root = responseSlime.setObject();
    Cursor docsums = root.setArray("docsums");
    for (Map<String, Object> docsumFields : docsumsToReturn) {
        Cursor docsumItem = docsums.addObject();
        Cursor docsum = docsumItem.setObject("docsum");
        for (Map.Entry<String, Object> field : docsumFields.entrySet()) {
            if (field.getValue() instanceof Integer)
                docsum.setLong(field.getKey(), (Integer) field.getValue());
            else if (field.getValue() instanceof String)
                docsum.setString(field.getKey(), (String) field.getValue());
            else
                throw new RuntimeException();
        }
    }
    byte[] slimeBytes = BinaryFormat.encode(responseSlime);
    Compressor.Compression compressionResult = compressor.compress(compression, slimeBytes);
    GetDocsumsResponse response = new GetDocsumsResponse(compressionResult.type().getCode(), slimeBytes.length, compressionResult.data(), hitsContext);
    responseReceiver.receive(GetDocsumsResponseOrError.fromResponse(response));
}

Example 5 with CompressionType

use of com.yahoo.compress.CompressionType in project vespa by vespa-engine.

the class RPCSendV2 method createReply.

@Override
protected Reply createReply(Values ret, String serviceName, Trace trace) {
    CompressionType compression = CompressionType.valueOf(ret.get(3).asInt8());
    byte[] slimeBytes = compressor.decompress(ret.get(5).asData(), compression, ret.get(4).asInt32());
    Slime slime = BinaryFormat.decode(slimeBytes);
    Inspector root = slime.get();
    Version version = new Version(root.field(VERSION_F).asString());
    byte[] payload = root.field(BLOB_F).asData();
    // Make sure that the owner understands the protocol.
    Reply reply = null;
    Error error = null;
    if (payload.length > 0) {
        Object retval = decode(new Utf8Array(root.field(PROTOCOL_F).asUtf8()), version, payload);
        if (retval instanceof Reply) {
            reply = (Reply) retval;
        } else {
            error = (Error) retval;
        }
    }
    if (reply == null) {
        reply = new EmptyReply();
    }
    if (error != null) {
        reply.addError(error);
    }
    reply.setRetryDelay(root.field(RETRYDELAY_F).asDouble());
    Inspector errors = root.field(ERRORS_F);
    for (int i = 0; i < errors.entries(); i++) {
        Inspector e = errors.entry(i);
        String service = e.field(SERVICE_F).asString();
        reply.addError(new Error((int) e.field(CODE_F).asLong(), e.field(MSG_F).asString(), (service != null && service.length() > 0) ? service : serviceName));
    }
    if (trace.getLevel() > 0) {
        trace.getRoot().addChild(TraceNode.decode(root.field(TRACE_F).asString()));
    }
    return reply;
}