Examples with FS4Channel com.yahoo.fs4.mplex.FS4Channel used on opensource projects

Example 1 with FS4Channel

use of com.yahoo.fs4.mplex.FS4Channel in project vespa by vespa-engine.

the class FastSearcher method fetchSummaries.

private Packet[] fetchSummaries(FS4Channel channel, Result result, String summaryClass) throws InvalidChannelException, ChannelTimeoutException, ClassCastException, IOException {
    BasicPacket[] receivedPackets;
    boolean summaryNeedsQuery = summaryNeedsQuery(result.getQuery());
    if (result.getQuery().getTraceLevel() >= 3)
        result.getQuery().trace((summaryNeedsQuery ? "Resending " : "Not resending ") + "query during document summary fetching", 3);
    GetDocSumsPacket docsumsPacket = GetDocSumsPacket.create(result, summaryClass, summaryNeedsQuery);
    int compressionLimit = result.getQuery().properties().getInteger(PACKET_COMPRESSION_LIMIT, 0);
    docsumsPacket.setCompressionLimit(compressionLimit);
    if (compressionLimit != 0) {
        docsumsPacket.setCompressionType(result.getQuery().properties().getString(PACKET_COMPRESSION_TYPE, "lz4"));
    }
    boolean couldSend = channel.sendPacket(docsumsPacket);
    if (isLoggingFine())
        getLogger().finest("Sent " + docsumsPacket + " on " + channel);
    if (!couldSend)
        throw new IOException("Could not successfully send GetDocSumsPacket.");
    receivedPackets = channel.receivePackets(result.getQuery().getTimeLeft(), docsumsPacket.getNumDocsums() + 1);
    if (isLoggingFine())
        getLogger().finest("got " + receivedPackets.length + "docsumPackets");
    return convertBasicPackets(receivedPackets);
}

Example 2 with FS4Channel

use of com.yahoo.fs4.mplex.FS4Channel in project vespa by vespa-engine.

the class FastSearcher method doPartialFill.

/**
 * Perform a partial docsum fill for a temporary result
 * representing a partition of the complete fill request.
 *
 * @param result result containing a partition of the unfilled hits
 * @param summaryClass the summary class we want to fill with
 */
protected void doPartialFill(Result result, String summaryClass) {
    if (result.isFilled(summaryClass))
        return;
    Query query = result.getQuery();
    traceQuery(getName(), "fill", query, query.getOffset(), query.getHits(), 2, quotedSummaryClass(summaryClass));
    if (wantsRPCSummaryFill(query)) {
        CompressionType compression = CompressionType.valueOf(query.properties().getString(dispatchCompression, "LZ4").toUpperCase());
        fillSDDocName(result);
        dispatcher.fill(result, summaryClass, compression);
        return;
    }
    CacheKey cacheKey = null;
    PacketWrapper packetWrapper = null;
    if (getCacheControl().useCache(query)) {
        cacheKey = fetchCacheKeyFromHits(result.hits(), summaryClass);
        if (cacheKey == null) {
            QueryPacket queryPacket = QueryPacket.create(query);
            cacheKey = new CacheKey(queryPacket);
        }
        packetWrapper = cacheLookupTwoPhase(cacheKey, result, summaryClass);
    }
    FS4Channel channel = chooseBackend(query).openChannel();
    channel.setQuery(query);
    Packet[] receivedPackets;
    try {
        DocsumPacketKey[] packetKeys;
        if (countFastHits(result) > 0) {
            packetKeys = getPacketKeys(result, summaryClass, false);
            if (packetKeys.length == 0) {
                receivedPackets = new Packet[0];
            } else {
                try {
                    receivedPackets = fetchSummaries(channel, result, summaryClass);
                } catch (InvalidChannelException e) {
                    result.hits().addError(ErrorMessage.createBackendCommunicationError("Invalid channel " + getName() + " (summary fetch)"));
                    return;
                } catch (ChannelTimeoutException e) {
                    result.hits().addError(ErrorMessage.createTimeout("timeout waiting for summaries from " + getName()));
                    return;
                } catch (IOException e) {
                    result.hits().addError(ErrorMessage.createBackendCommunicationError("IO error while talking on channel " + getName() + " (summary fetch): " + e.getMessage()));
                    return;
                }
                if (receivedPackets.length == 0) {
                    result.hits().addError(ErrorMessage.createBackendCommunicationError(getName() + " got no packets back (summary fetch)"));
                    return;
                }
            }
        } else {
            packetKeys = new DocsumPacketKey[0];
            receivedPackets = new Packet[0];
        }
        int skippedHits;
        try {
            FillHitsResult fillHitsResult = fillHits(result, receivedPackets, summaryClass);
            skippedHits = fillHitsResult.skippedHits;
            if (fillHitsResult.error != null) {
                result.hits().addError(ErrorMessage.createTimeout(fillHitsResult.error));
                return;
            }
        } catch (TimeoutException e) {
            result.hits().addError(ErrorMessage.createTimeout(e.getMessage()));
            return;
        } catch (IOException e) {
            result.hits().addError(ErrorMessage.createBackendCommunicationError("Error filling hits with summary fields, source: " + getName() + " Exception thrown: " + e.getMessage()));
            return;
        }
        if (skippedHits == 0 && packetWrapper != null) {
            cacheControl.updateCacheEntry(cacheKey, query, packetKeys, receivedPackets);
        }
        if (skippedHits > 0)
            result.hits().addError(com.yahoo.search.result.ErrorMessage.createEmptyDocsums("Missing hit data for summary '" + summaryClass + "' for " + skippedHits + " hits"));
        result.analyzeHits();
        if (query.getTraceLevel() >= 3) {
            int hitNumber = 0;
            for (Iterator<com.yahoo.search.result.Hit> i = hitIterator(result); i.hasNext(); ) {
                com.yahoo.search.result.Hit hit = i.next();
                if (!(hit instanceof FastHit))
                    continue;
                FastHit fastHit = (FastHit) hit;
                String traceMsg = "Hit: " + (hitNumber++) + " from " + (fastHit.isCached() ? "cache" : "backend");
                if (!fastHit.isFilled(summaryClass))
                    traceMsg += ". Error, hit, not filled";
                query.trace(traceMsg, false, 3);
            }
        }
    } finally {
        channel.close();
    }
}

Example 3 with FS4Channel

use of com.yahoo.fs4.mplex.FS4Channel in project vespa by vespa-engine.

the class FastSearcher method ping.

public static Pong ping(Ping ping, Backend backend, String name) {
    FS4Channel channel = backend.openPingChannel();
    // com.yahoo.prelude.cluster.TrafficNodeMonitor.failed(ErrorMessage)
    try {
        PingPacket pingPacket = new PingPacket();
        try {
            boolean couldSend = channel.sendPacket(pingPacket);
            if (!couldSend) {
                return new Pong(ErrorMessage.createBackendCommunicationError("Could not ping " + name));
            }
        } catch (InvalidChannelException e) {
            return new Pong(ErrorMessage.createBackendCommunicationError("Invalid channel " + name));
        } catch (IllegalStateException e) {
            return new Pong(ErrorMessage.createBackendCommunicationError("Illegal state in FS4: " + e.getMessage()));
        } catch (IOException e) {
            return new Pong(ErrorMessage.createBackendCommunicationError("IO error while sending ping: " + e.getMessage()));
        }
        // We should only get a single packet
        BasicPacket[] packets;
        try {
            packets = channel.receivePackets(ping.getTimeout(), 1);
        } catch (ChannelTimeoutException e) {
            return new Pong(ErrorMessage.createNoAnswerWhenPingingNode("timeout while waiting for fdispatch for " + name));
        } catch (InvalidChannelException e) {
            return new Pong(ErrorMessage.createBackendCommunicationError("Invalid channel for " + name));
        }
        if (packets.length == 0) {
            return new Pong(ErrorMessage.createBackendCommunicationError(name + " got no packets back"));
        }
        try {
            ensureInstanceOf(PongPacket.class, packets[0], name);
        } catch (TimeoutException e) {
            return new Pong(ErrorMessage.createTimeout(e.getMessage()));
        } catch (IOException e) {
            return new Pong(ErrorMessage.createBackendCommunicationError("Unexpected packet class returned after ping: " + e.getMessage()));
        }
        return new Pong((PongPacket) packets[0]);
    } finally {
        if (channel != null) {
            channel.close();
        }
    }
}

Example 4 with FS4Channel

use of com.yahoo.fs4.mplex.FS4Channel in project vespa by vespa-engine.

the class FastSearcher method doSearch2.

@Override
public Result doSearch2(Query query, QueryPacket queryPacket, CacheKey cacheKey, Execution execution) {
    FS4Channel channel = null;
    try {
        if (dispatcher.searchCluster().groupSize() == 1)
            forceSinglePassGrouping(query);
        channel = chooseBackend(query).openChannel();
        channel.setQuery(query);
        Result result = searchTwoPhase(channel, query, queryPacket, cacheKey);
        if (query.properties().getBoolean(Ranking.RANKFEATURES, false)) {
            // There is currently no correct choice for which
            // summary class we want to fetch at this point. If we
            // fetch the one selected by the user it may not
            // contain the data we need. If we fetch the default
            // one we end up fetching docsums twice unless the
            // user also requested the default one.
            // ARGH
            fill(result, query.getPresentation().getSummary(), execution);
        }
        return result;
    } catch (TimeoutException e) {
        return new Result(query, ErrorMessage.createTimeout(e.getMessage()));
    } catch (IOException e) {
        Result result = new Result(query);
        if (query.getTraceLevel() >= 1)
            query.trace(getName() + " error response: " + result, false, 1);
        result.hits().addError(ErrorMessage.createBackendCommunicationError(getName() + " failed: " + e.getMessage()));
        return result;
    } finally {
        if (channel != null)
            channel.close();
    }
}

Example 5 with FS4Channel

use of com.yahoo.fs4.mplex.FS4Channel in project vespa by vespa-engine.

the class FastSearcher method searchTwoPhase.

private Result searchTwoPhase(FS4Channel channel, Query query, QueryPacket queryPacket, CacheKey cacheKey) throws IOException {
    if (isLoggingFine())
        getLogger().finest("sending query packet");
    try {
        boolean couldSend = channel.sendPacket(queryPacket);
        if (!couldSend)
            return new Result(query, ErrorMessage.createBackendCommunicationError("Could not reach '" + getName() + "'"));
    } catch (InvalidChannelException e) {
        return new Result(query, ErrorMessage.createBackendCommunicationError("Invalid channel " + getName()));
    } catch (IllegalStateException e) {
        return new Result(query, ErrorMessage.createBackendCommunicationError("Illegal state in FS4: " + e.getMessage()));
    }
    BasicPacket[] basicPackets;
    try {
        basicPackets = channel.receivePackets(query.getTimeLeft(), 1);
    } catch (ChannelTimeoutException e) {
        return new Result(query, ErrorMessage.createTimeout("Timeout while waiting for " + getName()));
    } catch (InvalidChannelException e) {
        return new Result(query, ErrorMessage.createBackendCommunicationError("Invalid channel for " + getName()));
    }
    if (basicPackets.length == 0) {
        return new Result(query, ErrorMessage.createBackendCommunicationError(getName() + " got no packets back"));
    }
    if (isLoggingFine())
        getLogger().finest("got packets " + basicPackets.length + " packets");
    ensureInstanceOf(QueryResultPacket.class, basicPackets[0], getName());
    QueryResultPacket resultPacket = (QueryResultPacket) basicPackets[0];
    if (isLoggingFine())
        getLogger().finest("got query packet. " + "docsumClass=" + query.getPresentation().getSummary());
    if (query.getPresentation().getSummary() == null)
        query.getPresentation().setSummary(getDefaultDocsumClass());
    Result result = new Result(query);
    addMetaInfo(query, queryPacket.getQueryPacketData(), resultPacket, result, false);
    addUnfilledHits(result, resultPacket.getDocuments(), false, queryPacket.getQueryPacketData(), cacheKey);
    Packet[] packets;
    PacketWrapper packetWrapper = cacheControl.lookup(cacheKey, query);
    if (packetWrapper != null) {
        cacheControl.updateCacheEntry(cacheKey, query, resultPacket);
    } else {
        if (resultPacket.getCoverageFeature() && !resultPacket.getCoverageFull()) {
        // Don't add error here, it was done in first phase
        // No check if packetWrapper already exists, since incomplete
        // first phase data won't be cached anyway.
        } else {
            packets = new Packet[1];
            packets[0] = resultPacket;
            cacheControl.cache(cacheKey, query, new DocsumPacketKey[0], packets);
        }
    }
    return result;
}