Examples with QueryPacket com.yahoo.fs4.QueryPacket used on opensource projects

Example 1 with QueryPacket

use of com.yahoo.fs4.QueryPacket in project vespa by vespa-engine.

the class FastSearcher method doPartialFill.

/**
 * Perform a partial docsum fill for a temporary result
 * representing a partition of the complete fill request.
 *
 * @param result result containing a partition of the unfilled hits
 * @param summaryClass the summary class we want to fill with
 */
protected void doPartialFill(Result result, String summaryClass) {
    if (result.isFilled(summaryClass))
        return;
    Query query = result.getQuery();
    traceQuery(getName(), "fill", query, query.getOffset(), query.getHits(), 2, quotedSummaryClass(summaryClass));
    if (wantsRPCSummaryFill(query)) {
        CompressionType compression = CompressionType.valueOf(query.properties().getString(dispatchCompression, "LZ4").toUpperCase());
        fillSDDocName(result);
        dispatcher.fill(result, summaryClass, compression);
        return;
    }
    CacheKey cacheKey = null;
    PacketWrapper packetWrapper = null;
    if (getCacheControl().useCache(query)) {
        cacheKey = fetchCacheKeyFromHits(result.hits(), summaryClass);
        if (cacheKey == null) {
            QueryPacket queryPacket = QueryPacket.create(query);
            cacheKey = new CacheKey(queryPacket);
        }
        packetWrapper = cacheLookupTwoPhase(cacheKey, result, summaryClass);
    }
    FS4Channel channel = chooseBackend(query).openChannel();
    channel.setQuery(query);
    Packet[] receivedPackets;
    try {
        DocsumPacketKey[] packetKeys;
        if (countFastHits(result) > 0) {
            packetKeys = getPacketKeys(result, summaryClass, false);
            if (packetKeys.length == 0) {
                receivedPackets = new Packet[0];
            } else {
                try {
                    receivedPackets = fetchSummaries(channel, result, summaryClass);
                } catch (InvalidChannelException e) {
                    result.hits().addError(ErrorMessage.createBackendCommunicationError("Invalid channel " + getName() + " (summary fetch)"));
                    return;
                } catch (ChannelTimeoutException e) {
                    result.hits().addError(ErrorMessage.createTimeout("timeout waiting for summaries from " + getName()));
                    return;
                } catch (IOException e) {
                    result.hits().addError(ErrorMessage.createBackendCommunicationError("IO error while talking on channel " + getName() + " (summary fetch): " + e.getMessage()));
                    return;
                }
                if (receivedPackets.length == 0) {
                    result.hits().addError(ErrorMessage.createBackendCommunicationError(getName() + " got no packets back (summary fetch)"));
                    return;
                }
            }
        } else {
            packetKeys = new DocsumPacketKey[0];
            receivedPackets = new Packet[0];
        }
        int skippedHits;
        try {
            FillHitsResult fillHitsResult = fillHits(result, receivedPackets, summaryClass);
            skippedHits = fillHitsResult.skippedHits;
            if (fillHitsResult.error != null) {
                result.hits().addError(ErrorMessage.createTimeout(fillHitsResult.error));
                return;
            }
        } catch (TimeoutException e) {
            result.hits().addError(ErrorMessage.createTimeout(e.getMessage()));
            return;
        } catch (IOException e) {
            result.hits().addError(ErrorMessage.createBackendCommunicationError("Error filling hits with summary fields, source: " + getName() + " Exception thrown: " + e.getMessage()));
            return;
        }
        if (skippedHits == 0 && packetWrapper != null) {
            cacheControl.updateCacheEntry(cacheKey, query, packetKeys, receivedPackets);
        }
        if (skippedHits > 0)
            result.hits().addError(com.yahoo.search.result.ErrorMessage.createEmptyDocsums("Missing hit data for summary '" + summaryClass + "' for " + skippedHits + " hits"));
        result.analyzeHits();
        if (query.getTraceLevel() >= 3) {
            int hitNumber = 0;
            for (Iterator<com.yahoo.search.result.Hit> i = hitIterator(result); i.hasNext(); ) {
                com.yahoo.search.result.Hit hit = i.next();
                if (!(hit instanceof FastHit))
                    continue;
                FastHit fastHit = (FastHit) hit;
                String traceMsg = "Hit: " + (hitNumber++) + " from " + (fastHit.isCached() ? "cache" : "backend");
                if (!fastHit.isFilled(summaryClass))
                    traceMsg += ". Error, hit, not filled";
                query.trace(traceMsg, false, 3);
            }
        }
    } finally {
        channel.close();
    }
}

Example 2 with QueryPacket

use of com.yahoo.fs4.QueryPacket in project vespa by vespa-engine.

the class MockFSChannel method addDocsums.

/**
 * Adds the number of docsums requested in queryPacket.getHits()
 */
private void addDocsums(List packets, QueryPacket queryPacket) {
    int numHits = queryPacket.getHits();
    if (lastReceived instanceof GetDocSumsPacket) {
        numHits = ((GetDocSumsPacket) lastReceived).getNumDocsums();
    }
    for (int i = 0; i < numHits; i++) {
        ByteBuffer buffer;
        if (emptyDocsums) {
            buffer = createEmptyDocsumPacketData();
        } else {
            int[] docids = { 123, 456, 789, 789, 789, 789, 789, 789, 789, 789, 789, 789 };
            buffer = createDocsumPacketData(docids[i], DocsumDefinitionTestCase.makeDocsum());
        }
        buffer.position(0);
        packets.add(PacketDecoder.decode(buffer));
    }
    packets.add(EolPacket.create());
}

Example 3 with QueryPacket

use of com.yahoo.fs4.QueryPacket in project vespa by vespa-engine.

the class VdsStreamingSearcher method doSearch2.

@Override
public Result doSearch2(Query query, QueryPacket queryPacket, CacheKey cacheKey, Execution execution) {
    // TODO refactor this method into smaller methods, it's hard to see the actual code
    lazyTrace(query, 7, "Routing to storage cluster ", getStorageClusterRouteSpec());
    if (route == null) {
        route = Route.parse(getStorageClusterRouteSpec());
    }
    lazyTrace(query, 8, "Route is ", route);
    lazyTrace(query, 7, "doSearch2(): query docsum class=", query.getPresentation().getSummary(), ", default docsum class=", getDefaultDocsumClass());
    if (query.getPresentation().getSummary() == null) {
        lazyTrace(query, 6, "doSearch2(): No summary class specified in query, using default: ", getDefaultDocsumClass());
        query.getPresentation().setSummary(getDefaultDocsumClass());
    } else {
        lazyTrace(query, 6, "doSearch2(): Summary class has been specified in query: ", query.getPresentation().getSummary());
    }
    lazyTrace(query, 8, "doSearch2(): rank properties=", query.getRanking());
    lazyTrace(query, 8, "doSearch2(): sort specification=", query.getRanking().getSorting() == null ? null : query.getRanking().getSorting().fieldOrders());
    int documentSelectionQueryParameterCount = 0;
    if (query.properties().getString(streamingUserid) != null)
        documentSelectionQueryParameterCount++;
    if (query.properties().getString(streamingGroupname) != null)
        documentSelectionQueryParameterCount++;
    if (query.properties().getString(streamingSelection) != null)
        documentSelectionQueryParameterCount++;
    if (documentSelectionQueryParameterCount != 1) {
        return new Result(query, ErrorMessage.createBackendCommunicationError("Streaming search needs one and " + "only one of these query parameters to be set: streaming.userid, streaming.groupname, " + "streaming.selection"));
    }
    query.trace("Routing to search cluster " + getSearchClusterConfigId(), 4);
    Visitor visitor = visitorFactory.createVisitor(query, getSearchClusterConfigId(), route);
    try {
        visitor.doSearch();
    } catch (ParseException e) {
        return new Result(query, ErrorMessage.createBackendCommunicationError("Failed to parse document selection string: " + e.getMessage() + "'."));
    } catch (TokenMgrError e) {
        return new Result(query, ErrorMessage.createBackendCommunicationError("Failed to tokenize document selection string: " + e.getMessage() + "'."));
    } catch (TimeoutException e) {
        return new Result(query, ErrorMessage.createTimeout(e.getMessage()));
    } catch (InterruptedException | IllegalArgumentException e) {
        return new Result(query, ErrorMessage.createBackendCommunicationError(e.getMessage()));
    }
    lazyTrace(query, 8, "offset=", query.getOffset(), ", hits=", query.getHits());
    Result result = new Result(query);
    // Sorted on rank
    List<SearchResult.Hit> hits = visitor.getHits();
    Map<String, DocumentSummary.Summary> summaryMap = visitor.getSummaryMap();
    lazyTrace(query, 7, "total hit count = ", visitor.getTotalHitCount(), ", returned hit count = ", hits.size(), ", summary count = ", summaryMap.size());
    result.setTotalHitCount(visitor.getTotalHitCount());
    query.trace(visitor.getStatistics().toString(), false, 2);
    query.getContext(true).setProperty(STREAMING_STATISTICS, visitor.getStatistics());
    Packet[] summaryPackets = new Packet[hits.size()];
    int index = 0;
    boolean skippedEarlierResult = false;
    for (SearchResult.Hit hit : hits) {
        if (!verifyDocId(hit.getDocId(), query, skippedEarlierResult)) {
            skippedEarlierResult = true;
            continue;
        }
        FastHit fastHit = buildSummaryHit(query, hit);
        result.hits().add(fastHit);
        DocumentSummary.Summary summary = summaryMap.get(hit.getDocId());
        if (summary != null) {
            DocsumPacket dp = new DocsumPacket(summary.getSummary());
            // log.log(LogLevel.SPAM, "DocsumPacket: " + dp);
            summaryPackets[index] = dp;
        } else {
            return new Result(query, ErrorMessage.createBackendCommunicationError("Did not find summary for hit with document id " + hit.getDocId()));
        }
        index++;
    }
    if (result.isFilled(query.getPresentation().getSummary())) {
        lazyTrace(query, 8, "Result is filled for summary class ", query.getPresentation().getSummary());
    } else {
        lazyTrace(query, 8, "Result is not filled for summary class ", query.getPresentation().getSummary());
    }
    List<Grouping> groupingList = visitor.getGroupings();
    lazyTrace(query, 8, "Grouping list=", groupingList);
    if (!groupingList.isEmpty()) {
        GroupingListHit groupHit = new GroupingListHit(groupingList, getDocsumDefinitionSet(query));
        result.hits().add(groupHit);
    }
    int skippedHits;
    try {
        FillHitsResult fillHitsResult = fillHits(result, summaryPackets, query.getPresentation().getSummary());
        skippedHits = fillHitsResult.skippedHits;
        if (fillHitsResult.error != null) {
            result.hits().addError(ErrorMessage.createTimeout(fillHitsResult.error));
            return result;
        }
    } catch (TimeoutException e) {
        result.hits().addError(ErrorMessage.createTimeout(e.getMessage()));
        return result;
    } catch (IOException e) {
        return new Result(query, ErrorMessage.createBackendCommunicationError("Error filling hits with summary fields"));
    }
    if (skippedHits == 0) {
        // TODO: cache results or result.analyzeHits(); ?
        query.trace("All hits have been filled", 4);
    } else {
        lazyTrace(query, 8, "Skipping some hits for query: ", result.getQuery());
    }
    lazyTrace(query, 8, "Returning result ", result);
    if (skippedHits > 0) {
        getLogger().info("skipping " + skippedHits + " hits for query: " + result.getQuery());
        result.hits().addError(com.yahoo.search.result.ErrorMessage.createTimeout("Missing hit summary data for " + skippedHits + " hits"));
    }
    return result;
}

Example 4 with QueryPacket

use of com.yahoo.fs4.QueryPacket in project vespa by vespa-engine.

the class PacketTestCase method testLargeQueryFail.

/**
 * Make a query that is too large and then try to encode it
 * into a small ByteBuffer
 */
@Test
public void testLargeQueryFail() {
    StringBuilder queryBuffer = new StringBuilder(4008);
    queryBuffer.append("/?query=");
    for (int i = 0; i < 1000; i++) {
        queryBuffer.append("the%20");
    }
    Query query = new Query(queryBuffer.toString());
    assertNotNull(query);
    QueryPacket queryPacket = QueryPacket.create(query);
    assertNotNull(queryPacket);
    ByteBuffer buffer = ByteBuffer.allocate(100);
    int position = buffer.position();
    int limit = buffer.limit();
    try {
        queryPacket.encode(buffer, 0);
        fail();
    } catch (BufferTooSmallException e) {
    // success if exception is thrown
    }
    // make sure state of buffer is unchanged
    assertEquals(position, buffer.position());
    assertEquals(limit, buffer.limit());
}

Example 5 with QueryPacket

use of com.yahoo.fs4.QueryPacket in project vespa by vespa-engine.

the class PacketTestCase method requireThatPacketsCanTurnOnCompression.

@Test
public void requireThatPacketsCanTurnOnCompression() throws BufferTooSmallException {
    QueryPacket queryPacket = QueryPacket.create(new Query("/?query=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
    ByteBuffer buffer = ByteBuffer.allocate(1024);
    int channel = 32;
    queryPacket.encode(buffer, channel);
    buffer.flip();
    // size
    assertEquals(86, buffer.getInt());
    // code
    assertEquals(0xda, buffer.getInt());
    assertEquals(channel, buffer.getInt());
    queryPacket.setCompressionLimit(88);
    buffer.clear();
    queryPacket.encode(buffer, channel);
    buffer.flip();
    // size
    assertEquals(86, buffer.getInt());
    // code
    assertEquals(0xda, buffer.getInt());
    queryPacket.setCompressionLimit(84);
    buffer.clear();
    queryPacket.encode(buffer, channel);
    buffer.flip();
    // size
    assertEquals(57, buffer.getInt());
    // code
    assertEquals(0x060000da, buffer.getInt());
    assertEquals(channel, buffer.getInt());
}