Examples with Packet com.yahoo.fs4.Packet used on opensource projects

Example 16 with Packet

use of com.yahoo.fs4.Packet in project vespa by vespa-engine.

the class VdsStreamingSearcher method doSearch2.

@Override
public Result doSearch2(Query query, QueryPacket queryPacket, CacheKey cacheKey, Execution execution) {
    // TODO refactor this method into smaller methods, it's hard to see the actual code
    lazyTrace(query, 7, "Routing to storage cluster ", getStorageClusterRouteSpec());
    if (route == null) {
        route = Route.parse(getStorageClusterRouteSpec());
    }
    lazyTrace(query, 8, "Route is ", route);
    lazyTrace(query, 7, "doSearch2(): query docsum class=", query.getPresentation().getSummary(), ", default docsum class=", getDefaultDocsumClass());
    if (query.getPresentation().getSummary() == null) {
        lazyTrace(query, 6, "doSearch2(): No summary class specified in query, using default: ", getDefaultDocsumClass());
        query.getPresentation().setSummary(getDefaultDocsumClass());
    } else {
        lazyTrace(query, 6, "doSearch2(): Summary class has been specified in query: ", query.getPresentation().getSummary());
    }
    lazyTrace(query, 8, "doSearch2(): rank properties=", query.getRanking());
    lazyTrace(query, 8, "doSearch2(): sort specification=", query.getRanking().getSorting() == null ? null : query.getRanking().getSorting().fieldOrders());
    int documentSelectionQueryParameterCount = 0;
    if (query.properties().getString(streamingUserid) != null)
        documentSelectionQueryParameterCount++;
    if (query.properties().getString(streamingGroupname) != null)
        documentSelectionQueryParameterCount++;
    if (query.properties().getString(streamingSelection) != null)
        documentSelectionQueryParameterCount++;
    if (documentSelectionQueryParameterCount != 1) {
        return new Result(query, ErrorMessage.createBackendCommunicationError("Streaming search needs one and " + "only one of these query parameters to be set: streaming.userid, streaming.groupname, " + "streaming.selection"));
    }
    query.trace("Routing to search cluster " + getSearchClusterConfigId(), 4);
    Visitor visitor = visitorFactory.createVisitor(query, getSearchClusterConfigId(), route);
    try {
        visitor.doSearch();
    } catch (ParseException e) {
        return new Result(query, ErrorMessage.createBackendCommunicationError("Failed to parse document selection string: " + e.getMessage() + "'."));
    } catch (TokenMgrError e) {
        return new Result(query, ErrorMessage.createBackendCommunicationError("Failed to tokenize document selection string: " + e.getMessage() + "'."));
    } catch (TimeoutException e) {
        return new Result(query, ErrorMessage.createTimeout(e.getMessage()));
    } catch (InterruptedException | IllegalArgumentException e) {
        return new Result(query, ErrorMessage.createBackendCommunicationError(e.getMessage()));
    }
    lazyTrace(query, 8, "offset=", query.getOffset(), ", hits=", query.getHits());
    Result result = new Result(query);
    // Sorted on rank
    List<SearchResult.Hit> hits = visitor.getHits();
    Map<String, DocumentSummary.Summary> summaryMap = visitor.getSummaryMap();
    lazyTrace(query, 7, "total hit count = ", visitor.getTotalHitCount(), ", returned hit count = ", hits.size(), ", summary count = ", summaryMap.size());
    result.setTotalHitCount(visitor.getTotalHitCount());
    query.trace(visitor.getStatistics().toString(), false, 2);
    query.getContext(true).setProperty(STREAMING_STATISTICS, visitor.getStatistics());
    Packet[] summaryPackets = new Packet[hits.size()];
    int index = 0;
    boolean skippedEarlierResult = false;
    for (SearchResult.Hit hit : hits) {
        if (!verifyDocId(hit.getDocId(), query, skippedEarlierResult)) {
            skippedEarlierResult = true;
            continue;
        }
        FastHit fastHit = buildSummaryHit(query, hit);
        result.hits().add(fastHit);
        DocumentSummary.Summary summary = summaryMap.get(hit.getDocId());
        if (summary != null) {
            DocsumPacket dp = new DocsumPacket(summary.getSummary());
            // log.log(LogLevel.SPAM, "DocsumPacket: " + dp);
            summaryPackets[index] = dp;
        } else {
            return new Result(query, ErrorMessage.createBackendCommunicationError("Did not find summary for hit with document id " + hit.getDocId()));
        }
        index++;
    }
    if (result.isFilled(query.getPresentation().getSummary())) {
        lazyTrace(query, 8, "Result is filled for summary class ", query.getPresentation().getSummary());
    } else {
        lazyTrace(query, 8, "Result is not filled for summary class ", query.getPresentation().getSummary());
    }
    List<Grouping> groupingList = visitor.getGroupings();
    lazyTrace(query, 8, "Grouping list=", groupingList);
    if (!groupingList.isEmpty()) {
        GroupingListHit groupHit = new GroupingListHit(groupingList, getDocsumDefinitionSet(query));
        result.hits().add(groupHit);
    }
    int skippedHits;
    try {
        FillHitsResult fillHitsResult = fillHits(result, summaryPackets, query.getPresentation().getSummary());
        skippedHits = fillHitsResult.skippedHits;
        if (fillHitsResult.error != null) {
            result.hits().addError(ErrorMessage.createTimeout(fillHitsResult.error));
            return result;
        }
    } catch (TimeoutException e) {
        result.hits().addError(ErrorMessage.createTimeout(e.getMessage()));
        return result;
    } catch (IOException e) {
        return new Result(query, ErrorMessage.createBackendCommunicationError("Error filling hits with summary fields"));
    }
    if (skippedHits == 0) {
        // TODO: cache results or result.analyzeHits(); ?
        query.trace("All hits have been filled", 4);
    } else {
        lazyTrace(query, 8, "Skipping some hits for query: ", result.getQuery());
    }
    lazyTrace(query, 8, "Returning result ", result);
    if (skippedHits > 0) {
        getLogger().info("skipping " + skippedHits + " hits for query: " + result.getQuery());
        result.hits().addError(com.yahoo.search.result.ErrorMessage.createTimeout("Missing hit summary data for " + skippedHits + " hits"));
    }
    return result;
}

Example 17 with Packet

use of com.yahoo.fs4.Packet in project vespa by vespa-engine.

the class QueryTestCase method testPatchInChannelId.

@Test
public void testPatchInChannelId() {
    Query query = new Query("/?query=chain&timeout=0");
    query.setWindow(2, 8);
    QueryPacket packet = QueryPacket.create(query);
    assertEquals(2, packet.getOffset());
    assertEquals(8, packet.getHits());
    ByteBuffer buffer = packet.grantEncodingBuffer(0x07070707);
    byte[] correctBuffer = new byte[] { // Header
    0, // Header
    0, // Header
    0, // Header
    46, // Header
    0, // Header
    0, // Header
    0, // Header
    -38, // Header
    7, // Header
    7, // Header
    7, // Header
    7, // Features
    0, // Features
    0, // Features
    0, // Features
    6, 2, 8, // querytimeout
    0, // querytimeout
    0, // querytimeout
    0, // querytimeout
    1, // qflags
    0, // qflags
    0, // qflags
    0x40, // qflags
    0x03, 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', 0, 0, 0, 1, 0, 0, 0, 8, 4, 0, 5, 99, 104, 97, 105, 110 };
    byte[] encoded = new byte[buffer.limit()];
    buffer.get(encoded);
    assertEqualArrays(correctBuffer, encoded);
    packet.allocateAndEncode(0x07070707);
    buffer = packet.grantEncodingBuffer(0x09090909);
    correctBuffer = new byte[] { // Header
    0, // Header
    0, // Header
    0, // Header
    46, // Header
    0, // Header
    0, // Header
    0, // Header
    -38, // Header
    9, // Header
    9, // Header
    9, // Header
    9, // Features
    0, // Features
    0, // Features
    0, // Features
    6, 2, 8, // querytimeout
    0, // querytimeout
    0, // querytimeout
    0, // querytimeout
    1, // qflags
    0, // qflags
    0, // qflags
    0x40, // qflags
    0x03, 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', 0, 0, 0, 1, 0, 0, 0, 8, 4, 0, 5, 99, 104, 97, 105, 110 };
    encoded = new byte[buffer.limit()];
    buffer.get(encoded);
    assertEqualArrays(correctBuffer, encoded);
}

Example 18 with Packet

use of com.yahoo.fs4.Packet in project vespa by vespa-engine.

the class QueryTestCase method testEncodeQueryPacketWithManyFeatures.

/**
 * This test will tell you if you have screwed up the binary encoding, but it won't tell you how
 */
@Test
public void testEncodeQueryPacketWithManyFeatures() {
    Query query = new Query("/?query=chain" + "&ranking.features.query(foo)=30.3&ranking.features.query(bar)=0" + "&ranking.properties.property.p1=v1&ranking.properties.property.p2=v2" + "&pos.ll=S22.4532;W123.9887&pos.radius=3&pos.attribute=place&ranking.freshness=37" + "&model.searchPath=7/3");
    query.getRanking().setFreshness(new Freshness("123456"));
    query.getRanking().setSorting("+field1 -field2");
    query.getRanking().setProfile("two");
    Highlight highlight = new Highlight();
    highlight.addHighlightTerm("field1", "term1");
    highlight.addHighlightTerm("field1", "term2");
    query.getPresentation().setHighlight(highlight);
    query.prepare();
    QueryPacket packet = QueryPacket.create(query);
    byte[] encoded = packetToBytes(packet);
    byte[] correctBuffer = new byte[] { 0, 0, 1, 23, 0, 0, 0, -38, 0, 0, 0, 0, 0, 16, 0, -122, 0, 10, ignored, ignored, ignored, ignored, 0, 0, 0x40, 0x03, 3, 't', 'w', 'o', 0, 0, 0, 3, 0, 0, 0, 4, 'r', 'a', 'n', 'k', 0, 0, 0, 5, 0, 0, 0, 11, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'y', 46, 'p', '2', 0, 0, 0, 2, 'v', '2', 0, 0, 0, 11, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'y', 46, 'p', '1', 0, 0, 0, 2, 'v', '1', 0, 0, 0, 3, 'f', 'o', 'o', 0, 0, 0, 4, '3', '0', 46, '3', 0, 0, 0, 3, 'b', 'a', 'r', 0, 0, 0, 1, '0', 0, 0, 0, 9, 'v', 'e', 's', 'p', 'a', 46, 'n', 'o', 'w', 0, 0, 0, 6, '1', '2', '3', '4', '5', '6', 0, 0, 0, 14, 'h', 'i', 'g', 'h', 'l', 'i', 'g', 'h', 't', 't', 'e', 'r', 'm', 's', 0, 0, 0, 3, 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 1, '2', 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 5, 't', 'e', 'r', 'm', '1', 0, 0, 0, 6, 'f', 'i', 'e', 'l', 'd', '1', 0, 0, 0, 5, 't', 'e', 'r', 'm', '2', 0, 0, 0, 5, 'm', 'o', 'd', 'e', 'l', 0, 0, 0, 1, 0, 0, 0, 10, 's', 'e', 'a', 'r', 'c', 'h', 'p', 'a', 't', 'h', 0, 0, 0, 3, '7', 47, '3', 0, 0, 0, 15, 43, 'f', 'i', 'e', 'l', 'd', '1', 32, 45, 'f', 'i', 'e', 'l', 'd', '2', 0, 0, 0, 1, 0, 0, 0, 9, 68, 1, 0, 5, 'c', 'h', 'a', 'i', 'n' };
    assertEqualArrays(correctBuffer, encoded);
}

Example 19 with Packet

use of com.yahoo.fs4.Packet in project vespa by vespa-engine.

the class QueryTestCase method testEncodeQueryPacketWithSomeAdditionalFeatures.

@Test
public void testEncodeQueryPacketWithSomeAdditionalFeatures() {
    Query query = new Query("/?query=chain&dataset=10&type=phrase&timeout=0");
    // Because the rank mapping now needs config and a searcher,
    // we do the sledgehammer dance:
    query.getRanking().setProfile("two");
    query.setWindow(2, 8);
    QueryPacket packet = QueryPacket.create(query);
    byte[] encoded = packetToBytes(packet);
    byte[] correctBuffer = new byte[] { // Header
    0, // Header
    0, // Header
    0, // Header
    42, // Header
    0, // Header
    0, // Header
    0, // Header
    -38, // Header
    0, // Header
    0, // Header
    0, // Header
    0, // Features
    0, // Features
    0, // Features
    0, // Features
    6, 2, 8, // querytimeout
    0, // querytimeout
    0, // querytimeout
    0, // querytimeout
    1, // QFlags
    0, // QFlags
    0, // QFlags
    0x40, // QFlags
    0x03, 3, // Ranking
    't', // Ranking
    'w', // Ranking
    'o', 0, 0, 0, 1, 0, 0, 0, 8, 4, 0, 5, 99, 104, 97, 105, 110 };
    assertEqualArrays(correctBuffer, encoded);
}

Example 20 with Packet

use of com.yahoo.fs4.Packet in project vespa by vespa-engine.

the class QueryTestCase method testEncodeSortSpec.

@Test
public void testEncodeSortSpec() throws BufferTooSmallException {
    Query query = new Query("/?query=chain&sortspec=%2Ba+-b&timeout=0");
    query.setWindow(2, 8);
    QueryPacket packet = QueryPacket.create(query);
    ByteBuffer buffer = ByteBuffer.allocate(500);
    buffer.limit(0);
    packet.encode(buffer, 0);
    byte[] encoded = new byte[buffer.position()];
    buffer.rewind();
    buffer.get(encoded);
    byte[] correctBuffer = new byte[] { // Header
    0, // Header
    0, // Header
    0, // Header
    55, // Header
    0, // Header
    0, // Header
    0, // Header
    -38, // Header
    0, // Header
    0, // Header
    0, // Header
    0, // Features
    0, // Features
    0, // Features
    0, // Features
    -122, // offset
    2, // maxhits
    8, // querytimeout
    0, // querytimeout
    0, // querytimeout
    0, // querytimeout
    1, // qflags
    0, // qflags
    0, // qflags
    0x40, // qflags
    0x03, 7, 'd', 'e', 'f', 'a', 'u', 'l', 't', // sortspec length
    0, // sortspec length
    0, // sortspec length
    0, // sortspec length
    5, // sortspec
    43, // sortspec
    97, // sortspec
    32, // sortspec
    45, // sortspec
    98, // num stackitems
    0, // num stackitems
    0, // num stackitems
    0, // num stackitems
    1, 0, 0, 0, 8, 4, 0, 5, 99, 104, 97, 105, 110 };
    assertEqualArrays(correctBuffer, encoded);
    // Encode again to test grantEncodingBuffer
    buffer = packet.grantEncodingBuffer(0);
    encoded = new byte[buffer.limit()];
    buffer.get(encoded);
    assertEqualArrays(correctBuffer, encoded);
}