Examples with Document com.yahoo.vespa.http.client.core.Document used on opensource projects

Example 11 with Document

use of com.yahoo.vespa.http.client.core.Document in project vespa by vespa-engine.

the class IOThread method getNextDocsForFeeding.

List<Document> getNextDocsForFeeding(int maxWaitUnits, TimeUnit timeUnit) {
    final List<Document> docsForSendChunk = new ArrayList<>();
    int chunkSizeBytes = 0;
    try {
        drainFirstDocumentsInQueueIfOld();
        Document doc = documentQueue.poll(maxWaitUnits, timeUnit);
        if (doc != null) {
            docsForSendChunk.add(doc);
            chunkSizeBytes = doc.size();
        }
    } catch (InterruptedException ie) {
        log.fine("Got break signal while waiting for new documents to feed.");
        return docsForSendChunk;
    }
    int pendingSize = 1 + resultQueue.getPendingSize();
    // see if we can get more documents without blocking
    while (chunkSizeBytes < maxChunkSizeBytes && pendingSize < maxInFlightRequests) {
        drainFirstDocumentsInQueueIfOld();
        Document d = documentQueue.poll();
        if (d == null) {
            break;
        }
        docsForSendChunk.add(d);
        chunkSizeBytes += d.size();
        pendingSize++;
    }
    log.finest("Chunk has " + docsForSendChunk.size() + " docs with a size " + chunkSizeBytes + " bytes.");
    docsReceivedCounter.addAndGet(docsForSendChunk.size());
    return docsForSendChunk;
}

Example 12 with Document

use of com.yahoo.vespa.http.client.core.Document in project vespa by vespa-engine.

the class IOThread method pullAndProcessData.

private ProcessResponse pullAndProcessData(int maxWaitTimeMilliSecs) throws ServerResponseException, IOException {
    final int pendingResultQueueSize = resultQueue.getPendingSize();
    pendingDocumentStatusCount.set(pendingResultQueueSize);
    List<Document> nextDocsForFeeding = (pendingResultQueueSize > maxInFlightRequests) ? // The queue is full, will not send more documents.
    new ArrayList<>() : getNextDocsForFeeding(maxWaitTimeMilliSecs, TimeUnit.MILLISECONDS);
    if (nextDocsForFeeding.isEmpty() && pendingResultQueueSize == 0) {
        // we have no unfinished business with the server now.
        log.finest("No document awaiting feeding, not waiting for results.");
        return new ProcessResponse(0, 0);
    }
    log.finest("Awaiting " + pendingResultQueueSize + " results.");
    ProcessResponse processResponse = feedDocumentAndProcessResults(nextDocsForFeeding);
    if (pendingResultQueueSize > maxInFlightRequests && processResponse.processResultsCount == 0) {
        try {
            // Max outstanding document operations, no more results on server side, wait a bit
            // before asking again.
            Thread.sleep(300);
        } catch (InterruptedException e) {
        // Ignore
        }
    }
    return processResponse;
}

Example 13 with Document

use of com.yahoo.vespa.http.client.core.Document in project vespa by vespa-engine.

the class ApacheGatewayConnectionTest method testJsonDocumentHeader.

@Test
public void testJsonDocumentHeader() throws Exception {
    final Endpoint endpoint = Endpoint.create("hostname", 666, false);
    final FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.JSON_UTF8).build();
    final String clusterSpecificRoute = "";
    final ConnectionParams connectionParams = new ConnectionParams.Builder().setUseCompression(true).build();
    final List<Document> documents = new ArrayList<>();
    final CountDownLatch verifyContentSentLatch = new CountDownLatch(1);
    final String vespaDocContent = "Hello, I a JSON doc.";
    final String docId = "42";
    final AtomicInteger requestsReceived = new AtomicInteger(0);
    // This is the fake server, checks that DATA_FORMAT header is set properly.
    ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> {
        final Header header = post.getFirstHeader(Headers.DATA_FORMAT);
        if (requestsReceived.incrementAndGet() == 1) {
            // This is handshake, it is not json.
            assert (header == null);
            return httpResponse("clientId", "3");
        }
        assertNotNull(header);
        assertThat(header.getValue(), is(FeedParams.DataFormat.JSON_UTF8.name()));
        // Test is done.
        verifyContentSentLatch.countDown();
        return httpResponse("clientId", "3");
    });
    ApacheGatewayConnection apacheGatewayConnection = new ApacheGatewayConnection(endpoint, feedParams, clusterSpecificRoute, connectionParams, mockFactory, "clientId");
    apacheGatewayConnection.connect();
    apacheGatewayConnection.handshake();
    documents.add(createDoc(docId, vespaDocContent, true));
    apacheGatewayConnection.writeOperations(documents);
    assertTrue(verifyContentSentLatch.await(10, TimeUnit.SECONDS));
}

Example 14 with Document

use of com.yahoo.vespa.http.client.core.Document in project vespa by vespa-engine.

the class ApacheGatewayConnectionTest method testServerReturnsBadSessionInV3.

@Test(expected = IllegalArgumentException.class)
public void testServerReturnsBadSessionInV3() throws Exception {
    final Endpoint endpoint = Endpoint.create("hostname", 666, false);
    final FeedParams feedParams = new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.JSON_UTF8).build();
    final String clusterSpecificRoute = "";
    final ConnectionParams connectionParams = new ConnectionParams.Builder().setEnableV3Protocol(true).build();
    // This is the fake server, returns wrong session Id.
    ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> httpResponse("Wrong Id from server", "3"));
    ApacheGatewayConnection apacheGatewayConnection = new ApacheGatewayConnection(endpoint, feedParams, clusterSpecificRoute, connectionParams, mockFactory, "clientId");
    apacheGatewayConnection.connect();
    final List<Document> documents = new ArrayList<>();
    apacheGatewayConnection.writeOperations(documents);
}

Example 15 with Document

use of com.yahoo.vespa.http.client.core.Document in project vespa by vespa-engine.

the class ApacheGatewayConnectionTest method testCompressedWriteOperations.

/**
 *  Mocks the HttpClient, and verifies that the compressed data is sent.
 */
@Test
public void testCompressedWriteOperations() throws Exception {
    final Endpoint endpoint = Endpoint.create("hostname", 666, false);
    final FeedParams feedParams = new FeedParams.Builder().build();
    final String clusterSpecificRoute = "";
    final ConnectionParams connectionParams = new ConnectionParams.Builder().setUseCompression(true).build();
    final List<Document> documents = new ArrayList<>();
    final CountDownLatch verifyContentSentLatch = new CountDownLatch(1);
    final String vespaDocContent = "Hello, I am the document data.";
    final String docId = "42";
    final Document doc = createDoc(docId, vespaDocContent, false);
    // When sending data on http client, check if it is compressed. If compressed, unzip, check result,
    // and count down latch.
    ApacheGatewayConnection.HttpClientFactory mockFactory = mockHttpClientFactory(post -> {
        final Header header = post.getFirstHeader("Content-Encoding");
        if (header != null && header.getValue().equals("gzip")) {
            final String rawContent = TestUtils.zipStreamToString(post.getEntity().getContent());
            final String vespaHeaderText = "<vespafeed>\n";
            final String vespaFooterText = "</vespafeed>\n";
            assertThat(rawContent, is(doc.getOperationId() + " 38\n" + vespaHeaderText + vespaDocContent + "\n" + vespaFooterText));
            verifyContentSentLatch.countDown();
        }
        return httpResponse("clientId", "3");
    });
    StatusLine statusLineMock = mock(StatusLine.class);
    when(statusLineMock.getStatusCode()).thenReturn(200);
    ApacheGatewayConnection apacheGatewayConnection = new ApacheGatewayConnection(endpoint, feedParams, clusterSpecificRoute, connectionParams, mockFactory, "clientId");
    apacheGatewayConnection.connect();
    apacheGatewayConnection.handshake();
    documents.add(doc);
    apacheGatewayConnection.writeOperations(documents);
    assertTrue(verifyContentSentLatch.await(10, TimeUnit.SECONDS));
}