Examples with FeedClient com.yahoo.vespa.http.client.FeedClient used on opensource projects

Example 6 with FeedClient

use of com.yahoo.vespa.http.client.FeedClient in project vespa by vespa-engine.

the class Runner method main.

public static void main(String[] args) throws IOException, InterruptedException {
    final CommandLineArguments commandLineArgs = CommandLineArguments.build(args);
    if (commandLineArgs == null) {
        return;
    }
    FormatInputStream formatInputStream = new FormatInputStream(System.in, Optional.ofNullable(commandLineArgs.getFile()), commandLineArgs.getAddRootElementToXml());
    int intervalOfLogging = commandLineArgs.getVerbose() ? commandLineArgs.getWhenVerboseEnabledPrintMessageForEveryXDocuments() : Integer.MAX_VALUE;
    final AtomicInteger numSent = new AtomicInteger(0);
    final SimpleLoggerResultCallback callback = new SimpleLoggerResultCallback(numSent, intervalOfLogging);
    final FeedClient feedClient = FeedClientFactory.create(commandLineArgs.createSessionParams(formatInputStream.getFormat() == FormatInputStream.Format.JSON), callback);
    long sendTotalTimeMs = send(feedClient, formatInputStream.getInputStream(), formatInputStream.getFormat() == FormatInputStream.Format.JSON, numSent, commandLineArgs.getVerbose());
    if (commandLineArgs.getVerbose()) {
        System.err.println(feedClient.getStatsAsJson());
        double fileSizeMb = ((double) new File(commandLineArgs.getFile()).length()) / 1024.0 / 1024.0;
        double transferTimeSec = ((double) sendTotalTimeMs) / 1000.0;
        System.err.println("Sent " + fileSizeMb + " MB in " + transferTimeSec + " seconds.");
        System.err.println("Speed: " + ((fileSizeMb / transferTimeSec) * 8.0) + " Mbits/sec, + HTTP overhead " + "(not taking compression into account)");
        if (transferTimeSec > 0) {
            System.err.printf("Docs/sec %.3f%n\n", numSent.get() / transferTimeSec);
        }
    }
    callback.printProgress();
}

Example 7 with FeedClient

use of com.yahoo.vespa.http.client.FeedClient in project vespa by vespa-engine.

the class ExampleUsageFeedClientTest method exampleCode.

// Example usage of FeedClient
public static void exampleCode(String hostNameA, int portServerA, String hostNameB, int portServerB) {
    final boolean useSsl = false;
    final SessionParams sessionParams = new SessionParams.Builder().addCluster(new Cluster.Builder().addEndpoint(Endpoint.create(hostNameA, portServerA, useSsl)).build()).addCluster(new Cluster.Builder().addEndpoint(Endpoint.create(hostNameB, portServerB, useSsl)).build()).setFeedParams(new FeedParams.Builder().setDataFormat(FeedParams.DataFormat.JSON_UTF8).build()).build();
    final AtomicInteger resultsReceived = new AtomicInteger(0);
    final AtomicInteger errorsReceived = new AtomicInteger(0);
    FeedClient feedClient = FeedClientFactory.create(sessionParams, new FeedClient.ResultCallback() {

        @Override
        public void onCompletion(String docId, Result documentResult) {
            resultsReceived.incrementAndGet();
            if (!documentResult.getContext().equals(docId)) {
                System.err.println("Context does not work as expected.");
                errorsReceived.incrementAndGet();
            }
            if (!documentResult.isSuccess()) {
                System.err.println("Problems with docID " + docId + ":" + documentResult.toString());
                errorsReceived.incrementAndGet();
            }
        }
    });
    int sentCounter = 0;
    final List<String> docIds = Arrays.asList("1", "2", "3", "4");
    for (final String docId : docIds) {
        CharSequence docData = generateDocument(docId);
        feedClient.stream(docId, docData, docId);
        sentCounter++;
        System.out.println("Sent " + sentCounter + " received results from " + resultsReceived.get());
    }
    feedClient.close();
    System.out.println("Finished, got " + errorsReceived.get() + " errors from " + resultsReceived.get() + " results, sent " + sentCounter + " documents.");
}

Example 8 with FeedClient

use of com.yahoo.vespa.http.client.FeedClient in project vespa by vespa-engine.

the class XmlFeedReaderTest method testInsert.

@Test
public void testInsert() throws Exception {
    InputStream stream = new ByteArrayInputStream(insertDocOperation.getBytes(StandardCharsets.UTF_8));
    AtomicInteger numSent = new AtomicInteger(0);
    FeedClient feedClient = mock(FeedClient.class);
    XmlFeedReader.read(stream, feedClient, numSent);
    assertThat(numSent.get(), is(2));
}

Example 9 with FeedClient

use of com.yahoo.vespa.http.client.FeedClient in project vespa by vespa-engine.

the class XmlFeedReaderTest method testReadUpdate.

@Test
public void testReadUpdate() throws Exception {
    InputStream stream = new ByteArrayInputStream(updateDocUpdate.getBytes(StandardCharsets.UTF_8));
    AtomicInteger numSent = new AtomicInteger(0);
    FeedClient feedClient = mock(FeedClient.class);
    XmlFeedReader.read(stream, feedClient, numSent);
    assertThat(numSent.get(), is(1));
}

Example 10 with FeedClient

use of com.yahoo.vespa.http.client.FeedClient in project vespa by vespa-engine.

the class XmlFeedReaderTest method testNonDocument.

@Test
public void testNonDocument() throws Exception {
    InputStream stream = new ByteArrayInputStream(badperation.getBytes(StandardCharsets.UTF_8));
    AtomicInteger numSent = new AtomicInteger(0);
    FeedClient feedClient = mock(FeedClient.class);
    XmlFeedReader.read(stream, feedClient, numSent);
    assertThat(numSent.get(), is(0));
}