Examples with DataList com.yahoo.processing.response.DataList used on opensource projects

Example 1 with DataList

use of com.yahoo.processing.response.DataList in project vespa by vespa-engine.

the class ResponseTestCase method testRecursiveCompletionAndToString.

/**
 * Create a nested async tree of data elements, complete it recursively and check completion order.
 * Check the recursive toString printing along the way.
 * List variable names ends by numbers specifying the index of the list at each level.
 */
@SuppressWarnings("unchecked")
@Test
public void testRecursiveCompletionAndToString() throws InterruptedException, ExecutionException {
    // create lists
    Request request = new Request();
    DataList list1 = ArrayDataList.create(request);
    DataList list11 = ArrayDataList.create(request);
    DataList list12 = ArrayDataList.createAsync(request);
    DataList list13 = ArrayDataList.createAsync(request);
    DataList list14 = ArrayDataList.create(request);
    DataList list121 = ArrayDataList.createAsync(request);
    DataList list122 = ArrayDataList.create(request);
    DataList list123 = ArrayDataList.createAsync(request);
    DataList list1231 = ArrayDataList.createAsync(request);
    DataList list1232 = ArrayDataList.create(request);
    // wire tree
    list1.add(list11);
    list1.add(list12);
    list1.add(list13);
    list1.add(list14);
    list12.add(list121);
    list12.add(list122);
    list12.add(list123);
    list123.add(list1231);
    list123.add(list1232);
    // add sync data elements
    list1.add(new ProcessorLibrary.StringData(request, "list1"));
    list12.add(new ProcessorLibrary.StringData(request, "list12"));
    list14.add(new ProcessorLibrary.StringData(request, "list14"));
    list122.add(new ProcessorLibrary.StringData(request, "list122"));
    list1231.add(new ProcessorLibrary.StringData(request, "list1231"));
    assertEqualsIgnoreObjectNumbers("Uncompleted tree, no incoming", uncompletedTreeUncompletedIncoming, Responses.recursiveToString(list1));
    // provide all async incoming data
    list12.incoming().markComplete();
    list121.incoming().addLast(new ProcessorLibrary.StringData(request, "list121async1"));
    list123.incoming().markComplete();
    list1231.incoming().add(new ProcessorLibrary.StringData(request, "list13231async1"));
    list1231.incoming().addLast(new ProcessorLibrary.StringData(request, "list1231async2"));
    list13.incoming().add(new ProcessorLibrary.StringData(request, "list13async1"));
    list13.incoming().addLast(new ProcessorLibrary.StringData(request, "list13async2"));
    assertEqualsIgnoreObjectNumbers("Uncompleted tree, incoming complete", uncompletedTreeCompletedIncoming, Responses.recursiveToString(list1));
    // complete all
    Response.recursiveComplete(list1).get();
    assertEqualsIgnoreObjectNumbers("Completed tree", completedTree, Responses.recursiveToString(list1));
}

Example 2 with DataList

use of com.yahoo.processing.response.DataList in project vespa by vespa-engine.

the class FutureDataTestCase method testAsyncDataProcessingOfFederatedResult.

/**
 * Register a chain which federates over three sources, two of which are future.
 * When the first of the futures are done one additional chain is to be run.
 * When both are done another chain is to be run.
 */
@SuppressWarnings("unchecked")
@Test
public void testAsyncDataProcessingOfFederatedResult() throws InterruptedException, ExecutionException, TimeoutException {
    // Set up
    // Source 1 (async with completion chain)
    FutureDataSource futureSource1 = new FutureDataSource();
    Chain<Processor> asyncChainSource1 = new Chain<Processor>(new DataCounter("source1"));
    Chain<Processor> chainSource1 = new Chain<>(new AsyncDataProcessingInitiator(asyncChainSource1), futureSource1);
    // Source 2 (async source)
    FutureDataSource futureSource2 = new FutureDataSource();
    Chain<Processor> chainSource2 = new Chain<Processor>(futureSource2);
    // Source 3 (sync source)
    Chain<Processor> chainSource3 = new Chain<Processor>(new DataSource());
    // Main chain federating to the above - not waiting for source 1 and 2 but invoking asyncMain when both are complete
    Chain<Processor> asyncMain = new Chain<Processor>(new DataCounter("main"));
    Chain<Processor> main = new Chain<>(new AsyncDataProcessingInitiator(asyncMain), new Federator(chainSource1, chainSource2, chainSource3));
    // Execute
    Request request = new Request();
    Response response = Execution.createRoot(main, 0, Execution.Environment.createEmpty()).process(request);
    // Verify the result prior to completion of delayed data
    assertEquals("We have the sync data plus placeholders for the async lists", 3, response.data().asList().size());
    DataList source1Data = ((DataList) response.data().get(0));
    DataList source2Data = ((DataList) response.data().get(1));
    DataList source3Data = ((DataList) response.data().get(2));
    assertEquals("No data yet", 0, source1Data.asList().size());
    assertEquals("No data yet", 0, source2Data.asList().size());
    assertEquals(3, source3Data.asList().size());
    // complete async data in source1
    futureSource1.incomingData.get(0).addLast(new StringData(request, "source1Data"));
    assertEquals("Not visible yet", 0, source1Data.asList().size());
    source1Data.complete().get(1000, TimeUnit.MILLISECONDS);
    assertEquals(2, source1Data.asList().size());
    assertEquals("source1Data", source1Data.get(0).toString());
    assertEquals("Completion listener chain on this has run", "[source1] Data count: 1", source1Data.get(1).toString());
    // source2 & main completion
    assertEquals("Main completion listener has not run", 3, response.data().asList().size());
    futureSource2.incomingData.get(0).addLast(new StringData(request, "source2Data"));
    assertEquals("Main completion listener has not run", 3, response.data().asList().size());
    Response.recursiveComplete(response.data()).get();
    assertEquals("Main completion listener has run", 4, response.data().asList().size());
    assertEquals("The main data counter saw all sync data, but not source2 data as it executes after this", "[main] Data count: " + (2 + 0 + 3), response.data().get(3).toString());
}

Example 3 with DataList

use of com.yahoo.processing.response.DataList in project vespa by vespa-engine.

the class AsyncGroupPopulationTestCase method test.

@Test
public final void test() throws InterruptedException, ExecutionException, JsonParseException, JsonMappingException, IOException {
    String rawExpected = "{" + "    \"root\": {" + "        \"children\": [" + "            {" + "                \"id\": \"yahoo1\"," + "                \"relevance\": 1.0" + "            }," + "            {" + "                \"id\": \"yahoo2\"," + "                \"relevance\": 1.0" + "            }" + "        ]," + "        \"fields\": {" + "            \"totalCount\": 0" + "        }," + "        \"id\": \"yahoo\"," + "        \"relevance\": 1.0" + "    }" + "}";
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    HitGroup h = new InstrumentedGroup("yahoo");
    h.incoming().add(new Hit("yahoo1"));
    JsonRenderer renderer = new JsonRenderer();
    Result result = new Result(new Query(), h);
    renderer.init();
    ListenableFuture<Boolean> f = renderer.render(out, result, new Execution(Execution.Context.createContextStub()), result.getQuery());
    WrappedFuture<DataList<Hit>> x = (WrappedFuture<DataList<Hit>>) h.incoming().completed();
    x.isListening.get(86_400_000);
    h.incoming().add(new Hit("yahoo2"));
    h.incoming().markComplete();
    Boolean b = f.get();
    assertTrue(b);
    String rawGot = Utf8.toString(out.toByteArray());
    ObjectMapper m = new ObjectMapper();
    Map<?, ?> expected = m.readValue(rawExpected, Map.class);
    Map<?, ?> got = m.readValue(rawGot, Map.class);
    assertEquals(expected, got);
}

Example 4 with DataList

use of com.yahoo.processing.response.DataList in project vespa by vespa-engine.

the class AsynchronousSectionedRendererTest method testEmptyProcessingRendering.

@Test
public void testEmptyProcessingRendering() throws IOException, InterruptedException {
    Request request = new Request();
    DataList dataList = ArrayDataList.create(request);
    assertThat(render(dataList), equalTo("{\"datalist\":[" + "]}"));
}

Example 5 with DataList

use of com.yahoo.processing.response.DataList in project vespa by vespa-engine.

the class FutureDataTestCase method testFederateSyncAndAsyncData.

/**
 * Federate to one source which returns data immediately and one who return future data
 */
@SuppressWarnings("unchecked")
@Test
public void testFederateSyncAndAsyncData() throws InterruptedException, ExecutionException, TimeoutException {
    // Set up
    FutureDataSource futureDataSource = new FutureDataSource();
    Chain<Processor> chain = new Chain<>(new DataCounter(), new Federator(new Chain<>(new DataSource()), new Chain<>(futureDataSource)));
    // Execute
    Request request = new Request();
    request.properties().set("appendage", 1);
    Response response = Execution.createRoot(chain, 0, Execution.Environment.createEmpty()).process(request);
    // Verify the result prior to completion of delayed data
    // The sync data list + the (currently empty) future data list) + the data count
    assertEquals(3, response.data().asList().size());
    DataList syncData = (DataList) response.data().get(0);
    DataList asyncData = (DataList) response.data().get(1);
    StringData countData = (StringData) response.data().get(2);
    assertEquals("The sync data is available", 3, syncData.asList().size());
    assertEquals("first.1", syncData.get(0).toString());
    assertEquals("second.1", syncData.get(1).toString());
    assertEquals("third.1", syncData.get(2).toString());
    assertEquals("No async data yet", 0, asyncData.asList().size());
    assertEquals("The data counter has run and accessed the sync data", "Data count: 3", countData.toString());
    // complete async data
    futureDataSource.incomingData.get(0).add(new StringData(request, "d1"));
    futureDataSource.incomingData.get(0).addLast(new StringData(request, "d2"));
    assertEquals("New data is not visible because we haven't asked for it", 0, asyncData.asList().size());
    asyncData.complete().get(1000, TimeUnit.MILLISECONDS);
    assertEquals("Now the data is available", 2, asyncData.asList().size());
    assertEquals("d1", asyncData.get(0).toString().toString());
    assertEquals("d2", asyncData.get(1).toString().toString());
}