Search in sources :

Example 41 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class VdsVisitor method mergeGroupingMaps.

private void mergeGroupingMaps(Map<Integer, byte[]> newGroupingMap) {
    if (log.isLoggable(LogLevel.SPAM)) {
        log.log(LogLevel.SPAM, "mergeGroupingMaps: newGroupingMap = " + newGroupingMap);
    }
    for (Integer key : newGroupingMap.keySet()) {
        byte[] value = newGroupingMap.get(key);
        Grouping newGrouping = new Grouping();
        if (log.isLoggable(LogLevel.SPAM)) {
            log.log(LogLevel.SPAM, "Received group with key " + key + " and size " + value.length);
        }
        BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer(ByteBuffer.wrap(value)));
        newGrouping.deserialize(buf);
        if (buf.getBuf().hasRemaining()) {
            throw new IllegalArgumentException("Failed deserializing grouping. There are still data left. Position = " + buf.position() + ", limit = " + buf.getBuf().limit());
        }
        synchronized (groupingMap) {
            if (groupingMap.containsKey(key)) {
                Grouping grouping = groupingMap.get(key);
                grouping.merge(newGrouping);
            } else {
                groupingMap.put(key, newGrouping);
            }
        }
    }
}
Also used : BufferSerializer(com.yahoo.vespa.objects.BufferSerializer) GrowableByteBuffer(com.yahoo.io.GrowableByteBuffer) Grouping(com.yahoo.searchlib.aggregation.Grouping)

Example 42 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class RankFeaturesTestCase method decode.

private static Map<String, Object> decode(TensorType type, byte[] encodedProperties) {
    GrowableByteBuffer buffer = GrowableByteBuffer.wrap(encodedProperties);
    byte[] mapNameBytes = new byte[buffer.getInt()];
    buffer.get(mapNameBytes);
    int numEntries = buffer.getInt();
    Map<String, Object> result = new HashMap<>();
    for (int i = 0; i < numEntries; ++i) {
        byte[] keyBytes = new byte[buffer.getInt()];
        buffer.get(keyBytes);
        String key = Utf8.toString(keyBytes);
        byte[] value = new byte[buffer.getInt()];
        buffer.get(value);
        if (key.contains(".type")) {
            result.put(key, Utf8.toString(value));
        } else {
            result.put(key, TypedBinaryFormat.decode(Optional.of(type), GrowableByteBuffer.wrap(value)));
        }
    }
    return result;
}
Also used : GrowableByteBuffer(com.yahoo.io.GrowableByteBuffer)

Example 43 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class AggregationTestCase method serializeDeserialize1.

// --------------------------------------------------------------------------------
// 
// Everything below this point is helper functions.
// 
// --------------------------------------------------------------------------------
private static Identifiable serializeDeserialize1(Identifiable a) {
    BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer());
    a.serializeWithId(buf);
    buf.flip();
    Identifiable b = Identifiable.create(buf);
    assertEquals(a.getClass(), b.getClass());
    assertEquals(buf.getBuf().hasRemaining(), false);
    Identifiable c = b.clone();
    assertEquals(b.getClass(), c.getClass());
    BufferSerializer bb = new BufferSerializer(new GrowableByteBuffer());
    BufferSerializer cb = new BufferSerializer(new GrowableByteBuffer());
    b.serializeWithId(bb);
    c.serializeWithId(cb);
    assertEquals(bb.getBuf().limit(), cb.getBuf().limit());
    assertEquals(bb.position(), cb.position());
    bb.getBuf().flip();
    cb.getBuf().flip();
    for (int i = 0; i < bb.getBuf().limit(); i++) {
        assertEquals(bb.getBuf().get(), cb.getBuf().get());
    }
    return b;
}
Also used : BufferSerializer(com.yahoo.vespa.objects.BufferSerializer) GrowableByteBuffer(com.yahoo.io.GrowableByteBuffer) Identifiable(com.yahoo.vespa.objects.Identifiable)

Example 44 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class ExpressionTestCase method assertSerialize.

private static Identifiable assertSerialize(Identifiable node) {
    BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer());
    node.serializeWithId(buf);
    buf.flip();
    Identifiable created = Identifiable.create(buf);
    assertEquals(node, created);
    assertEquals(buf.getBuf().hasRemaining(), false);
    Identifiable cloned = created.clone();
    assertEquals(node, cloned);
    BufferSerializer createdBuffer = new BufferSerializer(new GrowableByteBuffer());
    BufferSerializer clonedBuffer = new BufferSerializer(new GrowableByteBuffer());
    created.serializeWithId(createdBuffer);
    cloned.serializeWithId(clonedBuffer);
    assertEquals(createdBuffer.getBuf().limit(), clonedBuffer.getBuf().limit());
    assertEquals(createdBuffer.position(), clonedBuffer.position());
    createdBuffer.getBuf().flip();
    clonedBuffer.getBuf().flip();
    for (int i = 0; i < createdBuffer.getBuf().limit(); i++) {
        assertEquals(createdBuffer.getBuf().get(), clonedBuffer.getBuf().get());
    }
    return created;
}
Also used : BufferSerializer(com.yahoo.vespa.objects.BufferSerializer) GrowableByteBuffer(com.yahoo.io.GrowableByteBuffer) Identifiable(com.yahoo.vespa.objects.Identifiable)

Example 45 with GrowableByteBuffer

use of com.yahoo.io.GrowableByteBuffer in project vespa by vespa-engine.

the class ExpressionTestCase method assertResultNode.

private static void assertResultNode(ResultNode node) {
    BufferSerializer buf = new BufferSerializer(new GrowableByteBuffer());
    long oldInteger = node.getInteger();
    double oldFloat = node.getFloat();
    String oldString = node.getString();
    byte[] oldRaw = node.getRaw();
    node.serialize(buf);
    buf.flip();
    node.deserialize(buf);
    assertEquals(oldInteger, node.getInteger());
    assertEquals(oldFloat, node.getFloat());
    assertEquals(oldString, node.getString());
    assertEquals(oldRaw.length, node.getRaw().length);
    buf = new BufferSerializer(new GrowableByteBuffer());
    node.serializeWithId(buf);
    buf.flip();
    node.deserializeWithId(buf);
    assertEquals(oldInteger, node.getInteger());
    assertEquals(oldFloat, node.getFloat());
    assertEquals(oldString, node.getString());
    assertEquals(oldRaw.length, node.getRaw().length);
    buf = new BufferSerializer(new GrowableByteBuffer());
    node.serializeWithId(buf);
    buf.flip();
    ResultNode obj = (ResultNode) Identifiable.create(buf);
    assertEquals(oldInteger, obj.getInteger());
    assertEquals(oldFloat, obj.getFloat());
    assertEquals(oldString, obj.getString());
    assertEquals(oldRaw.length, obj.getRaw().length);
    assertSerialize(node);
}
Also used : BufferSerializer(com.yahoo.vespa.objects.BufferSerializer) GrowableByteBuffer(com.yahoo.io.GrowableByteBuffer)

Aggregations

GrowableByteBuffer (com.yahoo.io.GrowableByteBuffer)74 Test (org.junit.Test)34 StringFieldValue (com.yahoo.document.datatypes.StringFieldValue)18 Document (com.yahoo.document.Document)9 BufferSerializer (com.yahoo.vespa.objects.BufferSerializer)9 Field (com.yahoo.document.Field)8 ByteBuffer (java.nio.ByteBuffer)8 FileOutputStream (java.io.FileOutputStream)6 DocumentType (com.yahoo.document.DocumentType)5 DocumentTypeManager (com.yahoo.document.DocumentTypeManager)5 IntegerFieldValue (com.yahoo.document.datatypes.IntegerFieldValue)5 AbstractTypesTest (com.yahoo.document.annotation.AbstractTypesTest)4 Raw (com.yahoo.document.datatypes.Raw)4 Struct (com.yahoo.document.datatypes.Struct)4 Grouping (com.yahoo.searchlib.aggregation.Grouping)4 Array (com.yahoo.document.datatypes.Array)3 DoubleFieldValue (com.yahoo.document.datatypes.DoubleFieldValue)3 MapFieldValue (com.yahoo.document.datatypes.MapFieldValue)3 FieldUpdate (com.yahoo.document.update.FieldUpdate)3 IOException (java.io.IOException)3