Search in sources :

Example 1 with FloatFieldValue

use of com.yahoo.document.datatypes.FloatFieldValue in project vespa by vespa-engine.

the class DocumentTestCase method testCppDocSplit.

@Test
public void testCppDocSplit() throws IOException {
    docMan = setUpCppDocType();
    byte[] headerData = readFile("src/test/document/serializecppsplit_header.dat");
    byte[] bodyData = readFile("src/test/document/serializecppsplit_body.dat");
    DocumentDeserializer header = DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(headerData), GrowableByteBuffer.wrap(bodyData));
    Document doc = new Document(header);
    assertEquals("doc:serializetest:http://test.doc.id/", doc.getId().toString());
    assertEquals(new IntegerFieldValue(5), doc.getFieldValue("intfield"));
    assertEquals(new FloatFieldValue((float) -9.23), doc.getFieldValue("floatfield"));
    assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield"));
    assertEquals(new LongFieldValue(398420092938472983L), doc.getFieldValue("longfield"));
    assertEquals(new DoubleFieldValue(98374532.398820d), doc.getFieldValue("doublefield"));
    assertEquals(new StringFieldValue("http://this.is.a.test/"), doc.getFieldValue("urifield"));
    // NOTE: The value really is unsigned 254, which becomes signed -2:
    assertEquals(new ByteFieldValue((byte) -2), doc.getFieldValue("bytefield"));
    ByteBuffer raw = ByteBuffer.wrap("RAW DATA".getBytes());
    assertEquals(new Raw(raw), doc.getFieldValue("rawfield"));
    Document docindoc = (Document) doc.getFieldValue("docfield");
    assertEquals(docMan.getDocumentType("docindoc"), docindoc.getDataType());
    assertEquals(new DocumentId("doc:docindoc:http://embedded"), docindoc.getId());
    WeightedSet wset = (WeightedSet) doc.getFieldValue("wsfield");
    assertEquals(new Integer(50), wset.get(new StringFieldValue("Weighted 0")));
    assertEquals(new Integer(199), wset.get(new StringFieldValue("Weighted 1")));
}
Also used : DoubleFieldValue(com.yahoo.document.datatypes.DoubleFieldValue) IntegerFieldValue(com.yahoo.document.datatypes.IntegerFieldValue) Raw(com.yahoo.document.datatypes.Raw) ByteBuffer(java.nio.ByteBuffer) GrowableByteBuffer(com.yahoo.io.GrowableByteBuffer) FloatFieldValue(com.yahoo.document.datatypes.FloatFieldValue) StringFieldValue(com.yahoo.document.datatypes.StringFieldValue) LongFieldValue(com.yahoo.document.datatypes.LongFieldValue) ByteFieldValue(com.yahoo.document.datatypes.ByteFieldValue) WeightedSet(com.yahoo.document.datatypes.WeightedSet) Test(org.junit.Test)

Example 2 with FloatFieldValue

use of com.yahoo.document.datatypes.FloatFieldValue in project vespa by vespa-engine.

the class DocumentTestCase method testTypeChecking.

@Test
public void testTypeChecking() {
    DocumentType type = new DocumentType("test");
    type.addField(new Field("double", DataType.DOUBLE));
    type.addField(new Field("float", DataType.FLOAT));
    type.addField(new Field("int", DataType.INT));
    type.addField(new Field("long", DataType.LONG));
    type.addField(new Field("string", DataType.STRING));
    Document doc = new Document(type, "doc:scheme:");
    FieldValue stringVal = new StringFieldValue("69");
    FieldValue doubleVal = new DoubleFieldValue(6.9);
    FieldValue floatVal = new FloatFieldValue(6.9f);
    FieldValue intVal = new IntegerFieldValue(69);
    FieldValue longVal = new LongFieldValue(69L);
    doc.setFieldValue("string", stringVal);
    doc.setFieldValue("string", doubleVal);
    doc.setFieldValue("string", floatVal);
    doc.setFieldValue("string", intVal);
    doc.setFieldValue("string", longVal);
    doc.setFieldValue("double", stringVal);
    doc.setFieldValue("double", doubleVal);
    doc.setFieldValue("double", floatVal);
    doc.setFieldValue("double", intVal);
    doc.setFieldValue("double", longVal);
    doc.setFieldValue("float", stringVal);
    doc.setFieldValue("float", doubleVal);
    doc.setFieldValue("float", floatVal);
    doc.setFieldValue("float", intVal);
    doc.setFieldValue("float", longVal);
    doc.setFieldValue("int", stringVal);
    doc.setFieldValue("int", doubleVal);
    doc.setFieldValue("int", floatVal);
    doc.setFieldValue("int", intVal);
    doc.setFieldValue("int", longVal);
    doc.setFieldValue("long", stringVal);
    doc.setFieldValue("long", doubleVal);
    doc.setFieldValue("long", floatVal);
    doc.setFieldValue("long", intVal);
    doc.setFieldValue("long", longVal);
}
Also used : StringFieldValue(com.yahoo.document.datatypes.StringFieldValue) DoubleFieldValue(com.yahoo.document.datatypes.DoubleFieldValue) IntegerFieldValue(com.yahoo.document.datatypes.IntegerFieldValue) LongFieldValue(com.yahoo.document.datatypes.LongFieldValue) ByteFieldValue(com.yahoo.document.datatypes.ByteFieldValue) DoubleFieldValue(com.yahoo.document.datatypes.DoubleFieldValue) StringFieldValue(com.yahoo.document.datatypes.StringFieldValue) FloatFieldValue(com.yahoo.document.datatypes.FloatFieldValue) FieldValue(com.yahoo.document.datatypes.FieldValue) IntegerFieldValue(com.yahoo.document.datatypes.IntegerFieldValue) MapFieldValue(com.yahoo.document.datatypes.MapFieldValue) LongFieldValue(com.yahoo.document.datatypes.LongFieldValue) FloatFieldValue(com.yahoo.document.datatypes.FloatFieldValue) Test(org.junit.Test)

Example 3 with FloatFieldValue

use of com.yahoo.document.datatypes.FloatFieldValue in project vespa by vespa-engine.

the class DocumentTestCase method testGenerateSerializedFile.

@Test
public void testGenerateSerializedFile() throws IOException {
    docMan = setUpCppDocType();
    Document doc = new Document(docMan.getDocumentType("serializetest"), new DocumentId("doc:serializetest:http://test.doc.id/"));
    Document docindoc = new Document(docMan.getDocumentType("docindoc"), new DocumentId("doc:serializetest:http://doc.in.doc/"));
    docindoc.setFieldValue("stringindocfield", "Elvis is dead");
    doc.setFieldValue("docfield", docindoc);
    Array<FloatFieldValue> l = new Array<>(doc.getField("arrayoffloatfield").getDataType());
    l.add(new FloatFieldValue((float) 1.0));
    l.add(new FloatFieldValue((float) 2.0));
    doc.setFieldValue("arrayoffloatfield", l);
    WeightedSet<StringFieldValue> wset = new WeightedSet<>(doc.getDataType().getField("wsfield").getDataType());
    wset.put(new StringFieldValue("Weighted 0"), 50);
    wset.put(new StringFieldValue("Weighted 1"), 199);
    doc.setFieldValue("wsfield", wset);
    MapFieldValue<StringFieldValue, StringFieldValue> map = new MapFieldValue<>((MapDataType) doc.getDataType().getField("mapfield").getDataType());
    map.put(new StringFieldValue("foo1"), new StringFieldValue("bar1"));
    map.put(new StringFieldValue("foo2"), new StringFieldValue("bar2"));
    doc.setFieldValue("mapfield", map);
    doc.setFieldValue("bytefield", new ByteFieldValue((byte) 254));
    doc.setFieldValue("rawfield", new Raw(ByteBuffer.wrap("RAW DATA".getBytes())));
    doc.setFieldValue("intfield", new IntegerFieldValue(5));
    doc.setFieldValue("floatfield", new FloatFieldValue(-9.23f));
    doc.setFieldValue("stringfield", new StringFieldValue("This is a string."));
    doc.setFieldValue("longfield", new LongFieldValue(398420092938472983L));
    doc.setFieldValue("doublefield", new DoubleFieldValue(98374532.398820d));
    doc.setFieldValue("urifield", new StringFieldValue("http://this.is.a.test/"));
    int size = doc.getSerializedSize();
    GrowableByteBuffer buf = new GrowableByteBuffer(size, 2.0f);
    doc.serialize(buf);
    assertEquals(size, buf.position());
    buf.position(0);
    FileOutputStream fos = new FileOutputStream("src/tests/data/serializejava.dat");
    fos.write(buf.array(), 0, size);
    fos.close();
    CompressionConfig noncomp = new CompressionConfig();
    CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4);
    doc.getDataType().getHeaderType().setCompressionConfig(lz4comp);
    doc.getDataType().getBodyType().setCompressionConfig(lz4comp);
    buf = new GrowableByteBuffer(size, 2.0f);
    doc.serialize(buf);
    doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
    doc.getDataType().getBodyType().setCompressionConfig(noncomp);
    fos = new FileOutputStream("src/tests/data/serializejava-compressed.dat");
    fos.write(buf.array(), 0, buf.position());
    fos.close();
}
Also used : MapFieldValue(com.yahoo.document.datatypes.MapFieldValue) DoubleFieldValue(com.yahoo.document.datatypes.DoubleFieldValue) Raw(com.yahoo.document.datatypes.Raw) IntegerFieldValue(com.yahoo.document.datatypes.IntegerFieldValue) GrowableByteBuffer(com.yahoo.io.GrowableByteBuffer) FloatFieldValue(com.yahoo.document.datatypes.FloatFieldValue) Array(com.yahoo.document.datatypes.Array) StringFieldValue(com.yahoo.document.datatypes.StringFieldValue) FileOutputStream(java.io.FileOutputStream) WeightedSet(com.yahoo.document.datatypes.WeightedSet) ByteFieldValue(com.yahoo.document.datatypes.ByteFieldValue) LongFieldValue(com.yahoo.document.datatypes.LongFieldValue) Test(org.junit.Test)

Example 4 with FloatFieldValue

use of com.yahoo.document.datatypes.FloatFieldValue in project vespa by vespa-engine.

the class DocumentSerializationTestCase method testSerializationAllVersions.

@Test
public void testSerializationAllVersions() throws IOException {
    DocumentType docInDocType = new DocumentType("docindoc");
    docInDocType.addField(new Field("stringindocfield", DataType.STRING, false));
    DocumentType docType = new DocumentType("serializetest");
    docType.addField(new Field("floatfield", DataType.FLOAT, true));
    docType.addField(new Field("stringfield", DataType.STRING, true));
    docType.addField(new Field("longfield", DataType.LONG, true));
    docType.addField(new Field("urifield", DataType.URI, true));
    docType.addField(new Field("intfield", DataType.INT, false));
    docType.addField(new Field("rawfield", DataType.RAW, false));
    docType.addField(new Field("doublefield", DataType.DOUBLE, false));
    docType.addField(new Field("bytefield", DataType.BYTE, false));
    DataType arrayOfFloatDataType = new ArrayDataType(DataType.FLOAT);
    docType.addField(new Field("arrayoffloatfield", arrayOfFloatDataType, false));
    DataType arrayOfArrayOfFloatDataType = new ArrayDataType(arrayOfFloatDataType);
    docType.addField(new Field("arrayofarrayoffloatfield", arrayOfArrayOfFloatDataType, false));
    docType.addField(new Field("docfield", DataType.DOCUMENT, false));
    DataType weightedSetDataType = DataType.getWeightedSet(DataType.STRING, false, false);
    docType.addField(new Field("wsfield", weightedSetDataType, false));
    DocumentTypeManager docMan = new DocumentTypeManager();
    docMan.register(docInDocType);
    docMan.register(docType);
    String path = "src/test/serializeddocuments/";
    {
        Document doc = new Document(docType, "doc:serializetest:http://test.doc.id/");
        doc.setFieldValue("intfield", 5);
        doc.setFieldValue("floatfield", -9.23);
        doc.setFieldValue("stringfield", "This is a string.");
        doc.setFieldValue("longfield", new LongFieldValue(398420092938472983l));
        doc.setFieldValue("doublefield", new DoubleFieldValue(98374532.398820));
        doc.setFieldValue("bytefield", new ByteFieldValue(254));
        byte[] rawData = "RAW DATA".getBytes();
        assertEquals(8, rawData.length);
        doc.setFieldValue(docType.getField("rawfield"), new Raw(ByteBuffer.wrap("RAW DATA".getBytes())));
        Document docInDoc = new Document(docInDocType, "doc:serializetest:http://doc.in.doc/");
        docInDoc.setFieldValue("stringindocfield", "Elvis is dead");
        doc.setFieldValue(docType.getField("docfield"), docInDoc);
        Array<FloatFieldValue> floatArray = new Array<>(arrayOfFloatDataType);
        floatArray.add(new FloatFieldValue(1.0f));
        floatArray.add(new FloatFieldValue(2.0f));
        doc.setFieldValue("arrayoffloatfield", floatArray);
        WeightedSet<StringFieldValue> weightedSet = new WeightedSet<>(weightedSetDataType);
        weightedSet.put(new StringFieldValue("Weighted 0"), 50);
        weightedSet.put(new StringFieldValue("Weighted 1"), 199);
        doc.setFieldValue("wsfield", weightedSet);
        CompressionConfig noncomp = new CompressionConfig();
        CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4);
        {
            doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
            doc.getDataType().getBodyType().setCompressionConfig(noncomp);
            FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-uncompressed.dat", false);
            doc.serialize(fout);
            fout.close();
        }
        {
            doc.getDataType().getHeaderType().setCompressionConfig(lz4comp);
            doc.getDataType().getBodyType().setCompressionConfig(lz4comp);
            FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-lz4-9.dat", false);
            doc.serialize(fout);
            doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
            doc.getDataType().getBodyType().setCompressionConfig(noncomp);
            fout.close();
        }
    }
    class TestDoc {

        String testFile;

        int version;

        TestDoc(String testFile, int version) {
            this.testFile = testFile;
            this.version = version;
        }
    }
    String cpppath = "src/tests/data/";
    List<TestDoc> tests = new ArrayList<>();
    tests.add(new TestDoc(path + "document-java-currentversion-uncompressed.dat", Document.SERIALIZED_VERSION));
    tests.add(new TestDoc(path + "document-java-currentversion-lz4-9.dat", Document.SERIALIZED_VERSION));
    tests.add(new TestDoc(path + "document-java-v8-uncompressed.dat", 8));
    tests.add(new TestDoc(cpppath + "document-cpp-currentversion-uncompressed.dat", 7));
    tests.add(new TestDoc(cpppath + "document-cpp-currentversion-lz4-9.dat", 7));
    tests.add(new TestDoc(cpppath + "document-cpp-v8-uncompressed.dat", 7));
    tests.add(new TestDoc(cpppath + "document-cpp-v7-uncompressed.dat", 7));
    tests.add(new TestDoc(cpppath + "serializev6.dat", 6));
    for (TestDoc test : tests) {
        File f = new File(test.testFile);
        FileInputStream fin = new FileInputStream(f);
        byte[] buffer = new byte[(int) f.length()];
        int pos = 0;
        int remaining = buffer.length;
        while (remaining > 0) {
            int read = fin.read(buffer, pos, remaining);
            assertFalse(read == -1);
            pos += read;
            remaining -= read;
        }
        System.err.println("Checking doc from file " + test.testFile);
        Document doc = new Document(DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(buffer)));
        System.err.println("Id: " + doc.getId());
        assertEquals(new IntegerFieldValue(5), doc.getFieldValue("intfield"));
        assertEquals(-9.23, ((FloatFieldValue) doc.getFieldValue("floatfield")).getFloat(), 1E-6);
        assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield"));
        assertEquals(new LongFieldValue(398420092938472983l), doc.getFieldValue("longfield"));
        assertEquals(98374532.398820, ((DoubleFieldValue) doc.getFieldValue("doublefield")).getDouble(), 1E-6);
        assertEquals(new ByteFieldValue((byte) 254), doc.getFieldValue("bytefield"));
        ByteBuffer bbuffer = ((Raw) doc.getFieldValue("rawfield")).getByteBuffer();
        if (!Arrays.equals("RAW DATA".getBytes(), bbuffer.array())) {
            System.err.println("Expected 'RAW DATA' but got '" + new String(bbuffer.array()) + "'.");
            assertTrue(false);
        }
        if (test.version > 6) {
            Document docInDoc = (Document) doc.getFieldValue("docfield");
            assertTrue(docInDoc != null);
            assertEquals(new StringFieldValue("Elvis is dead"), docInDoc.getFieldValue("stringindocfield"));
        }
        Array array = (Array) doc.getFieldValue("arrayoffloatfield");
        assertTrue(array != null);
        assertEquals(1.0f, ((FloatFieldValue) array.get(0)).getFloat(), 1E-6);
        assertEquals(2.0f, ((FloatFieldValue) array.get(1)).getFloat(), 1E-6);
        WeightedSet wset = (WeightedSet) doc.getFieldValue("wsfield");
        assertTrue(wset != null);
        assertEquals(Integer.valueOf(50), wset.get(new StringFieldValue("Weighted 0")));
        assertEquals(Integer.valueOf(199), wset.get(new StringFieldValue("Weighted 1")));
    }
}
Also used : ArrayList(java.util.ArrayList) Raw(com.yahoo.document.datatypes.Raw) IntegerFieldValue(com.yahoo.document.datatypes.IntegerFieldValue) FloatFieldValue(com.yahoo.document.datatypes.FloatFieldValue) StringFieldValue(com.yahoo.document.datatypes.StringFieldValue) ByteFieldValue(com.yahoo.document.datatypes.ByteFieldValue) DoubleFieldValue(com.yahoo.document.datatypes.DoubleFieldValue) ByteBuffer(java.nio.ByteBuffer) GrowableByteBuffer(com.yahoo.io.GrowableByteBuffer) FileInputStream(java.io.FileInputStream) Array(com.yahoo.document.datatypes.Array) FileOutputStream(java.io.FileOutputStream) File(java.io.File) LongFieldValue(com.yahoo.document.datatypes.LongFieldValue) WeightedSet(com.yahoo.document.datatypes.WeightedSet) Test(org.junit.Test) AbstractTypesTest(com.yahoo.document.annotation.AbstractTypesTest)

Example 5 with FloatFieldValue

use of com.yahoo.document.datatypes.FloatFieldValue in project vespa by vespa-engine.

the class DocumentListTestCase method testSelfSerializationAndWriteJavaFile.

@SuppressWarnings("deprecation")
public void testSelfSerializationAndWriteJavaFile() throws Exception {
    DocumentTypeManager docMan = new DocumentTypeManager();
    DocumentTypeManagerConfigurer.configure(docMan, "file:src/test/files/documentmanager.cfg");
    DocumentType bmType = docMan.getDocumentType("benchmark");
    DocumentPut put1 = new DocumentPut(bmType, "userdoc:foo:99999999:1");
    put1.getDocument().setFieldValue("headerstring", "foo");
    DocumentRemove doc2 = new DocumentRemove(new DocumentId("userdoc:foo:99999999:2"));
    DocumentPut put3 = new DocumentPut(bmType, "userdoc:foo:99999999:3");
    put3.getDocument().setFieldValue("bodyfloat", new FloatFieldValue(5.5f));
    DocumentUpdate docUp = new DocumentUpdate(docMan.getDocumentType("benchmark"), new DocumentId("userdoc:foo:99999999:4"));
    docUp.addFieldUpdate(FieldUpdate.createAssign(docUp.getType().getField("bodystring"), new StringFieldValue("ballooooo")));
    List<Entry> entries = new ArrayList<Entry>();
    entries.add(Entry.create(put1));
    entries.add(Entry.create(doc2));
    entries.add(Entry.create(put3));
    entries.add(Entry.create(docUp));
    DocumentList documentList = DocumentList.create(entries);
    DocumentSerializer gbuf = DocumentSerializerFactory.create42();
    // Add some data to avoid special case where position() is 0 for buffer used.
    gbuf.putInt(null, 1234);
    int startPos = gbuf.getBuf().position();
    documentList.serialize(gbuf);
    int size = gbuf.getBuf().position() - startPos;
    byte[] data = new byte[size];
    gbuf.getBuf().position(startPos);
    gbuf.getBuf().get(data);
    FileOutputStream stream = new FileOutputStream("./src/test/files/documentlist-java.dat");
    stream.write(data);
    stream.close();
    gbuf.getBuf().position(0);
    DocumentList documentList2 = DocumentList.create(docMan, data);
    assertEquals(4, documentList2.size());
    Entry entry1 = documentList2.get(0);
    assertEquals(0L, entry1.getTimestamp());
    assertFalse(entry1.isBodyStripped());
    assertFalse(entry1.isRemoveEntry());
    assertFalse(entry1.isUpdateEntry());
    assertTrue(entry1.getDocumentOperation() instanceof DocumentPut);
    assertEquals(new StringFieldValue("foo"), ((DocumentPut) entry1.getDocumentOperation()).getDocument().getFieldValue("headerstring"));
    Entry entry2 = documentList2.get(1);
    assertEquals(0L, entry2.getTimestamp());
    assertFalse(entry2.isBodyStripped());
    assertTrue(entry2.isRemoveEntry());
    assertFalse(entry2.isUpdateEntry());
    assertTrue(entry2.getDocumentOperation() instanceof DocumentRemove);
    Entry entry3 = documentList2.get(2);
    assertEquals(0L, entry3.getTimestamp());
    assertFalse(entry3.isBodyStripped());
    assertFalse(entry3.isRemoveEntry());
    assertFalse(entry3.isUpdateEntry());
    assertTrue(entry3.getDocumentOperation() instanceof DocumentPut);
    assertEquals(new FloatFieldValue(5.5f), ((DocumentPut) entry3.getDocumentOperation()).getDocument().getFieldValue("bodyfloat"));
    Entry entry4 = documentList2.get(3);
    assertEquals(0L, entry4.getTimestamp());
    assertFalse(entry4.isBodyStripped());
    assertFalse(entry4.isRemoveEntry());
    assertTrue(entry4.isUpdateEntry());
    assertTrue(entry4.getDocumentOperation() instanceof DocumentUpdate);
    assertEquals(new StringFieldValue("ballooooo"), ((DocumentUpdate) entry4.getDocumentOperation()).getFieldUpdate(0).getValueUpdate(0).getValue());
}
Also used : ArrayList(java.util.ArrayList) FloatFieldValue(com.yahoo.document.datatypes.FloatFieldValue) StringFieldValue(com.yahoo.document.datatypes.StringFieldValue) DocumentSerializer(com.yahoo.document.serialization.DocumentSerializer) FileOutputStream(java.io.FileOutputStream)

Aggregations

FloatFieldValue (com.yahoo.document.datatypes.FloatFieldValue)11 StringFieldValue (com.yahoo.document.datatypes.StringFieldValue)10 Test (org.junit.Test)7 IntegerFieldValue (com.yahoo.document.datatypes.IntegerFieldValue)6 LongFieldValue (com.yahoo.document.datatypes.LongFieldValue)6 ByteFieldValue (com.yahoo.document.datatypes.ByteFieldValue)5 DoubleFieldValue (com.yahoo.document.datatypes.DoubleFieldValue)5 Raw (com.yahoo.document.datatypes.Raw)5 WeightedSet (com.yahoo.document.datatypes.WeightedSet)4 GrowableByteBuffer (com.yahoo.io.GrowableByteBuffer)4 ByteBuffer (java.nio.ByteBuffer)4 Array (com.yahoo.document.datatypes.Array)3 FileOutputStream (java.io.FileOutputStream)3 ArrayList (java.util.ArrayList)3 FieldValue (com.yahoo.document.datatypes.FieldValue)2 MapFieldValue (com.yahoo.document.datatypes.MapFieldValue)2 AbstractTypesTest (com.yahoo.document.annotation.AbstractTypesTest)1 Struct (com.yahoo.document.datatypes.Struct)1 DocumentSerializer (com.yahoo.document.serialization.DocumentSerializer)1 SimpleTestAdapter (com.yahoo.vespa.indexinglanguage.SimpleTestAdapter)1