use of com.yahoo.document.datatypes.FloatFieldValue in project vespa by vespa-engine.
the class DocumentTestCase method testCppDocSplit.
@Test
public void testCppDocSplit() throws IOException {
docMan = setUpCppDocType();
byte[] headerData = readFile("src/test/document/serializecppsplit_header.dat");
byte[] bodyData = readFile("src/test/document/serializecppsplit_body.dat");
DocumentDeserializer header = DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(headerData), GrowableByteBuffer.wrap(bodyData));
Document doc = new Document(header);
assertEquals("doc:serializetest:http://test.doc.id/", doc.getId().toString());
assertEquals(new IntegerFieldValue(5), doc.getFieldValue("intfield"));
assertEquals(new FloatFieldValue((float) -9.23), doc.getFieldValue("floatfield"));
assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield"));
assertEquals(new LongFieldValue(398420092938472983L), doc.getFieldValue("longfield"));
assertEquals(new DoubleFieldValue(98374532.398820d), doc.getFieldValue("doublefield"));
assertEquals(new StringFieldValue("http://this.is.a.test/"), doc.getFieldValue("urifield"));
// NOTE: The value really is unsigned 254, which becomes signed -2:
assertEquals(new ByteFieldValue((byte) -2), doc.getFieldValue("bytefield"));
ByteBuffer raw = ByteBuffer.wrap("RAW DATA".getBytes());
assertEquals(new Raw(raw), doc.getFieldValue("rawfield"));
Document docindoc = (Document) doc.getFieldValue("docfield");
assertEquals(docMan.getDocumentType("docindoc"), docindoc.getDataType());
assertEquals(new DocumentId("doc:docindoc:http://embedded"), docindoc.getId());
WeightedSet wset = (WeightedSet) doc.getFieldValue("wsfield");
assertEquals(new Integer(50), wset.get(new StringFieldValue("Weighted 0")));
assertEquals(new Integer(199), wset.get(new StringFieldValue("Weighted 1")));
}
use of com.yahoo.document.datatypes.FloatFieldValue in project vespa by vespa-engine.
the class DocumentTestCase method testTypeChecking.
@Test
public void testTypeChecking() {
DocumentType type = new DocumentType("test");
type.addField(new Field("double", DataType.DOUBLE));
type.addField(new Field("float", DataType.FLOAT));
type.addField(new Field("int", DataType.INT));
type.addField(new Field("long", DataType.LONG));
type.addField(new Field("string", DataType.STRING));
Document doc = new Document(type, "doc:scheme:");
FieldValue stringVal = new StringFieldValue("69");
FieldValue doubleVal = new DoubleFieldValue(6.9);
FieldValue floatVal = new FloatFieldValue(6.9f);
FieldValue intVal = new IntegerFieldValue(69);
FieldValue longVal = new LongFieldValue(69L);
doc.setFieldValue("string", stringVal);
doc.setFieldValue("string", doubleVal);
doc.setFieldValue("string", floatVal);
doc.setFieldValue("string", intVal);
doc.setFieldValue("string", longVal);
doc.setFieldValue("double", stringVal);
doc.setFieldValue("double", doubleVal);
doc.setFieldValue("double", floatVal);
doc.setFieldValue("double", intVal);
doc.setFieldValue("double", longVal);
doc.setFieldValue("float", stringVal);
doc.setFieldValue("float", doubleVal);
doc.setFieldValue("float", floatVal);
doc.setFieldValue("float", intVal);
doc.setFieldValue("float", longVal);
doc.setFieldValue("int", stringVal);
doc.setFieldValue("int", doubleVal);
doc.setFieldValue("int", floatVal);
doc.setFieldValue("int", intVal);
doc.setFieldValue("int", longVal);
doc.setFieldValue("long", stringVal);
doc.setFieldValue("long", doubleVal);
doc.setFieldValue("long", floatVal);
doc.setFieldValue("long", intVal);
doc.setFieldValue("long", longVal);
}
use of com.yahoo.document.datatypes.FloatFieldValue in project vespa by vespa-engine.
the class DocumentTestCase method testGenerateSerializedFile.
@Test
public void testGenerateSerializedFile() throws IOException {
docMan = setUpCppDocType();
Document doc = new Document(docMan.getDocumentType("serializetest"), new DocumentId("doc:serializetest:http://test.doc.id/"));
Document docindoc = new Document(docMan.getDocumentType("docindoc"), new DocumentId("doc:serializetest:http://doc.in.doc/"));
docindoc.setFieldValue("stringindocfield", "Elvis is dead");
doc.setFieldValue("docfield", docindoc);
Array<FloatFieldValue> l = new Array<>(doc.getField("arrayoffloatfield").getDataType());
l.add(new FloatFieldValue((float) 1.0));
l.add(new FloatFieldValue((float) 2.0));
doc.setFieldValue("arrayoffloatfield", l);
WeightedSet<StringFieldValue> wset = new WeightedSet<>(doc.getDataType().getField("wsfield").getDataType());
wset.put(new StringFieldValue("Weighted 0"), 50);
wset.put(new StringFieldValue("Weighted 1"), 199);
doc.setFieldValue("wsfield", wset);
MapFieldValue<StringFieldValue, StringFieldValue> map = new MapFieldValue<>((MapDataType) doc.getDataType().getField("mapfield").getDataType());
map.put(new StringFieldValue("foo1"), new StringFieldValue("bar1"));
map.put(new StringFieldValue("foo2"), new StringFieldValue("bar2"));
doc.setFieldValue("mapfield", map);
doc.setFieldValue("bytefield", new ByteFieldValue((byte) 254));
doc.setFieldValue("rawfield", new Raw(ByteBuffer.wrap("RAW DATA".getBytes())));
doc.setFieldValue("intfield", new IntegerFieldValue(5));
doc.setFieldValue("floatfield", new FloatFieldValue(-9.23f));
doc.setFieldValue("stringfield", new StringFieldValue("This is a string."));
doc.setFieldValue("longfield", new LongFieldValue(398420092938472983L));
doc.setFieldValue("doublefield", new DoubleFieldValue(98374532.398820d));
doc.setFieldValue("urifield", new StringFieldValue("http://this.is.a.test/"));
int size = doc.getSerializedSize();
GrowableByteBuffer buf = new GrowableByteBuffer(size, 2.0f);
doc.serialize(buf);
assertEquals(size, buf.position());
buf.position(0);
FileOutputStream fos = new FileOutputStream("src/tests/data/serializejava.dat");
fos.write(buf.array(), 0, size);
fos.close();
CompressionConfig noncomp = new CompressionConfig();
CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4);
doc.getDataType().getHeaderType().setCompressionConfig(lz4comp);
doc.getDataType().getBodyType().setCompressionConfig(lz4comp);
buf = new GrowableByteBuffer(size, 2.0f);
doc.serialize(buf);
doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
doc.getDataType().getBodyType().setCompressionConfig(noncomp);
fos = new FileOutputStream("src/tests/data/serializejava-compressed.dat");
fos.write(buf.array(), 0, buf.position());
fos.close();
}
use of com.yahoo.document.datatypes.FloatFieldValue in project vespa by vespa-engine.
the class DocumentSerializationTestCase method testSerializationAllVersions.
@Test
public void testSerializationAllVersions() throws IOException {
DocumentType docInDocType = new DocumentType("docindoc");
docInDocType.addField(new Field("stringindocfield", DataType.STRING, false));
DocumentType docType = new DocumentType("serializetest");
docType.addField(new Field("floatfield", DataType.FLOAT, true));
docType.addField(new Field("stringfield", DataType.STRING, true));
docType.addField(new Field("longfield", DataType.LONG, true));
docType.addField(new Field("urifield", DataType.URI, true));
docType.addField(new Field("intfield", DataType.INT, false));
docType.addField(new Field("rawfield", DataType.RAW, false));
docType.addField(new Field("doublefield", DataType.DOUBLE, false));
docType.addField(new Field("bytefield", DataType.BYTE, false));
DataType arrayOfFloatDataType = new ArrayDataType(DataType.FLOAT);
docType.addField(new Field("arrayoffloatfield", arrayOfFloatDataType, false));
DataType arrayOfArrayOfFloatDataType = new ArrayDataType(arrayOfFloatDataType);
docType.addField(new Field("arrayofarrayoffloatfield", arrayOfArrayOfFloatDataType, false));
docType.addField(new Field("docfield", DataType.DOCUMENT, false));
DataType weightedSetDataType = DataType.getWeightedSet(DataType.STRING, false, false);
docType.addField(new Field("wsfield", weightedSetDataType, false));
DocumentTypeManager docMan = new DocumentTypeManager();
docMan.register(docInDocType);
docMan.register(docType);
String path = "src/test/serializeddocuments/";
{
Document doc = new Document(docType, "doc:serializetest:http://test.doc.id/");
doc.setFieldValue("intfield", 5);
doc.setFieldValue("floatfield", -9.23);
doc.setFieldValue("stringfield", "This is a string.");
doc.setFieldValue("longfield", new LongFieldValue(398420092938472983l));
doc.setFieldValue("doublefield", new DoubleFieldValue(98374532.398820));
doc.setFieldValue("bytefield", new ByteFieldValue(254));
byte[] rawData = "RAW DATA".getBytes();
assertEquals(8, rawData.length);
doc.setFieldValue(docType.getField("rawfield"), new Raw(ByteBuffer.wrap("RAW DATA".getBytes())));
Document docInDoc = new Document(docInDocType, "doc:serializetest:http://doc.in.doc/");
docInDoc.setFieldValue("stringindocfield", "Elvis is dead");
doc.setFieldValue(docType.getField("docfield"), docInDoc);
Array<FloatFieldValue> floatArray = new Array<>(arrayOfFloatDataType);
floatArray.add(new FloatFieldValue(1.0f));
floatArray.add(new FloatFieldValue(2.0f));
doc.setFieldValue("arrayoffloatfield", floatArray);
WeightedSet<StringFieldValue> weightedSet = new WeightedSet<>(weightedSetDataType);
weightedSet.put(new StringFieldValue("Weighted 0"), 50);
weightedSet.put(new StringFieldValue("Weighted 1"), 199);
doc.setFieldValue("wsfield", weightedSet);
CompressionConfig noncomp = new CompressionConfig();
CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4);
{
doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
doc.getDataType().getBodyType().setCompressionConfig(noncomp);
FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-uncompressed.dat", false);
doc.serialize(fout);
fout.close();
}
{
doc.getDataType().getHeaderType().setCompressionConfig(lz4comp);
doc.getDataType().getBodyType().setCompressionConfig(lz4comp);
FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-lz4-9.dat", false);
doc.serialize(fout);
doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
doc.getDataType().getBodyType().setCompressionConfig(noncomp);
fout.close();
}
}
class TestDoc {
String testFile;
int version;
TestDoc(String testFile, int version) {
this.testFile = testFile;
this.version = version;
}
}
String cpppath = "src/tests/data/";
List<TestDoc> tests = new ArrayList<>();
tests.add(new TestDoc(path + "document-java-currentversion-uncompressed.dat", Document.SERIALIZED_VERSION));
tests.add(new TestDoc(path + "document-java-currentversion-lz4-9.dat", Document.SERIALIZED_VERSION));
tests.add(new TestDoc(path + "document-java-v8-uncompressed.dat", 8));
tests.add(new TestDoc(cpppath + "document-cpp-currentversion-uncompressed.dat", 7));
tests.add(new TestDoc(cpppath + "document-cpp-currentversion-lz4-9.dat", 7));
tests.add(new TestDoc(cpppath + "document-cpp-v8-uncompressed.dat", 7));
tests.add(new TestDoc(cpppath + "document-cpp-v7-uncompressed.dat", 7));
tests.add(new TestDoc(cpppath + "serializev6.dat", 6));
for (TestDoc test : tests) {
File f = new File(test.testFile);
FileInputStream fin = new FileInputStream(f);
byte[] buffer = new byte[(int) f.length()];
int pos = 0;
int remaining = buffer.length;
while (remaining > 0) {
int read = fin.read(buffer, pos, remaining);
assertFalse(read == -1);
pos += read;
remaining -= read;
}
System.err.println("Checking doc from file " + test.testFile);
Document doc = new Document(DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(buffer)));
System.err.println("Id: " + doc.getId());
assertEquals(new IntegerFieldValue(5), doc.getFieldValue("intfield"));
assertEquals(-9.23, ((FloatFieldValue) doc.getFieldValue("floatfield")).getFloat(), 1E-6);
assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield"));
assertEquals(new LongFieldValue(398420092938472983l), doc.getFieldValue("longfield"));
assertEquals(98374532.398820, ((DoubleFieldValue) doc.getFieldValue("doublefield")).getDouble(), 1E-6);
assertEquals(new ByteFieldValue((byte) 254), doc.getFieldValue("bytefield"));
ByteBuffer bbuffer = ((Raw) doc.getFieldValue("rawfield")).getByteBuffer();
if (!Arrays.equals("RAW DATA".getBytes(), bbuffer.array())) {
System.err.println("Expected 'RAW DATA' but got '" + new String(bbuffer.array()) + "'.");
assertTrue(false);
}
if (test.version > 6) {
Document docInDoc = (Document) doc.getFieldValue("docfield");
assertTrue(docInDoc != null);
assertEquals(new StringFieldValue("Elvis is dead"), docInDoc.getFieldValue("stringindocfield"));
}
Array array = (Array) doc.getFieldValue("arrayoffloatfield");
assertTrue(array != null);
assertEquals(1.0f, ((FloatFieldValue) array.get(0)).getFloat(), 1E-6);
assertEquals(2.0f, ((FloatFieldValue) array.get(1)).getFloat(), 1E-6);
WeightedSet wset = (WeightedSet) doc.getFieldValue("wsfield");
assertTrue(wset != null);
assertEquals(Integer.valueOf(50), wset.get(new StringFieldValue("Weighted 0")));
assertEquals(Integer.valueOf(199), wset.get(new StringFieldValue("Weighted 1")));
}
}
use of com.yahoo.document.datatypes.FloatFieldValue in project vespa by vespa-engine.
the class DocumentListTestCase method testSelfSerializationAndWriteJavaFile.
@SuppressWarnings("deprecation")
public void testSelfSerializationAndWriteJavaFile() throws Exception {
DocumentTypeManager docMan = new DocumentTypeManager();
DocumentTypeManagerConfigurer.configure(docMan, "file:src/test/files/documentmanager.cfg");
DocumentType bmType = docMan.getDocumentType("benchmark");
DocumentPut put1 = new DocumentPut(bmType, "userdoc:foo:99999999:1");
put1.getDocument().setFieldValue("headerstring", "foo");
DocumentRemove doc2 = new DocumentRemove(new DocumentId("userdoc:foo:99999999:2"));
DocumentPut put3 = new DocumentPut(bmType, "userdoc:foo:99999999:3");
put3.getDocument().setFieldValue("bodyfloat", new FloatFieldValue(5.5f));
DocumentUpdate docUp = new DocumentUpdate(docMan.getDocumentType("benchmark"), new DocumentId("userdoc:foo:99999999:4"));
docUp.addFieldUpdate(FieldUpdate.createAssign(docUp.getType().getField("bodystring"), new StringFieldValue("ballooooo")));
List<Entry> entries = new ArrayList<Entry>();
entries.add(Entry.create(put1));
entries.add(Entry.create(doc2));
entries.add(Entry.create(put3));
entries.add(Entry.create(docUp));
DocumentList documentList = DocumentList.create(entries);
DocumentSerializer gbuf = DocumentSerializerFactory.create42();
// Add some data to avoid special case where position() is 0 for buffer used.
gbuf.putInt(null, 1234);
int startPos = gbuf.getBuf().position();
documentList.serialize(gbuf);
int size = gbuf.getBuf().position() - startPos;
byte[] data = new byte[size];
gbuf.getBuf().position(startPos);
gbuf.getBuf().get(data);
FileOutputStream stream = new FileOutputStream("./src/test/files/documentlist-java.dat");
stream.write(data);
stream.close();
gbuf.getBuf().position(0);
DocumentList documentList2 = DocumentList.create(docMan, data);
assertEquals(4, documentList2.size());
Entry entry1 = documentList2.get(0);
assertEquals(0L, entry1.getTimestamp());
assertFalse(entry1.isBodyStripped());
assertFalse(entry1.isRemoveEntry());
assertFalse(entry1.isUpdateEntry());
assertTrue(entry1.getDocumentOperation() instanceof DocumentPut);
assertEquals(new StringFieldValue("foo"), ((DocumentPut) entry1.getDocumentOperation()).getDocument().getFieldValue("headerstring"));
Entry entry2 = documentList2.get(1);
assertEquals(0L, entry2.getTimestamp());
assertFalse(entry2.isBodyStripped());
assertTrue(entry2.isRemoveEntry());
assertFalse(entry2.isUpdateEntry());
assertTrue(entry2.getDocumentOperation() instanceof DocumentRemove);
Entry entry3 = documentList2.get(2);
assertEquals(0L, entry3.getTimestamp());
assertFalse(entry3.isBodyStripped());
assertFalse(entry3.isRemoveEntry());
assertFalse(entry3.isUpdateEntry());
assertTrue(entry3.getDocumentOperation() instanceof DocumentPut);
assertEquals(new FloatFieldValue(5.5f), ((DocumentPut) entry3.getDocumentOperation()).getDocument().getFieldValue("bodyfloat"));
Entry entry4 = documentList2.get(3);
assertEquals(0L, entry4.getTimestamp());
assertFalse(entry4.isBodyStripped());
assertFalse(entry4.isRemoveEntry());
assertTrue(entry4.isUpdateEntry());
assertTrue(entry4.getDocumentOperation() instanceof DocumentUpdate);
assertEquals(new StringFieldValue("ballooooo"), ((DocumentUpdate) entry4.getDocumentOperation()).getFieldUpdate(0).getValueUpdate(0).getValue());
}
Aggregations