Examples with WeightedSet com.yahoo.document.datatypes.WeightedSet used on opensource projects

Example 1 with WeightedSet

use of com.yahoo.document.datatypes.WeightedSet in project vespa by vespa-engine.

the class ToWsetTestCase method assertConvert.

private static void assertConvert(boolean createIfNonExistent, boolean removeIfZero) {
    ExecutionContext ctx = new ExecutionContext(new SimpleTestAdapter());
    ctx.setValue(new StringFieldValue("69")).execute(new ToWsetExpression(createIfNonExistent, removeIfZero));
    FieldValue val = ctx.getValue();
    assertEquals(WeightedSet.class, val.getClass());
    WeightedSet wset = (WeightedSet) val;
    WeightedSetDataType type = wset.getDataType();
    assertEquals(DataType.STRING, type.getNestedType());
    assertEquals(createIfNonExistent, type.createIfNonExistent());
    assertEquals(removeIfZero, type.removeIfZero());
    assertEquals(1, wset.size());
    assertEquals(Integer.valueOf(1), wset.get(new StringFieldValue("69")));
}

Example 2 with WeightedSet

use of com.yahoo.document.datatypes.WeightedSet in project vespa by vespa-engine.

the class DocumentTestCase method testCppDocSplit.

@Test
public void testCppDocSplit() throws IOException {
    docMan = setUpCppDocType();
    byte[] headerData = readFile("src/test/document/serializecppsplit_header.dat");
    byte[] bodyData = readFile("src/test/document/serializecppsplit_body.dat");
    DocumentDeserializer header = DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(headerData), GrowableByteBuffer.wrap(bodyData));
    Document doc = new Document(header);
    assertEquals("doc:serializetest:http://test.doc.id/", doc.getId().toString());
    assertEquals(new IntegerFieldValue(5), doc.getFieldValue("intfield"));
    assertEquals(new FloatFieldValue((float) -9.23), doc.getFieldValue("floatfield"));
    assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield"));
    assertEquals(new LongFieldValue(398420092938472983L), doc.getFieldValue("longfield"));
    assertEquals(new DoubleFieldValue(98374532.398820d), doc.getFieldValue("doublefield"));
    assertEquals(new StringFieldValue("http://this.is.a.test/"), doc.getFieldValue("urifield"));
    // NOTE: The value really is unsigned 254, which becomes signed -2:
    assertEquals(new ByteFieldValue((byte) -2), doc.getFieldValue("bytefield"));
    ByteBuffer raw = ByteBuffer.wrap("RAW DATA".getBytes());
    assertEquals(new Raw(raw), doc.getFieldValue("rawfield"));
    Document docindoc = (Document) doc.getFieldValue("docfield");
    assertEquals(docMan.getDocumentType("docindoc"), docindoc.getDataType());
    assertEquals(new DocumentId("doc:docindoc:http://embedded"), docindoc.getId());
    WeightedSet wset = (WeightedSet) doc.getFieldValue("wsfield");
    assertEquals(new Integer(50), wset.get(new StringFieldValue("Weighted 0")));
    assertEquals(new Integer(199), wset.get(new StringFieldValue("Weighted 1")));
}

Example 3 with WeightedSet

use of com.yahoo.document.datatypes.WeightedSet in project vespa by vespa-engine.

the class DocumentTestCase method testGetRecursiveValue.

@Test
public void testGetRecursiveValue() {
    Document doc = new Document(testDocType, new DocumentId("doc:ns:testdoc"));
    doc.setFieldValue("primitive1", 1);
    Struct l1s1 = new Struct(doc.getField("l1s1").getDataType());
    l1s1.setFieldValue("primitive1", 2);
    Struct l2s1 = new Struct(doc.getField("struct2").getDataType());
    l2s1.setFieldValue("primitive1", 3);
    l2s1.setFieldValue("primitive2", 4);
    Array<IntegerFieldValue> iarr1 = new Array<>(l2s1.getField("iarray").getDataType());
    iarr1.add(new IntegerFieldValue(11));
    iarr1.add(new IntegerFieldValue(12));
    iarr1.add(new IntegerFieldValue(13));
    l2s1.setFieldValue("iarray", iarr1);
    ArrayDataType dt = (ArrayDataType) l2s1.getField("sarray").getDataType();
    Array<Struct> sarr1 = new Array<>(dt);
    {
        Struct l3s1 = new Struct(dt.getNestedType());
        l3s1.setFieldValue("primitive1", 1);
        l3s1.setFieldValue("primitive2", 2);
        sarr1.add(l3s1);
    }
    {
        Struct l3s1 = new Struct(dt.getNestedType());
        l3s1.setFieldValue("primitive1", 1);
        l3s1.setFieldValue("primitive2", 2);
        sarr1.add(l3s1);
    }
    l2s1.setFieldValue("sarray", sarr1);
    MapFieldValue<StringFieldValue, StringFieldValue> smap1 = new MapFieldValue<>((MapDataType) l2s1.getField("smap").getDataType());
    smap1.put(new StringFieldValue("leonardo"), new StringFieldValue("dicaprio"));
    smap1.put(new StringFieldValue("ellen"), new StringFieldValue("page"));
    smap1.put(new StringFieldValue("joseph"), new StringFieldValue("gordon-levitt"));
    l2s1.setFieldValue("smap", smap1);
    l1s1.setFieldValue("ss", l2s1.clone());
    MapFieldValue<StringFieldValue, Struct> structmap1 = new MapFieldValue<>((MapDataType) l1s1.getField("structmap").getDataType());
    structmap1.put(new StringFieldValue("test"), l2s1.clone());
    l1s1.setFieldValue("structmap", structmap1);
    WeightedSet<StringFieldValue> wset1 = new WeightedSet<>(l1s1.getField("wset").getDataType());
    wset1.add(new StringFieldValue("foo"));
    wset1.add(new StringFieldValue("bar"));
    wset1.add(new StringFieldValue("zoo"));
    l1s1.setFieldValue("wset", wset1);
    Struct l2s2 = new Struct(doc.getField("struct2").getDataType());
    l2s2.setFieldValue("primitive1", 5);
    l2s2.setFieldValue("primitive2", 6);
    WeightedSet<Struct> wset2 = new WeightedSet<>(l1s1.getField("structwset").getDataType());
    wset2.add(l2s1.clone());
    wset2.add(l2s2.clone());
    l1s1.setFieldValue("structwset", wset2);
    doc.setFieldValue("l1s1", l1s1.clone());
    {
        FieldValue fv = doc.getRecursiveValue("l1s1");
        assertEquals(l1s1, fv);
    }
    {
        FieldValue fv = doc.getRecursiveValue("l1s1.primitive1");
        assertEquals(new IntegerFieldValue(2), fv);
    }
    {
        FieldValue fv = doc.getRecursiveValue("l1s1.ss");
        assertEquals(l2s1, fv);
    }
    {
        FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray");
        assertEquals(iarr1, fv);
    }
    {
        FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray[2]");
        assertEquals(new IntegerFieldValue(13), fv);
    }
    {
        FieldValue fv = doc.getRecursiveValue("l1s1.ss.iarray[3]");
        assertNull(fv);
    }
    {
        FieldValue fv = doc.getRecursiveValue("l1s1.ss.sarray[0].primitive1");
        assertEquals(new IntegerFieldValue(1), fv);
    }
    {
        FieldValue fv = doc.getRecursiveValue("l1s1.ss.smap{joseph}");
        assertEquals(new StringFieldValue("gordon-levitt"), fv);
    }
    {
        FieldValue fv = doc.getRecursiveValue("l1s1.ss.smap.key");
        assertEquals(3, ((Array) fv).size());
    }
    {
        FieldValue fv = doc.getRecursiveValue("l1s1.structmap{test}.primitive1");
        assertEquals(new IntegerFieldValue(3), fv);
    }
    {
        FieldValue fv = doc.getRecursiveValue("l1s1.structmap.value.primitive1");
        assertEquals(new IntegerFieldValue(3), fv);
    }
    {
        FieldValue fv = doc.getRecursiveValue("l1s1.wset{foo}");
        assertEquals(new IntegerFieldValue(1), fv);
    }
    {
        FieldValue fv = doc.getRecursiveValue("l1s1.wset.key");
        assertEquals(3, ((Array) fv).size());
    }
    {
        FieldValue fv = doc.getRecursiveValue("l1s1.structwset.key.primitive1");
        assertEquals(DataType.INT, (((ArrayDataType) fv.getDataType()).getNestedType()));
        assertEquals(2, ((Array) fv).size());
    }
}

Example 4 with WeightedSet

use of com.yahoo.document.datatypes.WeightedSet in project vespa by vespa-engine.

the class DocumentTestCase method testGenerateSerializedFile.

@Test
public void testGenerateSerializedFile() throws IOException {
    docMan = setUpCppDocType();
    Document doc = new Document(docMan.getDocumentType("serializetest"), new DocumentId("doc:serializetest:http://test.doc.id/"));
    Document docindoc = new Document(docMan.getDocumentType("docindoc"), new DocumentId("doc:serializetest:http://doc.in.doc/"));
    docindoc.setFieldValue("stringindocfield", "Elvis is dead");
    doc.setFieldValue("docfield", docindoc);
    Array<FloatFieldValue> l = new Array<>(doc.getField("arrayoffloatfield").getDataType());
    l.add(new FloatFieldValue((float) 1.0));
    l.add(new FloatFieldValue((float) 2.0));
    doc.setFieldValue("arrayoffloatfield", l);
    WeightedSet<StringFieldValue> wset = new WeightedSet<>(doc.getDataType().getField("wsfield").getDataType());
    wset.put(new StringFieldValue("Weighted 0"), 50);
    wset.put(new StringFieldValue("Weighted 1"), 199);
    doc.setFieldValue("wsfield", wset);
    MapFieldValue<StringFieldValue, StringFieldValue> map = new MapFieldValue<>((MapDataType) doc.getDataType().getField("mapfield").getDataType());
    map.put(new StringFieldValue("foo1"), new StringFieldValue("bar1"));
    map.put(new StringFieldValue("foo2"), new StringFieldValue("bar2"));
    doc.setFieldValue("mapfield", map);
    doc.setFieldValue("bytefield", new ByteFieldValue((byte) 254));
    doc.setFieldValue("rawfield", new Raw(ByteBuffer.wrap("RAW DATA".getBytes())));
    doc.setFieldValue("intfield", new IntegerFieldValue(5));
    doc.setFieldValue("floatfield", new FloatFieldValue(-9.23f));
    doc.setFieldValue("stringfield", new StringFieldValue("This is a string."));
    doc.setFieldValue("longfield", new LongFieldValue(398420092938472983L));
    doc.setFieldValue("doublefield", new DoubleFieldValue(98374532.398820d));
    doc.setFieldValue("urifield", new StringFieldValue("http://this.is.a.test/"));
    int size = doc.getSerializedSize();
    GrowableByteBuffer buf = new GrowableByteBuffer(size, 2.0f);
    doc.serialize(buf);
    assertEquals(size, buf.position());
    buf.position(0);
    FileOutputStream fos = new FileOutputStream("src/tests/data/serializejava.dat");
    fos.write(buf.array(), 0, size);
    fos.close();
    CompressionConfig noncomp = new CompressionConfig();
    CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4);
    doc.getDataType().getHeaderType().setCompressionConfig(lz4comp);
    doc.getDataType().getBodyType().setCompressionConfig(lz4comp);
    buf = new GrowableByteBuffer(size, 2.0f);
    doc.serialize(buf);
    doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
    doc.getDataType().getBodyType().setCompressionConfig(noncomp);
    fos = new FileOutputStream("src/tests/data/serializejava-compressed.dat");
    fos.write(buf.array(), 0, buf.position());
    fos.close();
}

Example 5 with WeightedSet

use of com.yahoo.document.datatypes.WeightedSet in project vespa by vespa-engine.

the class DocumentSerializationTestCase method testSerializationAllVersions.

@Test
public void testSerializationAllVersions() throws IOException {
    DocumentType docInDocType = new DocumentType("docindoc");
    docInDocType.addField(new Field("stringindocfield", DataType.STRING, false));
    DocumentType docType = new DocumentType("serializetest");
    docType.addField(new Field("floatfield", DataType.FLOAT, true));
    docType.addField(new Field("stringfield", DataType.STRING, true));
    docType.addField(new Field("longfield", DataType.LONG, true));
    docType.addField(new Field("urifield", DataType.URI, true));
    docType.addField(new Field("intfield", DataType.INT, false));
    docType.addField(new Field("rawfield", DataType.RAW, false));
    docType.addField(new Field("doublefield", DataType.DOUBLE, false));
    docType.addField(new Field("bytefield", DataType.BYTE, false));
    DataType arrayOfFloatDataType = new ArrayDataType(DataType.FLOAT);
    docType.addField(new Field("arrayoffloatfield", arrayOfFloatDataType, false));
    DataType arrayOfArrayOfFloatDataType = new ArrayDataType(arrayOfFloatDataType);
    docType.addField(new Field("arrayofarrayoffloatfield", arrayOfArrayOfFloatDataType, false));
    docType.addField(new Field("docfield", DataType.DOCUMENT, false));
    DataType weightedSetDataType = DataType.getWeightedSet(DataType.STRING, false, false);
    docType.addField(new Field("wsfield", weightedSetDataType, false));
    DocumentTypeManager docMan = new DocumentTypeManager();
    docMan.register(docInDocType);
    docMan.register(docType);
    String path = "src/test/serializeddocuments/";
    {
        Document doc = new Document(docType, "doc:serializetest:http://test.doc.id/");
        doc.setFieldValue("intfield", 5);
        doc.setFieldValue("floatfield", -9.23);
        doc.setFieldValue("stringfield", "This is a string.");
        doc.setFieldValue("longfield", new LongFieldValue(398420092938472983l));
        doc.setFieldValue("doublefield", new DoubleFieldValue(98374532.398820));
        doc.setFieldValue("bytefield", new ByteFieldValue(254));
        byte[] rawData = "RAW DATA".getBytes();
        assertEquals(8, rawData.length);
        doc.setFieldValue(docType.getField("rawfield"), new Raw(ByteBuffer.wrap("RAW DATA".getBytes())));
        Document docInDoc = new Document(docInDocType, "doc:serializetest:http://doc.in.doc/");
        docInDoc.setFieldValue("stringindocfield", "Elvis is dead");
        doc.setFieldValue(docType.getField("docfield"), docInDoc);
        Array<FloatFieldValue> floatArray = new Array<>(arrayOfFloatDataType);
        floatArray.add(new FloatFieldValue(1.0f));
        floatArray.add(new FloatFieldValue(2.0f));
        doc.setFieldValue("arrayoffloatfield", floatArray);
        WeightedSet<StringFieldValue> weightedSet = new WeightedSet<>(weightedSetDataType);
        weightedSet.put(new StringFieldValue("Weighted 0"), 50);
        weightedSet.put(new StringFieldValue("Weighted 1"), 199);
        doc.setFieldValue("wsfield", weightedSet);
        CompressionConfig noncomp = new CompressionConfig();
        CompressionConfig lz4comp = new CompressionConfig(CompressionType.LZ4);
        {
            doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
            doc.getDataType().getBodyType().setCompressionConfig(noncomp);
            FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-uncompressed.dat", false);
            doc.serialize(fout);
            fout.close();
        }
        {
            doc.getDataType().getHeaderType().setCompressionConfig(lz4comp);
            doc.getDataType().getBodyType().setCompressionConfig(lz4comp);
            FileOutputStream fout = new FileOutputStream(path + "document-java-currentversion-lz4-9.dat", false);
            doc.serialize(fout);
            doc.getDataType().getHeaderType().setCompressionConfig(noncomp);
            doc.getDataType().getBodyType().setCompressionConfig(noncomp);
            fout.close();
        }
    }
    class TestDoc {

        String testFile;

        int version;

        TestDoc(String testFile, int version) {
            this.testFile = testFile;
            this.version = version;
        }
    }
    String cpppath = "src/tests/data/";
    List<TestDoc> tests = new ArrayList<>();
    tests.add(new TestDoc(path + "document-java-currentversion-uncompressed.dat", Document.SERIALIZED_VERSION));
    tests.add(new TestDoc(path + "document-java-currentversion-lz4-9.dat", Document.SERIALIZED_VERSION));
    tests.add(new TestDoc(path + "document-java-v8-uncompressed.dat", 8));
    tests.add(new TestDoc(cpppath + "document-cpp-currentversion-uncompressed.dat", 7));
    tests.add(new TestDoc(cpppath + "document-cpp-currentversion-lz4-9.dat", 7));
    tests.add(new TestDoc(cpppath + "document-cpp-v8-uncompressed.dat", 7));
    tests.add(new TestDoc(cpppath + "document-cpp-v7-uncompressed.dat", 7));
    tests.add(new TestDoc(cpppath + "serializev6.dat", 6));
    for (TestDoc test : tests) {
        File f = new File(test.testFile);
        FileInputStream fin = new FileInputStream(f);
        byte[] buffer = new byte[(int) f.length()];
        int pos = 0;
        int remaining = buffer.length;
        while (remaining > 0) {
            int read = fin.read(buffer, pos, remaining);
            assertFalse(read == -1);
            pos += read;
            remaining -= read;
        }
        System.err.println("Checking doc from file " + test.testFile);
        Document doc = new Document(DocumentDeserializerFactory.create42(docMan, GrowableByteBuffer.wrap(buffer)));
        System.err.println("Id: " + doc.getId());
        assertEquals(new IntegerFieldValue(5), doc.getFieldValue("intfield"));
        assertEquals(-9.23, ((FloatFieldValue) doc.getFieldValue("floatfield")).getFloat(), 1E-6);
        assertEquals(new StringFieldValue("This is a string."), doc.getFieldValue("stringfield"));
        assertEquals(new LongFieldValue(398420092938472983l), doc.getFieldValue("longfield"));
        assertEquals(98374532.398820, ((DoubleFieldValue) doc.getFieldValue("doublefield")).getDouble(), 1E-6);
        assertEquals(new ByteFieldValue((byte) 254), doc.getFieldValue("bytefield"));
        ByteBuffer bbuffer = ((Raw) doc.getFieldValue("rawfield")).getByteBuffer();
        if (!Arrays.equals("RAW DATA".getBytes(), bbuffer.array())) {
            System.err.println("Expected 'RAW DATA' but got '" + new String(bbuffer.array()) + "'.");
            assertTrue(false);
        }
        if (test.version > 6) {
            Document docInDoc = (Document) doc.getFieldValue("docfield");
            assertTrue(docInDoc != null);
            assertEquals(new StringFieldValue("Elvis is dead"), docInDoc.getFieldValue("stringindocfield"));
        }
        Array array = (Array) doc.getFieldValue("arrayoffloatfield");
        assertTrue(array != null);
        assertEquals(1.0f, ((FloatFieldValue) array.get(0)).getFloat(), 1E-6);
        assertEquals(2.0f, ((FloatFieldValue) array.get(1)).getFloat(), 1E-6);
        WeightedSet wset = (WeightedSet) doc.getFieldValue("wsfield");
        assertTrue(wset != null);
        assertEquals(Integer.valueOf(50), wset.get(new StringFieldValue("Weighted 0")));
        assertEquals(Integer.valueOf(199), wset.get(new StringFieldValue("Weighted 1")));
    }
}