Examples with LindenField com.xiaomi.linden.thrift.common.LindenField used on opensource projects

Example 1 with LindenField

use of com.xiaomi.linden.thrift.common.LindenField in project linden by XiaoMi.

the class TestLindenDynamicField method testBuildField.

@Test
public void testBuildField() throws IOException {
    LindenDocument lindenDocument = LindenDocumentBuilder.build(schema, JSON.parseObject(jsonStr));
    List<LindenField> fields = lindenDocument.getFields();
    Assert.assertEquals(true, fields.contains(new LindenField(new LindenFieldSchema().setName("mgroup").setType(LindenType.STRING).setIndexed(true).setStored(true), "misearch")));
    Assert.assertEquals(true, fields.contains(new LindenField(new LindenFieldSchema().setName("cost").setType(LindenType.LONG).setIndexed(true).setStored(true), "30")));
}

Example 2 with LindenField

use of com.xiaomi.linden.thrift.common.LindenField in project linden by XiaoMi.

the class LindenCoreImpl method updateIndexedFields.

protected Response updateIndexedFields(LindenDocument lindenDoc) throws IOException {
    JSONObject oldDoc = getInputDocument(new Term(idFieldName, lindenDoc.getId()));
    if (oldDoc == null) {
        // update failed for document not found.
        return ResponseUtils.FAILED;
    }
    for (LindenField field : lindenDoc.getFields()) {
        oldDoc.remove(field.getSchema().getName());
    }
    // merge new fields
    for (LindenField field : lindenDoc.getFields()) {
        // so we need convert these 2 parts back to raw JSONArray format in the specified schema
        if (field.getSchema().isMulti()) {
            if (field.getSchema().isDocValues()) {
                oldDoc.put(field.getSchema().getName(), JSON.parseArray(field.getValue()));
            }
            continue;
        }
        String fieldName = field.getSchema().getName();
        Object val = LindenUtil.parseLindenValue(field.getValue(), field.schema.getType());
        oldDoc.put(fieldName, val);
    }
    LindenDocument newDoc = LindenDocumentBuilder.build(config.getSchema(), oldDoc);
    Document doc = LindenDocParser.parse(newDoc, config);
    if (doc == null) {
        return ResponseUtils.FAILED;
    }
    trackingIndexWriter.updateDocument(new Term(idFieldName, lindenDoc.getId()), doc);
    return ResponseUtils.SUCCESS;
}

Example 3 with LindenField

use of com.xiaomi.linden.thrift.common.LindenField in project linden by XiaoMi.

the class LindenDocParser method parse.

public static Document parse(LindenDocument lindenDoc, LindenConfig config) {
    if (!lindenDoc.isSetFields()) {
        return null;
    }
    Document doc = new Document();
    doc.add(new StringField(config.getSchema().getId(), lindenDoc.getId(), Field.Store.YES));
    for (LindenField field : lindenDoc.getFields()) {
        LindenFieldSchema schema = field.getSchema();
        Field.Store isStored = schema.isStored() ? Field.Store.YES : Field.Store.NO;
        String name = field.getSchema().getName();
        Object value;
        if (!schema.isIndexed() && schema.isStored()) {
            doc.add(new Field(name, field.getValue(), STORED_ONLY));
        }
        switch(schema.getType()) {
            case INTEGER:
                value = Integer.valueOf(field.getValue());
                if (schema.isIndexed()) {
                    doc.add(new IntField(name, (Integer) value, isStored));
                }
                if (schema.isDocValues()) {
                    long docValuesBits = ((Integer) value).longValue();
                    doc.add(new NumericDocValuesField(name, docValuesBits));
                }
                break;
            case LONG:
                value = Long.valueOf(field.getValue());
                if (schema.isIndexed()) {
                    doc.add(new LongField(name, (Long) value, isStored));
                }
                if (schema.isDocValues()) {
                    doc.add(new NumericDocValuesField(name, (long) value));
                }
                break;
            case DOUBLE:
                value = Double.valueOf(field.getValue());
                if (schema.isIndexed()) {
                    doc.add(new DoubleField(name, (Double) value, isStored));
                }
                if (schema.isDocValues()) {
                    long docValuesBits = Double.doubleToLongBits((Double) value);
                    doc.add(new NumericDocValuesField(name, docValuesBits));
                }
                break;
            case FLOAT:
                value = Float.valueOf(field.getValue());
                if (schema.isIndexed()) {
                    doc.add(new FloatField(name, (Float) value, isStored));
                }
                if (schema.isDocValues()) {
                    long docValuesBits = Float.floatToIntBits((Float) value);
                    doc.add(new NumericDocValuesField(name, docValuesBits));
                }
                break;
            case STRING:
                if (Strings.isNullOrEmpty(field.getValue())) {
                    break;
                }
                if (schema.isIndexed()) {
                    FieldType type = new FieldType();
                    type.setTokenized(schema.isTokenized());
                    type.setIndexed(schema.isIndexed());
                    type.setStored(schema.isStored());
                    type.setOmitNorms(schema.isOmitNorms());
                    if (schema.isSnippet()) {
                        type.setIndexOptions(FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
                        // snippet will use the stored info.
                        type.setStored(true);
                    }
                    if (schema.isOmitFreqs()) {
                        type.setIndexOptions(FieldInfo.IndexOptions.DOCS_ONLY);
                    }
                    doc.add(new Field(name, field.getValue(), type));
                }
                if (schema.isDocValues()) {
                    BytesRef bytes = new BytesRef(field.getValue());
                    doc.add(new BinaryDocValuesField(name, bytes));
                }
                break;
            case FACET:
                String[] facetPath = field.getValue().split("/");
                doc.add(new FacetField(name, facetPath));
                if (schema.isIndexed()) {
                    doc.add(new StringField(name, field.getValue(), isStored));
                }
                if (schema.isDocValues()) {
                    doc.add(new BinaryDocValuesField(name, new BytesRef(field.getValue())));
                }
                break;
            default:
        }
    }
    if (lindenDoc.isSetCoordinate()) {
        Coordinate coord = lindenDoc.getCoordinate();
        Shape shape = SpatialContext.GEO.makePoint(coord.getLongitude(), coord.getLatitude());
        for (IndexableField field : config.getSpatialStrategy().createIndexableFields(shape)) {
            doc.add(field);
        }
    }
    return doc;
}

Example 4 with LindenField

use of com.xiaomi.linden.thrift.common.LindenField in project linden by XiaoMi.

the class LindenDocumentBuilder method buildDynamicField.

public static LindenField buildDynamicField(JSONObject jsonElement) throws IOException {
    LindenField field = new LindenField();
    // default field type is STRING
    // Linden dynamic field is always indexed and stored
    LindenFieldSchema fieldSchema = new LindenFieldSchema().setType(LindenType.STRING).setIndexed(true).setStored(true);
    for (Map.Entry<String, Object> entry : jsonElement.entrySet()) {
        String key = entry.getKey();
        String value = String.valueOf(entry.getValue());
        switch(key.toLowerCase()) {
            case "_tokenize":
                fieldSchema.setTokenized(value.equalsIgnoreCase("true"));
                break;
            case "_omitnorms":
                fieldSchema.setOmitNorms(value.equalsIgnoreCase("true"));
                break;
            case "_snippet":
                fieldSchema.setSnippet(value.equalsIgnoreCase("true"));
                break;
            case "_docvalues":
                fieldSchema.setDocValues(value.equalsIgnoreCase("true"));
                break;
            case "_multi":
                fieldSchema.setMulti(value.equalsIgnoreCase("true"));
                break;
            case "_omitfreqs":
                fieldSchema.setOmitFreqs(value.equalsIgnoreCase("true"));
                break;
            case "_type":
                switch(value.toLowerCase()) {
                    case "int":
                        fieldSchema.setType(LindenType.INTEGER);
                        break;
                    default:
                        fieldSchema.setType(LindenType.valueOf(value.toUpperCase()));
                        break;
                }
                break;
            default:
                if (fieldSchema.isSetName()) {
                    throw new IOException("Dynamic field name has already been set to " + fieldSchema.getName() + ", it can not be set to " + key);
                }
                fieldSchema.setName(key);
                field.setValue(value);
                break;
        }
    }
    LindenSchemaBuilder.verifyFieldSchema(fieldSchema);
    return field.setSchema(fieldSchema);
}

Example 5 with LindenField

use of com.xiaomi.linden.thrift.common.LindenField in project linden by XiaoMi.

the class LindenDocumentBuilder method build.

public static LindenDocument build(LindenSchema schema, JSONObject json) throws IOException {
    LindenDocument document = new LindenDocument();
    if (json.containsKey(schema.getId())) {
        document.setId(json.getString(schema.getId()));
    } else {
        throw new IOException(json.toJSONString() + " does not has id [" + schema.getId() + "]");
    }
    for (LindenFieldSchema fieldSchema : schema.getFields()) {
        Object value = json.get(fieldSchema.getName());
        if (value == null) {
            continue;
        }
        if (fieldSchema.isMulti()) {
            if (!(value instanceof JSONArray)) {
                throw new IOException("Multi value field " + fieldSchema.getName() + " must be in JSONArray format");
            }
            for (Object element : (JSONArray) value) {
                LindenField field = new LindenField().setSchema(fieldSchema).setValue(element.toString());
                document.addToFields(field);
            }
            LindenFieldSchema multiValueFieldSchema = new LindenFieldSchema(fieldSchema.getName(), LindenType.STRING);
            // both multi and docValues are true is forbidden in user defined schema.
            // this can only happen in multi-value source field, which is used for source data and score model
            multiValueFieldSchema.setMulti(true).setDocValues(true);
            LindenField multiValueField = new LindenField().setSchema(multiValueFieldSchema).setValue(value.toString());
            document.addToFields(multiValueField);
        } else {
            LindenField field = new LindenField().setSchema(fieldSchema).setValue(value.toString());
            document.addToFields(field);
        }
    }
    JSONArray dynamicFields = json.getJSONArray(LindenSchemaConf.DYNAMICS);
    if (dynamicFields != null) {
        for (Object element : dynamicFields) {
            JSONObject jsonElement = (JSONObject) element;
            document.addToFields(buildDynamicField(jsonElement));
        }
    }
    if (json.containsKey(LindenSchemaConf.LATITUDE) && json.containsKey(LindenSchemaConf.LONGITUDE)) {
        document.setCoordinate(new Coordinate().setLongitude(json.getDouble(LindenSchemaConf.LONGITUDE)).setLatitude(json.getDouble(LindenSchemaConf.LATITUDE)));
    }
    return document;
}