Examples with LindenFieldSchema com.xiaomi.linden.thrift.common.LindenFieldSchema used on opensource projects

Example 16 with LindenFieldSchema

use of com.xiaomi.linden.thrift.common.LindenFieldSchema in project linden by XiaoMi.

the class TestLindenMetricPlugin method init.

@Override
public void init() throws Exception {
    LindenSchema schema = new LindenSchema().setId("id");
    schema.addToFields(new LindenFieldSchema().setName("title").setIndexed(true).setStored(true).setTokenized(true));
    schema.addToFields(new LindenFieldSchema().setName("field1").setIndexed(true).setStored(false));
    schema.addToFields(new LindenFieldSchema().setName("field2").setIndexed(true).setTokenized(true).setStored(false).setOmitFreqs(true).setOmitNorms(true));
    schema.addToFields(new LindenFieldSchema().setName("rank").setType(LindenType.FLOAT).setIndexed(true).setStored(true));
    schema.addToFields(new LindenFieldSchema().setName("cat1").setType(LindenType.INTEGER).setIndexed(true).setStored(true));
    schema.addToFields(new LindenFieldSchema().setName("cat2").setType(LindenType.DOUBLE).setIndexed(true).setStored(true));
    schema.addToFields(new LindenFieldSchema().setName("tagstr").setIndexed(true));
    schema.addToFields(new LindenFieldSchema().setName("hotwords").setStored(true));
    schema.addToFields(new LindenFieldSchema().setName("tagnum").setType(LindenType.INTEGER).setIndexed(true));
    lindenConfig.setSchema(schema);
}

Example 17 with LindenFieldSchema

use of com.xiaomi.linden.thrift.common.LindenFieldSchema in project linden by XiaoMi.

the class TestLindenJiebaAnalyzerIndexMode method init.

@Override
public void init() {
    lindenConfig.setIndexType(LindenConfig.IndexType.RAM);
    lindenConfig.setClusterUrl("127.0.0.1:2181/mock");
    LindenSchema schema = new LindenSchema().setId("id");
    schema.addToFields(new LindenFieldSchema().setName("title").setIndexed(true).setTokenized(true).setSnippet(true).setMulti(true));
    lindenConfig.setSchema(schema);
    lindenConfig.putToProperties("search.analyzer.class", "com.xiaomi.linden.lucene.analyzer.LindenJiebaAnalyzerFactory");
    lindenConfig.putToProperties("index.analyzer.class", "com.xiaomi.linden.lucene.analyzer.LindenJiebaAnalyzerFactory");
    lindenConfig.putToProperties("index.analyzer.mode", "index");
}

Example 18 with LindenFieldSchema

use of com.xiaomi.linden.thrift.common.LindenFieldSchema in project linden by XiaoMi.

the class LindenConfig method setSchema.

public LindenConfig setSchema(com.xiaomi.linden.thrift.common.LindenSchema schema) {
    this.schema = schema;
    for (LindenFieldSchema fieldSchema : schema.getFields()) {
        fieldSchemaMap.put(fieldSchema.getName(), fieldSchema);
    }
    LindenFieldSchema idSchema = new LindenFieldSchema(schema.getId(), LindenType.STRING);
    idSchema.setIndexed(true).setStored(true).setOmitNorms(true);
    fieldSchemaMap.put(idSchema.getName(), idSchema);
    return this;
}

Example 19 with LindenFieldSchema

use of com.xiaomi.linden.thrift.common.LindenFieldSchema in project linden by XiaoMi.

the class LindenDocumentBuilder method buildDynamicField.

public static LindenField buildDynamicField(JSONObject jsonElement) throws IOException {
    LindenField field = new LindenField();
    // default field type is STRING
    // Linden dynamic field is always indexed and stored
    LindenFieldSchema fieldSchema = new LindenFieldSchema().setType(LindenType.STRING).setIndexed(true).setStored(true);
    for (Map.Entry<String, Object> entry : jsonElement.entrySet()) {
        String key = entry.getKey();
        String value = String.valueOf(entry.getValue());
        switch(key.toLowerCase()) {
            case "_tokenize":
                fieldSchema.setTokenized(value.equalsIgnoreCase("true"));
                break;
            case "_omitnorms":
                fieldSchema.setOmitNorms(value.equalsIgnoreCase("true"));
                break;
            case "_snippet":
                fieldSchema.setSnippet(value.equalsIgnoreCase("true"));
                break;
            case "_docvalues":
                fieldSchema.setDocValues(value.equalsIgnoreCase("true"));
                break;
            case "_multi":
                fieldSchema.setMulti(value.equalsIgnoreCase("true"));
                break;
            case "_omitfreqs":
                fieldSchema.setOmitFreqs(value.equalsIgnoreCase("true"));
                break;
            case "_type":
                switch(value.toLowerCase()) {
                    case "int":
                        fieldSchema.setType(LindenType.INTEGER);
                        break;
                    default:
                        fieldSchema.setType(LindenType.valueOf(value.toUpperCase()));
                        break;
                }
                break;
            default:
                if (fieldSchema.isSetName()) {
                    throw new IOException("Dynamic field name has already been set to " + fieldSchema.getName() + ", it can not be set to " + key);
                }
                fieldSchema.setName(key);
                field.setValue(value);
                break;
        }
    }
    LindenSchemaBuilder.verifyFieldSchema(fieldSchema);
    return field.setSchema(fieldSchema);
}

Example 20 with LindenFieldSchema

use of com.xiaomi.linden.thrift.common.LindenFieldSchema in project linden by XiaoMi.

the class LindenDocumentBuilder method build.

public static LindenDocument build(LindenSchema schema, JSONObject json) throws IOException {
    LindenDocument document = new LindenDocument();
    if (json.containsKey(schema.getId())) {
        document.setId(json.getString(schema.getId()));
    } else {
        throw new IOException(json.toJSONString() + " does not has id [" + schema.getId() + "]");
    }
    for (LindenFieldSchema fieldSchema : schema.getFields()) {
        Object value = json.get(fieldSchema.getName());
        if (value == null) {
            continue;
        }
        if (fieldSchema.isMulti()) {
            if (!(value instanceof JSONArray)) {
                throw new IOException("Multi value field " + fieldSchema.getName() + " must be in JSONArray format");
            }
            for (Object element : (JSONArray) value) {
                LindenField field = new LindenField().setSchema(fieldSchema).setValue(element.toString());
                document.addToFields(field);
            }
            LindenFieldSchema multiValueFieldSchema = new LindenFieldSchema(fieldSchema.getName(), LindenType.STRING);
            // both multi and docValues are true is forbidden in user defined schema.
            // this can only happen in multi-value source field, which is used for source data and score model
            multiValueFieldSchema.setMulti(true).setDocValues(true);
            LindenField multiValueField = new LindenField().setSchema(multiValueFieldSchema).setValue(value.toString());
            document.addToFields(multiValueField);
        } else {
            LindenField field = new LindenField().setSchema(fieldSchema).setValue(value.toString());
            document.addToFields(field);
        }
    }
    JSONArray dynamicFields = json.getJSONArray(LindenSchemaConf.DYNAMICS);
    if (dynamicFields != null) {
        for (Object element : dynamicFields) {
            JSONObject jsonElement = (JSONObject) element;
            document.addToFields(buildDynamicField(jsonElement));
        }
    }
    if (json.containsKey(LindenSchemaConf.LATITUDE) && json.containsKey(LindenSchemaConf.LONGITUDE)) {
        document.setCoordinate(new Coordinate().setLongitude(json.getDouble(LindenSchemaConf.LONGITUDE)).setLatitude(json.getDouble(LindenSchemaConf.LATITUDE)));
    }
    return document;
}