Examples with VisualType com.qlangtech.tis.runtime.module.misc.VisualType used on opensource projects

Example 1 with VisualType

use of com.qlangtech.tis.runtime.module.misc.VisualType in project plugins by qlangtech.

the class DataXElasticsearchWriter method projectionFromExpertModel.

@Override
public ISchema projectionFromExpertModel(JSONObject body) {
    Objects.requireNonNull(body, "request body can not be null");
    final String content = body.getString("content");
    if (StringUtils.isBlank(content)) {
        throw new IllegalStateException("content can not be null");
    }
    ESSchema schema = new ESSchema();
    JSONObject field = null;
    ESField esField = null;
    JSONObject b = JSON.parseObject(content);
    JSONArray fields = b.getJSONArray(ESTableAlias.KEY_COLUMN);
    for (int i = 0; i < fields.size(); i++) {
        field = fields.getJSONObject(i);
        esField = new ESField();
        esField.setName(field.getString(ISchemaField.KEY_NAME));
        final String type = field.getString(ISchemaField.KEY_TYPE);
        VisualType visualType = parseVisualType(type);
        esField.setType(visualType);
        if (visualType.isSplit()) {
            esField.setTokenizerType(StringUtils.equalsIgnoreCase(EsTokenizerType.NULL.getKey(), type) ? EsTokenizerType.NULL.getKey() : field.getString(ISchemaField.KEY_ANALYZER));
        }
        esField.setIndexed(field.getBooleanValue(ISchemaField.KEY_INDEX));
        esField.setMltiValued(field.getBooleanValue(ISchemaField.KEY_ARRAY));
        esField.setDocValue(field.getBooleanValue(ISchemaField.KEY_DOC_VALUES));
        esField.setStored(field.getBooleanValue(ISchemaField.KEY_STORE));
        esField.setUniqueKey(field.getBooleanValue(ISchemaField.KEY_PK));
        esField.setSharedKey(field.getBooleanValue(ISchemaField.KEY_SHARE_KEY));
        schema.fields.add(esField);
    }
    return schema;
}

Example 2 with VisualType

use of com.qlangtech.tis.runtime.module.misc.VisualType in project plugins by qlangtech.

the class ESField method serialVisualType2Json.

@Override
public void serialVisualType2Json(JSONObject f) {
    if (this.getType() == null) {
        throw new IllegalStateException("field:" + this.getName() + " 's fieldType is can not be null");
    }
    VisualType esType = this.getType();
    String type = esType.type;
    EsTokenizerType tokenizerType = EsTokenizerType.parse(this.getTokenizerType());
    if (tokenizerType == null) {
        // 非分词字段
        if (esType.isSplit()) {
            setStringType(f, type, EsTokenizerType.getTokenizerType().name());
        } else {
            f.put("split", false);
            VisualType vtype = EsTokenizerType.visualTypeMap.get(type);
            if (vtype != null) {
                f.put(ISchemaField.KEY_FIELD_TYPE, vtype.getType());
                return;
            }
            f.put(ISchemaField.KEY_FIELD_TYPE, type);
        }
    } else {
        // 分词字段
        setStringType(f, tokenizerType.getKey(), EsTokenizerType.getTokenizerType().name());
    }
}

Example 3 with VisualType

use of com.qlangtech.tis.runtime.module.misc.VisualType in project plugins by qlangtech.

the class ESSchema method serialTypes.

@Override
public JSONArray serialTypes() {
    JSONArray types = new JSONArray();
    com.alibaba.fastjson.JSONObject f = null;
    JSONArray tokens = null;
    com.alibaba.fastjson.JSONObject tt = null;
    SolrFieldsParser.SolrType solrType = null;
    // Set<String> typesSet = new HashSet<String>();
    for (Map.Entry<String, VisualType> t : EsTokenizerType.visualTypeMap.entrySet()) {
        f = new com.alibaba.fastjson.JSONObject();
        f.put("name", t.getKey());
        f.put("split", t.getValue().isSplit());
        tokens = new JSONArray();
        if (t.getValue().isSplit()) {
            // 默认类型
            for (ISearchEngineTokenizerType tokenType : t.getValue().getTokenerTypes()) {
                tt = new com.alibaba.fastjson.JSONObject();
                tt.put("key", tokenType.getKey());
                tt.put("value", tokenType.getDesc());
                tokens.add(tt);
            }
            // 外加类型
            // for (Map.Entry<String, SolrFieldsParser.SolrType> entry : this.types.entrySet()) {
            // solrType = entry.getValue();
            // if (solrType.tokenizerable) {
            // tt = new com.alibaba.fastjson.JSONObject();
            // tt.put("key", entry.getKey());
            // tt.put("value", entry.getKey());
            // tokens.add(tt);
            // }
            // }
            f.put("tokensType", tokens);
        }
        types.add(f);
    }
    // schema.put("fieldtypes", types);
    return types;
}

Example 4 with VisualType

use of com.qlangtech.tis.runtime.module.misc.VisualType in project tis by qlangtech.

the class PSchemaField method serialVisualType2Json.

@Override
public void serialVisualType2Json(JSONObject f) {
    if (this.getType() == null) {
        throw new IllegalStateException("field:" + this.getName() + " 's fieldType is can not be null");
    }
    // serialVisualType2Json(f, this.getType());
    SolrFieldsParser.SolrType solrType = this.getType();
    String type = solrType.getSType().getName();
    TokenizerType tokenizerType = TokenizerType.parse(type);
    if (tokenizerType == null) {
        // 非分词字段
        if (solrType.tokenizerable) {
            setStringType(f, type, ReflectSchemaFieldType.STRING.literia);
        } else {
            f.put("split", false);
            VisualType vtype = TokenizerType.visualTypeMap.get(type);
            if (vtype != null) {
                f.put(ISchemaField.KEY_FIELD_TYPE, vtype.getType());
                return;
            }
            f.put(ISchemaField.KEY_FIELD_TYPE, type);
        }
    } else {
        // 分词字段
        setStringType(f, tokenizerType.getKey(), ReflectSchemaFieldType.STRING.literia);
    }
// } else {
// throw new IllegalStateException("field:" + this.getName() + " 's fieldType is can not be null");
// }
}

Example 5 with VisualType

use of com.qlangtech.tis.runtime.module.misc.VisualType in project plugins by qlangtech.

the class DataXElasticsearchWriter method mergeFromStupidModel.

/**
 * 小白模式转专家模式，正好与方法projectionFromExpertModel相反
 *
 * @param schema
 * @param expertSchema
 * @return
 */
@Override
public JSONObject mergeFromStupidModel(ISchema schema, JSONObject expertSchema) {
    JSONArray mergeTarget = expertSchema.getJSONArray(ESTableAlias.KEY_COLUMN);
    Objects.requireNonNull(mergeTarget, "mergeTarget can not be null");
    JSONObject f = null;
    Map<String, JSONObject> mergeFields = Maps.newHashMap();
    for (int i = 0; i < mergeTarget.size(); i++) {
        f = mergeTarget.getJSONObject(i);
        mergeFields.put(f.getString("name"), f);
    }
    JSONArray jFields = new com.alibaba.fastjson.JSONArray();
    for (ISchemaField field : schema.getSchemaFields()) {
        if (StringUtils.isBlank(field.getName())) {
            throw new IllegalStateException("field name can not be null");
        }
        f = mergeFields.get(field.getName());
        if (f == null) {
            f = new JSONObject();
            f.put(ISchemaField.KEY_NAME, field.getName());
        }
        VisualType type = EsTokenizerType.visualTypeMap.get(field.getTisFieldTypeName());
        if (type.isSplit()) {
            if (StringUtils.isEmpty(field.getTokenizerType())) {
                throw new IllegalStateException("field:" + field.getName() + " relevant type is tokenizer but has not set analyzer");
            }
            if (StringUtils.endsWithIgnoreCase(field.getTokenizerType(), EsTokenizerType.NULL.getKey())) {
                f.put(ISchemaField.KEY_TYPE, EsTokenizerType.NULL.getKey());
                f.remove(ISchemaField.KEY_ANALYZER);
            } else {
                f.put(ISchemaField.KEY_TYPE, type.getType());
                f.put(ISchemaField.KEY_ANALYZER, field.getTokenizerType());
            }
        } else {
            f.put(ISchemaField.KEY_TYPE, type.getType());
            f.remove(ISchemaField.KEY_ANALYZER);
        }
        // TODO 还不确定array 是否对应multiValue的语义
        f.put(ISchemaField.KEY_ARRAY, field.isMultiValue());
        f.put(ISchemaField.KEY_DOC_VALUES, field.isDocValue());
        f.put(ISchemaField.KEY_INDEX, field.isIndexed());
        f.put(ISchemaField.KEY_STORE, field.isStored());
        if (field.isUniqueKey()) {
            f.put(ISchemaField.KEY_PK, true);
        }
        if (field.isSharedKey()) {
            f.put(ISchemaField.KEY_SHARE_KEY, true);
        }
        jFields.add(f);
    }
    expertSchema.put(ESTableAlias.KEY_COLUMN, jFields);
    return expertSchema;
}