Search in sources :

Example 1 with FieldTypeFactory

use of com.qlangtech.tis.plugin.solr.schema.FieldTypeFactory in project tis by qlangtech.

the class TestSchemaResult method testParseSchemaResultAlreadyContainFieldPlugin.

/**
 * 在schema配置文件中已经添加了 <br/>
 * <fieldType name="test" class="plugin:test" precisionStep="0" positionIncrementGap="0"/>
 * 该类型也要能自动加入到傻瓜模式下,string类型的联动下拉框 <br/>
 *
 * @throws Exception
 */
public void testParseSchemaResultAlreadyContainFieldPlugin() throws Exception {
    IMessageHandler msgHandler = EasyMock.createMock("msgHandler", IMessageHandler.class);
    Context context = EasyMock.createMock("context", Context.class);
    assertEquals(FieldTypeFactory.class, HeteroEnum.SOLR_FIELD_TYPE.extensionPoint);
    IPluginStore<FieldTypeFactory> fieldTypePluginStore = TIS.getPluginStore(collection, FieldTypeFactory.class);
    assertNotNull(fieldTypePluginStore);
    final String testFieldTypeName = "test";
    final List<FieldTypeFactory> plugins = fieldTypePluginStore.getPlugins();
    assertTrue(plugins.size() > 0);
    FieldTypeFactory fieldType = fieldTypePluginStore.find(testFieldTypeName);
    assertNotNull("fieldType can not be null", fieldType);
    try (InputStream schema = this.getClass().getResourceAsStream("s4totalpay-schema-already-contain-fieldtype-plugin.xml")) {
        EasyMock.replay(msgHandler, context);
        SchemaResult schemaResult = SchemaAction.parseSchemaResultWithPluginCfg(collection, msgHandler, context, IOUtils.toByteArray(schema));
        Collection<SolrFieldsParser.SolrType> fieldTypes = ((ParseResult) schemaResult.getParseResult()).getFieldTypes();
        assertEquals("fieldTypes size", 2, fieldTypes.size());
        // String content = (com.alibaba.fastjson.JSON.toJSONString(schemaResult.toJSON()
        // , SerializerFeature.DisableCircularReferenceDetect, SerializerFeature.PrettyFormat));
        // try (InputStream assertSchemaResultInput = this.getClass().getResourceAsStream("s4totalpay-schema-already-contain-fieldtype-plugin-schema-result.json")) {
        // assertNotNull(assertSchemaResultInput);
        // FileUtils.write(new File("test.json"), content, TisUTF8.get());
        // assertEquals(StringUtils.trim(IOUtils.toString(assertSchemaResultInput, TisUTF8.get())), content);
        // }
        JsonUtil.assertJSONEqual(this.getClass(), "s4totalpay-schema-already-contain-fieldtype-plugin-schema-result.json", schemaResult.toJSON(), (m, e, a) -> {
            assertEquals(m, e, a);
        });
    }
    EasyMock.verify(msgHandler, context);
}
Also used : Context(com.alibaba.citrus.turbine.Context) ParseResult(com.qlangtech.tis.solrdao.impl.ParseResult) IMessageHandler(com.qlangtech.tis.runtime.module.misc.IMessageHandler) InputStream(java.io.InputStream) FieldTypeFactory(com.qlangtech.tis.plugin.solr.schema.FieldTypeFactory)

Example 2 with FieldTypeFactory

use of com.qlangtech.tis.plugin.solr.schema.FieldTypeFactory in project tis by qlangtech.

the class TestSchemaResult method testParseSchemaResult.

/**
 * 模拟用户刚添加fieldtype plugin,则用户第一次家在schema中要自动加入 fieldtype
 *
 * @throws Exception
 */
public void testParseSchemaResult() throws Exception {
    IMessageHandler msgHandler = EasyMock.createMock("msgHandler", IMessageHandler.class);
    Context context = EasyMock.createMock("context", Context.class);
    assertEquals(FieldTypeFactory.class, HeteroEnum.SOLR_FIELD_TYPE.extensionPoint);
    final String collection = "search4totalpay";
    IPluginStore<FieldTypeFactory> fieldTypePluginStore = TIS.getPluginStore(collection, FieldTypeFactory.class);
    assertNotNull(fieldTypePluginStore);
    final String testFieldTypeName = "test";
    final List<FieldTypeFactory> plugins = fieldTypePluginStore.getPlugins();
    assertTrue(plugins.size() > 0);
    FieldTypeFactory fieldType = fieldTypePluginStore.find(testFieldTypeName);
    assertNotNull("fieldType can not be null", fieldType);
    try (InputStream schema = this.getClass().getResourceAsStream("s4totalpay-schema.xml")) {
        EasyMock.replay(msgHandler, context);
        SchemaResult schemaResult = SchemaAction.parseSchemaResultWithPluginCfg(collection, msgHandler, context, IOUtils.toByteArray(schema));
        Collection<SolrFieldsParser.SolrType> fieldTypes = ((ParseResult) schemaResult.getParseResult()).getFieldTypes();
        assertEquals("fieldTypes size", 16, fieldTypes.size());
        // String content = (com.alibaba.fastjson.JSON.toJSONString(schemaResult.toJSON()
        // , SerializerFeature.DisableCircularReferenceDetect, SerializerFeature.PrettyFormat));
        // 
        // try (InputStream assertSchemaResultInput = this.getClass().getResourceAsStream("assertSchemaResult.json")) {
        // assertNotNull(assertSchemaResultInput);
        // assertEquals(IOUtils.toString(assertSchemaResultInput, TisUTF8.get()), content);
        // }
        JsonUtil.assertJSONEqual(this.getClass(), "assertSchemaResult.json", schemaResult.toJSON(), (m, e, a) -> {
            assertEquals(m, e, a);
        });
    }
    EasyMock.verify(msgHandler, context);
}
Also used : Context(com.alibaba.citrus.turbine.Context) ParseResult(com.qlangtech.tis.solrdao.impl.ParseResult) IMessageHandler(com.qlangtech.tis.runtime.module.misc.IMessageHandler) InputStream(java.io.InputStream) FieldTypeFactory(com.qlangtech.tis.plugin.solr.schema.FieldTypeFactory)

Example 3 with FieldTypeFactory

use of com.qlangtech.tis.plugin.solr.schema.FieldTypeFactory in project tis by qlangtech.

the class SchemaAction method updateSchemaXML.

/**
 * @param fieldTypes 已经存在的字段类型
 * @param schemaForm
 * @param document2
 * @param modifier
 */
private static void updateSchemaXML(Map<String, SolrType> fieldTypes, ISchemaPluginContext schemaPlugin, ISchema schemaForm, Document document2, final XModifier modifier) {
    SolrType type = null;
    String fieldTypeRef = null;
    FieldTypeFactory fieldTypeFactory = null;
    String pluginName = null;
    for (ISchemaField field : schemaForm.getSchemaFields()) {
        modifySchemaProperty(modifier, field, "type", (fieldTypeRef = parseSolrFieldType(field)));
        modifySchemaProperty(modifier, field, "stored", field.isStored());
        modifySchemaProperty(modifier, field, "indexed", field.isIndexed());
        modifySchemaProperty(modifier, field, "docValues", field.isDocValue());
        modifySchemaProperty(modifier, field, "multiValued", field.isMultiValue());
        type = fieldTypes.get(fieldTypeRef);
        Objects.requireNonNull(type, "fieldName:" + field.getName() + " relevant fieldType can not be null");
        if (type.plugin) {
            pluginName = type.getPluginName();
            fieldTypeFactory = schemaPlugin.findFieldTypeFactory(pluginName);
            Objects.requireNonNull(fieldTypeFactory, "pluginName:" + pluginName + " relevant fieldTypeFactory can not be null");
            fieldTypeFactory.process(document2, modifier);
        }
    }
    if (StringUtils.isNotBlank(schemaForm.getUniqueKey())) {
        modifySchemaProperty("/uniqueKey/text()", schemaForm.getUniqueKey(), modifier);
    } else {
        modifier.deleteUniqueKey();
    }
    if (StringUtils.isNotBlank(schemaForm.getSharedKey())) {
        modifySchemaProperty("/sharedKey/text()", schemaForm.getSharedKey(), modifier);
    } else {
        modifier.deleteSharedKey();
    }
}
Also used : SolrType(com.qlangtech.tis.solrdao.SolrFieldsParser.SolrType) FieldTypeFactory(com.qlangtech.tis.plugin.solr.schema.FieldTypeFactory)

Example 4 with FieldTypeFactory

use of com.qlangtech.tis.plugin.solr.schema.FieldTypeFactory in project tis by qlangtech.

the class SchemaAction method parseSchemaResultWithPluginCfg.

public static SchemaResult parseSchemaResultWithPluginCfg(String collection, IMessageHandler msgHandler, Context context, byte[] resContent) throws Exception {
    final Map<String, Boolean> pluginTypeAddedMap = Maps.newHashMap();
    ISchemaPluginContext schemaPlugin = createSchemaPlugin(collection);
    SchemaResult schemaResult = SchemaResult.parseSchemaResult(msgHandler, context, resContent, false, schemaPlugin, (cols, sResult) -> {
        boolean pluginTypeAdded;
        String identityNameVal = null;
        List<FieldTypeFactory> fieldTypeFactories = schemaPlugin.getFieldTypeFactories();
        for (FieldTypeFactory plugin : fieldTypeFactories) {
            identityNameVal = plugin.identityValue();
            if (!(pluginTypeAdded = sResult.containType(identityNameVal))) {
                sResult.addFieldType(identityNameVal, SolrFieldsParser.parseFieldType(identityNameVal, identityNameVal, plugin.forStringTokenizer()));
            }
            pluginTypeAddedMap.put(identityNameVal, pluginTypeAdded);
        }
    });
    schemaResult.content = XModifier.modifySchemaContent(schemaResult.content, (document2, modifier) -> {
        // FieldTypeFactory fieldTypeFactory = null;
        for (Map.Entry<String, Boolean> entry : pluginTypeAddedMap.entrySet()) {
            if (!entry.getValue()) {
                // fieldTypeFactory = fieldTypePluginStore.find(entry.getKey());
                // Objects.requireNonNull(fieldTypeFactory, "the name:" + entry.getKey() + " relevant fieldTypeFactory can not be null");
                // fieldTypeFactory.
                modifier.addModify(String.format("/types/fieldType[@name='%s']/@class", entry.getKey()), "plugin:" + entry.getKey());
                // precisionStep="0" positionIncrementGap="0"
                // modifier.addModify(String.format("/types/fieldType[@name='%s']/@precisionStep", entry.getKey()), "0");
                modifier.addModify(String.format("/types/fieldType[@name='%s']/@positionIncrementGap", entry.getKey()), "0");
            }
        }
    });
    return schemaResult;
}
Also used : WorkFlow(com.qlangtech.tis.workflow.pojo.WorkFlow) StringUtils(org.apache.commons.lang.StringUtils) PluginStore(com.qlangtech.tis.plugin.PluginStore) PSchemaField(com.qlangtech.tis.solrdao.pojo.PSchemaField) TIS(com.qlangtech.tis.TIS) ISelectedTab(com.qlangtech.tis.plugin.ds.ISelectedTab) WorkFlowCriteria(com.qlangtech.tis.workflow.pojo.WorkFlowCriteria) IServerGroupDAO(com.qlangtech.tis.manage.biz.dal.dao.IServerGroupDAO) SaveFileContentAction(com.qlangtech.tis.runtime.module.action.jarcontent.SaveFileContentAction) Document(org.jdom2.Document) com.qlangtech.tis.solrdao(com.qlangtech.tis.solrdao) ByteArrayInputStream(java.io.ByteArrayInputStream) PermissionConstant(com.qlangtech.tis.manage.PermissionConstant) IDataxProcessor(com.qlangtech.tis.datax.IDataxProcessor) IPluginStore(com.qlangtech.tis.plugin.IPluginStore) EntityResolver(org.xml.sax.EntityResolver) Savefilecontent(com.qlangtech.tis.manage.Savefilecontent) com.qlangtech.tis.manage.biz.dal.pojo(com.qlangtech.tis.manage.biz.dal.pojo) Validator(com.qlangtech.tis.plugin.annotation.Validator) ISolrAppSource(com.qlangtech.tis.manage.ISolrAppSource) JSONException(com.alibaba.fastjson.JSONException) com.qlangtech.tis.manage.common(com.qlangtech.tis.manage.common) Collectors(java.util.stream.Collectors) IOUtils(org.apache.commons.io.IOUtils) ISearchEngineTypeTransfer(com.qlangtech.tis.datax.ISearchEngineTypeTransfer) DataxReader(com.qlangtech.tis.datax.impl.DataxReader) SAXException(org.xml.sax.SAXException) RunEnvironment(com.qlangtech.tis.pubhook.common.RunEnvironment) JSONObject(com.alibaba.fastjson.JSONObject) DocumentBuilderFactory(javax.xml.parsers.DocumentBuilderFactory) Pattern(java.util.regex.Pattern) UnsupportedEncodingException(java.io.UnsupportedEncodingException) LuceneVersion(com.qlangtech.tis.fullbuild.indexbuild.LuceneVersion) ParseResult(com.qlangtech.tis.solrdao.impl.ParseResult) java.util(java.util) BeanUtils(org.apache.commons.beanutils.BeanUtils) DocType(org.jdom2.DocType) Func(com.qlangtech.tis.manage.spring.aop.Func) ColumnMetaData(com.qlangtech.tis.plugin.ds.ColumnMetaData) SolrType(com.qlangtech.tis.solrdao.SolrFieldsParser.SolrType) Context(com.alibaba.citrus.turbine.Context) JsonUtil(com.qlangtech.tis.trigger.util.JsonUtil) JSONArray(com.alibaba.fastjson.JSONArray) com.qlangtech.tis.runtime.module.misc(com.qlangtech.tis.runtime.module.misc) DataxProcessor(com.qlangtech.tis.datax.impl.DataxProcessor) Charset(java.nio.charset.Charset) XModifier(com.yushu.tis.xmodifier.XModifier) DataxUtils(com.qlangtech.tis.offline.DataxUtils) InputSource(org.xml.sax.InputSource) ReflectSchemaFieldType(com.qlangtech.tis.plugin.ds.ReflectSchemaFieldType) FieldTypeFactory(com.qlangtech.tis.plugin.solr.schema.FieldTypeFactory) IOException(java.io.IOException) Maps(com.google.common.collect.Maps) JSON(com.alibaba.fastjson.JSON) StringReader(java.io.StringReader) DocumentBuilder(javax.xml.parsers.DocumentBuilder) Assert(com.qlangtech.tis.common.utils.Assert) FieldTypeFactory(com.qlangtech.tis.plugin.solr.schema.FieldTypeFactory)

Aggregations

FieldTypeFactory (com.qlangtech.tis.plugin.solr.schema.FieldTypeFactory)4 Context (com.alibaba.citrus.turbine.Context)3 ParseResult (com.qlangtech.tis.solrdao.impl.ParseResult)3 IMessageHandler (com.qlangtech.tis.runtime.module.misc.IMessageHandler)2 SolrType (com.qlangtech.tis.solrdao.SolrFieldsParser.SolrType)2 InputStream (java.io.InputStream)2 JSON (com.alibaba.fastjson.JSON)1 JSONArray (com.alibaba.fastjson.JSONArray)1 JSONException (com.alibaba.fastjson.JSONException)1 JSONObject (com.alibaba.fastjson.JSONObject)1 Maps (com.google.common.collect.Maps)1 TIS (com.qlangtech.tis.TIS)1 Assert (com.qlangtech.tis.common.utils.Assert)1 IDataxProcessor (com.qlangtech.tis.datax.IDataxProcessor)1 ISearchEngineTypeTransfer (com.qlangtech.tis.datax.ISearchEngineTypeTransfer)1 DataxProcessor (com.qlangtech.tis.datax.impl.DataxProcessor)1 DataxReader (com.qlangtech.tis.datax.impl.DataxReader)1 LuceneVersion (com.qlangtech.tis.fullbuild.indexbuild.LuceneVersion)1 ISolrAppSource (com.qlangtech.tis.manage.ISolrAppSource)1 PermissionConstant (com.qlangtech.tis.manage.PermissionConstant)1