Search in sources :

Example 1 with IPluginStore

use of com.qlangtech.tis.plugin.IPluginStore in project tis by qlangtech.

the class HeteroEnum method getPlugins.

/**
 * ref: PluginItems.save()
 *
 * @param pluginContext
 * @param pluginMeta
 * @param
 * @return
 */
public List<T> getPlugins(IPluginContext pluginContext, UploadPluginMeta pluginMeta) {
    IPluginStore store = getPluginStore(pluginContext, pluginMeta);
    if (store == null) {
        return Collections.emptyList();
    }
    // if (this == HeteroEnum.APP_SOURCE) {
    // final String dataxName = (pluginMeta.getExtraParam(DataxUtils.DATAX_NAME));
    // if (StringUtils.isEmpty(dataxName)) {
    // throw new IllegalArgumentException("plugin extra param 'DataxUtils.DATAX_NAME'" + DataxUtils.DATAX_NAME + " can not be null");
    // }
    // store = com.qlangtech.tis.manage.IAppSource.getPluginStore(pluginContext, dataxName);
    // } else if (this == HeteroEnum.DATAX_WRITER || this == HeteroEnum.DATAX_READER) {
    // final String dataxName = pluginMeta.getExtraParam(DataxUtils.DATAX_NAME);
    // if (StringUtils.isEmpty(dataxName)) {
    // throw new IllegalArgumentException("plugin extra param 'DataxUtils.DATAX_NAME': '" + DataxUtils.DATAX_NAME + "' can not be null");
    // }
    // store = (this == HeteroEnum.DATAX_READER) ? DataxReader.getPluginStore(pluginContext, dataxName) : DataxWriter.getPluginStore(pluginContext, dataxName);
    // } else if (pluginContext.isCollectionAware()) {
    // store = TIS.getPluginStore(pluginContext.getCollectionName(), this.extensionPoint);
    // } else if (pluginContext.isDataSourceAware()) {
    // PostedDSProp dsProp = PostedDSProp.parse(pluginMeta);
    // if (StringUtils.isEmpty(dsProp.getDbname())) {
    // return Collections.emptyList();
    // }
    // store = TIS.getDataBasePluginStore(dsProp);
    // } else {
    // store = TIS.getPluginStore(this.extensionPoint);
    // }
    // Objects.requireNonNull(store, "plugin store can not be null");
    List<T> plugins = store.getPlugins();
    if (pluginMeta != null && StringUtils.isNotEmpty(pluginMeta.getTargetPluginDesc())) {
        return plugins.stream().filter((p) -> StringUtils.equals(p.getDescriptor().getDisplayName(), pluginMeta.getTargetPluginDesc())).collect(Collectors.toList());
    }
    return plugins;
}
Also used : StringUtils(org.apache.commons.lang.StringUtils) FlatTableBuilder(com.qlangtech.tis.offline.FlatTableBuilder) IAppSource(com.qlangtech.tis.manage.IAppSource) TIS(com.qlangtech.tis.TIS) ParamsConfig(com.qlangtech.tis.config.ParamsConfig) PostedDSProp(com.qlangtech.tis.plugin.ds.PostedDSProp) ParamsConfigPluginStore(com.qlangtech.tis.plugin.credentials.ParamsConfigPluginStore) DataXJobWorker(com.qlangtech.tis.datax.job.DataXJobWorker) DataxUtils(com.qlangtech.tis.offline.DataxUtils) MQListenerFactory(com.qlangtech.tis.async.message.client.consumer.impl.MQListenerFactory) KeyedPluginStore(com.qlangtech.tis.plugin.KeyedPluginStore) TISTransformerFactory(com.qlangtech.tis.plugin.solr.config.TISTransformerFactory) IPluginStore(com.qlangtech.tis.plugin.IPluginStore) DataSourceFactory(com.qlangtech.tis.plugin.ds.DataSourceFactory) ExtensionList(com.qlangtech.tis.extension.ExtensionList) TISExtension(com.qlangtech.tis.extension.TISExtension) IncrStreamFactory(com.qlangtech.tis.plugin.incr.IncrStreamFactory) DataxWriter(com.qlangtech.tis.datax.impl.DataxWriter) Descriptor(com.qlangtech.tis.extension.Descriptor) IdentityName(com.qlangtech.tis.plugin.IdentityName) FieldTypeFactory(com.qlangtech.tis.plugin.solr.schema.FieldTypeFactory) K8sImage(com.qlangtech.tis.plugin.k8s.K8sImage) IPluginEnum(com.qlangtech.tis.IPluginEnum) QueryParserFactory(com.qlangtech.tis.plugin.solr.config.QueryParserFactory) Collectors(java.util.stream.Collectors) FileSystemFactory(com.qlangtech.tis.offline.FileSystemFactory) Objects(java.util.Objects) List(java.util.List) DataxReader(com.qlangtech.tis.datax.impl.DataxReader) SearchComponentFactory(com.qlangtech.tis.plugin.solr.config.SearchComponentFactory) Collections(java.util.Collections) Describable(com.qlangtech.tis.extension.Describable) IPluginStore(com.qlangtech.tis.plugin.IPluginStore)

Example 2 with IPluginStore

use of com.qlangtech.tis.plugin.IPluginStore in project tis by qlangtech.

the class OfflineDatasourceAction method doGetDsRelevantReaderDesc.

@Func(value = PermissionConstant.PERMISSION_DATASOURCE_EDIT, sideEffect = false)
public void doGetDsRelevantReaderDesc(Context context) {
    com.alibaba.fastjson.JSONObject form = this.parseJsonPost();
    Integer dbId = form.getInteger("dbId");
    if (dbId == null) {
        throw new IllegalStateException("dbId can not be null");
    }
    com.qlangtech.tis.workflow.pojo.DatasourceDb db = this.offlineDAOFacade.getDatasourceDbDAO().selectByPrimaryKey(dbId);
    // IPluginStore<DataSourceFactory> dbPlugin = TIS.getDataBasePluginStore(new PostedDSProp(db.getName(), DbScope.DETAILED));
    // DataSourceFactory.BaseDataSourceFactoryDescriptor descriptor = (DataSourceFactory.BaseDataSourceFactoryDescriptor) dbPlugin.getPlugin().getDescriptor();
    OfflineManager.DBDataXReaderDescName defaultDataXReaderDescName = offlineManager.getDBDataXReaderDescName(db.getName());
    Map<String, Object> result = Maps.newHashMap();
    if (!defaultDataXReaderDescName.readerDescName.isPresent()) {
        // throw new IllegalStateException("datasource:" + db.getName() + " desc:" + descriptor.getDisplayName() + " has not relevant DataXReader defined");
        // result.put(KEY_DATA_READER_SETTED + "NotSupport", descriptor.getId());
        this.addErrorMessage(context, "插件:" + defaultDataXReaderDescName.dsDescriptor.getId() + " 不支持表导入");
        return;
    }
    DataxReader dataxReader = OfflineManager.getDBDataxReader(this, db.getName());
    if (dataxReader != null) {
        result.put(KEY_DATA_READER_SETTED, true);
    // this.setBizResult(context, result);
    // return;
    }
    DescriptorExtensionList<DataxReader, Descriptor<DataxReader>> descriptorList = TIS.get().getDescriptorList(DataxReader.class);
    Optional<DataxReader.BaseDataxReaderDescriptor> dataXReaderDesc = descriptorList.stream().filter((de) -> {
        return defaultDataXReaderDescName.getReaderDescName().equals(de.getDisplayName());
    }).map((d) -> (DataxReader.BaseDataxReaderDescriptor) d).findFirst();
    if (!dataXReaderDesc.isPresent()) {
        throw new IllegalStateException("DataXReaderDescName:" + defaultDataXReaderDescName.getReaderDescName() + " can not find relevant DataXReader Descriptor");
    }
    result.put("readerDesc", new DescriptorsJSON(dataXReaderDesc.get()).getDescriptorsJSON());
    result.put("processMeta", DataxAction.getDataXBasicProcessMetaByReader(dataXReaderDesc.get()));
    this.setBizResult(context, result);
}
Also used : WorkFlow(com.qlangtech.tis.workflow.pojo.WorkFlow) StringUtils(org.apache.commons.lang.StringUtils) ERRules.$(com.qlangtech.tis.sql.parser.er.ERRules.$) SqlTaskNode(com.qlangtech.tis.sql.parser.SqlTaskNode) JoinRule(com.qlangtech.tis.git.GitUtils.JoinRule) TIS(com.qlangtech.tis.TIS) WorkFlowCriteria(com.qlangtech.tis.workflow.pojo.WorkFlowCriteria) Autowired(org.springframework.beans.factory.annotation.Autowired) Option(com.qlangtech.tis.manage.common.Option) DbScope(com.qlangtech.tis.offline.DbScope) DescriptorsJSON(com.qlangtech.tis.util.DescriptorsJSON) IFieldErrorHandler(com.qlangtech.tis.runtime.module.misc.IFieldErrorHandler) JSONObject(org.json.JSONObject) Matcher(java.util.regex.Matcher) DescriptorExtensionList(com.qlangtech.tis.extension.DescriptorExtensionList) PermissionConstant(com.qlangtech.tis.manage.PermissionConstant) FullbuildPhase(com.qlangtech.tis.assemble.FullbuildPhase) IPluginStore(com.qlangtech.tis.plugin.IPluginStore) PluginFormProperties(com.qlangtech.tis.extension.PluginFormProperties) PostParam(com.qlangtech.tis.manage.common.HttpUtils.PostParam) Validator(com.qlangtech.tis.plugin.annotation.Validator) GitRepositoryCommitPojo(com.qlangtech.tis.offline.pojo.GitRepositoryCommitPojo) DelegateControl4JsonPostMsgHandler(com.qlangtech.tis.runtime.module.misc.impl.DelegateControl4JsonPostMsgHandler) HeteroList(com.qlangtech.tis.util.HeteroList) Descriptor(com.qlangtech.tis.extension.Descriptor) TabCardinality(com.qlangtech.tis.sql.parser.er.TabCardinality) Collectors(java.util.stream.Collectors) name.fraser.neil.plaintext.diff_match_patch(name.fraser.neil.plaintext.diff_match_patch) IFullBuildContext(com.qlangtech.tis.fullbuild.IFullBuildContext) IOUtils(org.apache.commons.io.IOUtils) DataxReader(com.qlangtech.tis.datax.impl.DataxReader) IWorkFlowDAO(com.qlangtech.tis.workflow.dao.IWorkFlowDAO) IControlMsgHandler(com.qlangtech.tis.runtime.module.misc.IControlMsgHandler) TableRelation(com.qlangtech.tis.sql.parser.er.TableRelation) DataxAction(com.qlangtech.tis.coredefine.module.action.DataxAction) SqlDataFlowTopology(com.qlangtech.tis.sql.parser.SqlTaskNodeMeta.SqlDataFlowTopology) DBConfigSuit(com.qlangtech.tis.db.parser.DBConfigSuit) Pattern(java.util.regex.Pattern) SuFormProperties(com.qlangtech.tis.extension.impl.SuFormProperties) Types(java.sql.Types) Pager(com.koubei.web.tag.pager.Pager) java.util(java.util) Func(com.qlangtech.tis.manage.spring.aop.Func) UploadPluginMeta(com.qlangtech.tis.util.UploadPluginMeta) Context(com.alibaba.citrus.turbine.Context) TISDb(com.qlangtech.tis.offline.pojo.TISDb) JsonUtil(com.qlangtech.tis.trigger.util.JsonUtil) Lists(com.google.common.collect.Lists) PropertyType(com.qlangtech.tis.extension.impl.PropertyType) DatasourceTable(com.qlangtech.tis.workflow.pojo.DatasourceTable) CollectionUtils(org.apache.commons.collections.CollectionUtils) GitUtils(com.qlangtech.tis.git.GitUtils) OfflineManager(com.qlangtech.tis.offline.module.manager.impl.OfflineManager) FormFieldType(com.qlangtech.tis.plugin.annotation.FormFieldType) com.qlangtech.tis.sql.parser.meta(com.qlangtech.tis.sql.parser.meta) WorkflowPojo(com.qlangtech.tis.offline.pojo.WorkflowPojo) IWorkflowDAOFacade(com.qlangtech.tis.workflow.dao.IWorkflowDAOFacade) MapUtils(org.apache.commons.collections.MapUtils) IdentityName(com.qlangtech.tis.plugin.IdentityName) JSONTokener(org.json.JSONTokener) com.qlangtech.tis.plugin.ds(com.qlangtech.tis.plugin.ds) FileUtils(org.apache.commons.io.FileUtils) IOException(java.io.IOException) Maps(com.google.common.collect.Maps) File(java.io.File) TisSqlFormatException(com.qlangtech.tis.sql.parser.exception.TisSqlFormatException) JSON(com.alibaba.fastjson.JSON) ERRules(com.qlangtech.tis.sql.parser.er.ERRules) BasicModule(com.qlangtech.tis.runtime.module.action.BasicModule) AppDomainInfo(com.qlangtech.tis.manage.common.AppDomainInfo) CoreAction(com.qlangtech.tis.coredefine.module.action.CoreAction) Assert(com.qlangtech.tis.common.utils.Assert) SqlTaskNodeMeta(com.qlangtech.tis.sql.parser.SqlTaskNodeMeta) IUser(com.qlangtech.tis.manage.common.IUser) PrimaryTableMeta(com.qlangtech.tis.sql.parser.er.PrimaryTableMeta) JSONArray(org.json.JSONArray) PluginDescMeta(com.qlangtech.tis.coredefine.module.action.PluginDescMeta) OfflineManager(com.qlangtech.tis.offline.module.manager.impl.OfflineManager) DataxReader(com.qlangtech.tis.datax.impl.DataxReader) DescriptorsJSON(com.qlangtech.tis.util.DescriptorsJSON) Descriptor(com.qlangtech.tis.extension.Descriptor) JSONObject(org.json.JSONObject) Func(com.qlangtech.tis.manage.spring.aop.Func)

Example 3 with IPluginStore

use of com.qlangtech.tis.plugin.IPluginStore in project tis by qlangtech.

the class HeteroEnum method getPluginStore.

@Override
public IPluginStore getPluginStore(IPluginContext pluginContext, UploadPluginMeta pluginMeta) {
    IPluginStore store = null;
    if (this == HeteroEnum.APP_SOURCE) {
        final String dataxName = (pluginMeta.getExtraParam(DataxUtils.DATAX_NAME));
        if (StringUtils.isEmpty(dataxName)) {
            throw new IllegalArgumentException("plugin extra param 'DataxUtils.DATAX_NAME'" + DataxUtils.DATAX_NAME + " can not be null");
        }
        store = com.qlangtech.tis.manage.IAppSource.getPluginStore(pluginContext, dataxName);
    } else if (this == HeteroEnum.DATAX_WRITER || this == HeteroEnum.DATAX_READER) {
        // final String dataxName = pluginMeta.getExtraParam(DataxUtils.DATAX_NAME);
        // if (StringUtils.isEmpty(dataxName)) {
        // throw new IllegalArgumentException("plugin extra param 'DataxUtils.DATAX_NAME': '" + DataxUtils.DATAX_NAME + "' can not be null");
        // }
        // (this == HeteroEnum.DATAX_READER) ? DataxReader.getPluginStore(pluginContext, dataxName) : DataxWriter.getPluginStore(pluginContext, dataxName);
        store = getDataXReaderAndWriterStore(pluginContext, this == HeteroEnum.DATAX_READER, pluginMeta);
    } else if (this == PARAMS_CONFIG) {
        return new ParamsConfigPluginStore(pluginMeta);
    } else if (pluginContext.isDataSourceAware()) {
        PostedDSProp dsProp = PostedDSProp.parse(pluginMeta);
        if (StringUtils.isEmpty(dsProp.getDbname())) {
            // Collections.emptyList();
            return null;
        }
        store = TIS.getDataBasePluginStore(dsProp);
    } else {
        if (this.isAppNameAware()) {
            if (!pluginContext.isCollectionAware()) {
                throw new IllegalStateException(this.getExtensionPoint().getName() + " must be collection aware");
            }
            store = TIS.getPluginStore(pluginContext.getCollectionName(), this.extensionPoint);
        } else {
            store = TIS.getPluginStore(this.extensionPoint);
        }
    }
    Objects.requireNonNull(store, "plugin store can not be null");
    return store;
}
Also used : IPluginStore(com.qlangtech.tis.plugin.IPluginStore) PostedDSProp(com.qlangtech.tis.plugin.ds.PostedDSProp) ParamsConfigPluginStore(com.qlangtech.tis.plugin.credentials.ParamsConfigPluginStore)

Example 4 with IPluginStore

use of com.qlangtech.tis.plugin.IPluginStore in project tis by qlangtech.

the class GenerateDAOAndIncrScript method generateIncrScript.

public void generateIncrScript(Context context, IndexIncrStatus incrStatus, boolean compilerAndPackage, Map<DBNode, List<String>> dbNameMap) {
    try {
        // final Map<DBNode, List<String>> dbNameMap = Collections.unmodifiableMap(indexStreamCodeGenerator.getDbTables());
        File sourceRoot = StreamContextConstant.getStreamScriptRootDir(indexStreamCodeGenerator.collection, indexStreamCodeGenerator.incrScriptTimestamp);
        if (// 检查Faild Token文件是否存在
        !indexStreamCodeGenerator.isIncrScriptDirCreated() || incrStreamCodeCompileFaild(sourceRoot)) {
            /**
             * *********************************************************************************
             * 自动生成scala代码
             * ***********************************************************************************
             */
            indexStreamCodeGenerator.generateStreamScriptCode();
            // 生成依赖dao依赖元数据信息
            DBNode.dump(dbNameMap.keySet().stream().collect(Collectors.toList()), StreamContextConstant.getDbDependencyConfigMetaFile(indexStreamCodeGenerator.collection, indexStreamCodeGenerator.incrScriptTimestamp));
            /**
             * *********************************************************************************
             * 生成spring相关配置文件
             * ***********************************************************************************
             */
            indexStreamCodeGenerator.generateConfigFiles();
        }
        incrStatus.setIncrScriptMainFileContent(indexStreamCodeGenerator.readIncrScriptMainFileContent());
        // TODO 真实生产环境中需要 和 代码build阶段分成两步
        if (compilerAndPackage) {
            IPluginStore pluginStore = HeteroEnum.INCR_STREAM_CONFIG.getPluginStore(IPluginContext.namedContext(this.indexStreamCodeGenerator.collection), null);
            IncrStreamFactory streamFactory = (IncrStreamFactory) pluginStore.getPlugin();
            Objects.requireNonNull(streamFactory, "relevant streamFactory can not be null,collection:" + this.indexStreamCodeGenerator.collection);
            // CompileAndPackage packager = new CompileAndPackage();
            streamFactory.getCompileAndPackageManager().process(context, this.msgHandler, indexStreamCodeGenerator.collection, dbNameMap.entrySet().stream().collect(Collectors.toMap((e) -> e.getKey(), (e) -> e.getValue())), sourceRoot, indexStreamCodeGenerator.getSpringXmlConfigsObjectsContext());
        }
    } catch (Exception e) {
        // 将原始文件删除干净
        try {
            FileUtils.forceDelete(indexStreamCodeGenerator.getStreamCodeGenerator().getIncrScriptDir());
        } catch (Throwable ex) {
        // ex.printStackTrace();
        }
        throw new RuntimeException(e);
    }
}
Also used : IPluginStore(com.qlangtech.tis.plugin.IPluginStore) File(java.io.File) IncrStreamFactory(com.qlangtech.tis.plugin.incr.IncrStreamFactory) IOException(java.io.IOException)

Aggregations

IPluginStore (com.qlangtech.tis.plugin.IPluginStore)4 TIS (com.qlangtech.tis.TIS)2 DataxReader (com.qlangtech.tis.datax.impl.DataxReader)2 Descriptor (com.qlangtech.tis.extension.Descriptor)2 IdentityName (com.qlangtech.tis.plugin.IdentityName)2 ParamsConfigPluginStore (com.qlangtech.tis.plugin.credentials.ParamsConfigPluginStore)2 File (java.io.File)2 IOException (java.io.IOException)2 Context (com.alibaba.citrus.turbine.Context)1 JSON (com.alibaba.fastjson.JSON)1 Lists (com.google.common.collect.Lists)1 Maps (com.google.common.collect.Maps)1 Pager (com.koubei.web.tag.pager.Pager)1 IPluginEnum (com.qlangtech.tis.IPluginEnum)1 FullbuildPhase (com.qlangtech.tis.assemble.FullbuildPhase)1 MQListenerFactory (com.qlangtech.tis.async.message.client.consumer.impl.MQListenerFactory)1 Assert (com.qlangtech.tis.common.utils.Assert)1 ParamsConfig (com.qlangtech.tis.config.ParamsConfig)1 CoreAction (com.qlangtech.tis.coredefine.module.action.CoreAction)1 DataxAction (com.qlangtech.tis.coredefine.module.action.DataxAction)1