Examples with TargetResName com.qlangtech.tis.coredefine.module.action.TargetResName used on opensource projects

Example 16 with TargetResName

use of com.qlangtech.tis.coredefine.module.action.TargetResName in project plugins by qlangtech.

the class FlinkCDCOracleSourceFunction method start.

@Override
public JobExecutionResult start(TargetResName channalName, IDataxReader dataSource, List<ISelectedTab> tabs, IDataxProcessor dataXProcessor) throws MQConsumeException {
    try {
        BasicDataXRdbmsReader reader = (BasicDataXRdbmsReader) dataSource;
        BasicDataSourceFactory f = (BasicDataSourceFactory) reader.getDataSourceFactory();
        SourceChannel sourceChannel = new SourceChannel(SourceChannel.getSourceFunction(f, (tab) -> tab.getTabName(), tabs, (dbHost, dbs, tbs, debeziumProperties) -> {
            return dbs.stream().map((databaseName) -> {
                SourceFunction<DTO> sourceFunction = OracleSource.<DTO>builder().hostname(dbHost).debeziumProperties(debeziumProperties).port(f.port).startupOptions(sourceFactory.getStartupOptions()).database(// monitor XE database
                StringUtils.upperCase(f.dbName)).tableList(// monitor products table
                tbs.toArray(new String[tbs.size()])).username(f.getUserName()).password(f.getPassword()).deserializer(// converts SourceRecord to JSON String
                new TISDeserializationSchema()).build();
                return new ReaderSource(dbHost + ":" + f.port + "_" + databaseName, sourceFunction);
            }).collect(Collectors.toList());
        }));
        for (ISelectedTab tab : tabs) {
            sourceChannel.addFocusTab(tab.getName());
        }
        return (JobExecutionResult) getConsumerHandle().consume(channalName, sourceChannel, dataXProcessor);
    } catch (Exception e) {
        throw new MQConsumeException(e.getMessage(), e);
    }
}

Example 17 with TargetResName

use of com.qlangtech.tis.coredefine.module.action.TargetResName in project plugins by qlangtech.

the class FlinkCDCPostgreSQLSourceFunction method start.

@Override
public JobExecutionResult start(TargetResName dataxName, IDataxReader dataSource, List<ISelectedTab> tabs, IDataxProcessor dataXProcessor) throws MQConsumeException {
    try {
        BasicDataXRdbmsReader rdbmsReader = (BasicDataXRdbmsReader) dataSource;
        final BasicDataSourceFactory dsFactory = (BasicDataSourceFactory) rdbmsReader.getDataSourceFactory();
        BasicDataSourceFactory.ISchemaSupported schemaSupported = (BasicDataSourceFactory.ISchemaSupported) dsFactory;
        if (StringUtils.isEmpty(schemaSupported.getDBSchema())) {
            throw new IllegalStateException("dsFactory:" + dsFactory.dbName + " relevant dbSchema can not be null");
        }
        SourceChannel sourceChannel = new SourceChannel(SourceChannel.getSourceFunction(dsFactory, (tab) -> schemaSupported.getDBSchema() + "." + tab.getTabName(), tabs, (dbHost, dbs, tbs, debeziumProperties) -> {
            DateTimeConverter.setDatetimeConverters(PGDateTimeConverter.class.getName(), debeziumProperties);
            return dbs.stream().map((dbname) -> {
                SourceFunction<DTO> sourceFunction = PostgreSQLSource.<DTO>builder().hostname(dbHost).port(dsFactory.port).database(// monitor postgres database
                dbname).schemaList(// monitor inventory schema
                schemaSupported.getDBSchema()).tableList(// monitor products table
                tbs.toArray(new String[tbs.size()])).username(dsFactory.userName).password(dsFactory.password).debeziumProperties(debeziumProperties).deserializer(// converts SourceRecord to JSON String
                new TISDeserializationSchema()).build();
                return new ReaderSource(dbHost + ":" + dsFactory.port + "_" + dbname, sourceFunction);
            }).collect(Collectors.toList());
        }));
        for (ISelectedTab tab : tabs) {
            sourceChannel.addFocusTab(tab.getName());
        }
        return (JobExecutionResult) getConsumerHandle().consume(dataxName, sourceChannel, dataXProcessor);
    } catch (Exception e) {
        throw new MQConsumeException(e.getMessage(), e);
    }
}

Example 18 with TargetResName

use of com.qlangtech.tis.coredefine.module.action.TargetResName in project tis by qlangtech.

the class DataXJobWorker method getServerLaunchTokenFile.

protected File getServerLaunchTokenFile() {
    TargetResName workerType = ((BasicDescriptor) this.getDescriptor()).getWorkerType();
    IPluginStore<DataXJobWorker> workerStore = getJobWorkerStore(workerType);
    File target = workerStore.getTargetFile();
    return new File(target.getParentFile(), (workerType.getName() + ".launch_token"));
}

Example 19 with TargetResName

use of com.qlangtech.tis.coredefine.module.action.TargetResName in project tis by qlangtech.

the class PluginItems method save.

public List<Describable> save(Context context) {
    Objects.requireNonNull(this.pluginContext, "pluginContext can not be null");
    if (items == null) {
        throw new IllegalStateException("prop items can not be null");
    }
    Descriptor.ParseDescribable describable = null;
    AttrValMap attrValMap = null;
    List<Descriptor.ParseDescribable<?>> dlist = Lists.newArrayList();
    List<Describable> describableList = Lists.newArrayList();
    if (this.pluginMeta.isAppend()) {
        IPluginStore pluginStore = heteroEnum.getPluginStore(this.pluginContext, this.pluginMeta);
        if (pluginStore != null) {
            List<Describable> plugins = pluginStore.getPlugins();
            boolean firstSkip = false;
            for (Describable p : plugins) {
                if (!firstSkip) {
                    firstSkip = true;
                    Descriptor.ParseDescribable describablesWithMeta = PluginStore.getDescribablesWithMeta(pluginStore, p);
                    dlist.add(describablesWithMeta);
                } else {
                    dlist.add(new Descriptor.ParseDescribable(p));
                }
            }
        }
    }
    for (int i = 0; i < this.items.size(); i++) {
        attrValMap = this.items.get(i);
        /**
         *====================================================
         * 将客户端post数据包装
         *       ======================================================
         */
        describable = attrValMap.createDescribable(pluginContext);
        dlist.add(describable);
        describableList.add((Describable) describable.instance);
    }
    IPluginStoreSave<?> store = null;
    if (heteroEnum == HeteroEnum.APP_SOURCE) {
        for (Descriptor.ParseDescribable<?> d : dlist) {
            if (d.instance instanceof IdentityName) {
                store = IAppSource.getPluginStore(pluginContext, ((IdentityName) d.instance).identityValue());
                break;
            }
        }
        Objects.requireNonNull(store, "plugin type:" + heteroEnum.getIdentity() + " can not find relevant Store");
    } else if (this.pluginContext.isDataSourceAware()) {
        store = new IPluginStoreSave<DataSourceFactory>() {

            @Override
            public boolean setPlugins(IPluginContext pluginContext, Optional<Context> context, List<Descriptor.ParseDescribable<DataSourceFactory>> dlist, boolean update) {
                for (Descriptor.ParseDescribable<DataSourceFactory> plugin : dlist) {
                    PostedDSProp dbExtraProps = PostedDSProp.parse(pluginMeta);
                    if (DbScope.DETAILED == dbExtraProps.getDbType()) {
                        dbExtraProps.setDbname(plugin.instance.identityValue());
                    }
                    boolean success = TIS.getDataBasePluginStore(dbExtraProps).setPlugins(pluginContext, context, Collections.singletonList(plugin), dbExtraProps.isUpdate());
                    if (!success) {
                        return false;
                    }
                }
                return true;
            }
        };
    } else if (heteroEnum == HeteroEnum.DATAX_WRITER || heteroEnum == HeteroEnum.DATAX_READER) {
        // final String dataxName = pluginMeta.getExtraParam(DataxUtils.DATAX_NAME);
        // if (StringUtils.isEmpty(dataxName)) {
        // String saveDbName = pluginMeta.getExtraParam(DataxUtils.DATAX_DB_NAME);
        // if (StringUtils.isNotBlank(saveDbName)) {
        // store = DataxReader.getPluginStore(this.pluginContext, true, saveDbName);
        // } else {
        // throw new IllegalArgumentException("plugin extra param " + DataxUtils.DATAX_NAME + " can not be null");
        // }
        // } else {
        // KeyedPluginStore<?> keyStore = (heteroEnum == HeteroEnum.DATAX_READER)
        // ? DataxReader.getPluginStore(this.pluginContext, dataxName) : DataxWriter.getPluginStore(this.pluginContext, dataxName);
        // store = keyStore;
        // }
        // if ((heteroEnum == HeteroEnum.DATAX_READER)) {
        // for (Descriptor.ParseDescribable<?> dataXReader : dlist) {
        // DataSourceMeta sourceMeta = (DataSourceMeta) dataXReader.instance;
        // pluginContext.setBizResult(context, sourceMeta.getTablesInDB());
        // }
        // }
        store = HeteroEnum.getDataXReaderAndWriterStore(this.pluginContext, this.heteroEnum == HeteroEnum.DATAX_READER, this.pluginMeta);
        Optional<IPropertyType.SubFormFilter> subFormFilter = pluginMeta.getSubFormFilter();
        if (subFormFilter.isPresent()) {
            IPropertyType.SubFormFilter filter = subFormFilter.get();
            Optional<Descriptor> firstDesc = heteroEnum.descriptors().stream().filter((des) -> filter.match((Descriptor) des)).map((des) -> (Descriptor) des).findFirst();
            if (!firstDesc.isPresent()) {
                throw new IllegalStateException("can not find relevant descriptor:" + filter.uploadPluginMeta.toString());
            }
            // Describable p = keyStore.getPlugin();
            // Descriptor des = p.getDescriptor();
            PluginFormProperties pluginProps = firstDesc.get().getPluginFormPropertyTypes(subFormFilter);
            store = pluginProps.accept(new PluginFormProperties.IVisitor() {

                @Override
                public IPluginStoreSave<?> visit(SuFormProperties props) {
                    // 为了在更新插件时候不把plugin上的@SubForm标记的属性覆盖掉，需要先将老的plugin上的值覆盖到新http post过来的反序列化之后的plugin上
                    // IPluginContext pluginContext, String appname, SuFormProperties subfieldForm, Class<TT> clazz
                    Class<Describable> clazz = (Class<Describable>) heteroEnum.getExtensionPoint();
                    // DataxReader.SubFieldFormAppKey<Describable> key
                    // = new DataxReader.SubFieldFormAppKey<>(pluginContext, dataxName, props, clazz);
                    // 
                    DataxReader.SubFieldFormAppKey<Describable> key = HeteroEnum.createDataXReaderAndWriterRelevant(pluginContext, pluginMeta, new HeteroEnum.DataXReaderAndWriterRelevantCreator<DataxReader.SubFieldFormAppKey<Describable>>() {

                        @Override
                        public DataxReader.SubFieldFormAppKey<Describable> dbRelevant(IPluginContext pluginContext, String saveDbName) {
                            return new DataxReader.SubFieldFormAppKey<>(pluginContext, true, saveDbName, props, clazz);
                        }

                        @Override
                        public DataxReader.SubFieldFormAppKey<Describable> appRelevant(IPluginContext pluginContext, String dataxName) {
                            return new DataxReader.SubFieldFormAppKey<>(pluginContext, false, dataxName, props, clazz);
                        }
                    });
                    return KeyedPluginStore.getPluginStore(key);
                }
            });
        }
    } else if (heteroEnum == HeteroEnum.PARAMS_CONFIG) {
        store = new ParamsConfigPluginStore(this.pluginMeta);
    } else if (heteroEnum == HeteroEnum.DATAX_WORKER) {
        if (!this.pluginContext.isCollectionAware()) {
            throw new IllegalStateException("must be collection aware");
        }
        store = DataXJobWorker.getJobWorkerStore(new TargetResName(this.pluginContext.getCollectionName()));
    // } else if (this.pluginContext.isCollectionAware()) {
    // store = TIS.getPluginStore(this.pluginContext.getCollectionName(), heteroEnum.getExtensionPoint());
    } else {
        if (heteroEnum.isAppNameAware()) {
            if (!this.pluginContext.isCollectionAware()) {
                throw new IllegalStateException(heteroEnum.getExtensionPoint().getName() + " must be collection aware");
            }
            store = TIS.getPluginStore(this.pluginContext.getCollectionName(), heteroEnum.getExtensionPoint());
        } else {
            store = TIS.getPluginStore(heteroEnum.getExtensionPoint());
        }
    // if (heteroEnum.isAppNameAware()) {
    // throw new IllegalStateException(heteroEnum.getExtensionPoint().getName() + " must be app name aware");
    // }
    }
    // dlist
    if (!store.setPlugins(pluginContext, Optional.of(context), convert(dlist))) {
        return Collections.emptyList();
    }
    observable.notifyObservers(new PluginItemsSaveEvent(this.pluginContext, this.heteroEnum, describableList));
    return describableList;
}