Examples with DataxReader com.qlangtech.tis.datax.impl.DataxReader used on opensource projects

Example 6 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project tis by qlangtech.

the class OfflineManager method getDbConfig.

public DBConfigSuit getDbConfig(IPluginContext pluginContext, DatasourceDb db) {
    Objects.requireNonNull(db, "instance of DatasourceDb can not be null");
    DBDataXReaderDescName dbDataXReaderDesc = this.getDBDataXReaderDescName(db.getName());
    DataxReader dbDataxReader = null;
    if (dbDataXReaderDesc.isSupportDataXReader()) {
        dbDataxReader = getDBDataxReader(pluginContext, db.getName());
    }
    DBConfigSuit dbSuit = new DBConfigSuit(db, dbDataXReaderDesc.isSupportDataXReader(), dbDataxReader != null);
    if (dbDataxReader != null) {
        List<ISelectedTab> selectedTabs = dbDataxReader.getSelectedTabs();
        dbSuit.addTabs(selectedTabs.stream().map((t) -> t.getName()).collect(Collectors.toList()));
    }
    PostedDSProp dbProp = new PostedDSProp(db.getName(), DbScope.DETAILED);
    IPluginStore<DataSourceFactory> dbStore = TIS.getDataBasePluginStore(dbProp);
    DataSourceFactory dsPlugin = dbStore.getPlugin();
    dbSuit.setDetailed(dsPlugin);
    DataSourceFactoryPluginStore facadeStore = TIS.getDataBasePluginStore(new PostedDSProp(db.getName(), DbScope.FACADE));
    if (facadeStore.getPlugin() != null) {
        dbSuit.setFacade(facadeStore.getPlugin());
    }
    return dbSuit;
}

Example 7 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project tis by qlangtech.

the class DataxExecutor method startWork.

/**
 * 开始执行数据同步任务
 *
 * @param dataxName
 * @throws IOException
 * @throws Exception
 */
public void startWork(String dataxName, Integer jobId, String jobName, String execTimeStamp, IDataxProcessor dataxProcessor, final JarLoader uberClassLoader) throws IOException, Exception {
    try {
        Objects.requireNonNull(dataxProcessor, "dataxProcessor can not be null");
        KeyedPluginStore<DataxReader> readerStore = DataxReader.getPluginStore(null, dataxName);
        KeyedPluginStore<DataxWriter> writerStore = DataxWriter.getPluginStore(null, dataxName);
        File jobPath = new File(dataxProcessor.getDataxCfgDir(null), jobName);
        ;
        DataxReader reader = readerStore.getPlugin();
        Objects.requireNonNull(reader, "dataxName:" + dataxName + " relevant reader can not be null");
        DataxWriter writer = writerStore.getPlugin();
        Objects.requireNonNull(writer, "dataxName:" + dataxName + " relevant writer can not be null");
        this.readerMeta = reader.getDataxMeta();
        this.writerMeta = writer.getDataxMeta();
        Objects.requireNonNull(readerMeta, "readerMeta can not be null");
        Objects.requireNonNull(writerMeta, "writerMeta can not be null");
        initializeClassLoader(Sets.newHashSet(this.getPluginReaderKey(), this.getPluginWriterKey()), uberClassLoader);
        entry(new DataXJobArgs(jobPath, jobId, "standalone", execTimeStamp), jobName);
    } catch (Throwable e) {
        throw new Exception(e);
    } finally {
        cleanPerfTrace();
    }
}

Example 8 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project tis by qlangtech.

the class CoreAction method doGetDataXReaderWriterDesc.

private static IndexIncrStatus doGetDataXReaderWriterDesc(String appName) throws Exception {
    IndexIncrStatus incrStatus = new IndexIncrStatus();
    DataxProcessor dataxProcessor = DataxProcessor.load(null, appName);
    DataxWriter writer = (DataxWriter) dataxProcessor.getWriter(null);
    incrStatus.setWriterDesc(createDescVals(writer.getDescriptor()));
    DataxReader reader = (DataxReader) dataxProcessor.getReader(null);
    incrStatus.setReaderDesc(createDescVals(reader.getDescriptor()));
    return incrStatus;
}

Example 9 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project tis by qlangtech.

the class SchemaAction method doGetEsTplFields.

// private static Logger log = LoggerFactory.getLogger(SchemaAction.class);
/**
 * DataX 创建流程中取得es的默认字段
 *
 * @param context
 * @throws Exception
 */
public void doGetEsTplFields(Context context) throws Exception {
    String dataxName = this.getString(DataxUtils.DATAX_NAME);
    StepType stepType = StepType.parse(this.getString("stepType"));
    // DataxProcessor process = DataxProcessor.load(this, dataxName);
    // IDataxProcessor.TableMap tabMapper = null;
    // ESField field = null;
    ISearchEngineTypeTransfer typeTransfer = ISearchEngineTypeTransfer.load(this, dataxName);
    DataxReader dataxReader = DataxReader.load(this, dataxName);
    // ESField field = null;
    if (stepType.update) {
        DataxProcessor dataxProcessor = DataxProcessor.load(this, dataxName);
        for (Map.Entry<String, IDataxProcessor.TableAlias> e : dataxProcessor.getTabAlias().entrySet()) {
            writerStructFields(context, e.getValue(), typeTransfer);
            return;
        }
    } else {
        for (ISelectedTab tab : dataxReader.getSelectedTabs()) {
            // ESSchema parseResult = new ESSchema();
            SchemaMetaContent tplSchema = typeTransfer.initSchemaMetaContent(tab);
            this.setBizResult(context, tplSchema.toJSON());
            return;
        }
    }
    throw new IllegalStateException("have not find any tab in DataXReader");
}

Example 10 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project plugins by qlangtech.

the class TISFlinkCDCStart method deploy.

private static void deploy(TargetResName dataxName, BasicFlinkSourceHandle tableStreamHandle, ReplicasSpec incrSpec, long timestamp) throws Exception {
    // BasicFlinkSourceHandle tisFlinkSourceHandle = new TISFlinkSourceHandle();
    if (tableStreamHandle == null) {
        throw new IllegalStateException("tableStreamHandle has not been instantiated");
    }
    // ElasticSearchSinkFactory esSinkFactory = new ElasticSearchSinkFactory();
    // IPluginContext pluginContext = IPluginContext.namedContext(dataxName.getName());
    // List<TISSinkFactory> sinkFactories = TISSinkFactory.sinkFactory.getPlugins(pluginContext, null);
    // logger.info("sinkFactories size:" + sinkFactories.size());
    // for (TISSinkFactory factory : sinkFactories) {
    // sinkFactory = factory;
    // break;
    // }
    // Objects.requireNonNull(sinkFactory, "sinkFactories.size():" + sinkFactories.size());
    tableStreamHandle.setSinkFuncFactory(TISSinkFactory.getIncrSinKFactory(dataxName.getName()));
    // List<MQListenerFactory> mqFactories = HeteroEnum.MQ.getPlugins(pluginContext, null);
    MQListenerFactory mqFactory = HeteroEnum.getIncrSourceListenerFactory(dataxName.getName());
    mqFactory.setConsumerHandle(tableStreamHandle);
    // for (MQListenerFactory factory : mqFactories) {
    // factory.setConsumerHandle(tableStreamHandle);
    // mqFactory = factory;
    // }
    // Objects.requireNonNull(mqFactory, "mqFactory can not be null, mqFactories size:" + mqFactories.size());
    IMQListener mq = mqFactory.create();
    IDataxProcessor dataXProcess = DataxProcessor.load(null, dataxName.getName());
    DataxReader reader = (DataxReader) dataXProcess.getReader(null);
    if (reader == null) {
        throw new IllegalStateException("dataXReader is illegal");
    }
    // DBConfigGetter rdbmsReader = (DBConfigGetter) reader;
    List<ISelectedTab> tabs = reader.getSelectedTabs();
    mq.start(dataxName, reader, tabs, dataXProcess);
}