Examples with DataxProcessor com.qlangtech.tis.datax.impl.DataxProcessor used on opensource projects

Example 6 with DataxProcessor

use of com.qlangtech.tis.datax.impl.DataxProcessor in project plugins by qlangtech.

the class InitWriterTable method process.

/**
 * 初始化表RDBMS的表，如果表不存在就创建表
 *
 * @param
 * @throws Exception
 */
public static void process(String dataXName, String tableName, List<String> jdbcUrls) throws Exception {
    if (StringUtils.isEmpty(dataXName)) {
        throw new IllegalArgumentException("param dataXName can not be null");
    }
    BasicDataXRdbmsWriter<BasicDataSourceFactory> dataXWriter = (BasicDataXRdbmsWriter<BasicDataSourceFactory>) DataxWriter.load(null, dataXName);
    Objects.requireNonNull(dataXWriter, "dataXWriter can not be null,dataXName:" + dataXName);
    boolean autoCreateTable = dataXWriter.autoCreateTable;
    if (autoCreateTable) {
        DataxProcessor processor = DataxProcessor.load(null, dataXName);
        File createDDL = new File(processor.getDataxCreateDDLDir(null), tableName + IDataxProcessor.DATAX_CREATE_DDL_FILE_NAME_SUFFIX);
        if (!createDDL.exists()) {
            throw new IllegalStateException("create table script is not exist:" + createDDL.getAbsolutePath());
        }
        BasicDataSourceFactory dsFactory = dataXWriter.getDataSourceFactory();
        String createScript = FileUtils.readFileToString(createDDL, TisUTF8.get());
        for (String jdbcUrl : jdbcUrls) {
            try (Connection conn = dsFactory.getConnection(jdbcUrl)) {
                List<String> tabs = Lists.newArrayList();
                dsFactory.refectTableInDB(tabs, conn);
                if (!tabs.contains(tableName)) {
                    // 表不存在
                    try (Statement statement = conn.createStatement()) {
                        logger.info("create table:{}\n   script:{}", tableName, createScript);
                        statement.execute(createScript);
                    }
                } else {
                    logger.info("table:{} already exist ,skip the create table step", tableName);
                }
            }
        }
    }
}

Example 7 with DataxProcessor

use of com.qlangtech.tis.datax.impl.DataxProcessor in project tis by qlangtech.

the class DataxExecutor method synchronizeDataXPluginsFromRemoteRepository.

public static void synchronizeDataXPluginsFromRemoteRepository(String dataxName, String jobName) {
    if (CenterResource.notFetchFromCenterRepository()) {
        return;
    }
    TIS.permitInitialize = false;
    try {
        if (StringUtils.isBlank(dataxName)) {
            throw new IllegalArgumentException("param dataXName can not be null");
        }
        if (StringUtils.isBlank(jobName)) {
            throw new IllegalArgumentException("param jobName can not be null");
        }
        KeyedPluginStore<DataxProcessor> processStore = IAppSource.getPluginStore(null, dataxName);
        // Lists.newArrayList(DataxReader.getPluginStore(dataxName), DataxWriter.getPluginStore(dataxName));
        List<IRepositoryResource> keyedPluginStores = Lists.newArrayList();
        keyedPluginStores.add(TIS.getPluginStore(ParamsConfig.class));
        // keyedPluginStores.add(TIS.getPluginStore(FileSystemFactory.class));
        keyedPluginStores.add(processStore);
        keyedPluginStores.add(DataxReader.getPluginStore(null, dataxName));
        keyedPluginStores.add(DataxWriter.getPluginStore(null, dataxName));
        ComponentMeta dataxComponentMeta = new ComponentMeta(keyedPluginStores);
        dataxComponentMeta.synchronizePluginsFromRemoteRepository();
        CenterResource.copyFromRemote2Local(TIS.KEY_TIS_PLUGIN_CONFIG + "/" + processStore.key.getSubDirPath() + "/" + DataxProcessor.DATAX_CFG_DIR_NAME + "/" + jobName, true);
        CenterResource.synchronizeSubFiles(TIS.KEY_TIS_PLUGIN_CONFIG + "/" + processStore.key.getSubDirPath() + "/" + DataxProcessor.DATAX_CREATE_DDL_DIR_NAME);
    } finally {
        TIS.permitInitialize = true;
    }
}

Example 8 with DataxProcessor

use of com.qlangtech.tis.datax.impl.DataxProcessor in project tis by qlangtech.

the class TestDataxExecutorSynRes method testSynchronizeCreateDDLFromRemoteRepository.

/**
 * create DDL 下的文件是否同步过来
 */
public void testSynchronizeCreateDDLFromRemoteRepository() {
    String dataX = "mysql_clickhouse";
    DataxExecutor.synchronizeDataXPluginsFromRemoteRepository(dataX, jobName);
    DataxProcessor dataxProcessor = IAppSource.load(null, dataX);
    File dataxCreateDDLDir = dataxProcessor.getDataxCreateDDLDir(null);
    List<String> synFiles = Lists.newArrayList(dataxCreateDDLDir.list((dir, name) -> !StringUtils.endsWith(name, CenterResource.KEY_LAST_MODIFIED_EXTENDION)));
    assertEquals(1, synFiles.size());
    String synFilesStr = synFiles.stream().collect(Collectors.joining(","));
    assertTrue(synFilesStr, synFiles.contains("customer_order_relation.sql"));
// assertTrue(synFilesStr, synFiles.contains("instancedetail.sql"));
}

Example 9 with DataxProcessor

use of com.qlangtech.tis.datax.impl.DataxProcessor in project tis by qlangtech.

the class TestDataxExecutorSynRes method testSynchronizeDataXPluginsFromRemoteRepository.

public void testSynchronizeDataXPluginsFromRemoteRepository() {
    DataxExecutor.synchronizeDataXPluginsFromRemoteRepository(dataXName, jobName);
    DataxProcessor dataxProcessor = IAppSource.load(null, dataXName);
    File dataxCfgDir = dataxProcessor.getDataxCfgDir(null);
    assertTrue(dataxCfgDir.getAbsolutePath(), dataxCfgDir.exists());
    File jobCfgFile = new File(dataxCfgDir, jobName);
    assertTrue("jobCfgFile must exist:" + jobCfgFile.getAbsolutePath(), jobCfgFile.exists());
}

Example 10 with DataxProcessor

use of com.qlangtech.tis.datax.impl.DataxProcessor in project tis by qlangtech.

the class CoreAction method doGetDataXReaderWriterDesc.

private static IndexIncrStatus doGetDataXReaderWriterDesc(String appName) throws Exception {
    IndexIncrStatus incrStatus = new IndexIncrStatus();
    DataxProcessor dataxProcessor = DataxProcessor.load(null, appName);
    DataxWriter writer = (DataxWriter) dataxProcessor.getWriter(null);
    incrStatus.setWriterDesc(createDescVals(writer.getDescriptor()));
    DataxReader reader = (DataxReader) dataxProcessor.getReader(null);
    incrStatus.setReaderDesc(createDescVals(reader.getDescriptor()));
    return incrStatus;
}