Examples with DataxProcessor com.qlangtech.tis.datax.impl.DataxProcessor used on opensource projects

Example 11 with DataxProcessor

use of com.qlangtech.tis.datax.impl.DataxProcessor in project tis by qlangtech.

the class SchemaAction method doGetEsTplFields.

// private static Logger log = LoggerFactory.getLogger(SchemaAction.class);
/**
 * DataX 创建流程中取得es的默认字段
 *
 * @param context
 * @throws Exception
 */
public void doGetEsTplFields(Context context) throws Exception {
    String dataxName = this.getString(DataxUtils.DATAX_NAME);
    StepType stepType = StepType.parse(this.getString("stepType"));
    // DataxProcessor process = DataxProcessor.load(this, dataxName);
    // IDataxProcessor.TableMap tabMapper = null;
    // ESField field = null;
    ISearchEngineTypeTransfer typeTransfer = ISearchEngineTypeTransfer.load(this, dataxName);
    DataxReader dataxReader = DataxReader.load(this, dataxName);
    // ESField field = null;
    if (stepType.update) {
        DataxProcessor dataxProcessor = DataxProcessor.load(this, dataxName);
        for (Map.Entry<String, IDataxProcessor.TableAlias> e : dataxProcessor.getTabAlias().entrySet()) {
            writerStructFields(context, e.getValue(), typeTransfer);
            return;
        }
    } else {
        for (ISelectedTab tab : dataxReader.getSelectedTabs()) {
            // ESSchema parseResult = new ESSchema();
            SchemaMetaContent tplSchema = typeTransfer.initSchemaMetaContent(tab);
            this.setBizResult(context, tplSchema.toJSON());
            return;
        }
    }
    throw new IllegalStateException("have not find any tab in DataXReader");
}

Example 12 with DataxProcessor

use of com.qlangtech.tis.datax.impl.DataxProcessor in project plugins by qlangtech.

the class HudiSinkFactory method getTableMeta.

private Pair<HudiSelectedTab, HudiTableMeta> getTableMeta(String tableName) {
    if (StringUtils.isEmpty(tableName)) {
        throw new IllegalArgumentException("param tableName can not empty");
    }
    if (tableMetas == null) {
        if (StringUtils.isEmpty(this.dataXName)) {
            throw new IllegalStateException("prop dataXName can not be null");
        }
        tableMetas = Maps.newHashMap();
        DataxProcessor dataXProcessor = DataxProcessor.load(null, this.dataXName);
        IDataxReader reader = dataXProcessor.getReader(null);
        Map<String, HudiSelectedTab> selTabs = reader.getSelectedTabs().stream().map((tab) -> (HudiSelectedTab) tab).collect(Collectors.toMap((tab) -> tab.getName(), (tab) -> tab));
        List<File> dataxCfgFile = dataXProcessor.getDataxCfgFileNames(null);
        Configuration cfg = null;
        Configuration paramCfg = null;
        String table = null;
        HudiTableMeta tableMeta = null;
        for (File f : dataxCfgFile) {
            cfg = Configuration.from(f);
            paramCfg = cfg.getConfiguration("job.content[0].writer.parameter");
            if (paramCfg == null) {
                throw new NullPointerException("paramCfg can not be null,relevant path:" + f.getAbsolutePath());
            }
            table = paramCfg.getString("fileName");
            if (StringUtils.isEmpty(table)) {
                throw new IllegalStateException("table can not be null:" + paramCfg.toJSON());
            }
            tableMetas.put(table, Pair.of(Objects.requireNonNull(selTabs.get(table), "tab:" + table + " relevant 'HudiSelectedTab' can not be null"), new HudiTableMeta(paramCfg)));
        }
    }
    final Pair<HudiSelectedTab, HudiTableMeta> tabMeta = tableMetas.get(tableName);
    if (tabMeta == null || tabMeta.getRight().isColsEmpty()) {
        throw new IllegalStateException("table:" + tableName + " relevant colMetas can not be null,exist tables:" + tableMetas.keySet().stream().collect(Collectors.joining(",")));
    }
    return tabMeta;
}

Example 13 with DataxProcessor

use of com.qlangtech.tis.datax.impl.DataxProcessor in project plugins by qlangtech.

the class FlinkTaskNodeController method getIncrJobRecordFile.

private File getIncrJobRecordFile(TargetResName collection) {
    DataxProcessor processor = DataxProcessor.load(null, collection.getName());
    // assertTrue("launchResult must success", launchResult.get());
    // EasyMock.verify(jarLoader);
    File dataXWorkDir = processor.getDataXWorkDir(null);
    return new File(dataXWorkDir, "incrJob.log");
}

Example 14 with DataxProcessor

use of com.qlangtech.tis.datax.impl.DataxProcessor in project plugins by qlangtech.

the class TestDataXClickhouseWriter method testRealDump.

public void testRealDump() throws Exception {
    ClickHouseTest houseTest = createDataXWriter();
    houseTest.writer.autoCreateTable = true;
    DataxProcessor dataXProcessor = EasyMock.mock("dataXProcessor", DataxProcessor.class);
    File createDDLDir = new File(".");
    File createDDLFile = null;
    try {
        createDDLFile = new File(createDDLDir, targetTableName + IDataxProcessor.DATAX_CREATE_DDL_FILE_NAME_SUFFIX);
        FileUtils.write(createDDLFile, com.qlangtech.tis.extension.impl.IOUtils.loadResourceFromClasspath(DataXClickhouseWriter.class, "create_ddl_customer_order_relation.sql"), TisUTF8.get());
        EasyMock.expect(dataXProcessor.getDataxCreateDDLDir(null)).andReturn(createDDLDir);
        DataxWriter.dataxWriterGetter = (dataXName) -> {
            return houseTest.writer;
        };
        DataxProcessor.processorGetter = (dataXName) -> {
            assertEquals(testDataXName, dataXName);
            return dataXProcessor;
        };
        EasyMock.replay(dataXProcessor);
        DataXClickhouseWriter writer = new DataXClickhouseWriter();
        WriterTemplate.realExecuteDump(clickhouse_datax_writer_assert_without_optional, writer);
        EasyMock.verify(dataXProcessor);
    } finally {
        FileUtils.forceDelete(createDDLFile);
    }
}

Example 15 with DataxProcessor

use of com.qlangtech.tis.datax.impl.DataxProcessor in project plugins by qlangtech.

the class DataXHudiWriter method createPostTask.

@Override
public IRemoteTaskTrigger createPostTask(IExecChainContext execContext, ISelectedTab tab) {
    if (generateCfgs == null) {
        generateCfgs = new AtomicReference<>();
    }
    DataXCfgGenerator.GenerateCfgs genCfg = generateCfgs.updateAndGet((pre) -> {
        if (pre == null) {
            if (dataXName == null) {
                throw new IllegalStateException("prop dataXName can not be null");
            }
            DataxProcessor dataxProcessor = DataxProcessor.load(null, dataXName);
            pre = DataXCfgGenerator.GenerateCfgs.readFromGen(dataxProcessor.getDataxCfgDir(null));
            logger.info("create GenerateCfgs with genTime:" + pre.getGenTime());
            return pre;
        }
        return pre;
    });
    return new HudiDumpPostTask(execContext, (HudiSelectedTab) tab, this, genCfg);
}