Examples with DataxWriter com.qlangtech.tis.datax.impl.DataxWriter used on opensource projects

Example 1 with DataxWriter

use of com.qlangtech.tis.datax.impl.DataxWriter in project plugins by qlangtech.

the class TestDataxMySQLWriter method testFieldCount.

public void testFieldCount() throws Exception {
    DataxMySQLWriter mySQLWriter = new DataxMySQLWriter();
    Descriptor<DataxWriter> descriptor = mySQLWriter.getDescriptor();
    PluginFormProperties pluginFormPropertyTypes = descriptor.getPluginFormPropertyTypes();
    assertTrue(pluginFormPropertyTypes instanceof RootFormProperties);
    assertEquals(8, pluginFormPropertyTypes.getKVTuples().size());
}

Example 2 with DataxWriter

use of com.qlangtech.tis.datax.impl.DataxWriter in project plugins by qlangtech.

the class WriterTemplate method realExecuteDump.

/**
 * dataXWriter执行
 *
 * @param writerJson
 * @param dataxWriter
 * @throws IllegalAccessException
 */
public static void realExecuteDump(final String writerJson, IDataXPluginMeta dataxWriter, Function<Configuration, Configuration>... cfgSetter) throws IllegalAccessException {
    final JarLoader uberClassLoader = new JarLoader(new String[] { "." });
    DataxExecutor.initializeClassLoader(Sets.newHashSet("plugin.reader.streamreader", "plugin.writer." + dataxWriter.getDataxMeta().getName()), uberClassLoader);
    // Map<String, JarLoader> jarLoaderCenter = (Map<String, JarLoader>) jarLoaderCenterField.get(null);
    // jarLoaderCenter.clear();
    // 
    // 
    // jarLoaderCenter.put("plugin.reader.streamreader", uberClassLoader);
    // jarLoaderCenter.put("plugin.writer." + dataxWriter.getDataxMeta().getName(), uberClassLoader);
    Configuration allConf = // 
    IOUtils.loadResourceFromClasspath(// 
    MockDataxReaderContext.class, "container.json", true, (input) -> {
        Configuration cfg = Configuration.from(input);
        // "streamreader": {
        // "class": "com.alibaba.datax.plugin.reader.streamreader.StreamReader"
        // }
        cfg.set("plugin.reader.streamreader.class", "com.alibaba.datax.plugin.reader.streamreader.StreamReader");
        cfg.set("plugin.writer." + dataxWriter.getDataxMeta().getName() + ".class", dataxWriter.getDataxMeta().getImplClass());
        // 
        cfg.set(// 
        "job.content[0].writer", IOUtils.loadResourceFromClasspath(dataxWriter.getClass(), writerJson, true, (writerJsonInput) -> {
            Configuration c = Configuration.from(writerJsonInput);
            for (Function<Configuration, Configuration> setter : cfgSetter) {
                c = setter.apply(c);
            }
            return c;
        }));
        return cfg;
    });
    // 绑定column转换信息
    ColumnCast.bind(allConf);
    LoadUtil.bind(allConf);
    JobContainer container = new JobContainer(allConf);
    container.start();
}

Example 3 with DataxWriter

use of com.qlangtech.tis.datax.impl.DataxWriter in project tis by qlangtech.

the class DataxExecutor method startWork.

/**
 * 开始执行数据同步任务
 *
 * @param dataxName
 * @throws IOException
 * @throws Exception
 */
public void startWork(String dataxName, Integer jobId, String jobName, String execTimeStamp, IDataxProcessor dataxProcessor, final JarLoader uberClassLoader) throws IOException, Exception {
    try {
        Objects.requireNonNull(dataxProcessor, "dataxProcessor can not be null");
        KeyedPluginStore<DataxReader> readerStore = DataxReader.getPluginStore(null, dataxName);
        KeyedPluginStore<DataxWriter> writerStore = DataxWriter.getPluginStore(null, dataxName);
        File jobPath = new File(dataxProcessor.getDataxCfgDir(null), jobName);
        ;
        DataxReader reader = readerStore.getPlugin();
        Objects.requireNonNull(reader, "dataxName:" + dataxName + " relevant reader can not be null");
        DataxWriter writer = writerStore.getPlugin();
        Objects.requireNonNull(writer, "dataxName:" + dataxName + " relevant writer can not be null");
        this.readerMeta = reader.getDataxMeta();
        this.writerMeta = writer.getDataxMeta();
        Objects.requireNonNull(readerMeta, "readerMeta can not be null");
        Objects.requireNonNull(writerMeta, "writerMeta can not be null");
        initializeClassLoader(Sets.newHashSet(this.getPluginReaderKey(), this.getPluginWriterKey()), uberClassLoader);
        entry(new DataXJobArgs(jobPath, jobId, "standalone", execTimeStamp), jobName);
    } catch (Throwable e) {
        throw new Exception(e);
    } finally {
        cleanPerfTrace();
    }
}

Example 4 with DataxWriter

use of com.qlangtech.tis.datax.impl.DataxWriter in project tis by qlangtech.

the class CoreAction method doGetDataXReaderWriterDesc.

private static IndexIncrStatus doGetDataXReaderWriterDesc(String appName) throws Exception {
    IndexIncrStatus incrStatus = new IndexIncrStatus();
    DataxProcessor dataxProcessor = DataxProcessor.load(null, appName);
    DataxWriter writer = (DataxWriter) dataxProcessor.getWriter(null);
    incrStatus.setWriterDesc(createDescVals(writer.getDescriptor()));
    DataxReader reader = (DataxReader) dataxProcessor.getReader(null);
    incrStatus.setReaderDesc(createDescVals(reader.getDescriptor()));
    return incrStatus;
}