Examples with DataxReader com.qlangtech.tis.datax.impl.DataxReader used on opensource projects

Example 1 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project plugins by qlangtech.

the class TestDataxMySQLReader method testGetPluginFormPropertyTypes.

public void testGetPluginFormPropertyTypes() {
    DataxMySQLReader mySQLReader = new DataxMySQLReader();
    Descriptor<DataxReader> descriptor = mySQLReader.getDescriptor();
    assertNotNull(descriptor);
    PluginFormProperties propertyTypes = descriptor.getPluginFormPropertyTypes();
    assertEquals(4, propertyTypes.getKVTuples().size());
}

Example 2 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project plugins by qlangtech.

the class TestDataXMongodbWriter method testDescriptorsJSONGenerate.

public void testDescriptorsJSONGenerate() {
    DataxReader dataxReader = EasyMock.createMock("dataxReader", DataxReader.class);
    List<ISelectedTab> selectedTabs = TestSelectedTabs.createSelectedTabs(1).stream().map((t) -> t).collect(Collectors.toList());
    for (ISelectedTab tab : selectedTabs) {
        for (ISelectedTab.ColMeta cm : tab.getCols()) {
            cm.setType(DataXReaderColType.STRING.dataType);
        }
    }
    EasyMock.expect(dataxReader.getSelectedTabs()).andReturn(selectedTabs).times(4);
    DataxReader.dataxReaderThreadLocal.set(dataxReader);
    EasyMock.replay(dataxReader);
    DataXMongodbWriter writer = new DataXMongodbWriter();
    assertTrue(writer instanceof IDataSourceFactoryGetter);
    DescriptorsJSON descJson = new DescriptorsJSON(writer.getDescriptor());
    JsonUtil.assertJSONEqual(DataXMongodbWriter.class, "mongdodb-datax-writer-descriptor.json", descJson.getDescriptorsJSON(), (m, e, a) -> {
        assertEquals(m, e, a);
    });
    JsonUtil.assertJSONEqual(DataXMongodbWriter.class, "mongdodb-datax-writer-descriptor.json", descJson.getDescriptorsJSON(), (m, e, a) -> {
        assertEquals(m, e, a);
    });
    EasyMock.verify(dataxReader);
}

Example 3 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project plugins by qlangtech.

the class TisDataXHdfsReader method getHdfsReaderPlugin.

private static DataXHdfsReader getHdfsReaderPlugin(Configuration cfg) {
    String dataxName = cfg.getNecessaryValue(DataxUtils.DATAX_NAME, HdfsWriterErrorCode.REQUIRED_VALUE);
    DataxReader dataxReader = DataxReader.load(null, dataxName);
    if (!(dataxReader instanceof DataXHdfsReader)) {
        throw new BasicHdfsWriterJob.JobPropInitializeException("datax reader must be type of 'DataXHdfsReader',but now is:" + dataxReader.getClass());
    }
    return (DataXHdfsReader) dataxReader;
}

Example 4 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project plugins by qlangtech.

the class DataXMongodbWriter method getDftColumn.

/**
 * 取得默认的列内容
 *
 * @return
 */
public static String getDftColumn() {
    // [{"name":"user_id","type":"string"},{"name":"user_name","type":"array","splitter":","}]
    JSONArray fields = new JSONArray();
    DataxReader dataReader = DataxReader.getThreadBingDataXReader();
    if (dataReader == null) {
        return "[]";
    }
    try {
        List<ISelectedTab> selectedTabs = dataReader.getSelectedTabs();
        if (CollectionUtils.isEmpty(selectedTabs)) {
            return "[]";
        }
        for (ISelectedTab tab : selectedTabs) {
            tab.getCols().forEach((col) -> {
                JSONObject field = new JSONObject();
                field.put("name", col.getName());
                field.put("type", col.getType().getCollapse().getLiteria());
                fields.add(field);
            });
            break;
        }
    } catch (Exception e) {
        logger.warn(e.getMessage(), e);
        return "[]";
    }
    return JsonUtil.toString(fields);
}

Example 5 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project tis by qlangtech.

the class IDataxProcessor method getDataXWorkDir.

static File getDataXWorkDir(IPluginContext pluginContext, String appName) {
    KeyedPluginStore<DataxReader> readerStore = DataxReader.getPluginStore(pluginContext, appName);
    File targetFile = readerStore.getTargetFile();
    return targetFile.getParentFile();
}