Examples with DataxReader com.qlangtech.tis.datax.impl.DataxReader used on opensource projects

Example 16 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project plugins by qlangtech.

the class ReaderTemplate method realExecute.

/**
 * dataXWriter执行
 *
 * @param readerJson
 * @param dataxReader
 * @throws IllegalAccessException
 */
public static void realExecute(final String readerJson, IDataXPluginMeta dataxReader, Writer contentWriter) throws IllegalAccessException {
    final JarLoader uberClassLoader = new JarLoader(new String[] { "." });
    // DataxExecutor.initializeClassLoader(
    // Sets.newHashSet("plugin.reader.streamreader", "plugin.writer." + dataxReader.getDataxMeta().getName()), uberClassLoader);
    DataxExecutor.initializeClassLoader(Sets.newHashSet("plugin.reader." + dataxReader.getDataxMeta().getName(), "plugin.writer.streamwriter"), uberClassLoader);
    // Map<String, JarLoader> jarLoaderCenter = (Map<String, JarLoader>) jarLoaderCenterField.get(null);
    // jarLoaderCenter.clear();
    // 
    // 
    // jarLoaderCenter.put("plugin.reader.streamreader", uberClassLoader);
    // jarLoaderCenter.put("plugin.writer." + dataxWriter.getDataxMeta().getName(), uberClassLoader);
    Configuration allConf = // 
    IOUtils.loadResourceFromClasspath(// 
    MockDataxReaderContext.class, "container.json", true, (input) -> {
        Configuration cfg = Configuration.from(input);
        cfg.set("plugin.writer.streamwriter.class", "com.alibaba.datax.plugin.writer.streamwriter.StreamWriter");
        cfg.set("plugin.reader." + dataxReader.getDataxMeta().getName() + ".class", dataxReader.getDataxMeta().getImplClass());
        // 
        cfg.set(// 
        "job.content[0].reader", IOUtils.loadResourceFromClasspath(dataxReader.getClass(), readerJson, true, (writerJsonInput) -> {
            return Configuration.from(writerJsonInput);
        }));
        cfg.set("job.content[0].writer", Configuration.from("{\n" + "    \"name\": \"streamwriter\",\n" + "    \"parameter\": {\n" + "        \"print\": true\n" + "    }\n" + "}"));
        if (contentWriter != null) {
            cfg.set("job.content[0].writer.parameter." + com.alibaba.datax.plugin.writer.streamwriter.Key.CONTENT_WRITER, contentWriter);
        }
        return cfg;
    });
    // 绑定column转换信息
    ColumnCast.bind(allConf);
    LoadUtil.bind(allConf);
    JobContainer container = new JobContainer(allConf);
    container.start();
}

Example 17 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project tis by qlangtech.

the class TestDataXExecuteInterceptor method testExecute.

public void testExecute() throws Exception {
    DataxReader dataxReader = mock(AP_NAME + "DataXReader", DataxReader.class);
    ISelectedTab tab = new TestSelectedTab(tableName);
    EasyMock.expect(dataxReader.getSelectedTabs()).andReturn(Collections.singletonList(tab));
    DataxReader.dataxReaderGetter = (name) -> {
        return dataxReader;
    };
    BatchPostDataXWriter dataxWriter = new BatchPostDataXWriter(Collections.singletonList(dataCfgFileName));
    DataxWriter.dataxWriterGetter = (name) -> {
        // DataxWriter dataxWriter = mock(name + "DataXWriter", DataxWriter.class);
        return dataxWriter;
    };
    IRemoteTaskTrigger jobTrigger = mock(dataCfgTaskName + "_" + IRemoteTaskTrigger.class.getSimpleName(), IRemoteTaskTrigger.class);
    // 
    EasyMock.expect(jobTrigger.getTaskDependencies()).andReturn(Collections.emptyList()).anyTimes();
    EasyMock.expect(jobTrigger.getTaskName()).andReturn(dataCfgFileName).anyTimes();
    EasyMock.expect(jobTrigger.isAsyn()).andReturn(false).anyTimes();
    jobTrigger.run();
    // RunningStatus runningStatus = RunningStatus.SUCCESS;
    // EasyMock.expect(jobTrigger.getRunningStatus()).andReturn(runningStatus);
    executeJobTrigger(jobTrigger, true);
    dataxWriter.verify();
}

Example 18 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project tis by qlangtech.

the class OfflineDatasourceAction method doGetDsTabsVals.

/**
 * 批量设置用,初始化数据库表
 *
 * @param context
 * @throws IOException
 */
public void doGetDsTabsVals(Context context) throws IOException {
    com.alibaba.fastjson.JSONObject body = this.parseJsonPost();
    com.alibaba.fastjson.JSONArray tabs = body.getJSONArray("tabs");
    if (tabs == null) {
        throw new IllegalArgumentException("initialize Tabs can not be null");
    }
    List<String> selectedTabs = tabs.stream().map((tab) -> (String) tab).collect(Collectors.toList());
    String pluginName = body.getString("name");
    boolean require = body.getBooleanValue("require");
    String extraParam = body.getString("extraParam");
    UploadPluginMeta pluginMeta = null;
    List<UploadPluginMeta> pluginMetas = UploadPluginMeta.parse(new String[] { pluginName + ":" + (require ? "require" : StringUtils.EMPTY) + "," + extraParam });
    for (UploadPluginMeta m : pluginMetas) {
        pluginMeta = m;
    }
    Objects.requireNonNull(pluginMeta, "pluginMeta can not be null");
    HeteroList<DataxReader> heteroList = pluginMeta.getHeteroList(this);
    List<DataxReader> readers = heteroList.getItems();
    Map<String, List<ColumnMetaData>> mapCols = null;
    List<ISelectedTab> allNewTabs = Lists.newArrayList();
    PluginFormProperties pluginFormPropertyTypes = null;
    Map<String, Object> bizResult = Maps.newHashMap();
    for (DataxReader reader : readers) {
        mapCols = selectedTabs.stream().collect(Collectors.toMap((tab) -> tab, (tab) -> {
            return reader.getTableMetadata(tab);
        }));
        if (MapUtils.isEmpty(mapCols)) {
            throw new IllegalStateException("mapCols can not be empty");
        }
        pluginFormPropertyTypes = reader.getDescriptor().getPluginFormPropertyTypes(pluginMeta.getSubFormFilter());
        for (Map.Entry<String, List<ColumnMetaData>> tab2cols : mapCols.entrySet()) {
            SuFormProperties.setSuFormGetterContext(reader, pluginMeta, tab2cols.getKey());
            allNewTabs.add(createNewSelectedTab(pluginFormPropertyTypes, tab2cols));
        }
        DescriptorsJSON desc2Json = new DescriptorsJSON(reader.getDescriptor());
        bizResult.put("subformDescriptor", desc2Json.getDescriptorsJSON(pluginMeta.getSubFormFilter()));
        break;
    }
    Objects.requireNonNull(pluginFormPropertyTypes, "pluginFormPropertyTypes can not be null");
    if (allNewTabs.size() < 1) {
        throw new IllegalStateException("allNewTabs size can not small than 1");
    }
    bizResult.put("tabVals", pluginFormPropertyTypes.accept(new PluginFormProperties.IVisitor() {

        @Override
        public com.alibaba.fastjson.JSONObject visit(SuFormProperties props) {
            return props.createSubFormVals(allNewTabs.stream().map((t) -> (IdentityName) t).collect(Collectors.toList()));
        }
    }));
    this.setBizResult(context, bizResult);
}

Example 19 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project tis by qlangtech.

the class OfflineDatasourceAction method doGetDatasourceTableById.

/**
 * Do get datasource table. 获取一个table的git配置信息
 *
 * @param context the context
 */
public void doGetDatasourceTableById(Context context) throws IOException {
    Integer dbId = this.getInt("id");
    String tableName = this.getString("labelName");
    com.qlangtech.tis.workflow.pojo.DatasourceDb db = this.getWorkflowDAOFacade().getDatasourceDbDAO().selectByPrimaryKey(dbId);
    Objects.requireNonNull(db, "db can not be null");
    // TISTable tableConfig = this.offlineManager.getTableConfig(this, tableId);
    // this.setBizResult(context, tableConfig);
    DataxReader dbDataxReader = OfflineManager.getDBDataxReader(this, db.getName());
    this.setBizResult(context, new DescriptorsJSON(dbDataxReader.getDescriptor()).getDescriptorsJSON());
}

Example 20 with DataxReader

use of com.qlangtech.tis.datax.impl.DataxReader in project tis by qlangtech.

the class OfflineDatasourceAction method doGetDsRelevantReaderDesc.

@Func(value = PermissionConstant.PERMISSION_DATASOURCE_EDIT, sideEffect = false)
public void doGetDsRelevantReaderDesc(Context context) {
    com.alibaba.fastjson.JSONObject form = this.parseJsonPost();
    Integer dbId = form.getInteger("dbId");
    if (dbId == null) {
        throw new IllegalStateException("dbId can not be null");
    }
    com.qlangtech.tis.workflow.pojo.DatasourceDb db = this.offlineDAOFacade.getDatasourceDbDAO().selectByPrimaryKey(dbId);
    // IPluginStore<DataSourceFactory> dbPlugin = TIS.getDataBasePluginStore(new PostedDSProp(db.getName(), DbScope.DETAILED));
    // DataSourceFactory.BaseDataSourceFactoryDescriptor descriptor = (DataSourceFactory.BaseDataSourceFactoryDescriptor) dbPlugin.getPlugin().getDescriptor();
    OfflineManager.DBDataXReaderDescName defaultDataXReaderDescName = offlineManager.getDBDataXReaderDescName(db.getName());
    Map<String, Object> result = Maps.newHashMap();
    if (!defaultDataXReaderDescName.readerDescName.isPresent()) {
        // throw new IllegalStateException("datasource:" + db.getName() + " desc:" + descriptor.getDisplayName() + " has not relevant DataXReader defined");
        // result.put(KEY_DATA_READER_SETTED + "NotSupport", descriptor.getId());
        this.addErrorMessage(context, "插件:" + defaultDataXReaderDescName.dsDescriptor.getId() + " 不支持表导入");
        return;
    }
    DataxReader dataxReader = OfflineManager.getDBDataxReader(this, db.getName());
    if (dataxReader != null) {
        result.put(KEY_DATA_READER_SETTED, true);
    // this.setBizResult(context, result);
    // return;
    }
    DescriptorExtensionList<DataxReader, Descriptor<DataxReader>> descriptorList = TIS.get().getDescriptorList(DataxReader.class);
    Optional<DataxReader.BaseDataxReaderDescriptor> dataXReaderDesc = descriptorList.stream().filter((de) -> {
        return defaultDataXReaderDescName.getReaderDescName().equals(de.getDisplayName());
    }).map((d) -> (DataxReader.BaseDataxReaderDescriptor) d).findFirst();
    if (!dataXReaderDesc.isPresent()) {
        throw new IllegalStateException("DataXReaderDescName:" + defaultDataXReaderDescName.getReaderDescName() + " can not find relevant DataXReader Descriptor");
    }
    result.put("readerDesc", new DescriptorsJSON(dataXReaderDesc.get()).getDescriptorsJSON());
    result.put("processMeta", DataxAction.getDataXBasicProcessMetaByReader(dataXReaderDesc.get()));
    this.setBizResult(context, result);
}