Examples with ISelectedTab com.qlangtech.tis.plugin.ds.ISelectedTab used on opensource projects

Example 11 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project tis by qlangtech.

the class DataXCfgGenerator method createTableMap.

private IDataxProcessor.TableMap createTableMap(Map<String, IDataxProcessor.TableAlias> tabAlias, Map<String, ISelectedTab> selectedTabs, IDataxReaderContext readerContext) {
    IDataxProcessor.TableAlias tableAlias = tabAlias.get(readerContext.getSourceEntityName());
    if (tableAlias == null) {
        throw new IllegalStateException("sourceTable:" + readerContext.getSourceEntityName() + " can not find relevant 'tableAlias' keys:[" + tabAlias.keySet().stream().collect(Collectors.joining(",")) + "]");
    }
    ISelectedTab selectedTab = selectedTabs.get(readerContext.getSourceEntityName());
    IDataxProcessor.TableMap tableMap = new IDataxProcessor.TableMap(selectedTab);
    tableMap.setFrom(tableAlias.getFrom());
    tableMap.setTo(tableAlias.getTo());
    return tableMap;
}

Example 12 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project tis by qlangtech.

the class DataXCfgGenerator method startGenerateCfg.

public GenerateCfgs startGenerateCfg(IGenerateScriptFile scriptFileGenerator) throws Exception {
    GenerateCfgs cfgs = new GenerateCfgs();
    // FileUtils.forceMkdir(dataXCfgDir);
    // // 先清空文件
    // FileUtils.cleanDirectory(dataXCfgDir);
    boolean unStructedReader = dataxProcessor.isReaderUnStructed(this.pluginCtx);
    IDataxReader reader = dataxProcessor.getReader(this.pluginCtx);
    IDataxWriter writer = dataxProcessor.getWriter(this.pluginCtx);
    DataxWriter.BaseDataxWriterDescriptor writerDescriptor = writer.getWriterDescriptor();
    Map<String, IDataxProcessor.TableAlias> tabAlias = dataxProcessor.getTabAlias();
    AtomicReference<Map<String, ISelectedTab>> selectedTabsRef = new AtomicReference<>();
    java.util.concurrent.Callable<Map<String, ISelectedTab>> selectedTabsCall = () -> {
        if (selectedTabsRef.get() == null) {
            Map<String, ISelectedTab> selectedTabs = reader.getSelectedTabs().stream().collect(Collectors.toMap((t) -> t.getName(), (t) -> t));
            selectedTabsRef.set(selectedTabs);
        }
        return selectedTabsRef.get();
    };
    List<String> existDDLFiles = getExistDDLFiles();
    IGroupChildTaskIterator subTasks = reader.getSubTasks();
    IDataxReaderContext readerContext = null;
    File configFile = null;
    // List<String> subTaskName = Lists.newArrayList();
    Set<String> createDDLFiles = Sets.newHashSet();
    Optional<IDataxProcessor.TableMap> tableMapper = null;
    while (subTasks.hasNext()) {
        readerContext = subTasks.next();
        if (!dataxProcessor.isWriterSupportMultiTableInReader(this.pluginCtx)) {
            if (tabAlias.size() == 1) {
                // 针对ES的情况
                Optional<IDataxProcessor.TableMap> first = tabAlias.values().stream().filter((t) -> t instanceof IDataxProcessor.TableMap).map((t) -> (IDataxProcessor.TableMap) t).findFirst();
                if (first.isPresent()) {
                    tableMapper = first;
                }
            } else {
                // IDataxWriter writer = dataxProcessor.getWriter(this.pluginCtx);
                if (writer instanceof IDataxProcessor.INullTableMapCreator) {
                    tableMapper = Optional.empty();
                }
            }
            Objects.requireNonNull(tableMapper, "tabMapper can not be null,tabAlias.size()=" + tabAlias.size() + ",tabs:[" + tabAlias.keySet().stream().collect(Collectors.joining(",")) + "]");
        } else if (unStructedReader) {
            // 是在DataxAction的doSaveWriterColsMeta() 方法中持久化保存的
            for (IDataxProcessor.TableAlias tab : tabAlias.values()) {
                tableMapper = Optional.of((IDataxProcessor.TableMap) tab);
                break;
            }
            Objects.requireNonNull(tableMapper, "tableMap can not be null");
        } else if (dataxProcessor.isRDBMS2UnStructed(this.pluginCtx)) {
            // example: mysql -> oss
            Map<String, ISelectedTab> selectedTabs = selectedTabsCall.call();
            ISelectedTab tab = selectedTabs.get(readerContext.getSourceEntityName());
            Objects.requireNonNull(tab, readerContext.getSourceEntityName() + " relevant tab can not be null");
            IDataxProcessor.TableMap m = new IDataxProcessor.TableMap(tab);
            // m.setSourceCols(tab.getCols());
            m.setTo(tab.getName());
            m.setFrom(tab.getName());
            tableMapper = Optional.of(m);
        } else if (dataxProcessor.isRDBMS2RDBMS(this.pluginCtx)) {
            // example: mysql -> mysql
            tableMapper = Optional.of(createTableMap(tabAlias, selectedTabsCall.call(), readerContext));
        } else {
            // tableMapper = Optional.of(createTableMap(tabAlias, selectedTabsCall.call(), readerContext));
            throw new IllegalStateException("unexpect status");
        }
        scriptFileGenerator.generateScriptFile(reader, writer, readerContext, createDDLFiles, tableMapper);
    // generateScriptFile(dataXCfgDir, reader, writer, readerContext, subTaskName, createDDLFiles, tableMapper);
    }
    // 将老的已经没有用的ddl sql文件删除调
    File createDDLDir = this.dataxProcessor.getDataxCreateDDLDir(this.pluginCtx);
    for (String oldDDLFile : existDDLFiles) {
        if (!createDDLFiles.contains(oldDDLFile)) {
            FileUtils.deleteQuietly(new File(createDDLDir, oldDDLFile));
        }
    }
    long current = System.currentTimeMillis();
    // FileUtils.write(new File(dataXCfgDir, FILE_GEN), String.valueOf(current), TisUTF8.get(), false);
    cfgs.createDDLFiles = Lists.newArrayList(createDDLFiles);
    cfgs.groupedChildTask = subTasks.getGroupedInfo();
    // cfgs.dataxFiles = subTaskName;
    cfgs.genTime = current;
    return cfgs;
}

Example 13 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project tis by qlangtech.

the class Descriptor method getPluginFormPropertyTypes.

public PluginFormProperties getPluginFormPropertyTypes(Optional<IPropertyType.SubFormFilter> subFormFilter) {
    IPropertyType.SubFormFilter filter = null;
    if (subFormFilter.isPresent()) {
        filter = subFormFilter.get();
        if (filter.match(this)) {
            SuFormProperties subPluginFormPropertyTypes = (SuFormProperties) getSubPluginFormPropertyTypes(filter.subFieldName);
            try {
                // 类似Hudi的Writer需要覆盖Reader的subFieldName的在Reader的表设置表单中需要设置Hudi相关的属性
                // DataxWriter dataxWriter = DataxWriter.load(filter.uploadPluginMeta.getPluginContext(), dataXName);
                Descriptor writerDescriptor = // dataxWriter.getClass();
                IDataxProcessor.getWriterDescriptor(filter.uploadPluginMeta);
                if (writerDescriptor instanceof DataxWriter.IRewriteSuFormProperties) {
                    subPluginFormPropertyTypes = Objects.requireNonNull(((DataxWriter.IRewriteSuFormProperties) writerDescriptor).overwriteSubPluginFormPropertyTypes(subPluginFormPropertyTypes), "result can not be null " + PluginFormProperties.class.getSimpleName());
                }
            // String overwriteSubField = IOUtils.loadResourceFromClasspath(
            // writerClass, writerClass.getSimpleName() + "." + filter.subFieldName + ".json", false);
            // if (overwriteSubField != null) {
            // JSONObject subField = JSON.parseObject(overwriteSubField);
            // Class<?> clazz = writerClass.getClassLoader().loadClass(subField.getString(SubForm.FIELD_DES_CLASS));
            // return SuFormProperties.copy(filterFieldProp(buildPropertyTypes(this, clazz)), subPluginFormPropertyTypes);
            // }
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
            if (filter.subformDetailView) {
                final String subformDetailId = filter.subformDetailId;
                final SuFormProperties _subPluginFormPropertyTypes = subPluginFormPropertyTypes;
                return new AdapterPluginFormProperties(subPluginFormPropertyTypes) {

                    @Override
                    public JSON getInstancePropsJson(Object instance) {
                        Collection<IdentityName> subFormPropVal = _subPluginFormPropertyTypes.getSubFormPropVal(instance);
                        for (IdentityName subProp : subFormPropVal) {
                            if (StringUtils.equals(subformDetailId, subProp.identityValue())) {
                                return (new RootFormProperties(_subPluginFormPropertyTypes.fieldsType)).getInstancePropsJson(subProp);
                            }
                        }
                        ISelectedTab subDetailed = _subPluginFormPropertyTypes.newSubDetailed();
                        _subPluginFormPropertyTypes.pkPropertyType.setVal(subDetailed, subformDetailId);
                        return (new RootFormProperties(_subPluginFormPropertyTypes.fieldsType)).getInstancePropsJson(subDetailed);
                    // throw new IllegalStateException("subformDetailId:" + subformDetailId + " has not find subForm instance");
                    }
                };
            } else {
                return subPluginFormPropertyTypes;
            }
        }
    }
    return new RootFormProperties(filterFieldProp(getPropertyTypes()));
}

Example 14 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project tis by qlangtech.

the class DataxAction method doGetTableMapper.

/**
 * 取得表映射
 *
 * @param context
 */
@Func(value = PermissionConstant.DATAX_MANAGE, sideEffect = false)
public void doGetTableMapper(Context context) {
    String dataxName = this.getString(PARAM_KEY_DATAX_NAME);
    KeyedPluginStore<DataxReader> readerStore = DataxReader.getPluginStore(this, dataxName);
    DataxReader dataxReader = readerStore.getPlugin();
    Objects.requireNonNull(dataxReader, "dataReader:" + dataxName + " relevant instance can not be null");
    IDataxProcessor.TableAlias tableAlias;
    Optional<DataxProcessor> dataXAppSource = IAppSource.loadNullable(this, dataxName);
    Map<String, IDataxProcessor.TableAlias> tabMaps = Collections.emptyMap();
    if (dataXAppSource.isPresent()) {
        DataxProcessor dataxSource = dataXAppSource.get();
        tabMaps = dataxSource.getTabAlias();
    }
    if (!dataxReader.hasMulitTable()) {
        throw new IllegalStateException("reader has not set table at least");
    }
    List<IDataxProcessor.TableAlias> tmapList = Lists.newArrayList();
    for (ISelectedTab selectedTab : dataxReader.getSelectedTabs()) {
        tableAlias = tabMaps.get(selectedTab.getName());
        if (tableAlias == null) {
            tmapList.add(new IDataxProcessor.TableAlias(selectedTab.getName()));
        } else {
            tmapList.add(tableAlias);
        }
    }
    this.setBizResult(context, tmapList);
}

Example 15 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project tis by qlangtech.

the class SchemaAction method doGetEsTplFields.

// private static Logger log = LoggerFactory.getLogger(SchemaAction.class);
/**
 * DataX 创建流程中取得es的默认字段
 *
 * @param context
 * @throws Exception
 */
public void doGetEsTplFields(Context context) throws Exception {
    String dataxName = this.getString(DataxUtils.DATAX_NAME);
    StepType stepType = StepType.parse(this.getString("stepType"));
    // DataxProcessor process = DataxProcessor.load(this, dataxName);
    // IDataxProcessor.TableMap tabMapper = null;
    // ESField field = null;
    ISearchEngineTypeTransfer typeTransfer = ISearchEngineTypeTransfer.load(this, dataxName);
    DataxReader dataxReader = DataxReader.load(this, dataxName);
    // ESField field = null;
    if (stepType.update) {
        DataxProcessor dataxProcessor = DataxProcessor.load(this, dataxName);
        for (Map.Entry<String, IDataxProcessor.TableAlias> e : dataxProcessor.getTabAlias().entrySet()) {
            writerStructFields(context, e.getValue(), typeTransfer);
            return;
        }
    } else {
        for (ISelectedTab tab : dataxReader.getSelectedTabs()) {
            // ESSchema parseResult = new ESSchema();
            SchemaMetaContent tplSchema = typeTransfer.initSchemaMetaContent(tab);
            this.setBizResult(context, tplSchema.toJSON());
            return;
        }
    }
    throw new IllegalStateException("have not find any tab in DataXReader");
}