Examples with ISelectedTab com.qlangtech.tis.plugin.ds.ISelectedTab used on opensource projects

Example 26 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project tis by qlangtech.

the class DataxAction method doSaveWriterColsMeta.

/**
 * @param context
 */
@Func(value = PermissionConstant.DATAX_MANAGE)
public void doSaveWriterColsMeta(Context context) {
    String dataxName = this.getString(PARAM_KEY_DATAX_NAME);
    DataxProcessor.DataXCreateProcessMeta processMeta = DataxProcessor.getDataXCreateProcessMeta(this, dataxName);
    if ((processMeta.isReaderRDBMS())) {
        throw new IllegalStateException("can not process the flow with:" + processMeta.toString());
    }
    List<ISelectedTab.ColMeta> writerCols = Lists.newArrayList();
    IDataxProcessor.TableMap tableMapper = new IDataxProcessor.TableMap(new ISelectedTab() {

        @Override
        public List<ColMeta> getCols() {
            return writerCols;
        }
    });
    // tableMapper.setSourceCols(writerCols);
    // //////////////////
    final String keyColsMeta = "colsMeta";
    IControlMsgHandler handler = new DelegateControl4JsonPostMsgHandler(this, this.parseJsonPost());
    if (!Validator.validate(handler, context, // 
    Validator.fieldsValidator(// 
    "writerTargetTabName", new Validator.FieldValidators(Validator.require, Validator.db_col_name) {

        @Override
        public void setFieldVal(String val) {
            tableMapper.setTo(val);
        }
    }, "writerFromTabName", new Validator.FieldValidators(Validator.require, Validator.db_col_name) {

        @Override
        public void setFieldVal(String val) {
            tableMapper.setFrom(val);
        }
    }, // 
    keyColsMeta, new Validator.FieldValidators(Validator.require) {

        @Override
        public void setFieldVal(String val) {
        }
    }, new Validator.IFieldValidator() {

        @Override
        public boolean validate(IFieldErrorHandler msgHandler, Context context, String fieldKey, String fieldData) {
            ISelectedTab.ColMeta colMeta = null;
            JSONArray targetCols = JSON.parseArray(fieldData);
            JSONObject targetCol = null;
            int index;
            String targetColName = null;
            if (targetCols.size() < 1) {
                msgHandler.addFieldError(context, fieldKey, "Writer目标表列不能为空");
                return false;
            }
            Map<String, Integer> existCols = Maps.newHashMap();
            boolean validateFaild = false;
            Integer previousColIndex = null;
            for (int i = 0; i < targetCols.size(); i++) {
                targetCol = targetCols.getJSONObject(i);
                index = targetCol.getInteger("index");
                targetColName = targetCol.getString("name");
                if (StringUtils.isNotBlank(targetColName) && (previousColIndex = existCols.put(targetColName, index)) != null) {
                    msgHandler.addFieldError(context, keyColsMeta + "[" + previousColIndex + "]", "内容不能与第" + index + "行重复");
                    msgHandler.addFieldError(context, keyColsMeta + "[" + index + "]", "内容不能与第" + previousColIndex + "行重复");
                    return false;
                }
                if (!Validator.require.validate(DataxAction.this, context, keyColsMeta + "[" + index + "]", targetColName)) {
                    validateFaild = true;
                } else if (!Validator.db_col_name.validate(DataxAction.this, context, keyColsMeta + "[" + index + "]", targetColName)) {
                    validateFaild = true;
                }
                colMeta = new ISelectedTab.ColMeta();
                colMeta.setName(targetColName);
                DataType dataType = targetCol.getObject("type", DataType.class);
                // colMeta.setType(ISelectedTab.DataXReaderColType.parse(targetCol.getString("type")));
                colMeta.setType(dataType);
                writerCols.add(colMeta);
            }
            return !validateFaild;
        }
    }))) {
        return;
    }
    this.saveTableMapper(this, dataxName, Collections.singletonList(tableMapper));
}

Example 27 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project tis by qlangtech.

the class DataxAction method doGetWriterColsMeta.

/**
 * 当reader为非RDBMS，writer为RDBMS类型时 需要为writer设置表名称，以及各个列的名称
 *
 * @param context
 */
@Func(value = PermissionConstant.DATAX_MANAGE, sideEffect = false)
public void doGetWriterColsMeta(Context context) {
    final String dataxName = this.getString(PARAM_KEY_DATAX_NAME);
    DataxProcessor.DataXCreateProcessMeta processMeta = DataxProcessor.getDataXCreateProcessMeta(this, dataxName);
    DataxProcessor processor = DataxProcessor.load(this, dataxName);
    if (processMeta.isReaderRDBMS()) {
        throw new IllegalStateException("can not process the flow with:" + processMeta.toString());
    }
    int selectedTabsSize = processMeta.getReader().getSelectedTabs().size();
    if (selectedTabsSize != 1) {
        throw new IllegalStateException("dataX reader getSelectedTabs size must be 1 ,but now is :" + selectedTabsSize);
    }
    Map<String, IDataxProcessor.TableAlias> tabAlias = processor.getTabAlias();
    Optional<IDataxProcessor.TableAlias> findMapper = tabAlias.values().stream().findFirst();
    IDataxProcessor.TableMap tabMapper = null;
    for (ISelectedTab selectedTab : processMeta.getReader().getSelectedTabs()) {
        if (findMapper.isPresent()) {
            if (!(findMapper.get() instanceof IDataxProcessor.TableMap)) {
                throw new IllegalStateException("tableAlias must be type of " + IDataxProcessor.TableMap.class.getName());
            }
            tabMapper = (IDataxProcessor.TableMap) findMapper.get();
        } else {
            tabMapper = new IDataxProcessor.TableMap(selectedTab);
            // tabMapper.setSourceCols(selectedTab.getCols());
            tabMapper.setFrom(selectedTab.getName());
            tabMapper.setTo(selectedTab.getName());
        }
        this.setBizResult(context, tabMapper);
        return;
    }
}

Example 28 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project tis by qlangtech.

the class DataXExecuteInterceptor method execute.

@Override
protected ExecuteResult execute(IExecChainContext execChainContext) throws Exception {
    int nThreads = 2;
    final ExecutorService executorService = new ThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(DataXJobSubmit.MAX_TABS_NUM_IN_PER_JOB), Executors.defaultThreadFactory());
    final Map<String, TISReactor.TaskAndMilestone> taskMap = Maps.newHashMap();
    RpcServiceReference statusRpc = getDataXExecReporter();
    DataxProcessor appSource = execChainContext.getAppSource();
    IRemoteTaskTrigger jobTrigger = null;
    RunningStatus runningStatus = null;
    List<IRemoteTaskTrigger> triggers = Lists.newArrayList();
    List<File> cfgFileNames = appSource.getDataxCfgFileNames(null);
    if (CollectionUtils.isEmpty(cfgFileNames)) {
        throw new IllegalStateException("dataX cfgFileNames can not be empty");
    }
    DataXJobSubmit.InstanceType expectDataXJobSumit = getDataXTriggerType();
    Optional<DataXJobSubmit> jobSubmit = DataXJobSubmit.getDataXJobSubmit(expectDataXJobSumit);
    // 如果分布式worker ready的话
    if (!jobSubmit.isPresent()) {
        throw new IllegalStateException("can not find expect jobSubmit by type:" + expectDataXJobSumit);
    }
    DataXJobSubmit submit = jobSubmit.get();
    final DataXJobSubmit.IDataXJobContext dataXJobContext = submit.createJobContext(execChainContext);
    Objects.requireNonNull(dataXJobContext, "dataXJobContext can not be null");
    try {
        DumpPhaseStatus dumpStatus = this.getPhaseStatus(execChainContext, FullbuildPhase.FullDump);
        for (File fileName : cfgFileNames) {
            jobTrigger = createDataXJob(dataXJobContext, submit, expectDataXJobSumit, statusRpc, appSource, fileName.getName());
            triggers.add(jobTrigger);
            taskMap.put(fileName.getName(), new TISReactor.TaskAndMilestone(DataflowTask.createDumpTask(jobTrigger)));
            // StatusRpcClient.AssembleSvcCompsite svc = statusRpc.get();
            // 将任务注册，可供页面展示
            // svc.reportDumpJobStatus(false, false, true, execChainContext.getTaskId()
            // , fileName.getName(), 0, 0);
            dumpStatus.getTable(fileName.getName()).setWaiting(true);
        }
        logger.info("trigger dataX jobs by mode:{},with:{}", this.getDataXTriggerType(), cfgFileNames.stream().map((f) -> f.getName()).collect(Collectors.joining(",")));
        // for (IRemoteJobTrigger t : triggers) {
        // t.submitJob();
        // }
        IDataxReader reader = appSource.getReader(null);
        List<ISelectedTab> selectedTabs = reader.getSelectedTabs();
        // Map<String, IDataxProcessor.TableAlias> tabAlias = appSource.getTabAlias();
        IDataxWriter writer = appSource.getWriter(null);
        if (writer instanceof IDataXBatchPost) {
            IDataXBatchPost batchPostTask = (IDataXBatchPost) writer;
            JoinPhaseStatus phaseStatus = this.getPhaseStatus(execChainContext, FullbuildPhase.JOIN);
            for (ISelectedTab entry : selectedTabs) {
                IRemoteTaskTrigger postTaskTrigger = batchPostTask.createPostTask(execChainContext, entry);
                triggers.add(postTaskTrigger);
                JoinPhaseStatus.JoinTaskStatus taskStatus = phaseStatus.getTaskStatus(postTaskTrigger.getTaskName());
                taskStatus.setWaiting(true);
                taskMap.put(postTaskTrigger.getTaskName(), new TISReactor.TaskAndMilestone(createJoinTask(postTaskTrigger, taskStatus)));
            }
        }
        // example: "->a ->b a,b->c"
        String dagSessionSpec = triggers.stream().map((trigger) -> {
            List<String> dpts = trigger.getTaskDependencies();
            return dpts.stream().collect(Collectors.joining(",")) + "->" + trigger.getTaskName();
        }).collect(Collectors.joining(" "));
        logger.info("dataX:{} of dagSessionSpec:{}", execChainContext.getIndexName(), dagSessionSpec);
        ExecuteResult[] faildResult = new ExecuteResult[] { ExecuteResult.createSuccess() };
        this.executeDAG(executorService, execChainContext, dagSessionSpec, taskMap, new ReactorListener() {

            @Override
            public void onTaskCompleted(Task t) {
            // dumpPhaseStatus.isComplete();
            // joinPhaseStatus.isComplete();
            }

            @Override
            public void onTaskFailed(Task t, Throwable err, boolean fatal) {
                logger.error(t.getDisplayName(), err);
                faildResult[0] = ExecuteResult.createFaild().setMessage("status.runningStatus.isComplete():" + err.getMessage());
                if (err instanceof InterruptedException) {
                    logger.warn("DataX Name:{},taskid:{} has been canceled", execChainContext.getIndexName(), execChainContext.getTaskId());
                    // this job has been cancel, trigger from TisServlet.doDelete()
                    for (IRemoteTaskTrigger tt : triggers) {
                        try {
                            tt.cancel();
                        } catch (Throwable ex) {
                        }
                    }
                }
            }
        });
        // ExecuteResult result = new ExecuteResult(!faild);
        for (IRemoteTaskTrigger trigger : triggers) {
            if (trigger.isAsyn()) {
                execChainContext.addAsynSubJob(new IExecChainContext.AsynSubJob(trigger.getAsynJobName()));
            }
        }
        return faildResult[0];
    } finally {
        try {
            dataXJobContext.destroy();
        } catch (Throwable e) {
            logger.error(e.getMessage(), e);
        }
    }
}

Example 29 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project tis by qlangtech.

the class TestDataXExecuteInterceptor method testExecute.

public void testExecute() throws Exception {
    DataxReader dataxReader = mock(AP_NAME + "DataXReader", DataxReader.class);
    ISelectedTab tab = new TestSelectedTab(tableName);
    EasyMock.expect(dataxReader.getSelectedTabs()).andReturn(Collections.singletonList(tab));
    DataxReader.dataxReaderGetter = (name) -> {
        return dataxReader;
    };
    BatchPostDataXWriter dataxWriter = new BatchPostDataXWriter(Collections.singletonList(dataCfgFileName));
    DataxWriter.dataxWriterGetter = (name) -> {
        // DataxWriter dataxWriter = mock(name + "DataXWriter", DataxWriter.class);
        return dataxWriter;
    };
    IRemoteTaskTrigger jobTrigger = mock(dataCfgTaskName + "_" + IRemoteTaskTrigger.class.getSimpleName(), IRemoteTaskTrigger.class);
    // 
    EasyMock.expect(jobTrigger.getTaskDependencies()).andReturn(Collections.emptyList()).anyTimes();
    EasyMock.expect(jobTrigger.getTaskName()).andReturn(dataCfgFileName).anyTimes();
    EasyMock.expect(jobTrigger.isAsyn()).andReturn(false).anyTimes();
    jobTrigger.run();
    // RunningStatus runningStatus = RunningStatus.SUCCESS;
    // EasyMock.expect(jobTrigger.getRunningStatus()).andReturn(runningStatus);
    executeJobTrigger(jobTrigger, true);
    dataxWriter.verify();
}

Example 30 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project tis by qlangtech.

the class TestDataxReader method testGetDataxReader.

// public void testUpdateDataxReader() {
// KeyedPluginStore<DataxReader> readerStore = DataxReader.getPluginStore(null, dataXName);
// 
// DataxReader dataxReader = readerStore.getPlugin();
// assertNotNull(dataxReader);
// 
// 
// SuFormProperties props = EasyMock.createMock("subformProp", SuFormProperties.class);
// 
// EasyMock.expect(props.getSubFormFieldName()).andReturn("selectedTabs");
// 
// DataxReader.SubFieldFormAppKey<DataxReader> subFieldKey
// = new DataxReader.SubFieldFormAppKey<>(null, false, dataXName, props, DataxReader.class);
// KeyedPluginStore<DataxReader> subFieldStore = KeyedPluginStore.getPluginStore(subFieldKey);
// 
// List<Descriptor.ParseDescribable<DataxReader>> dlist = Lists.newArrayList();
// DataxReader subformReader =
// dlist.add(new Descriptor.ParseDescribable());
// subFieldStore.setPlugins(null, Optional.empty(), dlist);
// 
// }
public void testGetDataxReader() {
    KeyedPluginStore<DataxReader> readerStore = DataxReader.getPluginStore(null, dataXName);
    DataxReader dataxReader = readerStore.getPlugin();
    assertNotNull("dataxReader can not be null", dataxReader);
    List<ISelectedTab> selectedTabs = dataxReader.getSelectedTabs();
    assertNotNull(selectedTabs);
    assertTrue(selectedTabs.size() > 0);
    IPluginContext pluginContext = EasyMock.createMock("pluginContext", IPluginContext.class);
    // 
    String execId = "7b069200-9845-d60b-cef0-408c6940ffda";
    EasyMock.expect(pluginContext.getExecId()).andReturn(execId).times(2);
    EasyMock.expect(pluginContext.getRequestHeader(DataxReader.HEAD_KEY_REFERER)).andReturn("/x/" + dataXName + "/update").times(2);
    EasyMock.expect(pluginContext.isCollectionAware()).andReturn(true).times(2);
    EasyMock.replay(pluginContext);
    readerStore = DataxReader.getPluginStore(pluginContext, dataXName);
    assertNotNull("readerStore can not be null", readerStore);
    dataxReader = readerStore.getPlugin();
    assertNotNull("dataxReader can not be null", dataxReader);
    selectedTabs = dataxReader.getSelectedTabs();
    assertNotNull(selectedTabs);
    assertTrue(selectedTabs.size() > 0);
    for (ISelectedTab tab : selectedTabs) {
        assertTrue(tab.getCols().size() > 0);
        for (ISelectedTab.ColMeta col : tab.getCols()) {
            assertNotNull("tab:" + tab.getName() + ",col:" + col.getName() + " can not be null", col.getType());
        }
    }
    EasyMock.verify(pluginContext);
}