Examples with IDataxReader com.qlangtech.tis.datax.IDataxReader used on opensource projects

Example 11 with IDataxReader

use of com.qlangtech.tis.datax.IDataxReader in project plugins by qlangtech.

the class TestDataxMySQLWriter method validateConfigGenerate.

private void validateConfigGenerate(String assertFileName, DataxMySQLWriter mySQLWriter) throws IOException {
    // IDataxProcessor.TableMap tm = new IDataxProcessor.TableMap();
    // tm.setFrom("orderinfo");
    // tm.setTo("orderinfo_new");
    // tm.setSourceCols(Lists.newArrayList("col1", "col2", "col3").stream().map((c) -> {
    // ISelectedTab.ColMeta meta = new ISelectedTab.ColMeta();
    // meta.setName(c);
    // return meta;
    // }).collect(Collectors.toList()));
    Optional<IDataxProcessor.TableMap> tableMap = TestSelectedTabs.createTableMapper();
    IDataxContext subTaskCtx = mySQLWriter.getSubTask(tableMap);
    assertNotNull(subTaskCtx);
    RdbmsDataxContext mySQLDataxContext = (RdbmsDataxContext) subTaskCtx;
    assertEquals("\"`col1`\",\"`col2`\",\"`col3`\"", mySQLDataxContext.getColsQuotes());
    assertEquals(mysqlJdbcUrl, mySQLDataxContext.getJdbcUrl());
    assertEquals("123456", mySQLDataxContext.getPassword());
    assertEquals("orderinfo_new", mySQLDataxContext.tabName);
    assertEquals("root", mySQLDataxContext.getUsername());
    IDataxProcessor processor = EasyMock.mock("dataxProcessor", IDataxProcessor.class);
    IDataxGlobalCfg dataxGlobalCfg = EasyMock.mock("dataxGlobalCfg", IDataxGlobalCfg.class);
    IDataxReader dataxReader = EasyMock.mock("dataxReader", IDataxReader.class);
    EasyMock.expect(processor.getReader(null)).andReturn(dataxReader);
    EasyMock.expect(processor.getWriter(null)).andReturn(mySQLWriter);
    EasyMock.expect(processor.getDataXGlobalCfg()).andReturn(dataxGlobalCfg);
    EasyMock.replay(processor, dataxGlobalCfg, dataxReader);
    DataXCfgGenerator dataProcessor = new DataXCfgGenerator(null, "testDataXName", processor) {

        @Override
        public String getTemplateContent() {
            return mySQLWriter.getTemplate();
        }
    };
    String cfgResult = dataProcessor.generateDataxConfig(null, mySQLWriter, dataxReader, tableMap);
    JsonUtil.assertJSONEqual(this.getClass(), assertFileName, cfgResult, (m, e, a) -> {
        assertEquals(m, e, a);
    });
    EasyMock.verify(processor, dataxGlobalCfg, dataxReader);
}

Example 12 with IDataxReader

use of com.qlangtech.tis.datax.IDataxReader in project tis by qlangtech.

the class DataXExecuteInterceptor method execute.

@Override
protected ExecuteResult execute(IExecChainContext execChainContext) throws Exception {
    int nThreads = 2;
    final ExecutorService executorService = new ThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(DataXJobSubmit.MAX_TABS_NUM_IN_PER_JOB), Executors.defaultThreadFactory());
    final Map<String, TISReactor.TaskAndMilestone> taskMap = Maps.newHashMap();
    RpcServiceReference statusRpc = getDataXExecReporter();
    DataxProcessor appSource = execChainContext.getAppSource();
    IRemoteTaskTrigger jobTrigger = null;
    RunningStatus runningStatus = null;
    List<IRemoteTaskTrigger> triggers = Lists.newArrayList();
    List<File> cfgFileNames = appSource.getDataxCfgFileNames(null);
    if (CollectionUtils.isEmpty(cfgFileNames)) {
        throw new IllegalStateException("dataX cfgFileNames can not be empty");
    }
    DataXJobSubmit.InstanceType expectDataXJobSumit = getDataXTriggerType();
    Optional<DataXJobSubmit> jobSubmit = DataXJobSubmit.getDataXJobSubmit(expectDataXJobSumit);
    // 如果分布式worker ready的话
    if (!jobSubmit.isPresent()) {
        throw new IllegalStateException("can not find expect jobSubmit by type:" + expectDataXJobSumit);
    }
    DataXJobSubmit submit = jobSubmit.get();
    final DataXJobSubmit.IDataXJobContext dataXJobContext = submit.createJobContext(execChainContext);
    Objects.requireNonNull(dataXJobContext, "dataXJobContext can not be null");
    try {
        DumpPhaseStatus dumpStatus = this.getPhaseStatus(execChainContext, FullbuildPhase.FullDump);
        for (File fileName : cfgFileNames) {
            jobTrigger = createDataXJob(dataXJobContext, submit, expectDataXJobSumit, statusRpc, appSource, fileName.getName());
            triggers.add(jobTrigger);
            taskMap.put(fileName.getName(), new TISReactor.TaskAndMilestone(DataflowTask.createDumpTask(jobTrigger)));
            // StatusRpcClient.AssembleSvcCompsite svc = statusRpc.get();
            // 将任务注册，可供页面展示
            // svc.reportDumpJobStatus(false, false, true, execChainContext.getTaskId()
            // , fileName.getName(), 0, 0);
            dumpStatus.getTable(fileName.getName()).setWaiting(true);
        }
        logger.info("trigger dataX jobs by mode:{},with:{}", this.getDataXTriggerType(), cfgFileNames.stream().map((f) -> f.getName()).collect(Collectors.joining(",")));
        // for (IRemoteJobTrigger t : triggers) {
        // t.submitJob();
        // }
        IDataxReader reader = appSource.getReader(null);
        List<ISelectedTab> selectedTabs = reader.getSelectedTabs();
        // Map<String, IDataxProcessor.TableAlias> tabAlias = appSource.getTabAlias();
        IDataxWriter writer = appSource.getWriter(null);
        if (writer instanceof IDataXBatchPost) {
            IDataXBatchPost batchPostTask = (IDataXBatchPost) writer;
            JoinPhaseStatus phaseStatus = this.getPhaseStatus(execChainContext, FullbuildPhase.JOIN);
            for (ISelectedTab entry : selectedTabs) {
                IRemoteTaskTrigger postTaskTrigger = batchPostTask.createPostTask(execChainContext, entry);
                triggers.add(postTaskTrigger);
                JoinPhaseStatus.JoinTaskStatus taskStatus = phaseStatus.getTaskStatus(postTaskTrigger.getTaskName());
                taskStatus.setWaiting(true);
                taskMap.put(postTaskTrigger.getTaskName(), new TISReactor.TaskAndMilestone(createJoinTask(postTaskTrigger, taskStatus)));
            }
        }
        // example: "->a ->b a,b->c"
        String dagSessionSpec = triggers.stream().map((trigger) -> {
            List<String> dpts = trigger.getTaskDependencies();
            return dpts.stream().collect(Collectors.joining(",")) + "->" + trigger.getTaskName();
        }).collect(Collectors.joining(" "));
        logger.info("dataX:{} of dagSessionSpec:{}", execChainContext.getIndexName(), dagSessionSpec);
        ExecuteResult[] faildResult = new ExecuteResult[] { ExecuteResult.createSuccess() };
        this.executeDAG(executorService, execChainContext, dagSessionSpec, taskMap, new ReactorListener() {

            @Override
            public void onTaskCompleted(Task t) {
            // dumpPhaseStatus.isComplete();
            // joinPhaseStatus.isComplete();
            }

            @Override
            public void onTaskFailed(Task t, Throwable err, boolean fatal) {
                logger.error(t.getDisplayName(), err);
                faildResult[0] = ExecuteResult.createFaild().setMessage("status.runningStatus.isComplete():" + err.getMessage());
                if (err instanceof InterruptedException) {
                    logger.warn("DataX Name:{},taskid:{} has been canceled", execChainContext.getIndexName(), execChainContext.getTaskId());
                    // this job has been cancel, trigger from TisServlet.doDelete()
                    for (IRemoteTaskTrigger tt : triggers) {
                        try {
                            tt.cancel();
                        } catch (Throwable ex) {
                        }
                    }
                }
            }
        });
        // ExecuteResult result = new ExecuteResult(!faild);
        for (IRemoteTaskTrigger trigger : triggers) {
            if (trigger.isAsyn()) {
                execChainContext.addAsynSubJob(new IExecChainContext.AsynSubJob(trigger.getAsynJobName()));
            }
        }
        return faildResult[0];
    } finally {
        try {
            dataXJobContext.destroy();
        } catch (Throwable e) {
            logger.error(e.getMessage(), e);
        }
    }
}