Examples with DataXJobSubmit com.qlangtech.tis.datax.DataXJobSubmit used on opensource projects

Example 1 with DataXJobSubmit

use of com.qlangtech.tis.datax.DataXJobSubmit in project plugins by qlangtech.

the class TestDistributedOverseerDataXJobSubmit method testPushMsgToDistributeQueue.

public void testPushMsgToDistributeQueue() {
    DataXJobWorker dataxJobWorker = DataXJobWorker.getJobWorker(DataXJobWorker.K8S_DATAX_INSTANCE_NAME);
    assertEquals("/datax/jobs", dataxJobWorker.getZkQueuePath());
    assertEquals("192.168.28.200:2181/tis/cloud", dataxJobWorker.getZookeeperAddress());
    DataxProcessor dataxProcessor = IAppSource.load(DATAX_NAME);
    assertNotNull(dataxProcessor);
    // IDataxProcessor dataxProcessor = EasyMock.createMock("dataxProcessor", IDataxProcessor.class);
    // EasyMock.expect(dataxProcessor.getDataxCfgDir()).andReturn();
    IJoinTaskContext taskContext = EasyMock.createMock("joinTaskContext", IJoinTaskContext.class);
    EasyMock.expect(taskContext.getIndexName()).andReturn(DATAX_NAME);
    EasyMock.expect(taskContext.getTaskId()).andReturn(DATAX_TASK_ID);
    AtomicReference<ITISRpcService> ref = new AtomicReference<>();
    ref.set(StatusRpcClient.AssembleSvcCompsite.MOCK_PRC);
    RpcServiceReference svcRef = new RpcServiceReference(ref);
    Optional<DataXJobSubmit> jobSubmit = DataXJobSubmit.getDataXJobSubmit(DataXJobSubmit.InstanceType.DISTRIBUTE);
    assertTrue(jobSubmit.isPresent());
    DataXJobSubmit submit = jobSubmit.get();
    DataXJobSubmit.IDataXJobContext jobContext = submit.createJobContext(taskContext);
    EasyMock.replay(taskContext);
    // IJoinTaskContext taskContext
    // , RpcServiceReference statusRpc, IDataxProcessor dataxProcessor, String dataXfileName
    IRemoteTaskTrigger dataXJob = submit.createDataXJob(jobContext, svcRef, dataxProcessor, DATAX_JOB_FILE_NAME);
    dataXJob.run();
    EasyMock.verify(taskContext);
}

Example 2 with DataXJobSubmit

use of com.qlangtech.tis.datax.DataXJobSubmit in project plugins by qlangtech.

the class TestLocalDataXJobSubmit method testCreateDataXJob.

public void testCreateDataXJob() throws Exception {
    Optional<DataXJobSubmit> dataXJobSubmit = DataXJobSubmit.getDataXJobSubmit(DataXJobSubmit.InstanceType.LOCAL);
    Assert.assertTrue("dataXJobSubmit shall present", dataXJobSubmit.isPresent());
    LocalDataXJobSubmit jobSubmit = (LocalDataXJobSubmit) dataXJobSubmit.get();
    jobSubmit.setMainClassName(LocalDataXJobMainEntrypoint.class.getName());
    jobSubmit.setWorkingDirectory(new File("."));
    jobSubmit.setClasspath("target/classes:target/test-classes");
    AtomicReference<ITISRpcService> ref = new AtomicReference<>();
    ref.set(StatusRpcClient.AssembleSvcCompsite.MOCK_PRC);
    RpcServiceReference statusRpc = new RpcServiceReference(ref);
    DataXJobSubmit.IDataXJobContext dataXJobContext = EasyMock.createMock("dataXJobContext", DataXJobSubmit.IDataXJobContext.class);
    IExecChainContext taskContext = EasyMock.createMock("taskContext", IExecChainContext.class);
    EasyMock.expect(dataXJobContext.getTaskContext()).andReturn(taskContext).anyTimes();
    IDataxProcessor dataxProcessor = EasyMock.createMock("dataxProcessor", IDataxProcessor.class);
    EasyMock.expect(taskContext.getIndexName()).andReturn(dataXName).anyTimes();
    EasyMock.expect(taskContext.getTaskId()).andReturn(TaskId).anyTimes();
    int preSuccessTaskId = 99;
    PhaseStatusCollection preSuccessTask = new PhaseStatusCollection(preSuccessTaskId, new ExecutePhaseRange(FullbuildPhase.FullDump, FullbuildPhase.FullDump));
    DumpPhaseStatus preDumpStatus = new DumpPhaseStatus(preSuccessTaskId);
    DumpPhaseStatus.TableDumpStatus tableDumpStatus = preDumpStatus.getTable(dataXfileName);
    tableDumpStatus.setAllRows(LocalDataXJobMainEntrypoint.testAllRows);
    preSuccessTask.setDumpPhase(preDumpStatus);
    EasyMock.expect(taskContext.loadPhaseStatusFromLatest(dataXName)).andReturn(preSuccessTask).times(3);
    TisZkClient zkClient = EasyMock.createMock("TisZkClient", TisZkClient.class);
    String zkSubPath = "nodes0000000020";
    EasyMock.expect(zkClient.getChildren(ZkUtils.ZK_ASSEMBLE_LOG_COLLECT_PATH, null, true)).andReturn(Collections.singletonList(zkSubPath)).times(3);
    EasyMock.expect(zkClient.getData(EasyMock.eq(ZkUtils.ZK_ASSEMBLE_LOG_COLLECT_PATH + "/" + zkSubPath), EasyMock.isNull(), EasyMock.anyObject(Stat.class), EasyMock.eq(true))).andReturn(statusCollectorHost.getBytes(TisUTF8.get())).times(3);
    EasyMock.expect(taskContext.getZkClient()).andReturn(zkClient).anyTimes();
    EasyMock.replay(taskContext, dataxProcessor, zkClient, dataXJobContext);
    IRemoteTaskTrigger dataXJob = jobSubmit.createDataXJob(dataXJobContext, statusRpc, dataxProcessor, dataXfileName);
    RunningStatus running = getRunningStatus(dataXJob);
    assertTrue("running.isSuccess", running.isSuccess());
    jobSubmit.setMainClassName(LocalDataXJobMainEntrypointThrowException.class.getName());
    dataXJob = jobSubmit.createDataXJob(dataXJobContext, statusRpc, dataxProcessor, dataXfileName);
    running = getRunningStatus(dataXJob);
    assertFalse("shall faild", running.isSuccess());
    assertTrue("shall complete", running.isComplete());
    jobSubmit.setMainClassName(LocalDataXJobMainEntrypointCancellable.class.getName());
    dataXJob = jobSubmit.createDataXJob(dataXJobContext, statusRpc, dataxProcessor, dataXfileName);
    running = getRunningStatus(dataXJob, false);
    Thread.sleep(2000);
    dataXJob.cancel();
    int i = 0;
    while (i++ < 3 && !(running = dataXJob.getRunningStatus()).isComplete()) {
        Thread.sleep(1000);
    }
    assertFalse("shall faild", running.isSuccess());
    assertTrue("shall complete", running.isComplete());
    EasyMock.verify(taskContext, dataxProcessor, zkClient);
}

Example 3 with DataXJobSubmit

use of com.qlangtech.tis.datax.DataXJobSubmit in project tis by qlangtech.

the class DataXExecuteInterceptor method execute.

@Override
protected ExecuteResult execute(IExecChainContext execChainContext) throws Exception {
    int nThreads = 2;
    final ExecutorService executorService = new ThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(DataXJobSubmit.MAX_TABS_NUM_IN_PER_JOB), Executors.defaultThreadFactory());
    final Map<String, TISReactor.TaskAndMilestone> taskMap = Maps.newHashMap();
    RpcServiceReference statusRpc = getDataXExecReporter();
    DataxProcessor appSource = execChainContext.getAppSource();
    IRemoteTaskTrigger jobTrigger = null;
    RunningStatus runningStatus = null;
    List<IRemoteTaskTrigger> triggers = Lists.newArrayList();
    List<File> cfgFileNames = appSource.getDataxCfgFileNames(null);
    if (CollectionUtils.isEmpty(cfgFileNames)) {
        throw new IllegalStateException("dataX cfgFileNames can not be empty");
    }
    DataXJobSubmit.InstanceType expectDataXJobSumit = getDataXTriggerType();
    Optional<DataXJobSubmit> jobSubmit = DataXJobSubmit.getDataXJobSubmit(expectDataXJobSumit);
    // 如果分布式worker ready的话
    if (!jobSubmit.isPresent()) {
        throw new IllegalStateException("can not find expect jobSubmit by type:" + expectDataXJobSumit);
    }
    DataXJobSubmit submit = jobSubmit.get();
    final DataXJobSubmit.IDataXJobContext dataXJobContext = submit.createJobContext(execChainContext);
    Objects.requireNonNull(dataXJobContext, "dataXJobContext can not be null");
    try {
        DumpPhaseStatus dumpStatus = this.getPhaseStatus(execChainContext, FullbuildPhase.FullDump);
        for (File fileName : cfgFileNames) {
            jobTrigger = createDataXJob(dataXJobContext, submit, expectDataXJobSumit, statusRpc, appSource, fileName.getName());
            triggers.add(jobTrigger);
            taskMap.put(fileName.getName(), new TISReactor.TaskAndMilestone(DataflowTask.createDumpTask(jobTrigger)));
            // StatusRpcClient.AssembleSvcCompsite svc = statusRpc.get();
            // 将任务注册，可供页面展示
            // svc.reportDumpJobStatus(false, false, true, execChainContext.getTaskId()
            // , fileName.getName(), 0, 0);
            dumpStatus.getTable(fileName.getName()).setWaiting(true);
        }
        logger.info("trigger dataX jobs by mode:{},with:{}", this.getDataXTriggerType(), cfgFileNames.stream().map((f) -> f.getName()).collect(Collectors.joining(",")));
        // for (IRemoteJobTrigger t : triggers) {
        // t.submitJob();
        // }
        IDataxReader reader = appSource.getReader(null);
        List<ISelectedTab> selectedTabs = reader.getSelectedTabs();
        // Map<String, IDataxProcessor.TableAlias> tabAlias = appSource.getTabAlias();
        IDataxWriter writer = appSource.getWriter(null);
        if (writer instanceof IDataXBatchPost) {
            IDataXBatchPost batchPostTask = (IDataXBatchPost) writer;
            JoinPhaseStatus phaseStatus = this.getPhaseStatus(execChainContext, FullbuildPhase.JOIN);
            for (ISelectedTab entry : selectedTabs) {
                IRemoteTaskTrigger postTaskTrigger = batchPostTask.createPostTask(execChainContext, entry);
                triggers.add(postTaskTrigger);
                JoinPhaseStatus.JoinTaskStatus taskStatus = phaseStatus.getTaskStatus(postTaskTrigger.getTaskName());
                taskStatus.setWaiting(true);
                taskMap.put(postTaskTrigger.getTaskName(), new TISReactor.TaskAndMilestone(createJoinTask(postTaskTrigger, taskStatus)));
            }
        }
        // example: "->a ->b a,b->c"
        String dagSessionSpec = triggers.stream().map((trigger) -> {
            List<String> dpts = trigger.getTaskDependencies();
            return dpts.stream().collect(Collectors.joining(",")) + "->" + trigger.getTaskName();
        }).collect(Collectors.joining(" "));
        logger.info("dataX:{} of dagSessionSpec:{}", execChainContext.getIndexName(), dagSessionSpec);
        ExecuteResult[] faildResult = new ExecuteResult[] { ExecuteResult.createSuccess() };
        this.executeDAG(executorService, execChainContext, dagSessionSpec, taskMap, new ReactorListener() {

            @Override
            public void onTaskCompleted(Task t) {
            // dumpPhaseStatus.isComplete();
            // joinPhaseStatus.isComplete();
            }

            @Override
            public void onTaskFailed(Task t, Throwable err, boolean fatal) {
                logger.error(t.getDisplayName(), err);
                faildResult[0] = ExecuteResult.createFaild().setMessage("status.runningStatus.isComplete():" + err.getMessage());
                if (err instanceof InterruptedException) {
                    logger.warn("DataX Name:{},taskid:{} has been canceled", execChainContext.getIndexName(), execChainContext.getTaskId());
                    // this job has been cancel, trigger from TisServlet.doDelete()
                    for (IRemoteTaskTrigger tt : triggers) {
                        try {
                            tt.cancel();
                        } catch (Throwable ex) {
                        }
                    }
                }
            }
        });
        // ExecuteResult result = new ExecuteResult(!faild);
        for (IRemoteTaskTrigger trigger : triggers) {
            if (trigger.isAsyn()) {
                execChainContext.addAsynSubJob(new IExecChainContext.AsynSubJob(trigger.getAsynJobName()));
            }
        }
        return faildResult[0];
    } finally {
        try {
            dataXJobContext.destroy();
        } catch (Throwable e) {
            logger.error(e.getMessage(), e);
        }
    }
}