Examples with IRemoteTaskTrigger com.qlangtech.tis.fullbuild.indexbuild.IRemoteTaskTrigger used on opensource projects

Example 1 with IRemoteTaskTrigger

use of com.qlangtech.tis.fullbuild.indexbuild.IRemoteTaskTrigger in project plugins by qlangtech.

the class TestDistributedOverseerDataXJobSubmit method testPushMsgToDistributeQueue.

public void testPushMsgToDistributeQueue() {
    DataXJobWorker dataxJobWorker = DataXJobWorker.getJobWorker(DataXJobWorker.K8S_DATAX_INSTANCE_NAME);
    assertEquals("/datax/jobs", dataxJobWorker.getZkQueuePath());
    assertEquals("192.168.28.200:2181/tis/cloud", dataxJobWorker.getZookeeperAddress());
    DataxProcessor dataxProcessor = IAppSource.load(DATAX_NAME);
    assertNotNull(dataxProcessor);
    // IDataxProcessor dataxProcessor = EasyMock.createMock("dataxProcessor", IDataxProcessor.class);
    // EasyMock.expect(dataxProcessor.getDataxCfgDir()).andReturn();
    IJoinTaskContext taskContext = EasyMock.createMock("joinTaskContext", IJoinTaskContext.class);
    EasyMock.expect(taskContext.getIndexName()).andReturn(DATAX_NAME);
    EasyMock.expect(taskContext.getTaskId()).andReturn(DATAX_TASK_ID);
    AtomicReference<ITISRpcService> ref = new AtomicReference<>();
    ref.set(StatusRpcClient.AssembleSvcCompsite.MOCK_PRC);
    RpcServiceReference svcRef = new RpcServiceReference(ref);
    Optional<DataXJobSubmit> jobSubmit = DataXJobSubmit.getDataXJobSubmit(DataXJobSubmit.InstanceType.DISTRIBUTE);
    assertTrue(jobSubmit.isPresent());
    DataXJobSubmit submit = jobSubmit.get();
    DataXJobSubmit.IDataXJobContext jobContext = submit.createJobContext(taskContext);
    EasyMock.replay(taskContext);
    // IJoinTaskContext taskContext
    // , RpcServiceReference statusRpc, IDataxProcessor dataxProcessor, String dataXfileName
    IRemoteTaskTrigger dataXJob = submit.createDataXJob(jobContext, svcRef, dataxProcessor, DATAX_JOB_FILE_NAME);
    dataXJob.run();
    EasyMock.verify(taskContext);
}

Example 2 with IRemoteTaskTrigger

use of com.qlangtech.tis.fullbuild.indexbuild.IRemoteTaskTrigger in project plugins by qlangtech.

the class TestLocalDataXJobSubmit method testCreateDataXJob.

public void testCreateDataXJob() throws Exception {
    Optional<DataXJobSubmit> dataXJobSubmit = DataXJobSubmit.getDataXJobSubmit(DataXJobSubmit.InstanceType.LOCAL);
    Assert.assertTrue("dataXJobSubmit shall present", dataXJobSubmit.isPresent());
    LocalDataXJobSubmit jobSubmit = (LocalDataXJobSubmit) dataXJobSubmit.get();
    jobSubmit.setMainClassName(LocalDataXJobMainEntrypoint.class.getName());
    jobSubmit.setWorkingDirectory(new File("."));
    jobSubmit.setClasspath("target/classes:target/test-classes");
    AtomicReference<ITISRpcService> ref = new AtomicReference<>();
    ref.set(StatusRpcClient.AssembleSvcCompsite.MOCK_PRC);
    RpcServiceReference statusRpc = new RpcServiceReference(ref);
    DataXJobSubmit.IDataXJobContext dataXJobContext = EasyMock.createMock("dataXJobContext", DataXJobSubmit.IDataXJobContext.class);
    IExecChainContext taskContext = EasyMock.createMock("taskContext", IExecChainContext.class);
    EasyMock.expect(dataXJobContext.getTaskContext()).andReturn(taskContext).anyTimes();
    IDataxProcessor dataxProcessor = EasyMock.createMock("dataxProcessor", IDataxProcessor.class);
    EasyMock.expect(taskContext.getIndexName()).andReturn(dataXName).anyTimes();
    EasyMock.expect(taskContext.getTaskId()).andReturn(TaskId).anyTimes();
    int preSuccessTaskId = 99;
    PhaseStatusCollection preSuccessTask = new PhaseStatusCollection(preSuccessTaskId, new ExecutePhaseRange(FullbuildPhase.FullDump, FullbuildPhase.FullDump));
    DumpPhaseStatus preDumpStatus = new DumpPhaseStatus(preSuccessTaskId);
    DumpPhaseStatus.TableDumpStatus tableDumpStatus = preDumpStatus.getTable(dataXfileName);
    tableDumpStatus.setAllRows(LocalDataXJobMainEntrypoint.testAllRows);
    preSuccessTask.setDumpPhase(preDumpStatus);
    EasyMock.expect(taskContext.loadPhaseStatusFromLatest(dataXName)).andReturn(preSuccessTask).times(3);
    TisZkClient zkClient = EasyMock.createMock("TisZkClient", TisZkClient.class);
    String zkSubPath = "nodes0000000020";
    EasyMock.expect(zkClient.getChildren(ZkUtils.ZK_ASSEMBLE_LOG_COLLECT_PATH, null, true)).andReturn(Collections.singletonList(zkSubPath)).times(3);
    EasyMock.expect(zkClient.getData(EasyMock.eq(ZkUtils.ZK_ASSEMBLE_LOG_COLLECT_PATH + "/" + zkSubPath), EasyMock.isNull(), EasyMock.anyObject(Stat.class), EasyMock.eq(true))).andReturn(statusCollectorHost.getBytes(TisUTF8.get())).times(3);
    EasyMock.expect(taskContext.getZkClient()).andReturn(zkClient).anyTimes();
    EasyMock.replay(taskContext, dataxProcessor, zkClient, dataXJobContext);
    IRemoteTaskTrigger dataXJob = jobSubmit.createDataXJob(dataXJobContext, statusRpc, dataxProcessor, dataXfileName);
    RunningStatus running = getRunningStatus(dataXJob);
    assertTrue("running.isSuccess", running.isSuccess());
    jobSubmit.setMainClassName(LocalDataXJobMainEntrypointThrowException.class.getName());
    dataXJob = jobSubmit.createDataXJob(dataXJobContext, statusRpc, dataxProcessor, dataXfileName);
    running = getRunningStatus(dataXJob);
    assertFalse("shall faild", running.isSuccess());
    assertTrue("shall complete", running.isComplete());
    jobSubmit.setMainClassName(LocalDataXJobMainEntrypointCancellable.class.getName());
    dataXJob = jobSubmit.createDataXJob(dataXJobContext, statusRpc, dataxProcessor, dataXfileName);
    running = getRunningStatus(dataXJob, false);
    Thread.sleep(2000);
    dataXJob.cancel();
    int i = 0;
    while (i++ < 3 && !(running = dataXJob.getRunningStatus()).isComplete()) {
        Thread.sleep(1000);
    }
    assertFalse("shall faild", running.isSuccess());
    assertTrue("shall complete", running.isComplete());
    EasyMock.verify(taskContext, dataxProcessor, zkClient);
}

Example 3 with IRemoteTaskTrigger

use of com.qlangtech.tis.fullbuild.indexbuild.IRemoteTaskTrigger in project plugins by qlangtech.

the class TaskExec method getRemoteJobTrigger.

static IRemoteTaskTrigger getRemoteJobTrigger(DataXJobSubmit.IDataXJobContext jobContext, LocalDataXJobSubmit localDataXJobSubmit, String dataXfileName) {
    // final JarLoader uberClassLoader = new TISJarLoader(pluginManager);
    IJoinTaskContext taskContext = jobContext.getTaskContext();
    AtomicBoolean complete = new AtomicBoolean(false);
    AtomicBoolean success = new AtomicBoolean(false);
    return new IRemoteTaskTrigger() {

        DataXJobSingleProcessorExecutor jobConsumer;

        boolean hasCanceled;

        // final ExecutorService dataXExecutor = jobContext.getContextInstance();
        @Override
        public void run() {
            // dataXExecutor.submit(() -> {
            try {
                MDC.put(IParamContext.KEY_TASK_ID, String.valueOf(taskContext.getTaskId()));
                MDC.put(TISCollectionUtils.KEY_COLLECTION, taskContext.getIndexName());
                jobConsumer = new DataXJobSingleProcessorExecutor() {

                    @Override
                    protected DataXJobSubmit.InstanceType getExecMode() {
                        return DataXJobSubmit.InstanceType.LOCAL;
                    }

                    @Override
                    protected String getClasspath() {
                        return localDataXJobSubmit.getClasspath();
                    }

                    @Override
                    protected boolean useRuntimePropEnvProps() {
                        return false;
                    }

                    @Override
                    protected String[] getExtraJavaSystemPrams() {
                        return new String[] { // "-D" + SYSTEM_KEY_LOGBACK_PATH_KEY + "=" + SYSTEM_KEY_LOGBACK_PATH_VALUE
                        "-D" + CenterResource.KEY_notFetchFromCenterRepository + "=true" };
                    }

                    @Override
                    protected String getIncrStateCollectAddress() {
                        return ZkUtils.getFirstChildValue(((IExecChainContext) taskContext).getZkClient(), ZkUtils.ZK_ASSEMBLE_LOG_COLLECT_PATH);
                    }

                    @Override
                    protected String getMainClassName() {
                        return localDataXJobSubmit.getMainClassName();
                    }

                    @Override
                    protected File getWorkingDirectory() {
                        return localDataXJobSubmit.getWorkingDirectory();
                    }
                };
                CuratorDataXTaskMessage dataXJob = localDataXJobSubmit.getDataXJobDTO(taskContext, dataXfileName);
                // new CuratorDataXTaskMessage();
                // dataXJob.setJobId(taskContext.getTaskId());
                // dataXJob.setJobName(dataXfileName);
                // dataXJob.setDataXName(taskContext.getIndexName());
                jobConsumer.consumeMessage(dataXJob);
                success.set(true);
            } catch (Throwable e) {
                // e.printStackTrace();
                success.set(false);
                if (this.hasCanceled) {
                    logger.warn("datax:" + taskContext.getIndexName() + " has been canceled");
                } else {
                    logger.error("datax:" + taskContext.getIndexName() + ",jobName:" + dataXfileName, e);
                    if (!(e instanceof DataXJobSingleProcessorException)) {
                        throw new RuntimeException(e);
                    }
                }
            } finally {
                complete.set(true);
            // shutdownExecutor();
            }
        // });
        }

        // private void shutdownExecutor() {
        // try {
        // dataXExecutor.shutdownNow();
        // } catch (Throwable e) {
        // logger.error(e.getMessage(), e);
        // }
        // }
        @Override
        public String getTaskName() {
            return dataXfileName;
        }

        @Override
        public void cancel() {
            if (jobConsumer == null) {
                return;
            }
            jobConsumer.runningTask.forEach((taskId, watchdog) -> {
                watchdog.destroyProcess();
                logger.info("taskId:{} relevant task has been canceled", taskId);
            });
            // shutdownExecutor();
            this.hasCanceled = true;
        }

        @Override
        public RunningStatus getRunningStatus() {
            return new RunningStatus(0, complete.get(), success.get());
        }
    };
}

Example 4 with IRemoteTaskTrigger

use of com.qlangtech.tis.fullbuild.indexbuild.IRemoteTaskTrigger in project tis by qlangtech.

the class DataXExecuteInterceptor method execute.

@Override
protected ExecuteResult execute(IExecChainContext execChainContext) throws Exception {
    int nThreads = 2;
    final ExecutorService executorService = new ThreadPoolExecutor(nThreads, nThreads, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(DataXJobSubmit.MAX_TABS_NUM_IN_PER_JOB), Executors.defaultThreadFactory());
    final Map<String, TISReactor.TaskAndMilestone> taskMap = Maps.newHashMap();
    RpcServiceReference statusRpc = getDataXExecReporter();
    DataxProcessor appSource = execChainContext.getAppSource();
    IRemoteTaskTrigger jobTrigger = null;
    RunningStatus runningStatus = null;
    List<IRemoteTaskTrigger> triggers = Lists.newArrayList();
    List<File> cfgFileNames = appSource.getDataxCfgFileNames(null);
    if (CollectionUtils.isEmpty(cfgFileNames)) {
        throw new IllegalStateException("dataX cfgFileNames can not be empty");
    }
    DataXJobSubmit.InstanceType expectDataXJobSumit = getDataXTriggerType();
    Optional<DataXJobSubmit> jobSubmit = DataXJobSubmit.getDataXJobSubmit(expectDataXJobSumit);
    // 如果分布式worker ready的话
    if (!jobSubmit.isPresent()) {
        throw new IllegalStateException("can not find expect jobSubmit by type:" + expectDataXJobSumit);
    }
    DataXJobSubmit submit = jobSubmit.get();
    final DataXJobSubmit.IDataXJobContext dataXJobContext = submit.createJobContext(execChainContext);
    Objects.requireNonNull(dataXJobContext, "dataXJobContext can not be null");
    try {
        DumpPhaseStatus dumpStatus = this.getPhaseStatus(execChainContext, FullbuildPhase.FullDump);
        for (File fileName : cfgFileNames) {
            jobTrigger = createDataXJob(dataXJobContext, submit, expectDataXJobSumit, statusRpc, appSource, fileName.getName());
            triggers.add(jobTrigger);
            taskMap.put(fileName.getName(), new TISReactor.TaskAndMilestone(DataflowTask.createDumpTask(jobTrigger)));
            // StatusRpcClient.AssembleSvcCompsite svc = statusRpc.get();
            // 将任务注册，可供页面展示
            // svc.reportDumpJobStatus(false, false, true, execChainContext.getTaskId()
            // , fileName.getName(), 0, 0);
            dumpStatus.getTable(fileName.getName()).setWaiting(true);
        }
        logger.info("trigger dataX jobs by mode:{},with:{}", this.getDataXTriggerType(), cfgFileNames.stream().map((f) -> f.getName()).collect(Collectors.joining(",")));
        // for (IRemoteJobTrigger t : triggers) {
        // t.submitJob();
        // }
        IDataxReader reader = appSource.getReader(null);
        List<ISelectedTab> selectedTabs = reader.getSelectedTabs();
        // Map<String, IDataxProcessor.TableAlias> tabAlias = appSource.getTabAlias();
        IDataxWriter writer = appSource.getWriter(null);
        if (writer instanceof IDataXBatchPost) {
            IDataXBatchPost batchPostTask = (IDataXBatchPost) writer;
            JoinPhaseStatus phaseStatus = this.getPhaseStatus(execChainContext, FullbuildPhase.JOIN);
            for (ISelectedTab entry : selectedTabs) {
                IRemoteTaskTrigger postTaskTrigger = batchPostTask.createPostTask(execChainContext, entry);
                triggers.add(postTaskTrigger);
                JoinPhaseStatus.JoinTaskStatus taskStatus = phaseStatus.getTaskStatus(postTaskTrigger.getTaskName());
                taskStatus.setWaiting(true);
                taskMap.put(postTaskTrigger.getTaskName(), new TISReactor.TaskAndMilestone(createJoinTask(postTaskTrigger, taskStatus)));
            }
        }
        // example: "->a ->b a,b->c"
        String dagSessionSpec = triggers.stream().map((trigger) -> {
            List<String> dpts = trigger.getTaskDependencies();
            return dpts.stream().collect(Collectors.joining(",")) + "->" + trigger.getTaskName();
        }).collect(Collectors.joining(" "));
        logger.info("dataX:{} of dagSessionSpec:{}", execChainContext.getIndexName(), dagSessionSpec);
        ExecuteResult[] faildResult = new ExecuteResult[] { ExecuteResult.createSuccess() };
        this.executeDAG(executorService, execChainContext, dagSessionSpec, taskMap, new ReactorListener() {

            @Override
            public void onTaskCompleted(Task t) {
            // dumpPhaseStatus.isComplete();
            // joinPhaseStatus.isComplete();
            }

            @Override
            public void onTaskFailed(Task t, Throwable err, boolean fatal) {
                logger.error(t.getDisplayName(), err);
                faildResult[0] = ExecuteResult.createFaild().setMessage("status.runningStatus.isComplete():" + err.getMessage());
                if (err instanceof InterruptedException) {
                    logger.warn("DataX Name:{},taskid:{} has been canceled", execChainContext.getIndexName(), execChainContext.getTaskId());
                    // this job has been cancel, trigger from TisServlet.doDelete()
                    for (IRemoteTaskTrigger tt : triggers) {
                        try {
                            tt.cancel();
                        } catch (Throwable ex) {
                        }
                    }
                }
            }
        });
        // ExecuteResult result = new ExecuteResult(!faild);
        for (IRemoteTaskTrigger trigger : triggers) {
            if (trigger.isAsyn()) {
                execChainContext.addAsynSubJob(new IExecChainContext.AsynSubJob(trigger.getAsynJobName()));
            }
        }
        return faildResult[0];
    } finally {
        try {
            dataXJobContext.destroy();
        } catch (Throwable e) {
            logger.error(e.getMessage(), e);
        }
    }
}

Example 5 with IRemoteTaskTrigger

use of com.qlangtech.tis.fullbuild.indexbuild.IRemoteTaskTrigger in project tis by qlangtech.

the class TestDataXExecuteInterceptor method testExecuteWithExcpetionWhenSubmitJob.

public void testExecuteWithExcpetionWhenSubmitJob() throws Exception {
    IRemoteTaskTrigger jobTrigger = mock("remoteJobTrigger", IRemoteTaskTrigger.class);
    // 
    EasyMock.expect(jobTrigger.isAsyn()).andReturn(false);
    jobTrigger.run();
    EasyMock.expectLastCall().andThrow(new RuntimeException("throw a exception"));
    RunningStatus runningStatus = RunningStatus.SUCCESS;
    EasyMock.expect(jobTrigger.getRunningStatus()).andReturn(runningStatus);
    try {
        executeJobTrigger(jobTrigger, true);
        fail("shall throw an exception");
    } catch (Exception e) {
    }
}