Examples with RunningStatus com.qlangtech.tis.fullbuild.indexbuild.RunningStatus used on opensource projects

Example 1 with RunningStatus

use of com.qlangtech.tis.fullbuild.indexbuild.RunningStatus in project plugins by qlangtech.

the class TestLocalTableDumpAndIndex method waitJobTerminatorAndAssert.

public static void waitJobTerminatorAndAssert(IRemoteJobTrigger triggerJob) throws InterruptedException {
    final RunningStatus runningStatus = triggerJob.getRunningStatus();
    assertNotNull(runningStatus);
    CountDownLatch countDown = new CountDownLatch(1);
    Runnable waitThread = () -> {
        try {
            RunningStatus s = runningStatus;
            while (!s.isComplete()) {
                Thread.sleep(1000);
                s = triggerJob.getRunningStatus();
            }
            assertTrue("s.isComplete()", s.isComplete());
            assertTrue("s.isSuccess()", s.isSuccess());
            countDown.countDown();
        } catch (InterruptedException e) {
            throw new RuntimeException(e);
        } finally {
        }
    };
    Thread t = new Thread(waitThread);
    Object[] errs = new Object[1];
    t.setUncaughtExceptionHandler((thread, e) -> {
        errs[0] = e;
        countDown.countDown();
    });
    t.start();
    if (!countDown.await(600, TimeUnit.SECONDS)) {
        fail("execute table dump expire 6s");
    }
    assertNull("errs[0] shall be null", errs[0]);
}

Example 2 with RunningStatus

use of com.qlangtech.tis.fullbuild.indexbuild.RunningStatus in project plugins by qlangtech.

the class LocalTableDumpFactory method triggerTask.

public static IRemoteJobTrigger triggerTask(TaskContext context, ILocalTask task) {
    TisZkClient zk = context.getCoordinator().unwrap();
    Objects.requireNonNull(zk, "zk(TisZkClient) can not be null");
    AtomicReference<Throwable> errRef = new AtomicReference<>();
    CountDownLatch countDown = new CountDownLatch(1);
    final ExecutorService executor = Executors.newSingleThreadExecutor((r) -> {
        Thread t = new Thread(r);
        t.setUncaughtExceptionHandler((thread, e) -> {
            errRef.set(e);
            logger.error(e.getMessage(), e);
            countDown.countDown();
        });
        return t;
    });
    return new IRemoteJobTrigger() {

        @Override
        public void submitJob() {
            executor.execute(() -> {
                RpcServiceReference statusRpc = null;
                try {
                    statusRpc = StatusRpcClient.getService(zk);
                    task.process(statusRpc);
                    countDown.countDown();
                } catch (Exception e) {
                    throw new RuntimeException(e);
                } finally {
                    try {
                        statusRpc.get().close();
                    } catch (Throwable e) {
                    }
                }
            });
        }

        @Override
        public RunningStatus getRunningStatus() {
            RunningStatus runningStatus = null;
            // 反馈执行状态
            if (countDown.getCount() > 0) {
                runningStatus = new RunningStatus(0, false, false);
            } else {
                executor.shutdown();
                runningStatus = new RunningStatus(1, true, errRef.get() == null);
            }
            return runningStatus;
        }
    };
}

Example 3 with RunningStatus

use of com.qlangtech.tis.fullbuild.indexbuild.RunningStatus in project plugins by qlangtech.

the class TestLocalDataXJobSubmit method getRunningStatus.

protected RunningStatus getRunningStatus(IRemoteTaskTrigger dataXJob, boolean waitting) {
    dataXJob.run();
    RunningStatus running = null;
    while ((running = dataXJob.getRunningStatus()) != null && waitting) {
        if (running.isComplete()) {
            break;
        }
    }
    assertNotNull(running);
    return running;
}

Example 4 with RunningStatus

use of com.qlangtech.tis.fullbuild.indexbuild.RunningStatus in project plugins by qlangtech.

the class TestLocalDataXJobSubmit method testCreateDataXJob.

public void testCreateDataXJob() throws Exception {
    Optional<DataXJobSubmit> dataXJobSubmit = DataXJobSubmit.getDataXJobSubmit(DataXJobSubmit.InstanceType.LOCAL);
    Assert.assertTrue("dataXJobSubmit shall present", dataXJobSubmit.isPresent());
    LocalDataXJobSubmit jobSubmit = (LocalDataXJobSubmit) dataXJobSubmit.get();
    jobSubmit.setMainClassName(LocalDataXJobMainEntrypoint.class.getName());
    jobSubmit.setWorkingDirectory(new File("."));
    jobSubmit.setClasspath("target/classes:target/test-classes");
    AtomicReference<ITISRpcService> ref = new AtomicReference<>();
    ref.set(StatusRpcClient.AssembleSvcCompsite.MOCK_PRC);
    RpcServiceReference statusRpc = new RpcServiceReference(ref);
    DataXJobSubmit.IDataXJobContext dataXJobContext = EasyMock.createMock("dataXJobContext", DataXJobSubmit.IDataXJobContext.class);
    IExecChainContext taskContext = EasyMock.createMock("taskContext", IExecChainContext.class);
    EasyMock.expect(dataXJobContext.getTaskContext()).andReturn(taskContext).anyTimes();
    IDataxProcessor dataxProcessor = EasyMock.createMock("dataxProcessor", IDataxProcessor.class);
    EasyMock.expect(taskContext.getIndexName()).andReturn(dataXName).anyTimes();
    EasyMock.expect(taskContext.getTaskId()).andReturn(TaskId).anyTimes();
    int preSuccessTaskId = 99;
    PhaseStatusCollection preSuccessTask = new PhaseStatusCollection(preSuccessTaskId, new ExecutePhaseRange(FullbuildPhase.FullDump, FullbuildPhase.FullDump));
    DumpPhaseStatus preDumpStatus = new DumpPhaseStatus(preSuccessTaskId);
    DumpPhaseStatus.TableDumpStatus tableDumpStatus = preDumpStatus.getTable(dataXfileName);
    tableDumpStatus.setAllRows(LocalDataXJobMainEntrypoint.testAllRows);
    preSuccessTask.setDumpPhase(preDumpStatus);
    EasyMock.expect(taskContext.loadPhaseStatusFromLatest(dataXName)).andReturn(preSuccessTask).times(3);
    TisZkClient zkClient = EasyMock.createMock("TisZkClient", TisZkClient.class);
    String zkSubPath = "nodes0000000020";
    EasyMock.expect(zkClient.getChildren(ZkUtils.ZK_ASSEMBLE_LOG_COLLECT_PATH, null, true)).andReturn(Collections.singletonList(zkSubPath)).times(3);
    EasyMock.expect(zkClient.getData(EasyMock.eq(ZkUtils.ZK_ASSEMBLE_LOG_COLLECT_PATH + "/" + zkSubPath), EasyMock.isNull(), EasyMock.anyObject(Stat.class), EasyMock.eq(true))).andReturn(statusCollectorHost.getBytes(TisUTF8.get())).times(3);
    EasyMock.expect(taskContext.getZkClient()).andReturn(zkClient).anyTimes();
    EasyMock.replay(taskContext, dataxProcessor, zkClient, dataXJobContext);
    IRemoteTaskTrigger dataXJob = jobSubmit.createDataXJob(dataXJobContext, statusRpc, dataxProcessor, dataXfileName);
    RunningStatus running = getRunningStatus(dataXJob);
    assertTrue("running.isSuccess", running.isSuccess());
    jobSubmit.setMainClassName(LocalDataXJobMainEntrypointThrowException.class.getName());
    dataXJob = jobSubmit.createDataXJob(dataXJobContext, statusRpc, dataxProcessor, dataXfileName);
    running = getRunningStatus(dataXJob);
    assertFalse("shall faild", running.isSuccess());
    assertTrue("shall complete", running.isComplete());
    jobSubmit.setMainClassName(LocalDataXJobMainEntrypointCancellable.class.getName());
    dataXJob = jobSubmit.createDataXJob(dataXJobContext, statusRpc, dataxProcessor, dataXfileName);
    running = getRunningStatus(dataXJob, false);
    Thread.sleep(2000);
    dataXJob.cancel();
    int i = 0;
    while (i++ < 3 && !(running = dataXJob.getRunningStatus()).isComplete()) {
        Thread.sleep(1000);
    }
    assertFalse("shall faild", running.isSuccess());
    assertTrue("shall complete", running.isComplete());
    EasyMock.verify(taskContext, dataxProcessor, zkClient);
}

Example 5 with RunningStatus

use of com.qlangtech.tis.fullbuild.indexbuild.RunningStatus in project plugins by qlangtech.

the class TaskExec method getRemoteJobTrigger.

static IRemoteTaskTrigger getRemoteJobTrigger(DataXJobSubmit.IDataXJobContext jobContext, LocalDataXJobSubmit localDataXJobSubmit, String dataXfileName) {
    // final JarLoader uberClassLoader = new TISJarLoader(pluginManager);
    IJoinTaskContext taskContext = jobContext.getTaskContext();
    AtomicBoolean complete = new AtomicBoolean(false);
    AtomicBoolean success = new AtomicBoolean(false);
    return new IRemoteTaskTrigger() {

        DataXJobSingleProcessorExecutor jobConsumer;

        boolean hasCanceled;

        // final ExecutorService dataXExecutor = jobContext.getContextInstance();
        @Override
        public void run() {
            // dataXExecutor.submit(() -> {
            try {
                MDC.put(IParamContext.KEY_TASK_ID, String.valueOf(taskContext.getTaskId()));
                MDC.put(TISCollectionUtils.KEY_COLLECTION, taskContext.getIndexName());
                jobConsumer = new DataXJobSingleProcessorExecutor() {

                    @Override
                    protected DataXJobSubmit.InstanceType getExecMode() {
                        return DataXJobSubmit.InstanceType.LOCAL;
                    }

                    @Override
                    protected String getClasspath() {
                        return localDataXJobSubmit.getClasspath();
                    }

                    @Override
                    protected boolean useRuntimePropEnvProps() {
                        return false;
                    }

                    @Override
                    protected String[] getExtraJavaSystemPrams() {
                        return new String[] { // "-D" + SYSTEM_KEY_LOGBACK_PATH_KEY + "=" + SYSTEM_KEY_LOGBACK_PATH_VALUE
                        "-D" + CenterResource.KEY_notFetchFromCenterRepository + "=true" };
                    }

                    @Override
                    protected String getIncrStateCollectAddress() {
                        return ZkUtils.getFirstChildValue(((IExecChainContext) taskContext).getZkClient(), ZkUtils.ZK_ASSEMBLE_LOG_COLLECT_PATH);
                    }

                    @Override
                    protected String getMainClassName() {
                        return localDataXJobSubmit.getMainClassName();
                    }

                    @Override
                    protected File getWorkingDirectory() {
                        return localDataXJobSubmit.getWorkingDirectory();
                    }
                };
                CuratorDataXTaskMessage dataXJob = localDataXJobSubmit.getDataXJobDTO(taskContext, dataXfileName);
                // new CuratorDataXTaskMessage();
                // dataXJob.setJobId(taskContext.getTaskId());
                // dataXJob.setJobName(dataXfileName);
                // dataXJob.setDataXName(taskContext.getIndexName());
                jobConsumer.consumeMessage(dataXJob);
                success.set(true);
            } catch (Throwable e) {
                // e.printStackTrace();
                success.set(false);
                if (this.hasCanceled) {
                    logger.warn("datax:" + taskContext.getIndexName() + " has been canceled");
                } else {
                    logger.error("datax:" + taskContext.getIndexName() + ",jobName:" + dataXfileName, e);
                    if (!(e instanceof DataXJobSingleProcessorException)) {
                        throw new RuntimeException(e);
                    }
                }
            } finally {
                complete.set(true);
            // shutdownExecutor();
            }
        // });
        }

        // private void shutdownExecutor() {
        // try {
        // dataXExecutor.shutdownNow();
        // } catch (Throwable e) {
        // logger.error(e.getMessage(), e);
        // }
        // }
        @Override
        public String getTaskName() {
            return dataXfileName;
        }

        @Override
        public void cancel() {
            if (jobConsumer == null) {
                return;
            }
            jobConsumer.runningTask.forEach((taskId, watchdog) -> {
                watchdog.destroyProcess();
                logger.info("taskId:{} relevant task has been canceled", taskId);
            });
            // shutdownExecutor();
            this.hasCanceled = true;
        }

        @Override
        public RunningStatus getRunningStatus() {
            return new RunningStatus(0, complete.get(), success.get());
        }
    };
}