Examples with DefaultChainContext com.qlangtech.tis.exec.impl.DefaultChainContext used on opensource projects

Example 1 with DefaultChainContext

use of com.qlangtech.tis.exec.impl.DefaultChainContext in project tis by qlangtech.

the class TisServlet method doPost.

@Override
protected void doPost(HttpServletRequest req, HttpServletResponse res) throws ServletException, IOException {
    final HttpExecContext execContext = createHttpExecContext(req);
    final MDCParamContext mdcContext = this.getMDCParam(execContext, res);
    try {
        if (!isValidParams(execContext, req, res)) {
            return;
        }
        if (!mdcContext.validateParam()) {
            return;
        }
        final ExecuteLock lock = mdcContext.getExecLock();
        // getLog().info("start to execute index swap work flow");
        final CountDownLatch countDown = new CountDownLatch(1);
        DefaultChainContext chainContext = new DefaultChainContext(execContext);
        chainContext.setMdcParamContext(mdcContext);
        chainContext.setAppSourcePipelineController(IncrStatusUmbilicalProtocolImpl.getInstance());
        final Integer newTaskId = createNewTask(chainContext);
        lock.addTaskFuture(newTaskId, executeService.submit(() -> {
            // MDC.put("app", indexName);
            getLog().info("index swap start to work");
            try {
                while (true) {
                    try {
                        if (lock.lock()) {
                            try {
                                String msg = "trigger task" + mdcContext.getExecLockKey() + " successful";
                                getLog().info(msg);
                                mdcContext.resetParam(newTaskId);
                                writeResult(true, msg, res, new KV(IExecChainContext.KEY_TASK_ID, String.valueOf(newTaskId)));
                                // IndexBuilderTriggerFactory builderFactory = HeteroEnum.INDEX_BUILD_CONTAINER.getPlugin();
                                // Objects.requireNonNull(builderFactory, "builderFactory can not be null");
                                // chainContext.setIndexBuildFileSystem(builderFactory.getFsFactory());
                                // chainContext.setIndexBuilderTriggerFactory(builderFactory);
                                // IPluginStore<TableDumpFactory> tableDumpFactory = TIS.getPluginStore(TableDumpFactory.class);
                                // if (tableDumpFactory.getPlugin() != null) {
                                // Objects.requireNonNull(tableDumpFactory.getPlugin(), "tableDumpFactory can not be null");
                                // chainContext.setTableDumpFactory(tableDumpFactory.getPlugin());
                                // }
                                countDown.countDown();
                                /**
                                 **********************************************************
                                 * 开始执行内部任务
                                 ***********************************************************
                                 */
                                ExecResult execResult = startWork(chainContext).isSuccess() ? ExecResult.SUCCESS : ExecResult.FAILD;
                                DagTaskUtils.createTaskComplete(newTaskId, chainContext, execResult);
                            } catch (InterruptedException e) {
                                // 说明当前任务被 终止了
                                logger.info("taskid:{} has been canceled", newTaskId);
                                return;
                            } catch (Throwable e) {
                                DagTaskUtils.createTaskComplete(newTaskId, chainContext, ExecResult.FAILD);
                                getLog().error(e.getMessage(), e);
                                throw new RuntimeException(e);
                            } finally {
                                lock.clearLockFutureQueue();
                            }
                        } else {
                            if (lock.isExpire()) {
                                getLog().warn("this lock has expire,this lock will cancel");
                                // 执行已經超時
                                lock.clearLockFutureQueue();
                                // while (lock.futureQueue.size() >= 1)
                                // {
                                // lock.futureQueue.poll().cancel(true);
                                // }
                                getLog().warn("this lock[" + lock.getTaskOwnerUniqueName() + "] has expire,has unlocked");
                                continue;
                            } else {
                                String msg = "pre task[" + lock.getTaskOwnerUniqueName() + "] is executing ,so this commit will be ignore";
                                getLog().warn(msg);
                                writeResult(false, msg, res);
                            }
                            countDown.countDown();
                        }
                        // }
                        break;
                    } catch (Throwable e) {
                        getLog().error(e.getMessage(), e);
                        try {
                            if (countDown.getCount() > 0) {
                                writeResult(false, ExceptionUtils.getMessage(e), res);
                            }
                        } catch (Exception e1) {
                        } finally {
                            try {
                                countDown.countDown();
                            } catch (Throwable ee) {
                            }
                        }
                        break;
                    }
                }
            } finally {
                mdcContext.removeParam();
            }
        // end run
        }));
        try {
            countDown.await(30, TimeUnit.SECONDS);
        } catch (InterruptedException e) {
        }
    } finally {
        mdcContext.removeParam();
    }
}

Example 2 with DefaultChainContext

use of com.qlangtech.tis.exec.impl.DefaultChainContext in project tis by qlangtech.

the class TestDefaultChainContext method testLoadPhaseStatusFromLatest.

public void testLoadPhaseStatusFromLatest() {
    IParamContext paramContext = this.mock("paramContext", IParamContext.class);
    DefaultChainContext chainContext = new DefaultChainContext(paramContext);
    PhaseStatusCollection statusCollection = chainContext.loadPhaseStatusFromLatest(dataXname);
    assertNull(statusCollection);
    // ./src/test/resources/com/qlangtech/tis/full/dump/cfg_repo/df-logs/66/dump
    statusCollection = chainContext.loadPhaseStatusFromLatest(dataXname);
    assertNotNull(statusCollection);
    DumpPhaseStatus dumpPhase = statusCollection.getDumpPhase();
    assertNotNull(dumpPhase);
    assertEquals(62, dumpPhase.getTaskId());
    String dataXFileName = "instancedetail_0.json";
    DumpPhaseStatus.TableDumpStatus dataXExecStatus = dumpPhase.getTable(dataXFileName);
    assertNotNull(dataXFileName + " relevant dataX instance can be null", dataXExecStatus);
    assertEquals(524525, dataXExecStatus.getReadRows());
    assertEquals(1000001, dataXExecStatus.getAllRows());
}

Example 3 with DefaultChainContext

use of com.qlangtech.tis.exec.impl.DefaultChainContext in project tis by qlangtech.

the class TestIndexSwapTaskflowLauncher method createRangeChainContext.

// 
public static DefaultChainContext createRangeChainContext(String collectionName, FullbuildPhase start, FullbuildPhase end, String... pts) throws Exception {
    TestParamContext params = new TestParamContext();
    params.set(IFullBuildContext.KEY_APP_SHARD_COUNT, String.valueOf(shardCount));
    params.set(IFullBuildContext.KEY_APP_NAME, collectionName);
    params.set(IFullBuildContext.KEY_WORKFLOW_NAME, "totalpay");
    params.set(IFullBuildContext.KEY_WORKFLOW_ID, WF_ID);
    params.set(IExecChainContext.COMPONENT_START, start.getName());
    params.set(IExecChainContext.COMPONENT_END, end.getName());
    final DefaultChainContext chainContext = new DefaultChainContext(params);
    ExecutePhaseRange range = chainContext.getExecutePhaseRange();
    Assert.assertEquals(start, range.getStart());
    Assert.assertEquals(end, range.getEnd());
    Map<IDumpTable, ITabPartition> dateParams = Maps.newHashMap();
    chainContext.setAttribute(ExecChainContextUtils.PARTITION_DATA_PARAMS, new TabPartitions(dateParams));
    chainContext.setAttribute(IExecChainContext.KEY_TASK_ID, TASK_ID);
    // final PluginStore<IndexBuilderTriggerFactory> buildTriggerFactory = TIS.getPluginStore(IndexBuilderTriggerFactory.class);
    // assertNotNull(buildTriggerFactory.getPlugin());
    // if (pts.length > 0) {
    // chainContext.setPs(pts[0]);
    // } else {
    // SimpleDateFormat format = new SimpleDateFormat("yyyyMMddHHmmss");
    // String pt = format.format(new Date());
    // chainContext.setPs(pt);
    // }
    chainContext.setMdcParamContext(() -> {
    });
    return chainContext;
}

Example 4 with DefaultChainContext

use of com.qlangtech.tis.exec.impl.DefaultChainContext in project tis by qlangtech.

the class TestIndexSwapTaskflowLauncherWithDataXTrigger method testDataXProcessTrigger.

public void testDataXProcessTrigger() throws Exception {
    IndexSwapTaskflowLauncher taskflowLauncher = new IndexSwapTaskflowLauncher();
    DefaultChainContext chainContext = createRangeChainContext(FullbuildPhase.FullDump, FullbuildPhase.FullDump);
    ExecuteResult executeResult = taskflowLauncher.startWork(chainContext);
    assertTrue(executeResult.isSuccess());
}

Example 5 with DefaultChainContext

use of com.qlangtech.tis.exec.impl.DefaultChainContext in project tis by qlangtech.

the class TestIndexSwapTaskflowLauncherWithDataXTrigger method createRangeChainContext.

public static DefaultChainContext createRangeChainContext(FullbuildPhase start, FullbuildPhase end) throws Exception {
    TestParamContext params = new TestParamContext();
    params.set(IFullBuildContext.KEY_APP_NAME, DATAX_NAME);
    params.set(IExecChainContext.COMPONENT_START, start.getName());
    params.set(IExecChainContext.COMPONENT_END, end.getName());
    final DefaultChainContext chainContext = new DefaultChainContext(params);
    ExecutePhaseRange range = chainContext.getExecutePhaseRange();
    Assert.assertEquals(start, range.getStart());
    Assert.assertEquals(end, range.getEnd());
    chainContext.setAttribute(IExecChainContext.KEY_TASK_ID, TASK_ID);
    chainContext.setMdcParamContext(() -> {
    });
    return chainContext;
}