Examples with DumpPhaseStatus com.qlangtech.tis.fullbuild.phasestatus.impl.DumpPhaseStatus used on opensource projects

Example 1 with DumpPhaseStatus

use of com.qlangtech.tis.fullbuild.phasestatus.impl.DumpPhaseStatus in project tis by qlangtech.

the class FullBuildStatCollectorServer method convertPP.

public PPhaseStatusCollection convertPP(PhaseStatusCollection phaseStatusSet) {
    PPhaseStatusCollection.Builder scBuilder = PPhaseStatusCollection.newBuilder();
    scBuilder.setTaskId(phaseStatusSet.getTaskid());
    DumpPhaseStatus dumpPhase = phaseStatusSet.getDumpPhase();
    JoinPhaseStatus joinPhase = phaseStatusSet.getJoinPhase();
    BuildPhaseStatus buildPhase = phaseStatusSet.getBuildPhase();
    IndexBackFlowPhaseStatus indexBackFlowPhase = phaseStatusSet.getIndexBackFlowPhaseStatus();
    if (dumpPhase != null) {
        PDumpPhaseStatus.Builder builder = PDumpPhaseStatus.newBuilder();
        com.qlangtech.tis.rpc.grpc.log.common.TableDumpStatus.Builder tabDump = null;
        DumpPhaseStatus.TableDumpStatus s = null;
        for (Map.Entry<String, DumpPhaseStatus.TableDumpStatus> entry : dumpPhase.tablesDump.entrySet()) {
            s = entry.getValue();
            tabDump = com.qlangtech.tis.rpc.grpc.log.common.TableDumpStatus.newBuilder();
            tabDump.setAllRows(s.getAllRows());
            tabDump.setTableName(s.getName());
            tabDump.setTaskid(s.getTaskid());
            tabDump.setReadRows(s.getReadRows());
            tabDump.setFaild(s.isFaild());
            tabDump.setComplete(s.isComplete());
            tabDump.setWaiting(s.isWaiting());
            builder.putTablesDump(entry.getKey(), tabDump.build());
        }
        scBuilder.setDumpPhase(builder);
    }
    if (joinPhase != null) {
        PJoinPhaseStatus.Builder builder = PJoinPhaseStatus.newBuilder();
        com.qlangtech.tis.rpc.grpc.log.common.JoinTaskStatus.Builder pbuilder = null;
        JoinPhaseStatus.JoinTaskStatus j = null;
        com.qlangtech.tis.rpc.grpc.log.common.JobLog.Builder jlog = null;
        JobLog jl = null;
        for (Map.Entry<String, JoinPhaseStatus.JoinTaskStatus> entry : joinPhase.taskStatus.entrySet()) {
            pbuilder = com.qlangtech.tis.rpc.grpc.log.common.JoinTaskStatus.newBuilder();
            j = entry.getValue();
            pbuilder.setJoinTaskName(j.getName());
            pbuilder.setFaild(j.isFaild());
            pbuilder.setComplete(j.isComplete());
            pbuilder.setWaiting(j.isWaiting());
            for (Map.Entry<Integer, JobLog> e : j.jobsStatus.entrySet()) {
                jl = e.getValue();
                jlog = com.qlangtech.tis.rpc.grpc.log.common.JobLog.newBuilder();
                jlog.setMapper(jl.getMapper());
                jlog.setReducer(jl.getReducer());
                jlog.setWaiting(jl.isWaiting());
                pbuilder.putJobStatus(e.getKey(), jlog.build());
            }
            builder.putTaskStatus(entry.getKey(), pbuilder.build());
        }
        scBuilder.setJoinPhase(builder);
    }
    if (buildPhase != null) {
        PBuildPhaseStatus.Builder builder = PBuildPhaseStatus.newBuilder();
        buildPhase.nodeBuildStatus.entrySet().stream().forEach((e) -> {
            com.qlangtech.tis.fullbuild.phasestatus.impl.BuildSharedPhaseStatus bf = null;
            com.qlangtech.tis.rpc.grpc.log.common.BuildSharedPhaseStatus.Builder bfBuilder = com.qlangtech.tis.rpc.grpc.log.common.BuildSharedPhaseStatus.newBuilder();
            bf = e.getValue();
            bfBuilder.setAllBuildSize(bf.getAllBuildSize());
            bfBuilder.setBuildReaded(bf.getBuildReaded());
            bfBuilder.setTaskid(bf.getTaskid());
            bfBuilder.setSharedName(bf.getSharedName());
            bfBuilder.setFaild(bf.isFaild());
            bfBuilder.setComplete(bf.isComplete());
            bfBuilder.setWaiting(bf.isWaiting());
            builder.putNodeBuildStatus(e.getKey(), bfBuilder.build());
        });
        scBuilder.setBuildPhase(builder);
    }
    if (indexBackFlowPhase != null) {
        PIndexBackFlowPhaseStatus.Builder builder = PIndexBackFlowPhaseStatus.newBuilder();
        indexBackFlowPhase.nodesStatus.entrySet().stream().forEach((e) -> {
            IndexBackFlowPhaseStatus.NodeBackflowStatus ib = e.getValue();
            com.qlangtech.tis.rpc.grpc.log.common.NodeBackflowStatus.Builder ibBuilder = com.qlangtech.tis.rpc.grpc.log.common.NodeBackflowStatus.newBuilder();
            ibBuilder.setNodeName(ib.getName());
            ibBuilder.setAllSize(ib.getAllSize());
            ibBuilder.setReaded(ib.getReaded());
            ibBuilder.setFaild(ib.isFaild());
            ibBuilder.setComplete(ib.isComplete());
            ibBuilder.setWaiting(ib.isWaiting());
            builder.putNodesStatus(e.getKey(), ibBuilder.build());
        });
        scBuilder.setIndexBackFlowPhaseStatus(builder);
    }
    return scBuilder.build();
}

Example 2 with DumpPhaseStatus

use of com.qlangtech.tis.fullbuild.phasestatus.impl.DumpPhaseStatus in project tis by qlangtech.

the class SingleTableAppSource method getProcessDataResults.

@Override
public ExecuteResult getProcessDataResults(IExecChainContext execChainContext, ISingleTableDumpFactory singleTableDumpFactory, IDataProcessFeedback dataProcessFeedback, ITaskPhaseInfo taskPhaseInfo) throws Exception {
    // 复杂数据导出
    DumpPhaseStatus dumpPhaseStatus = taskPhaseInfo.getPhaseStatus(execChainContext, FullbuildPhase.FullDump);
    DataflowTask tabDump = null;
    DependencyNode dump = new DependencyNode();
    dump.setId(db.getName() + "." + tabName);
    dump.setName(tabName);
    dump.setDbName(db.getName());
    dump.setTabid(String.valueOf(tabId));
    dump.setDbid(String.valueOf(db.getId()));
    // for (DependencyNode dump : topology.getDumpNodes()) {
    tabDump = singleTableDumpFactory.createSingleTableDump(dump, false, /* isHasValidTableDump */
    "tableDump.getPt()", execChainContext.getZkClient(), execChainContext, dumpPhaseStatus);
    tabDump.run();
    return ExecuteResult.SUCCESS;
}

Example 3 with DumpPhaseStatus

use of com.qlangtech.tis.fullbuild.phasestatus.impl.DumpPhaseStatus in project tis by qlangtech.

the class DataFlowAppSource method executeDAG.

private ExecuteResult executeDAG(IExecChainContext execChainContext, SqlTaskNodeMeta.SqlDataFlowTopology topology, IDataProcessFeedback dataProcessFeedback, Map<String, TISReactor.TaskAndMilestone> taskMap) {
    final ExecuteResult[] faildResult = new ExecuteResult[1];
    try {
        TISReactor reactor = new TISReactor(execChainContext, taskMap);
        String dagSessionSpec = topology.getDAGSessionSpec();
        logger.info("dagSessionSpec:" + dagSessionSpec);
        // final PrintWriter w = new PrintWriter(sw, true);
        ReactorListener listener = new ReactorListener() {

            // TODO: Does it really needs handlers to be synchronized?
            @Override
            public synchronized void onTaskCompleted(Task t) {
                processTaskResult(execChainContext, (TISReactor.TaskImpl) t, dataProcessFeedback, new ITaskResultProcessor() {

                    @Override
                    public void process(DumpPhaseStatus dumpPhase, TISReactor.TaskImpl task) {
                    }

                    @Override
                    public void process(JoinPhaseStatus joinPhase, TISReactor.TaskImpl task) {
                    }
                });
            }

            @Override
            public synchronized void onTaskFailed(Task t, Throwable err, boolean fatal) {
                // w.println("Failed " + t.getDisplayName() + " with " + err);
                processTaskResult(execChainContext, (TISReactor.TaskImpl) t, dataProcessFeedback, new ITaskResultProcessor() {

                    @Override
                    public void process(DumpPhaseStatus dumpPhase, TISReactor.TaskImpl task) {
                        dataProcessFeedback.reportDumpTableStatusError(execChainContext, task);
                    }

                    @Override
                    public void process(JoinPhaseStatus joinPhase, TISReactor.TaskImpl task) {
                        JoinPhaseStatus.JoinTaskStatus stat = joinPhase.getTaskStatus(task.getIdentityName());
                        // statReceiver.reportBuildIndexStatErr(execContext.getTaskId(),task.getIdentityName());
                        stat.setWaiting(false);
                        stat.setFaild(true);
                        stat.setComplete(true);
                    }
                });
            }
        };
        // 执行DAG地调度
        reactor.execute(executorService, reactor.buildSession(dagSessionSpec), listener, new ReactorListener() {

            @Override
            public void onTaskCompleted(Task t) {
            // dumpPhaseStatus.isComplete();
            // joinPhaseStatus.isComplete();
            }

            @Override
            public void onTaskFailed(Task t, Throwable err, boolean fatal) {
                logger.error(t.getDisplayName(), err);
                faildResult[0] = ExecuteResult.createFaild().setMessage("status.runningStatus.isComplete():" + err.getMessage());
            }
        });
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    return faildResult[0];
}

Example 4 with DumpPhaseStatus

use of com.qlangtech.tis.fullbuild.phasestatus.impl.DumpPhaseStatus in project plugins by qlangtech.

the class TestLocalDataXJobSubmit method testCreateDataXJob.

public void testCreateDataXJob() throws Exception {
    Optional<DataXJobSubmit> dataXJobSubmit = DataXJobSubmit.getDataXJobSubmit(DataXJobSubmit.InstanceType.LOCAL);
    Assert.assertTrue("dataXJobSubmit shall present", dataXJobSubmit.isPresent());
    LocalDataXJobSubmit jobSubmit = (LocalDataXJobSubmit) dataXJobSubmit.get();
    jobSubmit.setMainClassName(LocalDataXJobMainEntrypoint.class.getName());
    jobSubmit.setWorkingDirectory(new File("."));
    jobSubmit.setClasspath("target/classes:target/test-classes");
    AtomicReference<ITISRpcService> ref = new AtomicReference<>();
    ref.set(StatusRpcClient.AssembleSvcCompsite.MOCK_PRC);
    RpcServiceReference statusRpc = new RpcServiceReference(ref);
    DataXJobSubmit.IDataXJobContext dataXJobContext = EasyMock.createMock("dataXJobContext", DataXJobSubmit.IDataXJobContext.class);
    IExecChainContext taskContext = EasyMock.createMock("taskContext", IExecChainContext.class);
    EasyMock.expect(dataXJobContext.getTaskContext()).andReturn(taskContext).anyTimes();
    IDataxProcessor dataxProcessor = EasyMock.createMock("dataxProcessor", IDataxProcessor.class);
    EasyMock.expect(taskContext.getIndexName()).andReturn(dataXName).anyTimes();
    EasyMock.expect(taskContext.getTaskId()).andReturn(TaskId).anyTimes();
    int preSuccessTaskId = 99;
    PhaseStatusCollection preSuccessTask = new PhaseStatusCollection(preSuccessTaskId, new ExecutePhaseRange(FullbuildPhase.FullDump, FullbuildPhase.FullDump));
    DumpPhaseStatus preDumpStatus = new DumpPhaseStatus(preSuccessTaskId);
    DumpPhaseStatus.TableDumpStatus tableDumpStatus = preDumpStatus.getTable(dataXfileName);
    tableDumpStatus.setAllRows(LocalDataXJobMainEntrypoint.testAllRows);
    preSuccessTask.setDumpPhase(preDumpStatus);
    EasyMock.expect(taskContext.loadPhaseStatusFromLatest(dataXName)).andReturn(preSuccessTask).times(3);
    TisZkClient zkClient = EasyMock.createMock("TisZkClient", TisZkClient.class);
    String zkSubPath = "nodes0000000020";
    EasyMock.expect(zkClient.getChildren(ZkUtils.ZK_ASSEMBLE_LOG_COLLECT_PATH, null, true)).andReturn(Collections.singletonList(zkSubPath)).times(3);
    EasyMock.expect(zkClient.getData(EasyMock.eq(ZkUtils.ZK_ASSEMBLE_LOG_COLLECT_PATH + "/" + zkSubPath), EasyMock.isNull(), EasyMock.anyObject(Stat.class), EasyMock.eq(true))).andReturn(statusCollectorHost.getBytes(TisUTF8.get())).times(3);
    EasyMock.expect(taskContext.getZkClient()).andReturn(zkClient).anyTimes();
    EasyMock.replay(taskContext, dataxProcessor, zkClient, dataXJobContext);
    IRemoteTaskTrigger dataXJob = jobSubmit.createDataXJob(dataXJobContext, statusRpc, dataxProcessor, dataXfileName);
    RunningStatus running = getRunningStatus(dataXJob);
    assertTrue("running.isSuccess", running.isSuccess());
    jobSubmit.setMainClassName(LocalDataXJobMainEntrypointThrowException.class.getName());
    dataXJob = jobSubmit.createDataXJob(dataXJobContext, statusRpc, dataxProcessor, dataXfileName);
    running = getRunningStatus(dataXJob);
    assertFalse("shall faild", running.isSuccess());
    assertTrue("shall complete", running.isComplete());
    jobSubmit.setMainClassName(LocalDataXJobMainEntrypointCancellable.class.getName());
    dataXJob = jobSubmit.createDataXJob(dataXJobContext, statusRpc, dataxProcessor, dataXfileName);
    running = getRunningStatus(dataXJob, false);
    Thread.sleep(2000);
    dataXJob.cancel();
    int i = 0;
    while (i++ < 3 && !(running = dataXJob.getRunningStatus()).isComplete()) {
        Thread.sleep(1000);
    }
    assertFalse("shall faild", running.isSuccess());
    assertTrue("shall complete", running.isComplete());
    EasyMock.verify(taskContext, dataxProcessor, zkClient);
}

Example 5 with DumpPhaseStatus

use of com.qlangtech.tis.fullbuild.phasestatus.impl.DumpPhaseStatus in project tis by qlangtech.

the class IncrStatusUmbilicalProtocolImpl method reportDumpTableStatus.

@Override
public void reportDumpTableStatus(com.qlangtech.tis.rpc.grpc.log.common.TableDumpStatus tableDumpStatus, StreamObserver<Empty> responseObserver) {
    Integer taskid = tableDumpStatus.getTaskid();
    if (taskid == null || taskid < 1) {
        throw new IllegalArgumentException("taskid illegal:" + taskid);
    }
    PhaseStatusCollection phaseStatusSet = TrackableExecuteInterceptor.getTaskPhaseReference(taskid);
    if (phaseStatusSet == null) {
        returnEmpty(responseObserver);
        return;
    }
    log.info("taskid:" + taskid + ",tablename:" + tableDumpStatus.getTableName() + ",read:" + tableDumpStatus.getReadRows() + ",all:" + tableDumpStatus.getAllRows());
    DumpPhaseStatus dumpPhase = phaseStatusSet.getDumpPhase();
    TableDumpStatus dumpStatus = phaseStatusSet.getDumpPhase().getTable(tableDumpStatus.getTableName());
    // }
    if (tableDumpStatus.getComplete()) {
        // 成功
        dumpStatus.setComplete(true);
    }
    if (tableDumpStatus.getFaild()) {
        // 失败了
        dumpStatus.setFaild(true);
    }
    dumpStatus.setReadRows(tableDumpStatus.getReadRows());
    dumpStatus.setAllRows(tableDumpStatus.getAllRows());
    if (!tableDumpStatus.getWaiting()) {
        dumpStatus.setWaiting(tableDumpStatus.getWaiting());
    }
    dumpPhase.isComplete();
    returnEmpty(responseObserver);
}