Examples with PhaseStatusCollection com.qlangtech.tis.fullbuild.phasestatus.PhaseStatusCollection used on opensource projects

Example 11 with PhaseStatusCollection

use of com.qlangtech.tis.fullbuild.phasestatus.PhaseStatusCollection in project tis by qlangtech.

the class TestDefaultChainContext method testLoadPhaseStatusFromLatest.

public void testLoadPhaseStatusFromLatest() {
    IParamContext paramContext = this.mock("paramContext", IParamContext.class);
    DefaultChainContext chainContext = new DefaultChainContext(paramContext);
    PhaseStatusCollection statusCollection = chainContext.loadPhaseStatusFromLatest(dataXname);
    assertNull(statusCollection);
    // ./src/test/resources/com/qlangtech/tis/full/dump/cfg_repo/df-logs/66/dump
    statusCollection = chainContext.loadPhaseStatusFromLatest(dataXname);
    assertNotNull(statusCollection);
    DumpPhaseStatus dumpPhase = statusCollection.getDumpPhase();
    assertNotNull(dumpPhase);
    assertEquals(62, dumpPhase.getTaskId());
    String dataXFileName = "instancedetail_0.json";
    DumpPhaseStatus.TableDumpStatus dataXExecStatus = dumpPhase.getTable(dataXFileName);
    assertNotNull(dataXFileName + " relevant dataX instance can be null", dataXExecStatus);
    assertEquals(524525, dataXExecStatus.getReadRows());
    assertEquals(1000001, dataXExecStatus.getAllRows());
}

Example 12 with PhaseStatusCollection

use of com.qlangtech.tis.fullbuild.phasestatus.PhaseStatusCollection in project tis by qlangtech.

the class DataXJobSubmit method getDataXJobDTO.

protected CuratorDataXTaskMessage getDataXJobDTO(IJoinTaskContext taskContext, String dataXfileName) {
    CuratorDataXTaskMessage msg = new CuratorDataXTaskMessage();
    msg.setDataXName(taskContext.getIndexName());
    msg.setJobId(taskContext.getTaskId());
    msg.setJobName(dataXfileName);
    msg.setExecTimeStamp(taskContext.getPartitionTimestamp());
    PhaseStatusCollection preTaskStatus = taskContext.loadPhaseStatusFromLatest(taskContext.getIndexName());
    DumpPhaseStatus.TableDumpStatus dataXJob = null;
    if (preTaskStatus != null && (dataXJob = preTaskStatus.getDumpPhase().getTable(dataXfileName)) != null && dataXJob.getAllRows() > 0) {
        msg.setAllRowsApproximately(dataXJob.getReadRows());
    } else {
        msg.setAllRowsApproximately(-1);
    }
    return msg;
}

Example 13 with PhaseStatusCollection

use of com.qlangtech.tis.fullbuild.phasestatus.PhaseStatusCollection in project tis by qlangtech.

the class LogCollectorClient method convert.

public static PhaseStatusCollection convert(PPhaseStatusCollection stat, ExecutePhaseRange executePhaseRange) {
    PDumpPhaseStatus dumpPhase = stat.getDumpPhase();
    PJoinPhaseStatus joinPhase = stat.getJoinPhase();
    PBuildPhaseStatus buildPhase = stat.getBuildPhase();
    PIndexBackFlowPhaseStatus backflow = stat.getIndexBackFlowPhaseStatus();
    PhaseStatusCollection result = new PhaseStatusCollection(stat.getTaskId(), executePhaseRange);
    if (executePhaseRange.contains(FullbuildPhase.FullDump) && dumpPhase != null) {
        DumpPhaseStatus dump = result.getDumpPhase();
        dumpPhase.getTablesDumpMap().forEach((k, v) -> {
            DumpPhaseStatus.TableDumpStatus s = new DumpPhaseStatus.TableDumpStatus(v.getTableName(), v.getTaskid());
            s.setAllRows(v.getAllRows());
            s.setReadRows(v.getReadRows());
            s.setComplete(v.getComplete());
            s.setFaild(v.getFaild());
            s.setWaiting(v.getWaiting());
            dump.tablesDump.put(k, s);
        });
    }
    if (executePhaseRange.contains(FullbuildPhase.JOIN) && joinPhase != null) {
        JoinPhaseStatus join = result.getJoinPhase();
        Map<String, JoinPhaseStatus.JoinTaskStatus> sm = join.taskStatus;
        joinPhase.getTaskStatusMap().forEach((k, v) -> {
            JoinPhaseStatus.JoinTaskStatus s = new JoinPhaseStatus.JoinTaskStatus(v.getJoinTaskName());
            s.setComplete(v.getComplete());
            s.setFaild(v.getFaild());
            s.setWaiting(v.getWaiting());
            v.getJobStatusMap().forEach((jk, jv) -> {
                JobLog jl = new JobLog();
                jl.setMapper(jv.getMapper());
                jl.setReducer(jv.getReducer());
                jl.setWaiting(jv.getWaiting());
                s.jobsStatus.put(jk, jl);
            });
            sm.put(k, s);
        });
    }
    if (executePhaseRange.contains(FullbuildPhase.BUILD) && buildPhase != null) {
        BuildPhaseStatus build = result.getBuildPhase();
        buildPhase.getNodeBuildStatusMap().forEach((k, v) -> {
            com.qlangtech.tis.fullbuild.phasestatus.impl.BuildSharedPhaseStatus s = new com.qlangtech.tis.fullbuild.phasestatus.impl.BuildSharedPhaseStatus();
            s.setAllBuildSize(v.getAllBuildSize());
            s.setBuildReaded(v.getBuildReaded());
            s.setTaskid(v.getTaskid());
            s.setSharedName(v.getSharedName());
            s.setComplete(v.getComplete());
            s.setFaild(v.getFaild());
            s.setWaiting(v.getWaiting());
            build.nodeBuildStatus.put(k, s);
        });
    }
    if (executePhaseRange.contains(FullbuildPhase.IndexBackFlow) && backflow != null) {
        IndexBackFlowPhaseStatus bf = result.getIndexBackFlowPhaseStatus();
        backflow.getNodesStatusMap().forEach((k, v) -> {
            IndexBackFlowPhaseStatus.NodeBackflowStatus s = new IndexBackFlowPhaseStatus.NodeBackflowStatus(v.getNodeName());
            s.setAllSize((int) v.getAllSize());
            s.setReaded((int) v.getReaded());
            s.setComplete(v.getComplete());
            s.setFaild(v.getFaild());
            s.setWaiting(v.getWaiting());
            bf.nodesStatus.put(k, s);
        });
    }
    return result;
}

Example 14 with PhaseStatusCollection

use of com.qlangtech.tis.fullbuild.phasestatus.PhaseStatusCollection in project tis by qlangtech.

the class DataFlowAppSource method processTaskResult.

private void processTaskResult(IExecChainContext execContext, TISReactor.TaskImpl t, IDataProcessFeedback dataProcessFeedback, ITaskResultProcessor resultProcessor) {
    TISReactor.TaskImpl task = t;
    // TrackableExecuteInterceptor.taskPhaseReference.get(execContext.getTaskId());
    PhaseStatusCollection pstats = dataProcessFeedback.getPhaseStatusSet(execContext);
    if (pstats != null) {
        switch(task.getPhase()) {
            case FullDump:
                // pstats.getDumpPhase()
                // IncrStatusUmbilicalProtocolImpl statReceiver = IncrStatusUmbilicalProtocolImpl.getInstance();
                // statReceiver.reportDumpTableStatusError(execContext.getTaskId(), task.getIdentityName());
                pstats.getDumpPhase().isComplete();
                resultProcessor.process(pstats.getDumpPhase(), task);
                return;
            case JOIN:
                // JoinPhaseStatus.JoinTaskStatus stat
                // = pstats.getJoinPhase().getTaskStatus(task.getIdentityName());
                // //statReceiver.reportBuildIndexStatErr(execContext.getTaskId(),task.getIdentityName());
                // stat.setWaiting(false);
                // stat.setFaild(true);
                // stat.setComplete(true);
                pstats.getJoinPhase().isComplete();
                resultProcessor.process(pstats.getJoinPhase(), task);
                return;
            default:
                throw new IllegalStateException("taskphase:" + task.getPhase() + " is illegal");
        }
    }
}