Examples with JobLog com.qlangtech.tis.fullbuild.phasestatus.JobLog used on opensource projects

Example 1 with JobLog

use of com.qlangtech.tis.fullbuild.phasestatus.JobLog in project plugins by qlangtech.

the class SparkExecLiveLogParser method process.

@Override
public void process(String log) {
    if (!StringUtils.startsWith(log, STATUS_PREFIX)) {
        return;
    }
    Integer jobId = null;
    Matcher matcher = RUNNING_STATUS.matcher(log);
    Integer percent = null;
    if (matcher.matches()) {
        jobId = Integer.parseInt(matcher.group(1));
        percent = Integer.parseInt(matcher.group(5));
        JobLog job = joinTaskStatus.getJoblog(jobId);
        if (job == null) {
            return;
        }
        job.setWaiting(false);
        job.setReducer(percent);
        job.setMapper(percent);
        return;
    }
    matcher = START_STATUS.matcher(log);
    if (matcher.matches()) {
        jobId = Integer.parseInt(matcher.group(1));
        joinTaskStatus.createJobStatus(jobId);
        joinTaskStatus.setStart();
        return;
    }
    matcher = COMPLETE_STATUS.matcher(log);
    if (matcher.matches()) {
        this.execOver = true;
        jobId = Integer.parseInt(matcher.group(1));
        String result = matcher.group(2);
        joinTaskStatus.setComplete(true);
        joinTaskStatus.setFaild(!EXEC_RESULT_SUCCESS.equals(result));
        return;
    }
}

Example 2 with JobLog

use of com.qlangtech.tis.fullbuild.phasestatus.JobLog in project plugins by qlangtech.

the class TestHiveFlatTableBuilder method testTotalpaySummary.

public void testTotalpaySummary() throws Exception {
    HiveFlatTableBuilder flatTableBuilder = (HiveFlatTableBuilder) flatTableBuilderStore.getPlugin();
    // ISqlTask nodeMeta, boolean isFinalNode
    // , ITemplateContext tplContext, ITaskContext taskContext, IFs2Table fs2Table, IJoinTaskStatus joinTaskStatus
    IFs2Table fs2Table = new MockFs2Table();
    IJoinTaskStatus joinTaskStatus = EasyMock.createMock("joinTaskStatus", IJoinTaskStatus.class);
    joinTaskStatus.setComplete(true);
    joinTaskStatus.createJobStatus(EasyMock.anyInt());
    JobLog jobLog = new JobLog();
    EasyMock.expect(joinTaskStatus.getJoblog(EasyMock.anyInt())).andReturn(jobLog).anyTimes();
    joinTaskStatus.setStart();
    IJoinTaskContext joinTaskContext = EasyMock.createMock("joinTaskContext", IJoinTaskContext.class);
    Map<IDumpTable, ITabPartition> dateParams = Maps.newHashMap();
    EasyMock.expect(joinTaskContext.getAttribute(ExecChainContextUtils.PARTITION_DATA_PARAMS)).andReturn(dateParams).anyTimes();
    Map<String, Boolean> taskWorkStatus = Maps.newHashMap();
    EasyMock.expect(joinTaskContext.getAttribute(AdapterTask.KEY_TASK_WORK_STATUS)).andReturn(taskWorkStatus);
    ERRules erRules = EasyMock.createMock("erRules", ERRules.class);
    EasyMock.expect(joinTaskContext.getAttribute("er_rules")).andReturn(erRules);
    EasyMock.replay(joinTaskStatus, joinTaskContext, erRules);
    MockTemplateContext tplContext = new MockTemplateContext(joinTaskContext);
    flatTableBuilder.startTask((context) -> {
        try (InputStream input = TestHiveFlatTableBuilder.class.getResourceAsStream("groupby_totalpay.sql")) {
            // try (InputStream input = TestHiveFlatTableBuilder.class.getResourceAsStream("totalpay_summary.sql")) {
            ISqlTask sqlTask = new DefaultSqlTask(IOUtils.toString(input, TisUTF8.get()));
            // ISqlTask nodeMeta, boolean isFinalNode
            // , ITemplateContext tplContext, ITaskContext taskContext, //
            // IJoinTaskStatus joinTaskStatus
            DataflowTask joinTask = flatTableBuilder.createTask(sqlTask, true, tplContext, context, joinTaskStatus);
            joinTask.run();
        }
    });
}

Example 3 with JobLog

use of com.qlangtech.tis.fullbuild.phasestatus.JobLog in project tis by qlangtech.

the class FullBuildStatCollectorServer method convertPP.

public PPhaseStatusCollection convertPP(PhaseStatusCollection phaseStatusSet) {
    PPhaseStatusCollection.Builder scBuilder = PPhaseStatusCollection.newBuilder();
    scBuilder.setTaskId(phaseStatusSet.getTaskid());
    DumpPhaseStatus dumpPhase = phaseStatusSet.getDumpPhase();
    JoinPhaseStatus joinPhase = phaseStatusSet.getJoinPhase();
    BuildPhaseStatus buildPhase = phaseStatusSet.getBuildPhase();
    IndexBackFlowPhaseStatus indexBackFlowPhase = phaseStatusSet.getIndexBackFlowPhaseStatus();
    if (dumpPhase != null) {
        PDumpPhaseStatus.Builder builder = PDumpPhaseStatus.newBuilder();
        com.qlangtech.tis.rpc.grpc.log.common.TableDumpStatus.Builder tabDump = null;
        DumpPhaseStatus.TableDumpStatus s = null;
        for (Map.Entry<String, DumpPhaseStatus.TableDumpStatus> entry : dumpPhase.tablesDump.entrySet()) {
            s = entry.getValue();
            tabDump = com.qlangtech.tis.rpc.grpc.log.common.TableDumpStatus.newBuilder();
            tabDump.setAllRows(s.getAllRows());
            tabDump.setTableName(s.getName());
            tabDump.setTaskid(s.getTaskid());
            tabDump.setReadRows(s.getReadRows());
            tabDump.setFaild(s.isFaild());
            tabDump.setComplete(s.isComplete());
            tabDump.setWaiting(s.isWaiting());
            builder.putTablesDump(entry.getKey(), tabDump.build());
        }
        scBuilder.setDumpPhase(builder);
    }
    if (joinPhase != null) {
        PJoinPhaseStatus.Builder builder = PJoinPhaseStatus.newBuilder();
        com.qlangtech.tis.rpc.grpc.log.common.JoinTaskStatus.Builder pbuilder = null;
        JoinPhaseStatus.JoinTaskStatus j = null;
        com.qlangtech.tis.rpc.grpc.log.common.JobLog.Builder jlog = null;
        JobLog jl = null;
        for (Map.Entry<String, JoinPhaseStatus.JoinTaskStatus> entry : joinPhase.taskStatus.entrySet()) {
            pbuilder = com.qlangtech.tis.rpc.grpc.log.common.JoinTaskStatus.newBuilder();
            j = entry.getValue();
            pbuilder.setJoinTaskName(j.getName());
            pbuilder.setFaild(j.isFaild());
            pbuilder.setComplete(j.isComplete());
            pbuilder.setWaiting(j.isWaiting());
            for (Map.Entry<Integer, JobLog> e : j.jobsStatus.entrySet()) {
                jl = e.getValue();
                jlog = com.qlangtech.tis.rpc.grpc.log.common.JobLog.newBuilder();
                jlog.setMapper(jl.getMapper());
                jlog.setReducer(jl.getReducer());
                jlog.setWaiting(jl.isWaiting());
                pbuilder.putJobStatus(e.getKey(), jlog.build());
            }
            builder.putTaskStatus(entry.getKey(), pbuilder.build());
        }
        scBuilder.setJoinPhase(builder);
    }
    if (buildPhase != null) {
        PBuildPhaseStatus.Builder builder = PBuildPhaseStatus.newBuilder();
        buildPhase.nodeBuildStatus.entrySet().stream().forEach((e) -> {
            com.qlangtech.tis.fullbuild.phasestatus.impl.BuildSharedPhaseStatus bf = null;
            com.qlangtech.tis.rpc.grpc.log.common.BuildSharedPhaseStatus.Builder bfBuilder = com.qlangtech.tis.rpc.grpc.log.common.BuildSharedPhaseStatus.newBuilder();
            bf = e.getValue();
            bfBuilder.setAllBuildSize(bf.getAllBuildSize());
            bfBuilder.setBuildReaded(bf.getBuildReaded());
            bfBuilder.setTaskid(bf.getTaskid());
            bfBuilder.setSharedName(bf.getSharedName());
            bfBuilder.setFaild(bf.isFaild());
            bfBuilder.setComplete(bf.isComplete());
            bfBuilder.setWaiting(bf.isWaiting());
            builder.putNodeBuildStatus(e.getKey(), bfBuilder.build());
        });
        scBuilder.setBuildPhase(builder);
    }
    if (indexBackFlowPhase != null) {
        PIndexBackFlowPhaseStatus.Builder builder = PIndexBackFlowPhaseStatus.newBuilder();
        indexBackFlowPhase.nodesStatus.entrySet().stream().forEach((e) -> {
            IndexBackFlowPhaseStatus.NodeBackflowStatus ib = e.getValue();
            com.qlangtech.tis.rpc.grpc.log.common.NodeBackflowStatus.Builder ibBuilder = com.qlangtech.tis.rpc.grpc.log.common.NodeBackflowStatus.newBuilder();
            ibBuilder.setNodeName(ib.getName());
            ibBuilder.setAllSize(ib.getAllSize());
            ibBuilder.setReaded(ib.getReaded());
            ibBuilder.setFaild(ib.isFaild());
            ibBuilder.setComplete(ib.isComplete());
            ibBuilder.setWaiting(ib.isWaiting());
            builder.putNodesStatus(e.getKey(), ibBuilder.build());
        });
        scBuilder.setIndexBackFlowPhaseStatus(builder);
    }
    return scBuilder.build();
}

Example 4 with JobLog

use of com.qlangtech.tis.fullbuild.phasestatus.JobLog in project tis by qlangtech.

the class LogCollectorClient method convert.

public static PhaseStatusCollection convert(PPhaseStatusCollection stat, ExecutePhaseRange executePhaseRange) {
    PDumpPhaseStatus dumpPhase = stat.getDumpPhase();
    PJoinPhaseStatus joinPhase = stat.getJoinPhase();
    PBuildPhaseStatus buildPhase = stat.getBuildPhase();
    PIndexBackFlowPhaseStatus backflow = stat.getIndexBackFlowPhaseStatus();
    PhaseStatusCollection result = new PhaseStatusCollection(stat.getTaskId(), executePhaseRange);
    if (executePhaseRange.contains(FullbuildPhase.FullDump) && dumpPhase != null) {
        DumpPhaseStatus dump = result.getDumpPhase();
        dumpPhase.getTablesDumpMap().forEach((k, v) -> {
            DumpPhaseStatus.TableDumpStatus s = new DumpPhaseStatus.TableDumpStatus(v.getTableName(), v.getTaskid());
            s.setAllRows(v.getAllRows());
            s.setReadRows(v.getReadRows());
            s.setComplete(v.getComplete());
            s.setFaild(v.getFaild());
            s.setWaiting(v.getWaiting());
            dump.tablesDump.put(k, s);
        });
    }
    if (executePhaseRange.contains(FullbuildPhase.JOIN) && joinPhase != null) {
        JoinPhaseStatus join = result.getJoinPhase();
        Map<String, JoinPhaseStatus.JoinTaskStatus> sm = join.taskStatus;
        joinPhase.getTaskStatusMap().forEach((k, v) -> {
            JoinPhaseStatus.JoinTaskStatus s = new JoinPhaseStatus.JoinTaskStatus(v.getJoinTaskName());
            s.setComplete(v.getComplete());
            s.setFaild(v.getFaild());
            s.setWaiting(v.getWaiting());
            v.getJobStatusMap().forEach((jk, jv) -> {
                JobLog jl = new JobLog();
                jl.setMapper(jv.getMapper());
                jl.setReducer(jv.getReducer());
                jl.setWaiting(jv.getWaiting());
                s.jobsStatus.put(jk, jl);
            });
            sm.put(k, s);
        });
    }
    if (executePhaseRange.contains(FullbuildPhase.BUILD) && buildPhase != null) {
        BuildPhaseStatus build = result.getBuildPhase();
        buildPhase.getNodeBuildStatusMap().forEach((k, v) -> {
            com.qlangtech.tis.fullbuild.phasestatus.impl.BuildSharedPhaseStatus s = new com.qlangtech.tis.fullbuild.phasestatus.impl.BuildSharedPhaseStatus();
            s.setAllBuildSize(v.getAllBuildSize());
            s.setBuildReaded(v.getBuildReaded());
            s.setTaskid(v.getTaskid());
            s.setSharedName(v.getSharedName());
            s.setComplete(v.getComplete());
            s.setFaild(v.getFaild());
            s.setWaiting(v.getWaiting());
            build.nodeBuildStatus.put(k, s);
        });
    }
    if (executePhaseRange.contains(FullbuildPhase.IndexBackFlow) && backflow != null) {
        IndexBackFlowPhaseStatus bf = result.getIndexBackFlowPhaseStatus();
        backflow.getNodesStatusMap().forEach((k, v) -> {
            IndexBackFlowPhaseStatus.NodeBackflowStatus s = new IndexBackFlowPhaseStatus.NodeBackflowStatus(v.getNodeName());
            s.setAllSize((int) v.getAllSize());
            s.setReaded((int) v.getReaded());
            s.setComplete(v.getComplete());
            s.setFaild(v.getFaild());
            s.setWaiting(v.getWaiting());
            bf.nodesStatus.put(k, s);
        });
    }
    return result;
}