Examples with JudgeResult com.dtstack.taier.pluginapi.pojo.JudgeResult used on opensource projects

Example 1 with JudgeResult

use of com.dtstack.taier.pluginapi.pojo.JudgeResult in project Taier by DTStack.

the class JobSubmitDealer method submitJob.

private void submitJob(JobClient jobClient) {
    JobResult jobResult = null;
    try {
        // 判断资源
        JudgeResult judgeResult = workerOperator.judgeSlots(jobClient);
        if (JudgeResult.JudgeType.OK == judgeResult.getResult()) {
            LOGGER.info("jobId:{} taskType:{} submit to engine start.", jobClient.getJobId(), jobClient.getTaskType());
            jobClient.doStatusCallBack(TaskStatus.COMPUTING.getStatus());
            // 提交任务
            jobResult = workerOperator.submitJob(jobClient);
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug("jobId:{} taskType:{} submit jobResult:{}.", jobClient.getJobId(), jobClient.getTaskType(), jobResult);
            }
            jobClient.setEngineTaskId(jobResult.getData(JobResult.EXT_ID_KEY));
            jobClient.setApplicationId(jobResult.getData(JobResult.JOB_ID_KEY));
            addToTaskListener(jobClient, jobResult);
            LOGGER.info("jobId:{} taskType:{} submit to engine end.", jobClient.getJobId(), jobClient.getTaskType());
        } else if (JudgeResult.JudgeType.LIMIT_ERROR == judgeResult.getResult()) {
            LOGGER.info("jobId:{} taskType:{} submitJob happens system limitError:{}", jobClient.getJobId(), jobClient.getTaskType(), judgeResult.getReason());
            jobClient.setEngineTaskId(null);
            jobResult = JobResult.createErrorResult(false, judgeResult.getReason());
            addToTaskListener(jobClient, jobResult);
        } else if (JudgeResult.JudgeType.EXCEPTION == judgeResult.getResult()) {
            LOGGER.info("jobId:{} taskType:{} judgeSlots result is exception {}", jobClient.getJobId(), jobClient.getTaskType(), judgeResult.getReason());
            handlerFailedWithRetry(jobClient, true, new Exception(judgeResult.getReason()));
        } else {
            LOGGER.info("jobId:{} taskType:{} judgeSlots result is false.", jobClient.getJobId(), jobClient.getTaskType());
            handlerNoResource(jobClient, judgeResult);
        }
    } catch (WorkerAccessException e) {
        LOGGER.info(" jobId:{} taskType:{} worker not find.", jobClient.getJobId(), jobClient.getTaskType());
        handlerNoResource(jobClient, workerNotFindResult);
    } catch (ClientAccessException | ClientArgumentException e) {
        handlerFailedWithRetry(jobClient, false, e);
    } catch (Throwable e) {
        handlerFailedWithRetry(jobClient, true, e);
    }
}

Example 2 with JudgeResult

use of com.dtstack.taier.pluginapi.pojo.JudgeResult in project Taier by DTStack.

the class AbstractRdbsClientTest method testJudgeSlots.

@Test
public void testJudgeSlots() throws Exception {
    JobClient jobClient = new JobClient();
    JudgeResult result = testRdbsClient.judgeSlots(jobClient);
    Assert.assertEquals(JudgeResult.JudgeType.NOT_OK, result.getResult());
    EngineResourceInfo info = PowerMockito.mock(EngineResourceInfo.class);
    when(info.judgeSlots(any(JobClient.class))).thenReturn(JudgeResult.ok());
    MemberModifier.field(TestRdbsClient.class, "resourceInfo").set(testRdbsClient, info);
    JudgeResult judgeResult = testRdbsClient.judgeSlots(jobClient);
    Assert.assertEquals(JudgeResult.JudgeType.OK, judgeResult.getResult());
}

Example 3 with JudgeResult

use of com.dtstack.taier.pluginapi.pojo.JudgeResult in project Taier by DTStack.

the class RdbsResourceInfoTest method testJudgeSlots.

@Test
public void testJudgeSlots() throws Exception {
    RdbsExeQueue rdbsExeQueue = PowerMockito.mock(RdbsExeQueue.class);
    when(rdbsExeQueue.checkCanSubmit()).thenReturn(true);
    MemberModifier.field(RdbsResourceInfo.class, "rdbsExeQueue").set(rdbsResourceInfo, rdbsExeQueue);
    JobClient jobClient = new JobClient();
    JudgeResult judgeResult = rdbsResourceInfo.judgeSlots(jobClient);
    Assert.assertTrue(judgeResult.available());
}

Example 4 with JudgeResult

use of com.dtstack.taier.pluginapi.pojo.JudgeResult in project Taier by DTStack.

the class AbstractYarnResourceInfo method getYarnSlots.

public JudgeResult getYarnSlots(YarnClient yarnClient, String queueName, int yarnAccepterTaskNumber) {
    try {
        EnumSet<YarnApplicationState> enumSet = EnumSet.noneOf(YarnApplicationState.class);
        enumSet.add(YarnApplicationState.ACCEPTED);
        List<ApplicationReport> acceptedApps = yarnClient.getApplications(enumSet).stream().filter(report -> report.getQueue().endsWith(queueName)).collect(Collectors.toList());
        if (acceptedApps.size() > yarnAccepterTaskNumber) {
            logger.info("queueName:{} acceptedApps:{} >= yarnAccepterTaskNumber:{}", queueName, acceptedApps.size(), yarnAccepterTaskNumber);
            return JudgeResult.notOk("queueName:" + queueName + " acceptedApps:" + acceptedApps.size() + " >= yarnAccepterTaskNumber:" + yarnAccepterTaskNumber);
        }
        List<NodeReport> nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
        if (!elasticCapacity) {
            getQueueRemainCapacity(1, queueName, yarnClient.getRootQueueInfos());
        }
        for (NodeReport report : nodeReports) {
            Resource capability = report.getCapability();
            Resource used = report.getUsed();
            int totalMem = capability.getMemory();
            int totalCores = capability.getVirtualCores();
            int usedMem = used.getMemory();
            int usedCores = used.getVirtualCores();
            int freeCores = totalCores - usedCores;
            int freeMem = totalMem - usedMem;
            if (freeCores > containerCoreMax) {
                containerCoreMax = freeCores;
            }
            if (freeMem > containerMemoryMax) {
                containerMemoryMax = freeMem;
            }
            this.addNodeResource(new NodeResourceDetail(report.getNodeId().toString(), totalCores, usedCores, freeCores, totalMem, usedMem, freeMem));
        }
        calc();
        return JudgeResult.ok();
    } catch (IOException | YarnException e) {
        throw new RuntimeException(e);
    }
}

Example 5 with JudgeResult

use of com.dtstack.taier.pluginapi.pojo.JudgeResult in project Taier by DTStack.

the class SparkYarnResourceInfo method judgeSlots.

@Override
public JudgeResult judgeSlots(JobClient jobClient) {
    JudgeResult jr = getYarnSlots(yarnClient, queueName, yarnAccepterTaskNumber);
    if (!jr.available()) {
        return jr;
    }
    Properties properties = jobClient.getConfProperties();
    int driverCores = DEFAULT_CORES;
    if (properties != null && properties.containsKey(DRIVER_CORE_KEY)) {
        driverCores = MathUtil.getIntegerVal(properties.get(DRIVER_CORE_KEY));
    }
    int driverMem = DEFAULT_MEM;
    if (properties != null && properties.containsKey(DRIVER_MEM_KEY)) {
        String setMemStr = properties.getProperty(DRIVER_MEM_KEY);
        driverMem = UnitConvertUtil.convert2Mb(setMemStr);
    }
    int driverMemOverhead = DEFAULT_MEM_OVERHEAD;
    if (properties != null && properties.containsKey(DRIVER_MEM_OVERHEAD_KEY)) {
        String setMemStr = properties.getProperty(DRIVER_MEM_OVERHEAD_KEY);
        driverMemOverhead = UnitConvertUtil.convert2Mb(setMemStr);
    }
    driverMem += driverMemOverhead;
    int executorNum = DEFAULT_INSTANCES;
    if (properties != null && properties.containsKey(EXECUTOR_INSTANCES_KEY)) {
        executorNum = MathUtil.getIntegerVal(properties.get(EXECUTOR_INSTANCES_KEY));
    }
    int executorCores = DEFAULT_CORES;
    if (properties != null && properties.containsKey(EXECUTOR_CORES_KEY)) {
        executorCores = MathUtil.getIntegerVal(properties.get(EXECUTOR_CORES_KEY));
    }
    int executorMem = DEFAULT_MEM;
    if (properties != null && properties.containsKey(EXECUTOR_MEM_KEY)) {
        String setMemStr = properties.getProperty(EXECUTOR_MEM_KEY);
        executorMem = UnitConvertUtil.convert2Mb(setMemStr);
    }
    int executorMemOverhead = DEFAULT_MEM_OVERHEAD;
    if (properties != null && properties.containsKey(EXECUTOR_MEM_OVERHEAD_KEY)) {
        String setMemStr = properties.getProperty(EXECUTOR_MEM_OVERHEAD_KEY);
        executorMemOverhead = UnitConvertUtil.convert2Mb(setMemStr);
    }
    executorMem += executorMemOverhead;
    List<InstanceInfo> instanceInfos = Lists.newArrayList(InstanceInfo.newRecord(1, driverCores, driverMem), InstanceInfo.newRecord(executorNum, executorCores, executorMem));
    return judgeYarnResource(instanceInfos);
}