Examples with RdosDefineException com.dtstack.taier.common.exception.RdosDefineException used on opensource projects

Example 21 with RdosDefineException

use of com.dtstack.taier.common.exception.RdosDefineException in project Taier by DTStack.

the class BatchResourceService method addResource.

/**
 * 添加资源
 */
public CatalogueVO addResource(BatchResourceAddDTO batchResourceAddDTO) {
    Long tenantId = batchResourceAddDTO.getTenantId();
    Long userId = batchResourceAddDTO.getUserId();
    String resourceName;
    Long resourceId = null;
    BatchResource resourceDB = null;
    Integer resourceType = null;
    if (batchResourceAddDTO.getId() != null && batchResourceAddDTO.getId() != 0L) {
        resourceId = batchResourceAddDTO.getId();
        resourceDB = this.developResourceDao.getOne(resourceId);
        resourceName = resourceDB.getResourceName();
        resourceType = resourceDB.getResourceType();
    } else {
        if (StringUtils.isEmpty(batchResourceAddDTO.getResourceName())) {
            throw new RdosDefineException("需要设置参数 resourceName.", ErrorCode.INVALID_PARAMETERS);
        }
        resourceName = batchResourceAddDTO.getResourceName();
        resourceType = batchResourceAddDTO.getResourceType() == null ? ResourceType.OTHER.getType() : batchResourceAddDTO.getResourceType();
    }
    String hdfsPath = uploadHDFSFileWithResource(tenantId, resourceName, batchResourceAddDTO.getOriginalFilename(), batchResourceAddDTO.getTmpPath());
    BatchResource batchResource = null;
    // 重新上传资源
    if (Objects.nonNull(resourceId)) {
        batchResource = resourceDB;
        if (Deleted.DELETED.getStatus().equals(batchResource.getIsDeleted())) {
            throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_RESOURCE);
        }
        batchResource.setResourceDesc(batchResourceAddDTO.getResourceDesc());
        batchResource.setOriginFileName(batchResourceAddDTO.getOriginalFilename());
        batchResource.setUrl(hdfsPath);
    } else {
        // 判断是否已经存在相同的资源了
        batchTaskService.checkName(resourceName, CatalogueType.RESOURCE_MANAGER.name(), null, 1, tenantId);
        batchResourceAddDTO.setUrl(hdfsPath);
        batchResourceAddDTO.setCreateUserId(userId);
        batchResource = PublicUtil.objectToObject(batchResourceAddDTO, BatchResource.class);
        if (Objects.isNull(batchResource)) {
            throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_RESOURCE);
        }
        batchResource.setOriginFileName(batchResourceAddDTO.getOriginalFilename());
    }
    // resourceType 设置默认值
    resourceType = resourceType != null ? resourceType : ResourceType.OTHER.getType();
    batchResource.setResourceType(resourceType);
    batchResource.setModifyUserId(userId);
    addOrUpdate(batchResource);
    BatchCatalogue catalogue = batchCatalogueService.getOne(batchResource.getNodePid());
    CatalogueVO catalogueVO = new CatalogueVO();
    catalogueVO.setId(batchResource.getId());
    catalogueVO.setName(batchResource.getResourceName());
    catalogueVO.setType("file");
    catalogueVO.setLevel(catalogue.getLevel() + 1);
    catalogueVO.setChildren(null);
    catalogueVO.setParentId(catalogue.getId());
    catalogueVO.setResourceType(resourceType);
    String username = userService.getUserName(catalogue.getCreateUserId());
    catalogueVO.setCreateUser(username);
    return catalogueVO;
}

Example 22 with RdosDefineException

use of com.dtstack.taier.common.exception.RdosDefineException in project Taier by DTStack.

the class BatchServerLogService method getLogsByJobId.

public BatchServerLogVO getLogsByJobId(String jobId, Integer pageInfo) {
    if (StringUtils.isBlank(jobId)) {
        return null;
    }
    final ScheduleJob job = scheduleJobService.getByJobId(jobId);
    if (Objects.isNull(job)) {
        LOGGER.info("can not find job by id:{}.", jobId);
        throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_JOB);
    }
    final Long tenantId = job.getTenantId();
    final ScheduleTaskShade scheduleTaskShade = this.taskService.findTaskByTaskId(job.getTaskId());
    if (Objects.isNull(scheduleTaskShade)) {
        LOGGER.info("can not find task shade  by jobId:{}.", jobId);
        throw new RdosDefineException(ErrorCode.SERVER_EXCEPTION);
    }
    final BatchServerLogVO batchServerLogVO = new BatchServerLogVO();
    // 日志从engine获取
    final JSONObject logsBody = new JSONObject(2);
    logsBody.put("jobId", jobId);
    logsBody.put("computeType", ComputeType.BATCH.getType());
    ActionLogVO actionLogVO = actionService.log(jobId);
    JSONObject info = new JSONObject();
    if (!Strings.isNullOrEmpty(actionLogVO.getLogInfo())) {
        try {
            info = JSON.parseObject(actionLogVO.getLogInfo());
        } catch (final Exception e) {
            LOGGER.error(String.format("parse jobId： %s  logInfo：%s", jobId, actionLogVO.getLogInfo()), e);
            info.put("msg_info", actionLogVO.getLogInfo());
        }
    }
    if (Objects.nonNull(job.getVersionId())) {
        // 需要获取执行任务时候版本对应的sql
        BatchTaskVersionDetailDTO taskVersion = this.batchTaskVersionService.getByVersionId((long) job.getVersionId());
        if (Objects.nonNull(taskVersion)) {
            if (StringUtils.isEmpty(taskVersion.getOriginSql())) {
                String jsonSql = StringUtils.isEmpty(taskVersion.getSqlText()) ? "{}" : taskVersion.getSqlText();
                scheduleTaskShade.setSqlText(jsonSql);
            } else {
                scheduleTaskShade.setSqlText(taskVersion.getOriginSql());
            }
        }
    }
    info.put("status", job.getStatus());
    if (EScheduleJobType.SPARK_SQL.getVal().equals(scheduleTaskShade.getTaskType())) {
        // 处理sql注释，先把注释base64编码，再处理非注释的自定义参数
        String sql = SqlFormatterUtil.dealAnnotationBefore(scheduleTaskShade.getSqlText());
        final List<BatchTaskParamShade> taskParamsToReplace = this.batchTaskParamShadeService.getTaskParam(scheduleTaskShade.getId());
        sql = this.jobParamReplace.paramReplace(sql, taskParamsToReplace, job.getCycTime());
        sql = SqlFormatterUtil.dealAnnotationAfter(sql);
        info.put("sql", sql);
    } else if (EScheduleJobType.SYNC.getVal().equals(scheduleTaskShade.getTaskType())) {
        final JSONObject jobJson;
        // taskShade 需要解码
        JSONObject sqlJson = null;
        try {
            sqlJson = JSON.parseObject(Base64Util.baseDecode(scheduleTaskShade.getSqlText()));
        } catch (final Exception e) {
            sqlJson = JSON.parseObject(scheduleTaskShade.getSqlText());
        }
        jobJson = sqlJson.getJSONObject("job");
        // 密码脱敏
        DataFilter.passwordFilter(jobJson);
        String jobStr = jobJson.toJSONString();
        final List<BatchTaskParamShade> taskParamsToReplace = this.batchTaskParamShadeService.getTaskParam(scheduleTaskShade.getId());
        jobStr = this.jobParamReplace.paramReplace(jobStr, taskParamsToReplace, job.getCycTime());
        info.put("sql", JsonUtils.formatJSON(jobStr));
        if (Objects.nonNull(job.getExecEndTime()) && Objects.nonNull(job.getExecStartTime())) {
            List<ActionJobEntityVO> engineEntities = actionService.entitys(Collections.singletonList(logsBody.getString("jobId")));
            String engineJobId = "";
            if (CollectionUtils.isNotEmpty(engineEntities)) {
                engineJobId = engineEntities.get(0).getEngineJobId();
            }
            this.parseIncreInfo(info, jobStr, tenantId, engineJobId, job.getExecStartTime().getTime(), job.getExecEndTime().getTime(), "");
        }
    }
    if (job.getJobId() != null) {
        try {
            if (StringUtils.isNotBlank(actionLogVO.getEngineLog())) {
                final Map<String, Object> engineLogMap = BatchServerLogService.objectMapper.readValue(actionLogVO.getEngineLog(), Map.class);
                this.dealPerfLog(engineLogMap);
                info.putAll(engineLogMap);
                // 去掉统计信息，界面不展示，调度端统计使用
                info.remove("countInfo");
            }
        } catch (Exception e) {
            // 非json格式的日志也返回
            info.put("msg_info", actionLogVO.getEngineLog());
            LOGGER.error("", e);
        }
    }
    // 增加重试日志
    final String retryLog = this.buildRetryLog(jobId, pageInfo, batchServerLogVO);
    this.formatForLogInfo(info, job.getType(), scheduleTaskShade.getTaskType(), retryLog, null, null, null, batchServerLogVO, tenantId, jobId);
    if (!scheduleTaskShade.getTaskType().equals(EScheduleJobType.SYNC.getVal()) && !scheduleTaskShade.getTaskType().equals(EScheduleJobType.VIRTUAL.getVal()) && !scheduleTaskShade.getTaskType().equals(EScheduleJobType.WORK_FLOW.getVal()) && TaskStatus.getStoppedStatus().contains(job.getStatus())) {
        batchServerLogVO.setDownloadLog(String.format(DOWNLOAD_LOG, jobId, scheduleTaskShade.getTaskType(), 0L));
    }
    batchServerLogVO.setName(scheduleTaskShade.getName());
    batchServerLogVO.setComputeType(scheduleTaskShade.getComputeType());
    batchServerLogVO.setTaskType(scheduleTaskShade.getTaskType());
    return batchServerLogVO;
}

Example 23 with RdosDefineException

use of com.dtstack.taier.common.exception.RdosDefineException in project Taier by DTStack.

the class BatchServerLogService method getLogsByAppLogType.

public BatchServerLogByAppLogTypeResultVO getLogsByAppLogType(Long tenantId, Integer taskType, String jobId, String logType, Long projectId) {
    if (EScheduleJobType.SYNC.getVal().equals(taskType) || EScheduleJobType.VIRTUAL.getVal().equals(taskType) || EScheduleJobType.WORK_FLOW.getVal().equals(taskType)) {
        throw new RdosDefineException("数据同步、虚节点、工作流的任务日志不支持下载");
    }
    if (YarnAppLogType.getType(logType) == null) {
        throw new RdosDefineException("not support the logType:" + logType);
    }
    final String msg = this.batchDownloadService.downloadAppTypeLog(tenantId, jobId, 100, logType.toUpperCase(), taskType);
    BatchServerLogByAppLogTypeResultVO resultVO = new BatchServerLogByAppLogTypeResultVO();
    resultVO.setMsg(msg);
    resultVO.setDownload(String.format(BatchServerLogService.DOWNLOAD_TYPE_LOG, jobId, logType, projectId));
    return resultVO;
}

Example 24 with RdosDefineException

use of com.dtstack.taier.common.exception.RdosDefineException in project Taier by DTStack.

the class BatchServerLogService method formatPerfLogInfo.

public String formatPerfLogInfo(final String engineJobId, final String jobId, final long startTime, final long endTime, final Long tenantId) {
    final ScheduleJob job = scheduleJobService.getByJobId(jobId);
    if (Objects.isNull(job)) {
        LOGGER.info("can not find job by id:{}.", jobId);
        throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_JOB);
    }
    if (job.getTaskId() == null || job.getTaskId() == -1) {
        throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_TASK);
    }
    BatchTask batchTaskById = batchTaskService.getBatchTaskById(job.getTaskId());
    // prometheus的配置信息 从控制台获取
    final Pair<String, String> prometheusHostAndPort = this.getPrometheusHostAndPort(tenantId, batchTaskById.getTaskParams());
    if (prometheusHostAndPort == null) {
        return "promethues配置为空";
    }
    final PrometheusMetricQuery prometheusMetricQuery = new PrometheusMetricQuery(String.format("%s:%s", prometheusHostAndPort.getKey(), prometheusHostAndPort.getValue()));
    // 之后查询是可以直接获取最后一条记录的方法
    // 防止数据同步执行时间太长 查询prometheus的时候返回exceeded maximum resolution of 11,000 points per timeseries
    final long maxGapTime = 60 * 1000 * 60 * (long) 8;
    long gapStartTime = startTime;
    if (endTime - startTime >= maxGapTime) {
        // 超过11,000 points 查询1小时间隔内
        gapStartTime = endTime - 60 * 1000 * 60;
    }
    final IMetric numReadMetric = MetricBuilder.buildMetric("numRead", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric byteReadMetric = MetricBuilder.buildMetric("byteRead", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric readDurationMetric = MetricBuilder.buildMetric("readDuration", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric numWriteMetric = MetricBuilder.buildMetric("numWrite", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric byteWriteMetric = MetricBuilder.buildMetric("byteWrite", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric writeDurationMetric = MetricBuilder.buildMetric("writeDuration", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric numErrorMetric = MetricBuilder.buildMetric("nErrors", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final SyncStatusLogInfoVO formatPerfLogInfo = this.getFormatPerfLogInfo(numReadMetric, byteReadMetric, readDurationMetric, numWriteMetric, byteWriteMetric, writeDurationMetric, numErrorMetric);
    return formatPerfLogInfo.buildReadableLog();
}

Example 25 with RdosDefineException

use of com.dtstack.taier.common.exception.RdosDefineException in project Taier by DTStack.

the class HBaseWriter method checkRowkey.

private String checkRowkey() {
    Matcher matcher = PATTERN.matcher(rowkey);
    Boolean isMatcher = false;
    while (matcher.find()) {
        isMatcher = true;
        String varName = matcher.group(1);
        try {
            Integer index = srcColumns.indexOf(varName.split(":")[1]);
            if (index == -1) {
                throw new RdosDefineException("rowkey column not found: " + varName);
            }
        } catch (Exception e) {
            if (e instanceof RdosDefineException) {
                throw e;
            } else {
                throw new RdosDefineException(String.format("请检查rowkey格式，原因是：%s", e.getMessage()));
            }
        }
    }
    if (!isMatcher) {
        throw new RdosDefineException("请输入正确的rowkey格式");
    }
    return this.rowkey;
}