Examples with BatchTask com.dtstack.taier.dao.domain.BatchTask used on opensource projects

Example 1 with BatchTask

use of com.dtstack.taier.dao.domain.BatchTask in project Taier by DTStack.

the class BatchSelectSqlService method beforeGetResult.

/**
 * sql查询前置处理
 *
 * @param jobId
 * @param taskId
 * @param tenantId
 * @param type
 * @param sqlId
 * @return
 */
private ExecuteSelectSqlData beforeGetResult(String jobId, Long taskId, Long tenantId, Integer type, String sqlId) {
    BatchSelectSql batchHiveSelectSql = developHiveSelectSqlDao.getByJobId(StringUtils.isNotEmpty(sqlId) ? sqlId : jobId, tenantId, null);
    Preconditions.checkNotNull(batchHiveSelectSql, "不存在该临时查询");
    if (StringUtils.isNotEmpty(sqlId)) {
        batchHiveSelectSql.setFatherJobId(jobId);
        batchHiveSelectSql.setJobId(sqlId);
    }
    IBatchSelectSqlService selectSqlService = multiEngineServiceFactory.getBatchSelectSqlService(batchHiveSelectSql.getTaskType());
    Preconditions.checkNotNull(selectSqlService, String.format("不支持此任务类型 %d", batchHiveSelectSql.getTaskType()));
    BatchTask batchTask = batchTaskService.getOneWithError(taskId);
    ;
    Integer taskType = null;
    if (Objects.nonNull(batchTask)) {
        taskType = batchTask.getTaskType();
    }
    if (Objects.isNull(taskType)) {
        throw new DtCenterDefException("任务类型为空");
    }
    return new ExecuteSelectSqlData(batchHiveSelectSql, batchTask, taskType, selectSqlService);
}

Example 2 with BatchTask

use of com.dtstack.taier.dao.domain.BatchTask in project Taier by DTStack.

the class BatchServerLogService method formatPerfLogInfo.

public String formatPerfLogInfo(final String engineJobId, final String jobId, final long startTime, final long endTime, final Long tenantId) {
    final ScheduleJob job = scheduleJobService.getByJobId(jobId);
    if (Objects.isNull(job)) {
        LOGGER.info("can not find job by id:{}.", jobId);
        throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_JOB);
    }
    if (job.getTaskId() == null || job.getTaskId() == -1) {
        throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_TASK);
    }
    BatchTask batchTaskById = batchTaskService.getBatchTaskById(job.getTaskId());
    // prometheus的配置信息 从控制台获取
    final Pair<String, String> prometheusHostAndPort = this.getPrometheusHostAndPort(tenantId, batchTaskById.getTaskParams());
    if (prometheusHostAndPort == null) {
        return "promethues配置为空";
    }
    final PrometheusMetricQuery prometheusMetricQuery = new PrometheusMetricQuery(String.format("%s:%s", prometheusHostAndPort.getKey(), prometheusHostAndPort.getValue()));
    // 之后查询是可以直接获取最后一条记录的方法
    // 防止数据同步执行时间太长 查询prometheus的时候返回exceeded maximum resolution of 11,000 points per timeseries
    final long maxGapTime = 60 * 1000 * 60 * (long) 8;
    long gapStartTime = startTime;
    if (endTime - startTime >= maxGapTime) {
        // 超过11,000 points 查询1小时间隔内
        gapStartTime = endTime - 60 * 1000 * 60;
    }
    final IMetric numReadMetric = MetricBuilder.buildMetric("numRead", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric byteReadMetric = MetricBuilder.buildMetric("byteRead", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric readDurationMetric = MetricBuilder.buildMetric("readDuration", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric numWriteMetric = MetricBuilder.buildMetric("numWrite", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric byteWriteMetric = MetricBuilder.buildMetric("byteWrite", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric writeDurationMetric = MetricBuilder.buildMetric("writeDuration", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric numErrorMetric = MetricBuilder.buildMetric("nErrors", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final SyncStatusLogInfoVO formatPerfLogInfo = this.getFormatPerfLogInfo(numReadMetric, byteReadMetric, readDurationMetric, numWriteMetric, byteWriteMetric, writeDurationMetric, numErrorMetric);
    return formatPerfLogInfo.buildReadableLog();
}

Example 3 with BatchTask

use of com.dtstack.taier.dao.domain.BatchTask in project Taier by DTStack.

the class BatchTaskService method allProductGlobalSearch.

/**
 * 查找所有产品提交的任务
 *
 * @param searchVO
 * @return
 */
public List<BatchAllProductGlobalReturnVO> allProductGlobalSearch(AllProductGlobalSearchVO searchVO) {
    BatchTask batchTask = developTaskDao.getOne(searchVO.getTaskId());
    if (batchTask == null) {
        throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_TASK);
    }
    if (batchTask.getTaskType().intValue() == EScheduleJobType.VIRTUAL.getVal().intValue()) {
        throw new RdosDefineException(ErrorCode.VIRTUAL_TASK_UNSUPPORTED_OPERATION);
    }
    // 过滤掉已经依赖的任务
    final List<BatchTaskTask> taskTasks = this.batchTaskTaskService.getByParentTaskId(searchVO.getTaskId());
    final List<Long> excludeIds = new ArrayList<>(taskTasks.size());
    excludeIds.add(searchVO.getTaskId());
    taskTasks.forEach(taskTask -> excludeIds.add(taskTask.getTaskId()));
    List<ScheduleTaskShade> scheduleTaskShadeList = taskService.findTaskByTaskName(searchVO.getTaskName(), searchVO.getSelectTenantId(), searchVO.getUserId());
    List<ScheduleTaskShade> filterTask = scheduleTaskShadeList.stream().filter(scheduleTask -> !excludeIds.contains(scheduleTask.getTaskId())).collect(Collectors.toList());
    Map<Long, Tenant> tenantMap = tenantService.listAllTenant().stream().collect(Collectors.toMap(Tenant::getId, g -> (g)));
    List<BatchAllProductGlobalReturnVO> voList = Lists.newArrayList();
    for (ScheduleTaskShade scheduleTaskShade : filterTask) {
        BatchAllProductGlobalReturnVO vo = new BatchAllProductGlobalReturnVO();
        vo.setTaskId(scheduleTaskShade.getTaskId());
        vo.setTaskName(scheduleTaskShade.getName());
        Tenant tenant = tenantMap.get(scheduleTaskShade.getTenantId());
        if (tenant != null) {
            vo.setTenantId(tenant.getId());
            vo.setTenantName(tenant.getTenantName());
        }
        voList.add(vo);
    }
    return voList;
}

Example 4 with BatchTask

use of com.dtstack.taier.dao.domain.BatchTask in project Taier by DTStack.

the class BatchTaskService method frozenTask.

/**
 * 冻结任务
 *
 * @param taskId         任务编号
 * @param scheduleStatus 调度状态
 * @param userId         用户ID
 */
public void frozenTask(Long taskId, Integer scheduleStatus, Long userId) {
    BatchTask batchTask = getOneWithError(taskId);
    EScheduleStatus targetStatus = EScheduleStatus.getStatus(scheduleStatus);
    if (Objects.isNull(targetStatus)) {
        throw new RdosDefineException("任务状态参数非法", ErrorCode.INVALID_PARAMETERS);
    }
    batchTask.setModifyUserId(userId);
    batchTask.setScheduleStatus(scheduleStatus);
    developTaskDao.update(batchTask);
    taskService.frozenTask(Lists.newArrayList(taskId), scheduleStatus);
}

Example 5 with BatchTask

use of com.dtstack.taier.dao.domain.BatchTask in project Taier by DTStack.

the class BatchTaskService method getTaskById.

/**
 * 数据开发-根据任务id，查询详情
 *
 * @return
 * @author toutian
 */
public BatchTaskBatchVO getTaskById(final ScheduleTaskVO scheduleTaskVO) {
    final BatchTask task = this.developTaskDao.getOne(scheduleTaskVO.getId());
    if (task == null) {
        throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_TASK);
    }
    final List<BatchResource> resources = this.batchTaskResourceService.getResources(scheduleTaskVO.getId(), ResourceRefType.MAIN_RES.getType());
    final List<BatchResource> refResourceIdList = this.batchTaskResourceService.getResources(scheduleTaskVO.getId(), ResourceRefType.DEPENDENCY_RES.getType());
    final BatchTaskBatchVO taskVO = new BatchTaskBatchVO(this.batchTaskTaskService.getForefathers(task));
    taskVO.setVersion(task.getVersion());
    if (task.getTaskType().intValue() == EScheduleJobType.SYNC.getVal().intValue()) {
        // 同步任务类型
        final String taskJson = Base64Util.baseDecode(task.getSqlText());
        if (StringUtils.isBlank(taskJson)) {
            // 向导模式存在为空的情况
            taskVO.setCreateModel(Constant.CREATE_MODEL_GUIDE);
            taskVO.setSqlText("");
        } else {
            final JSONObject obj = JSON.parseObject(taskJson);
            taskVO.setCreateModel(obj.get("createModel") == null ? Constant.CREATE_MODEL_GUIDE : Integer.parseInt(String.valueOf(obj.get("createModel"))));
            formatSqlText(taskVO, obj);
        }
    }
    this.setTaskOperatorModelAndOptions(taskVO, task);
    if (task.getFlowId() != null && task.getFlowId() > 0) {
        taskVO.setFlowId(task.getFlowId());
        final BatchTask flow = this.developTaskDao.getOne(task.getFlowId());
        if (flow != null) {
            taskVO.setFlowName(flow.getName());
        }
    }
    final BatchCatalogue catalogue = batchCatalogueService.getOne(task.getNodePid());
    if (catalogue != null) {
        taskVO.setNodePName(catalogue.getNodeName());
    }
    taskVO.setResourceList(resources);
    taskVO.setRefResourceList(refResourceIdList);
    PageQuery pageQuery = new PageQuery(1, 5, "gmt_create", Sort.DESC.name());
    List<BatchTaskVersionDetailDTO> taskVersions = batchTaskVersionService.listByTaskId(scheduleTaskVO.getId(), pageQuery).stream().map(ver -> {
        if (StringUtils.isNotBlank(ver.getOriginSql())) {
            if (task.getTaskType().intValue() == EScheduleJobType.SYNC.getVal().intValue()) {
                ver.setSqlText(ver.getSqlText());
            } else {
                ver.setSqlText(ver.getOriginSql());
            }
        }
        // 填充用户名称
        ver.setUserName(userService.getUserName(ver.getCreateUserId()));
        return ver;
    }).collect(Collectors.toList());
    taskVO.setTaskVersions(taskVersions);
    // 密码脱敏 --2019/10/25 茂茂-- 同步任务 密码脱敏 仅 向导模式 修改成 全部模式
    if (task.getTaskType().intValue() == EScheduleJobType.SYNC.getVal().intValue()) {
        try {
            taskVO.setSqlText(JsonUtils.formatJSON(DataFilter.passwordFilter(taskVO.getSqlText())));
        } catch (final Exception e) {
            LOGGER.error("同步任务json解析失败 taskId = {}", task.getId(), e);
            taskVO.setSqlText(DataFilter.passwordFilter(taskVO.getSqlText()));
        }
        for (BatchTaskVersionDetailDTO taskVersion : taskVO.getTaskVersions()) {
            try {
                taskVersion.setSqlText(JsonUtils.formatJSON(DataFilter.passwordFilter(taskVersion.getSqlText())));
            } catch (final Exception e) {
                LOGGER.error("同步任务json解析失败 taskVersionId = {}", taskVersion.getId(), e);
                taskVersion.setSqlText(DataFilter.passwordFilter(taskVersion.getSqlText()));
            }
        }
    }
    final ReadWriteLockVO readWriteLockVO = this.readWriteLockService.getDetail(scheduleTaskVO.getTenantId(), task.getId(), ReadWriteLockType.BATCH_TASK, scheduleTaskVO.getUserId(), task.getModifyUserId(), task.getGmtModified());
    taskVO.setReadWriteLockVO(readWriteLockVO);
    taskVO.setUserId(scheduleTaskVO.getUserId());
    setTaskVariables(taskVO, scheduleTaskVO.getId());
    final List<Long> userIds = new ArrayList<>();
    userIds.add(task.getCreateUserId());
    final Map<Long, User> userMap = userService.getUserMap(userIds);
    buildUserDTOInfo(userMap, taskVO);
    return taskVO;
}