Examples with IDownload com.dtstack.taier.develop.utils.develop.common.IDownload used on opensource projects

Example 6 with IDownload

use of com.dtstack.taier.develop.utils.develop.common.IDownload in project Taier by DTStack.

the class BatchHadoopSelectSqlService method queryData.

/**
 * 查询表数据 - 简单查询
 *
 * @param tenantId 租户id
 * @param sql 查询sql
 * @param taskType 任务类型
 * @return 查询数据
 * @throws Exception
 */
private List<Object> queryData(Long tenantId, String sql, Integer taskType) throws Exception {
    List<Object> queryResult = Lists.newArrayList();
    IDownload resultDownload = hadoopDataDownloadService.getSimpleSelectDownLoader(tenantId, sql, taskType);
    Integer num = getMaxQueryNum(sql, tenantId, taskType);
    int readCounter = 0;
    // 第一行插入传字段信息
    queryResult.add(resultDownload.getMetaInfo());
    while (!resultDownload.reachedEnd()) {
        if (readCounter >= num) {
            break;
        }
        queryResult.add(resultDownload.readNext());
        readCounter++;
    }
    return queryResult;
}

Example 7 with IDownload

use of com.dtstack.taier.develop.utils.develop.common.IDownload in project Taier by DTStack.

the class HadoopDataDownloadService method buildIDownLoad.

@Override
public IDownload buildIDownLoad(String jobId, Integer taskType, Long tenantId, Integer limitNum) {
    if (StringUtils.isBlank(jobId)) {
        throw new RdosDefineException("jobId 不能为空");
    }
    if (EScheduleJobType.SYNC.getVal().equals(taskType)) {
        // standalone模式的不支持日志下载直接返回null
        Boolean isStandalone = clusterService.hasStandalone(tenantId, EComponentType.FLINK.getTypeCode());
        if (isStandalone) {
            return null;
        }
        // 同步任务
        StringBuilder syncLog = new StringBuilder();
        SyncDownload syncDownload = new SyncDownload();
        ActionLogVO log = actionService.log(jobId);
        if (!Objects.isNull(log)) {
            String engineLogStr = log.getEngineLog();
            String logInfoStr = log.getLogInfo();
            if (StringUtils.isNotBlank(engineLogStr)) {
                try {
                    JSONObject engineLogJson = JSON.parseObject(engineLogStr);
                    engineLogJson.remove("increConf");
                    engineLogStr = engineLogJson.toJSONString();
                } catch (Exception e) {
                    LOGGER.info("engineLog非json", e);
                }
                syncLog.append("engineLog:\n").append(engineLogStr).append("\n");
            }
            try {
                JSONObject logInfo = JSON.parseObject(logInfoStr);
                syncLog.append("logInfo:\n").append(logInfo.getString("msg_info"));
            } catch (Exception e) {
                LOGGER.error("同步任务日志下载失败", e);
            }
        }
        syncDownload.setLogInfo(syncLog.toString());
        return syncDownload;
    }
    String applicationId = batchJobService.getApplicationId(jobId);
    if (StringUtils.isBlank(applicationId)) {
        return null;
    }
    IDownload iDownload = null;
    try {
        iDownload = RetryUtil.executeWithRetry(() -> {
            final Map<String, Object> hadoopConf = Engine2DTOService.getHdfs(tenantId);
            final JSONObject yarnConf = Engine2DTOService.getComponentConfig(tenantId, EComponentType.YARN);
            String submitUserName = getSubmitUserNameByJobId(jobId);
            final LogPluginDownload downloader = new LogPluginDownload(applicationId, yarnConf, hadoopConf, submitUserName, limitNum);
            return downloader;
        }, 3, 1000L, false);
    } catch (Exception e) {
        LOGGER.error("downloadJobLog {}  失败:{}", jobId, e);
        return null;
    }
    return iDownload;
}