Examples with IDownload com.dtstack.taier.develop.utils.develop.common.IDownload used on opensource projects

Example 1 with IDownload

use of com.dtstack.taier.develop.utils.develop.common.IDownload in project Taier by DTStack.

the class HadoopDataDownloadService method queryDataFromTempTable.

/**
 * 从临时表获取表数据
 *
 * @param tenantId
 * @param tableName
 * @param db
 * @return
 * @throws Exception
 */
public List<Object> queryDataFromTempTable(Long tenantId, String tableName, String db) throws Exception {
    DataSourceType dataSourceType = datasourceService.getHadoopDefaultDataSourceByTenantId(tenantId);
    IDownload downloader = getDownloader(tenantId, tableName, db, dataSourceType.getVal());
    List<Object> result = new ArrayList<>();
    List<String> alias = downloader.getMetaInfo();
    result.add(alias);
    JdbcInfo jdbcInfo = Engine2DTOService.getJdbcInfo(tenantId, null, DataSourceTypeJobTypeMapping.getTaskTypeByDataSourceType(dataSourceType.getVal()));
    int readCounter = 0;
    while (!downloader.reachedEnd() && readCounter < jdbcInfo.getMaxRows()) {
        List<String> row = (List<String>) downloader.readNext();
        result.add(row);
        readCounter++;
    }
    return result;
}

Example 2 with IDownload

use of com.dtstack.taier.develop.utils.develop.common.IDownload in project Taier by DTStack.

the class BatchDownloadService method loadJobLog.

/**
 * 按行数获取job的log
 *
 * @param tenantId
 * @param taskType      除数据同步和虚节点都可以导出jobLog
 * @param jobId
 * @param byteNum
 * @return
 * @throws Exception
 */
public String loadJobLog(Long tenantId, Integer taskType, String jobId, Integer byteNum) {
    LOGGER.info("获取job日志下载器-->jobId:{}", jobId);
    IDownload downloader = buildIDownLoad(jobId, taskType, tenantId, byteNum == null ? DEFAULT_LOG_PREVIEW_BYTES : byteNum);
    LOGGER.info("获取job日志下载器完成-->jobId:{}", jobId);
    if (Objects.isNull(downloader)) {
        LOGGER.error("-----日志文件导出失败-----");
        return "";
    }
    StringBuilder result = new StringBuilder();
    while (!downloader.reachedEnd()) {
        Object row = downloader.readNext();
        result.append(row);
    }
    return result.toString();
}

Example 3 with IDownload

use of com.dtstack.taier.develop.utils.develop.common.IDownload in project Taier by DTStack.

the class BatchDownloadService method downloadAppTypeLog.

public String downloadAppTypeLog(Long tenantId, String jobId, Integer limitNum, String logType, Integer taskType) {
    IDataDownloadService dataDownloadService = multiEngineServiceFactory.getDataDownloadService(taskType);
    IDownload downloader = dataDownloadService.typeLogDownloader(tenantId, jobId, limitNum == null ? Integer.MAX_VALUE : limitNum, logType);
    if (Objects.isNull(downloader)) {
        LOGGER.error("-----日志文件导出失败-----");
        return "-----日志文件不存在-----";
    }
    StringBuilder result = new StringBuilder();
    while (!downloader.reachedEnd()) {
        Object row = downloader.readNext();
        result.append(row);
    }
    return result.toString();
}

Example 4 with IDownload

use of com.dtstack.taier.develop.utils.develop.common.IDownload in project Taier by DTStack.

the class HadoopDataDownloadService method typeLogDownloader.

@Override
public IDownload typeLogDownloader(Long tenantId, String jobId, Integer limitNum, String logType) {
    String applicationId = batchJobService.getApplicationId(jobId);
    if (StringUtils.isBlank(applicationId)) {
        throw new RdosDefineException("任务尚未执行完成或提交失败，请稍后再试");
    }
    IDownload iDownload = null;
    try {
        iDownload = RetryUtil.executeWithRetry(() -> {
            final Map<String, Object> hadoopConf = Engine2DTOService.getHdfs(tenantId);
            JSONObject yarnConf = Engine2DTOService.getComponentConfig(tenantId, EComponentType.YARN);
            String submitUserName = getSubmitUserNameByJobId(jobId);
            final LogPluginDownload downloader = new LogPluginDownload(applicationId, yarnConf, hadoopConf, submitUserName, limitNum);
            downloader.configure();
            return downloader;
        }, 3, 1000L, false);
    } catch (Exception e) {
        throw new RdosDefineException(String.format("typeLogDownloader 失败，原因是：%s", e.getMessage()), e);
    }
    return iDownload;
}

Example 5 with IDownload

use of com.dtstack.taier.develop.utils.develop.common.IDownload in project Taier by DTStack.

the class DevelopDownloadController method downloadJobLog.

@GetMapping(value = "downloadJobLog")
@ApiOperation("下载job日志")
public void downloadJobLog(BatchDownloadJobLogVO vo, HttpServletResponse response) {
    IDownload iDownload = batchDownloadService.downloadJobLog(vo.getJobId(), vo.getTaskType(), vo.getTenantId());
    batchDownloadService.handleDownload(response, iDownload, DownloadType.DEVELOP_LOG, vo.getJobId());
}