Examples with BatchSysParameter com.dtstack.taier.dao.domain.BatchSysParameter used on opensource projects

Example 1 with BatchSysParameter

use of com.dtstack.taier.dao.domain.BatchSysParameter in project Taier by DTStack.

the class BatchTaskParamService method getTaskParam.

public List<BatchTaskParam> getTaskParam(final long taskId) {
    List<BatchTaskParam> taskParams = developTaskParamDao.listByTaskId(taskId);
    // 特殊处理 TaskParam 系统参数
    for (BatchTaskParam taskParamShade : taskParams) {
        if (!EParamType.SYS_TYPE.getType().equals(taskParamShade.getType())) {
            continue;
        }
        // 将 command 属性设置为系统表的 command
        BatchSysParameter sysParameter = batchSysParamService.getBatchSysParamByName(taskParamShade.getParamName());
        taskParamShade.setParamCommand(sysParameter.getParamCommand());
    }
    return taskParams;
}

Example 2 with BatchSysParameter

use of com.dtstack.taier.dao.domain.BatchSysParameter in project Taier by DTStack.

the class BatchTaskParamShadeService method getTaskParam.

public List<BatchTaskParamShade> getTaskParam(long taskId) {
    List<BatchTaskParamShade> taskParamShades = developTaskParamShadeDao.listByTaskId(taskId);
    // 特殊处理 TaskParam 系统参数
    for (BatchTaskParamShade taskParamShade : taskParamShades) {
        if (EParamType.SYS_TYPE.getType() != taskParamShade.getType()) {
            continue;
        }
        // 将 command 属性设置为系统表的 command
        BatchSysParameter sysParameter = batchSysParamService.getBatchSysParamByName(taskParamShade.getParamName());
        taskParamShade.setParamCommand(sysParameter.getParamCommand());
    }
    return taskParamShades;
}

Example 3 with BatchSysParameter

use of com.dtstack.taier.dao.domain.BatchSysParameter in project Taier by DTStack.

the class BatchTaskService method checkTrace.

private JSONObject checkTrace(final JSONObject jsonObject) {
    final JSONObject keymap = jsonObject.getJSONObject("keymap");
    final JSONArray source = keymap.getJSONArray("source");
    final JSONArray target = keymap.getJSONArray("target");
    final JSONObject sourceMap = jsonObject.getJSONObject("sourceMap");
    final Integer fromId = (Integer) sourceMap.get("sourceId");
    final JSONObject targetMap = jsonObject.getJSONObject("targetMap");
    final Integer toId = (Integer) targetMap.get("sourceId");
    final JSONObject sourceType = sourceMap.getJSONObject("type");
    final List<String> sourceTables = this.getTables(sourceType);
    final JSONObject targetType = targetMap.getJSONObject("type");
    final List<String> targetTables = this.getTables(targetType);
    final BatchDataSource fromDs = dataSourceService.getOne(fromId.longValue());
    final BatchDataSource toDs = dataSourceService.getOne(toId.longValue());
    int fromSourceType = DataSourceType.getSourceType(fromDs.getType()).getVal();
    int toSourceType = DataSourceType.getSourceType(toDs.getType()).getVal();
    if (DataSourceType.HBASE.getVal() == fromSourceType || DataSourceType.HBASE.getVal() == toSourceType) {
        return jsonObject;
    }
    // 处理分库分表的信息
    this.addSourceList(sourceMap);
    if (CollectionUtils.isNotEmpty(sourceTables)) {
        getMetaDataColumns(sourceMap, sourceTables, fromDs);
    }
    if (CollectionUtils.isNotEmpty(targetTables)) {
        getMetaDataColumns(targetMap, targetTables, toDs);
    }
    // 因为下面要对keyMap中target中的字段类型进行更新 所以遍历一次目标map 拿出字段和类型的映射
    Map<String, String> newTargetColumnTypeMap = targetMap.getJSONArray(COLUMN).stream().map(column -> (JSONObject) column).collect(Collectors.toMap(column -> column.getString(KEY), column -> column.getString(TYPE)));
    final Collection<BatchSysParameter> sysParams = this.getSysParams();
    final JSONArray newSource = new JSONArray();
    final JSONArray newTarget = new JSONArray();
    for (int i = 0; i < source.size(); ++i) {
        boolean srcTag = true;
        final JSONArray srcColumns = sourceMap.getJSONArray("column");
        if (CollectionUtils.isNotEmpty(sourceTables)) {
            int j = 0;
            final String srcColName;
            String colValue = "";
            if (!(source.get(i) instanceof JSONObject)) {
                srcColName = source.getString(i);
            } else {
                // source 可能含有系统变量
                srcColName = source.getJSONObject(i).getString("key");
                colValue = source.getJSONObject(i).getString("value");
            }
            // srcColumns 源表中的字段
            for (; j < srcColumns.size(); ++j) {
                final JSONObject srcColumn = srcColumns.getJSONObject(j);
                if (srcColumn.getString("key").equals(srcColName)) {
                    break;
                }
            }
            boolean isSysParam = false;
            for (final BatchSysParameter sysParam : sysParams) {
                if (sysParam.strIsSysParam(colValue)) {
                    isSysParam = true;
                    break;
                }
            }
            // 没有系统变量 还需要判断是否有自定义变量
            if (!isSysParam) {
                isSysParam = StringUtils.isNotBlank(colValue);
            }
            // 兼容系统变量
            if (isSysParam) {
                boolean hasThisKey = false;
                for (int k = 0; k < srcColumns.size(); ++k) {
                    final JSONObject srcColumn = srcColumns.getJSONObject(k);
                    if (srcColumn.getString("key").equals(srcColName)) {
                        hasThisKey = true;
                        break;
                    }
                }
                if (!hasThisKey) {
                    // 创建出系统变量colume
                    final JSONObject jsonColumn = new JSONObject();
                    jsonColumn.put("key", srcColName);
                    jsonColumn.put("value", colValue);
                    jsonColumn.put("type", source.getJSONObject(i).getString("type"));
                    jsonColumn.put("format", source.getJSONObject(i).getString("format"));
                    srcColumns.add(jsonColumn);
                }
            }
            if (j == srcColumns.size() && !isSysParam) {
                srcTag = false;
            }
        }
        boolean destTag = true;
        final JSONArray destColumns = targetMap.getJSONArray("column");
        if (CollectionUtils.isNotEmpty(targetTables)) {
            int k = 0;
            final String destColName;
            if (!(target.get(i) instanceof JSONObject)) {
                destColName = target.getString(i);
            } else {
                destColName = target.getJSONObject(i).getString("key");
                // 更新dest表中字段类型
                final String newType = newTargetColumnTypeMap.get(destColName);
                if (StringUtils.isNotEmpty(newType)) {
                    target.getJSONObject(i).put("type", newType);
                }
            }
            for (; k < destColumns.size(); ++k) {
                final JSONObject destColumn = destColumns.getJSONObject(k);
                if (destColumn.getString("key").equals(destColName)) {
                    break;
                }
            }
            if (k == destColumns.size()) {
                destTag = false;
            }
        }
        if (srcTag && destTag) {
            newSource.add(source.get(i));
            newTarget.add(target.get(i));
        }
    }
    keymap.put("source", newSource);
    keymap.put("target", newTarget);
    return jsonObject;
}

Example 4 with BatchSysParameter

use of com.dtstack.taier.dao.domain.BatchSysParameter in project Taier by DTStack.

the class BatchSysParamService method loadSystemParam.

@Forbidden
public void loadSystemParam() {
    cache = Maps.newHashMap();
    List<BatchSysParameter> sysParamList = developSysParamDao.listAll();
    for (BatchSysParameter tmp : sysParamList) {
        cache.put(tmp.getParamName(), tmp);
    }
}

Example 5 with BatchSysParameter

use of com.dtstack.taier.dao.domain.BatchSysParameter in project Taier by DTStack.

the class JobParamReplace method convertParam.

public String convertParam(Integer type, String paramName, String paramCommand, String cycTime) {
    String command = null;
    if (type == EParamType.SYS_TYPE.getType()) {
        BatchSysParameter sysParameter = batchSysParamService.getBatchSysParamByName(paramName);
        command = sysParameter.getParamCommand();
        // 特殊处理 bdp.system.currenttime
        if ("bdp.system.runtime".equals(sysParameter.getParamName())) {
            return TimeParamOperator.dealCustomizeTimeOperator(command, cycTime);
        }
    } else {
        command = paramCommand;
        return TimeParamOperator.dealCustomizeTimeOperator(command, cycTime);
    }
    command = TimeParamOperator.transform(command, cycTime);
    return command;
}