Examples with EDeployMode com.dtstack.taier.pluginapi.enums.EDeployMode used on opensource projects

Example 1 with EDeployMode

use of com.dtstack.taier.pluginapi.enums.EDeployMode in project Taier by DTStack.

the class BatchServerLogService method parseIncreInfo.

/**
 * 解析增量同步信息
 */
private void parseIncreInfo(final JSONObject info, final String job, final Long tenantId, final String jobId, final long startTime, final long endTime, final String taskParams) {
    if (StringUtils.isEmpty(jobId)) {
        return;
    }
    try {
        final EDeployMode deployModeEnum = TaskParamsUtils.parseDeployTypeByTaskParams(taskParams, ComputeType.BATCH.getType());
        JSONObject flinkJsonObject = Engine2DTOService.getComponentConfig(tenantId, EComponentType.FLINK);
        final String prometheusHost = flinkJsonObject.getJSONObject(deployModeEnum.name()).getString("prometheusHost");
        final String prometheusPort = flinkJsonObject.getJSONObject(deployModeEnum.name()).getString("prometheusPort");
        // prometheus的配置信息 从控制台获取
        final PrometheusMetricQuery prometheusMetricQuery = new PrometheusMetricQuery(String.format("%s:%s", prometheusHost, prometheusPort));
        final IMetric startLocationMetric = MetricBuilder.buildMetric("startLocation", jobId, startTime, endTime, prometheusMetricQuery);
        final IMetric endLocationMetric = MetricBuilder.buildMetric("endLocation", jobId, startTime, endTime, prometheusMetricQuery);
        String startLocation = null;
        String endLocation = null;
        if (startLocationMetric != null) {
            startLocation = String.valueOf(startLocationMetric.getMetric());
        }
        if (Objects.nonNull(endLocationMetric)) {
            endLocation = String.valueOf(endLocationMetric.getMetric());
        }
        if (StringUtils.isBlank(job)) {
            return;
        }
        final JSONObject jobJson = JSON.parseObject(job);
        final String increColumn = (String) JSONPath.eval(jobJson, "$.job.content[0].reader.parameter.increColumn");
        final String table = (String) JSONPath.eval(jobJson, "$.job.content[0].reader.parameter.connection[0].table[0]");
        final StringBuilder increStrBuild = new StringBuilder();
        increStrBuild.append("数据表:  \t").append(table).append("\n");
        increStrBuild.append("增量标识:\t").append(increColumn).append("\n");
        if (StringUtils.isEmpty(endLocation) || endLocation.startsWith("-")) {
            increStrBuild.append("开始位置:\t").append("同步数据条数为0").append("\n");
            info.put("increInfo", increStrBuild.toString());
            return;
        }
        boolean isDateCol = false;
        final JSONArray columns = (JSONArray) JSONPath.eval(jobJson, "$.job.content[0].reader.parameter.column");
        for (final Object column : columns) {
            if (column instanceof JSONObject) {
                final String name = ((JSONObject) column).getString("name");
                if (name != null && name.equals(increColumn)) {
                    final String type = ((JSONObject) column).getString("type");
                    Boolean typeCheck = type != null && (type.matches("(?i)date|datetime|time") || type.toLowerCase().contains("timestamp"));
                    if (typeCheck) {
                        isDateCol = true;
                    }
                }
            }
        }
        if (StringUtils.isEmpty(startLocation)) {
            startLocation = "全量同步";
        }
        if (isDateCol) {
            startLocation = this.formatLongStr(startLocation);
            endLocation = this.formatLongStr(endLocation);
        }
        increStrBuild.append("开始位置:\t").append(startLocation).append("\n");
        increStrBuild.append("结束位置:\t").append(endLocation).append("\n");
        info.put("increInfo", increStrBuild.toString());
    } catch (Exception e) {
        LOGGER.warn("{}", e);
    }
}

Example 2 with EDeployMode

use of com.dtstack.taier.pluginapi.enums.EDeployMode in project Taier by DTStack.

the class BatchServerLogService method getPrometheusHostAndPort.

private Pair<String, String> getPrometheusHostAndPort(final Long tenantId, final String taskParams) {
    Boolean hasStandAlone = clusterService.hasStandalone(tenantId, EComponentType.FLINK.getTypeCode());
    JSONObject flinkJsonObject;
    if (hasStandAlone) {
        flinkJsonObject = clusterService.getConfigByKey(tenantId, EComponentType.FLINK.getConfName(), null);
    } else {
        JSONObject jsonObject = Engine2DTOService.getComponentConfig(tenantId, EComponentType.FLINK);
        if (null == jsonObject) {
            LOGGER.info("console tenantId {} pluginInfo is null", tenantId);
            return null;
        }
        EDeployMode deployModeEnum = TaskParamsUtils.parseDeployTypeByTaskParams(taskParams, ComputeType.BATCH.getType());
        flinkJsonObject = jsonObject.getJSONObject(deployModeEnum.name().toLowerCase(Locale.ROOT));
    }
    String prometheusHost = flinkJsonObject.getString("prometheusHost");
    String prometheusPort = flinkJsonObject.getString("prometheusPort");
    if (StringUtils.isBlank(prometheusHost) || StringUtils.isBlank(prometheusPort)) {
        LOGGER.info("prometheus http info is blank prometheusHost：{} prometheusPort：{}", prometheusHost, prometheusPort);
        return null;
    }
    return new Pair<>(prometheusHost, prometheusPort);
}

Example 3 with EDeployMode

use of com.dtstack.taier.pluginapi.enums.EDeployMode in project Taier by DTStack.

the class ClusterService method buildDeployMode.

private JSONObject buildDeployMode(JSONObject clusterConfigJson, EComponentType componentType, Long clusterId, Integer deployMode) {
    JSONObject pluginInfo;
    // 默认为session
    EDeployMode deploy = EComponentType.FLINK.equals(componentType) ? EDeployMode.SESSION : EDeployMode.PERJOB;
    // spark 暂时全部为perjob
    if (Objects.nonNull(deployMode) && !EComponentType.SPARK.equals(componentType)) {
        deploy = EDeployMode.getByType(deployMode);
    }
    JSONObject confConfig = null;
    if (EComponentType.FLINK.equals(componentType) && EDeployMode.STANDALONE.getType().equals(deployMode)) {
        confConfig = clusterConfigJson.getJSONObject(EComponentType.FLINK.getConfName());
        return confConfig;
    } else {
        confConfig = clusterConfigJson.getJSONObject(componentType.getConfName());
    }
    pluginInfo = confConfig.getJSONObject(deploy.getMode());
    if (Objects.isNull(pluginInfo)) {
        throw new RdosDefineException(String.format("Corresponding mode [%s] no information is configured", deploy.name()));
    }
    String typeName = confConfig.getString(TYPE_NAME);
    if (!StringUtils.isBlank(typeName)) {
        pluginInfo.put(TYPE_NAME_KEY, typeName);
    }
    if (EComponentType.SPARK.equals(componentType)) {
        JSONObject sftpConfig = clusterConfigJson.getJSONObject(EComponentType.SFTP.getConfName());
        if (Objects.nonNull(sftpConfig)) {
            String confHdfsPath = sftpConfig.getString("path") + File.separator + componentService.buildConfRemoteDir(clusterId);
            pluginInfo.put("confHdfsPath", confHdfsPath);
        }
    }
    return pluginInfo;
}

Example 4 with EDeployMode

use of com.dtstack.taier.pluginapi.enums.EDeployMode in project Taier by DTStack.

the class SyncOperatorPipeline method queryLastLocation.

public String queryLastLocation(Long tenantId, String engineJobId, long startTime, long endTime, EDeployMode deployMode, String jobId, String componentVersion) {
    endTime = endTime + 1000 * 60;
    List<Component> components = componentService.listComponentsByComponentType(tenantId, EComponentType.FLINK.getTypeCode());
    if (CollectionUtils.isEmpty(components)) {
        return null;
    }
    Optional<Component> componentOptional = components.stream().filter(c -> c.getVersionValue().equals(componentVersion)).findFirst();
    if (!componentOptional.isPresent()) {
        return null;
    }
    Map<String, Object> componentConfigToMap = componentConfigService.convertComponentConfigToMap(componentOptional.get().getId(), true);
    Map<String, Object> flinkConfig = (Map<String, Object>) componentConfigToMap.get(deployMode.getMode());
    String prometheusHost = (String) flinkConfig.get("prometheusHost");
    String prometheusPort = (String) flinkConfig.get("prometheusPort");
    LOGGER.info("last job {} deployMode {} prometheus host {} port {}", jobId, deployMode.getType(), prometheusHost, prometheusPort);
    // prometheus的配置信息 从控制台获取
    PrometheusMetricQuery prometheusMetricQuery = new PrometheusMetricQuery(String.format("%s:%s", prometheusHost, prometheusPort));
    IMetric numReadMetric = MetricBuilder.buildMetric("endLocation", engineJobId, startTime, endTime, prometheusMetricQuery);
    if (numReadMetric != null) {
        String startLocation = String.valueOf(numReadMetric.getMetric());
        LOGGER.info("job {} deployMode {} startLocation [{}]", jobId, deployMode.getType(), startLocation);
        if (StringUtils.isEmpty(startLocation) || "0".equalsIgnoreCase(startLocation)) {
            return null;
        }
        return String.valueOf(numReadMetric.getMetric());
    }
    return null;
}

Example 5 with EDeployMode

use of com.dtstack.taier.pluginapi.enums.EDeployMode in project Taier by DTStack.

the class SyncOperatorPipeline method pipeline.

@Override
public void pipeline(Map<String, Object> actionParam, Map<String, Object> pipelineParam) throws Exception {
    ScheduleTaskShade taskShade = (ScheduleTaskShade) pipelineParam.get(taskShadeKey);
    ScheduleJob scheduleJob = (ScheduleJob) pipelineParam.get(scheduleJobKey);
    List<ScheduleTaskParamShade> taskParamShades = (List) pipelineParam.get(taskParamsToReplaceKey);
    String taskParams = (String) actionParam.get("taskParams");
    String job = (String) actionParam.get("job");
    EDeployMode deployMode = TaskParamsUtils.parseDeployTypeByTaskParams(taskParams, taskShade.getComputeType());
    job = this.replaceSyncJobString(actionParam, taskShade, scheduleJob, taskParamShades, job, deployMode);
    // 构造savepoint参数
    String savepointArgs = null;
    String taskExeArgs = null;
    if (isRestore(job)) {
        String savepointPath = this.getSavepointPath(taskShade.getTenantId(), deployMode, taskShade.getComponentVersion());
        savepointArgs = buildSyncTaskExecArgs(savepointPath, taskParams);
        taskParams += String.format(" \n %s=%s", KEY_OPEN_CHECKPOINT, Boolean.TRUE);
    }
    job = URLEncoder.encode(job.replace(CommonConstant.JOB_ID, scheduleJob.getJobId()), Charsets.UTF_8.name());
    taskExeArgs = String.format(JOB_ARGS_TEMPLATE, scheduleJob.getJobName(), job);
    if (savepointArgs != null) {
        taskExeArgs += " " + savepointArgs;
    }
    actionParam.put("exeArgs", taskExeArgs);
    actionParam.put("taskParams", taskParams);
}