Examples with PrometheusMetricQuery com.dtstack.taier.common.metric.prometheus.PrometheusMetricQuery used on opensource projects

Example 1 with PrometheusMetricQuery

use of com.dtstack.taier.common.metric.prometheus.PrometheusMetricQuery in project Taier by DTStack.

the class BatchServerLogService method parseIncreInfo.

/**
 * 解析增量同步信息
 */
private void parseIncreInfo(final JSONObject info, final String job, final Long tenantId, final String jobId, final long startTime, final long endTime, final String taskParams) {
    if (StringUtils.isEmpty(jobId)) {
        return;
    }
    try {
        final EDeployMode deployModeEnum = TaskParamsUtils.parseDeployTypeByTaskParams(taskParams, ComputeType.BATCH.getType());
        JSONObject flinkJsonObject = Engine2DTOService.getComponentConfig(tenantId, EComponentType.FLINK);
        final String prometheusHost = flinkJsonObject.getJSONObject(deployModeEnum.name()).getString("prometheusHost");
        final String prometheusPort = flinkJsonObject.getJSONObject(deployModeEnum.name()).getString("prometheusPort");
        // prometheus的配置信息 从控制台获取
        final PrometheusMetricQuery prometheusMetricQuery = new PrometheusMetricQuery(String.format("%s:%s", prometheusHost, prometheusPort));
        final IMetric startLocationMetric = MetricBuilder.buildMetric("startLocation", jobId, startTime, endTime, prometheusMetricQuery);
        final IMetric endLocationMetric = MetricBuilder.buildMetric("endLocation", jobId, startTime, endTime, prometheusMetricQuery);
        String startLocation = null;
        String endLocation = null;
        if (startLocationMetric != null) {
            startLocation = String.valueOf(startLocationMetric.getMetric());
        }
        if (Objects.nonNull(endLocationMetric)) {
            endLocation = String.valueOf(endLocationMetric.getMetric());
        }
        if (StringUtils.isBlank(job)) {
            return;
        }
        final JSONObject jobJson = JSON.parseObject(job);
        final String increColumn = (String) JSONPath.eval(jobJson, "$.job.content[0].reader.parameter.increColumn");
        final String table = (String) JSONPath.eval(jobJson, "$.job.content[0].reader.parameter.connection[0].table[0]");
        final StringBuilder increStrBuild = new StringBuilder();
        increStrBuild.append("数据表:  \t").append(table).append("\n");
        increStrBuild.append("增量标识:\t").append(increColumn).append("\n");
        if (StringUtils.isEmpty(endLocation) || endLocation.startsWith("-")) {
            increStrBuild.append("开始位置:\t").append("同步数据条数为0").append("\n");
            info.put("increInfo", increStrBuild.toString());
            return;
        }
        boolean isDateCol = false;
        final JSONArray columns = (JSONArray) JSONPath.eval(jobJson, "$.job.content[0].reader.parameter.column");
        for (final Object column : columns) {
            if (column instanceof JSONObject) {
                final String name = ((JSONObject) column).getString("name");
                if (name != null && name.equals(increColumn)) {
                    final String type = ((JSONObject) column).getString("type");
                    Boolean typeCheck = type != null && (type.matches("(?i)date|datetime|time") || type.toLowerCase().contains("timestamp"));
                    if (typeCheck) {
                        isDateCol = true;
                    }
                }
            }
        }
        if (StringUtils.isEmpty(startLocation)) {
            startLocation = "全量同步";
        }
        if (isDateCol) {
            startLocation = this.formatLongStr(startLocation);
            endLocation = this.formatLongStr(endLocation);
        }
        increStrBuild.append("开始位置:\t").append(startLocation).append("\n");
        increStrBuild.append("结束位置:\t").append(endLocation).append("\n");
        info.put("increInfo", increStrBuild.toString());
    } catch (Exception e) {
        LOGGER.warn("{}", e);
    }
}

Example 2 with PrometheusMetricQuery

use of com.dtstack.taier.common.metric.prometheus.PrometheusMetricQuery in project Taier by DTStack.

the class BatchServerLogService method formatPerfLogInfo.

public String formatPerfLogInfo(final String engineJobId, final String jobId, final long startTime, final long endTime, final Long tenantId) {
    final ScheduleJob job = scheduleJobService.getByJobId(jobId);
    if (Objects.isNull(job)) {
        LOGGER.info("can not find job by id:{}.", jobId);
        throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_JOB);
    }
    if (job.getTaskId() == null || job.getTaskId() == -1) {
        throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_TASK);
    }
    BatchTask batchTaskById = batchTaskService.getBatchTaskById(job.getTaskId());
    // prometheus的配置信息 从控制台获取
    final Pair<String, String> prometheusHostAndPort = this.getPrometheusHostAndPort(tenantId, batchTaskById.getTaskParams());
    if (prometheusHostAndPort == null) {
        return "promethues配置为空";
    }
    final PrometheusMetricQuery prometheusMetricQuery = new PrometheusMetricQuery(String.format("%s:%s", prometheusHostAndPort.getKey(), prometheusHostAndPort.getValue()));
    // 之后查询是可以直接获取最后一条记录的方法
    // 防止数据同步执行时间太长 查询prometheus的时候返回exceeded maximum resolution of 11,000 points per timeseries
    final long maxGapTime = 60 * 1000 * 60 * (long) 8;
    long gapStartTime = startTime;
    if (endTime - startTime >= maxGapTime) {
        // 超过11,000 points 查询1小时间隔内
        gapStartTime = endTime - 60 * 1000 * 60;
    }
    final IMetric numReadMetric = MetricBuilder.buildMetric("numRead", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric byteReadMetric = MetricBuilder.buildMetric("byteRead", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric readDurationMetric = MetricBuilder.buildMetric("readDuration", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric numWriteMetric = MetricBuilder.buildMetric("numWrite", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric byteWriteMetric = MetricBuilder.buildMetric("byteWrite", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric writeDurationMetric = MetricBuilder.buildMetric("writeDuration", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric numErrorMetric = MetricBuilder.buildMetric("nErrors", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final SyncStatusLogInfoVO formatPerfLogInfo = this.getFormatPerfLogInfo(numReadMetric, byteReadMetric, readDurationMetric, numWriteMetric, byteWriteMetric, writeDurationMetric, numErrorMetric);
    return formatPerfLogInfo.buildReadableLog();
}

Example 3 with PrometheusMetricQuery

use of com.dtstack.taier.common.metric.prometheus.PrometheusMetricQuery in project Taier by DTStack.

the class SyncOperatorPipeline method queryLastLocation.

public String queryLastLocation(Long tenantId, String engineJobId, long startTime, long endTime, EDeployMode deployMode, String jobId, String componentVersion) {
    endTime = endTime + 1000 * 60;
    List<Component> components = componentService.listComponentsByComponentType(tenantId, EComponentType.FLINK.getTypeCode());
    if (CollectionUtils.isEmpty(components)) {
        return null;
    }
    Optional<Component> componentOptional = components.stream().filter(c -> c.getVersionValue().equals(componentVersion)).findFirst();
    if (!componentOptional.isPresent()) {
        return null;
    }
    Map<String, Object> componentConfigToMap = componentConfigService.convertComponentConfigToMap(componentOptional.get().getId(), true);
    Map<String, Object> flinkConfig = (Map<String, Object>) componentConfigToMap.get(deployMode.getMode());
    String prometheusHost = (String) flinkConfig.get("prometheusHost");
    String prometheusPort = (String) flinkConfig.get("prometheusPort");
    LOGGER.info("last job {} deployMode {} prometheus host {} port {}", jobId, deployMode.getType(), prometheusHost, prometheusPort);
    // prometheus的配置信息 从控制台获取
    PrometheusMetricQuery prometheusMetricQuery = new PrometheusMetricQuery(String.format("%s:%s", prometheusHost, prometheusPort));
    IMetric numReadMetric = MetricBuilder.buildMetric("endLocation", engineJobId, startTime, endTime, prometheusMetricQuery);
    if (numReadMetric != null) {
        String startLocation = String.valueOf(numReadMetric.getMetric());
        LOGGER.info("job {} deployMode {} startLocation [{}]", jobId, deployMode.getType(), startLocation);
        if (StringUtils.isEmpty(startLocation) || "0".equalsIgnoreCase(startLocation)) {
            return null;
        }
        return String.valueOf(numReadMetric.getMetric());
    }
    return null;
}