Examples with IMetric com.dtstack.taier.common.metric.batch.IMetric used on opensource projects

Example 1 with IMetric

use of com.dtstack.taier.common.metric.batch.IMetric in project Taier by DTStack.

the class BatchServerLogService method parseIncreInfo.

/**
 * 解析增量同步信息
 */
private void parseIncreInfo(final JSONObject info, final String job, final Long tenantId, final String jobId, final long startTime, final long endTime, final String taskParams) {
    if (StringUtils.isEmpty(jobId)) {
        return;
    }
    try {
        final EDeployMode deployModeEnum = TaskParamsUtils.parseDeployTypeByTaskParams(taskParams, ComputeType.BATCH.getType());
        JSONObject flinkJsonObject = Engine2DTOService.getComponentConfig(tenantId, EComponentType.FLINK);
        final String prometheusHost = flinkJsonObject.getJSONObject(deployModeEnum.name()).getString("prometheusHost");
        final String prometheusPort = flinkJsonObject.getJSONObject(deployModeEnum.name()).getString("prometheusPort");
        // prometheus的配置信息 从控制台获取
        final PrometheusMetricQuery prometheusMetricQuery = new PrometheusMetricQuery(String.format("%s:%s", prometheusHost, prometheusPort));
        final IMetric startLocationMetric = MetricBuilder.buildMetric("startLocation", jobId, startTime, endTime, prometheusMetricQuery);
        final IMetric endLocationMetric = MetricBuilder.buildMetric("endLocation", jobId, startTime, endTime, prometheusMetricQuery);
        String startLocation = null;
        String endLocation = null;
        if (startLocationMetric != null) {
            startLocation = String.valueOf(startLocationMetric.getMetric());
        }
        if (Objects.nonNull(endLocationMetric)) {
            endLocation = String.valueOf(endLocationMetric.getMetric());
        }
        if (StringUtils.isBlank(job)) {
            return;
        }
        final JSONObject jobJson = JSON.parseObject(job);
        final String increColumn = (String) JSONPath.eval(jobJson, "$.job.content[0].reader.parameter.increColumn");
        final String table = (String) JSONPath.eval(jobJson, "$.job.content[0].reader.parameter.connection[0].table[0]");
        final StringBuilder increStrBuild = new StringBuilder();
        increStrBuild.append("数据表:  \t").append(table).append("\n");
        increStrBuild.append("增量标识:\t").append(increColumn).append("\n");
        if (StringUtils.isEmpty(endLocation) || endLocation.startsWith("-")) {
            increStrBuild.append("开始位置:\t").append("同步数据条数为0").append("\n");
            info.put("increInfo", increStrBuild.toString());
            return;
        }
        boolean isDateCol = false;
        final JSONArray columns = (JSONArray) JSONPath.eval(jobJson, "$.job.content[0].reader.parameter.column");
        for (final Object column : columns) {
            if (column instanceof JSONObject) {
                final String name = ((JSONObject) column).getString("name");
                if (name != null && name.equals(increColumn)) {
                    final String type = ((JSONObject) column).getString("type");
                    Boolean typeCheck = type != null && (type.matches("(?i)date|datetime|time") || type.toLowerCase().contains("timestamp"));
                    if (typeCheck) {
                        isDateCol = true;
                    }
                }
            }
        }
        if (StringUtils.isEmpty(startLocation)) {
            startLocation = "全量同步";
        }
        if (isDateCol) {
            startLocation = this.formatLongStr(startLocation);
            endLocation = this.formatLongStr(endLocation);
        }
        increStrBuild.append("开始位置:\t").append(startLocation).append("\n");
        increStrBuild.append("结束位置:\t").append(endLocation).append("\n");
        info.put("increInfo", increStrBuild.toString());
    } catch (Exception e) {
        LOGGER.warn("{}", e);
    }
}

Example 2 with IMetric

use of com.dtstack.taier.common.metric.batch.IMetric in project Taier by DTStack.

the class BatchServerLogService method formatPerfLogInfo.

public String formatPerfLogInfo(final String engineJobId, final String jobId, final long startTime, final long endTime, final Long tenantId) {
    final ScheduleJob job = scheduleJobService.getByJobId(jobId);
    if (Objects.isNull(job)) {
        LOGGER.info("can not find job by id:{}.", jobId);
        throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_JOB);
    }
    if (job.getTaskId() == null || job.getTaskId() == -1) {
        throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_TASK);
    }
    BatchTask batchTaskById = batchTaskService.getBatchTaskById(job.getTaskId());
    // prometheus的配置信息 从控制台获取
    final Pair<String, String> prometheusHostAndPort = this.getPrometheusHostAndPort(tenantId, batchTaskById.getTaskParams());
    if (prometheusHostAndPort == null) {
        return "promethues配置为空";
    }
    final PrometheusMetricQuery prometheusMetricQuery = new PrometheusMetricQuery(String.format("%s:%s", prometheusHostAndPort.getKey(), prometheusHostAndPort.getValue()));
    // 之后查询是可以直接获取最后一条记录的方法
    // 防止数据同步执行时间太长 查询prometheus的时候返回exceeded maximum resolution of 11,000 points per timeseries
    final long maxGapTime = 60 * 1000 * 60 * (long) 8;
    long gapStartTime = startTime;
    if (endTime - startTime >= maxGapTime) {
        // 超过11,000 points 查询1小时间隔内
        gapStartTime = endTime - 60 * 1000 * 60;
    }
    final IMetric numReadMetric = MetricBuilder.buildMetric("numRead", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric byteReadMetric = MetricBuilder.buildMetric("byteRead", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric readDurationMetric = MetricBuilder.buildMetric("readDuration", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric numWriteMetric = MetricBuilder.buildMetric("numWrite", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric byteWriteMetric = MetricBuilder.buildMetric("byteWrite", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric writeDurationMetric = MetricBuilder.buildMetric("writeDuration", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final IMetric numErrorMetric = MetricBuilder.buildMetric("nErrors", engineJobId, gapStartTime, endTime, prometheusMetricQuery);
    final SyncStatusLogInfoVO formatPerfLogInfo = this.getFormatPerfLogInfo(numReadMetric, byteReadMetric, readDurationMetric, numWriteMetric, byteWriteMetric, writeDurationMetric, numErrorMetric);
    return formatPerfLogInfo.buildReadableLog();
}

Example 3 with IMetric

use of com.dtstack.taier.common.metric.batch.IMetric in project Taier by DTStack.

the class SyncOperatorPipeline method queryLastLocation.

public String queryLastLocation(Long tenantId, String engineJobId, long startTime, long endTime, EDeployMode deployMode, String jobId, String componentVersion) {
    endTime = endTime + 1000 * 60;
    List<Component> components = componentService.listComponentsByComponentType(tenantId, EComponentType.FLINK.getTypeCode());
    if (CollectionUtils.isEmpty(components)) {
        return null;
    }
    Optional<Component> componentOptional = components.stream().filter(c -> c.getVersionValue().equals(componentVersion)).findFirst();
    if (!componentOptional.isPresent()) {
        return null;
    }
    Map<String, Object> componentConfigToMap = componentConfigService.convertComponentConfigToMap(componentOptional.get().getId(), true);
    Map<String, Object> flinkConfig = (Map<String, Object>) componentConfigToMap.get(deployMode.getMode());
    String prometheusHost = (String) flinkConfig.get("prometheusHost");
    String prometheusPort = (String) flinkConfig.get("prometheusPort");
    LOGGER.info("last job {} deployMode {} prometheus host {} port {}", jobId, deployMode.getType(), prometheusHost, prometheusPort);
    // prometheus的配置信息 从控制台获取
    PrometheusMetricQuery prometheusMetricQuery = new PrometheusMetricQuery(String.format("%s:%s", prometheusHost, prometheusPort));
    IMetric numReadMetric = MetricBuilder.buildMetric("endLocation", engineJobId, startTime, endTime, prometheusMetricQuery);
    if (numReadMetric != null) {
        String startLocation = String.valueOf(numReadMetric.getMetric());
        LOGGER.info("job {} deployMode {} startLocation [{}]", jobId, deployMode.getType(), startLocation);
        if (StringUtils.isEmpty(startLocation) || "0".equalsIgnoreCase(startLocation)) {
            return null;
        }
        return String.valueOf(numReadMetric.getMetric());
    }
    return null;
}