Search in sources :

Example 1 with AuditInfo

use of org.apache.inlong.manager.common.pojo.audit.AuditInfo in project incubator-inlong by apache.

the class AuditServiceTest method testQueryFromMySQL.

@Test
public void testQueryFromMySQL() throws IOException {
    AuditRequest request = new AuditRequest();
    request.setAuditIds(Arrays.asList("3", "4"));
    request.setInlongGroupId("g1");
    request.setInlongStreamId("s1");
    request.setDt("2022-01-01");
    List<AuditVO> result = new ArrayList<>();
    AuditVO auditVO = new AuditVO();
    auditVO.setAuditId("3");
    auditVO.setAuditSet(Arrays.asList(new AuditInfo("2022-01-01 00:00:00", 123L), new AuditInfo("2022-01-01 00:01:00", 124L)));
    result.add(auditVO);
    Assert.assertNotNull(result);
// close real test for testQueryFromMySQL due to date_format function not support in h2
// Assert.assertNotNull(auditService.listByCondition(request));
}
Also used : AuditInfo(org.apache.inlong.manager.common.pojo.audit.AuditInfo) AuditRequest(org.apache.inlong.manager.common.pojo.audit.AuditRequest) ArrayList(java.util.ArrayList) AuditVO(org.apache.inlong.manager.common.pojo.audit.AuditVO) ServiceBaseTest(org.apache.inlong.manager.service.ServiceBaseTest) Test(org.junit.Test)

Example 2 with AuditInfo

use of org.apache.inlong.manager.common.pojo.audit.AuditInfo in project incubator-inlong by apache.

the class AuditServiceImpl method listByCondition.

/**
 * Query audit data for list by condition
 *
 * @param request The audit request of query condition
 * @return The result of query
 */
@Override
public List<AuditVO> listByCondition(AuditRequest request) throws IOException {
    LOGGER.info("begin query audit list request={}", request);
    Preconditions.checkNotNull(request, "request is null");
    List<AuditVO> result = new ArrayList<>();
    AuditQuerySource querySource = AuditQuerySource.valueOf(auditQuerySource);
    for (String auditId : request.getAuditIds()) {
        if (AuditQuerySource.MYSQL == querySource) {
            String format = "%Y-%m-%d %H:%i:00";
            // Support min agg at now
            DateTimeFormatter forPattern = DateTimeFormat.forPattern("yyyy-MM-dd");
            DateTime dtDate = forPattern.parseDateTime(request.getDt());
            String eDate = dtDate.plusDays(1).toString(forPattern);
            List<Map<String, Object>> sumList = auditEntityMapper.sumByLogTs(request.getInlongGroupId(), request.getInlongStreamId(), auditId, request.getDt(), eDate, format);
            List<AuditInfo> auditSet = sumList.stream().map(s -> {
                AuditInfo vo = new AuditInfo();
                vo.setLogTs((String) s.get("logTs"));
                vo.setCount(((BigDecimal) s.get("total")).longValue());
                return vo;
            }).collect(Collectors.toList());
            result.add(new AuditVO(auditId, auditSet));
        } else if (AuditQuerySource.ELASTICSEARCH == querySource) {
            String index = String.format("%s_%s", request.getDt().replaceAll("-", ""), auditId);
            if (elasticsearchApi.indexExists(index)) {
                SearchResponse response = elasticsearchApi.search(toAuditSearchRequest(index, request.getInlongGroupId(), request.getInlongStreamId()));
                final List<Aggregation> aggregations = response.getAggregations().asList();
                if (CollectionUtils.isNotEmpty(aggregations)) {
                    ParsedTerms terms = (ParsedTerms) aggregations.get(0);
                    if (CollectionUtils.isNotEmpty(terms.getBuckets())) {
                        List<AuditInfo> auditSet = terms.getBuckets().stream().map(bucket -> {
                            AuditInfo vo = new AuditInfo();
                            vo.setLogTs(bucket.getKeyAsString());
                            vo.setCount((long) ((ParsedSum) bucket.getAggregations().asList().get(0)).getValue());
                            return vo;
                        }).collect(Collectors.toList());
                        result.add(new AuditVO(auditId, auditSet));
                    }
                }
            } else {
                LOGGER.warn("Elasticsearch index={} not exists", index);
            }
        }
    }
    LOGGER.info("success to query audit list for request={}", request);
    return result;
}
Also used : AuditService(org.apache.inlong.manager.service.core.AuditService) LoggerFactory(org.slf4j.LoggerFactory) AuditInfo(org.apache.inlong.manager.common.pojo.audit.AuditInfo) Autowired(org.springframework.beans.factory.annotation.Autowired) SearchRequest(org.elasticsearch.action.search.SearchRequest) AuditEntityMapper(org.apache.inlong.manager.dao.mapper.AuditEntityMapper) ArrayList(java.util.ArrayList) AuditVO(org.apache.inlong.manager.common.pojo.audit.AuditVO) Value(org.springframework.beans.factory.annotation.Value) BigDecimal(java.math.BigDecimal) Preconditions(org.apache.inlong.manager.common.util.Preconditions) CollectionUtils(org.apache.commons.collections.CollectionUtils) Service(org.springframework.stereotype.Service) Map(java.util.Map) SearchResponse(org.elasticsearch.action.search.SearchResponse) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) Aggregation(org.elasticsearch.search.aggregations.Aggregation) TermsAggregationBuilder(org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder) DateTimeFormat(org.joda.time.format.DateTimeFormat) ParsedTerms(org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms) Logger(org.slf4j.Logger) ParsedSum(org.elasticsearch.search.aggregations.metrics.sum.ParsedSum) DateTimeFormatter(org.joda.time.format.DateTimeFormatter) DateTime(org.joda.time.DateTime) IOException(java.io.IOException) AggregationBuilders(org.elasticsearch.search.aggregations.AggregationBuilders) Collectors(java.util.stream.Collectors) ElasticsearchApi(org.apache.inlong.manager.service.thirdparty.elasticsearch.ElasticsearchApi) QueryBuilders.termQuery(org.elasticsearch.index.query.QueryBuilders.termQuery) List(java.util.List) AuditRequest(org.apache.inlong.manager.common.pojo.audit.AuditRequest) SortOrder(org.elasticsearch.search.sort.SortOrder) AuditQuerySource(org.apache.inlong.manager.common.enums.AuditQuerySource) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) AuditQuerySource(org.apache.inlong.manager.common.enums.AuditQuerySource) AuditInfo(org.apache.inlong.manager.common.pojo.audit.AuditInfo) ParsedTerms(org.elasticsearch.search.aggregations.bucket.terms.ParsedTerms) ArrayList(java.util.ArrayList) ParsedSum(org.elasticsearch.search.aggregations.metrics.sum.ParsedSum) DateTime(org.joda.time.DateTime) BigDecimal(java.math.BigDecimal) SearchResponse(org.elasticsearch.action.search.SearchResponse) AuditVO(org.apache.inlong.manager.common.pojo.audit.AuditVO) ArrayList(java.util.ArrayList) List(java.util.List) DateTimeFormatter(org.joda.time.format.DateTimeFormatter) Map(java.util.Map)

Aggregations

ArrayList (java.util.ArrayList)2 AuditInfo (org.apache.inlong.manager.common.pojo.audit.AuditInfo)2 AuditRequest (org.apache.inlong.manager.common.pojo.audit.AuditRequest)2 AuditVO (org.apache.inlong.manager.common.pojo.audit.AuditVO)2 IOException (java.io.IOException)1 BigDecimal (java.math.BigDecimal)1 List (java.util.List)1 Map (java.util.Map)1 Collectors (java.util.stream.Collectors)1 CollectionUtils (org.apache.commons.collections.CollectionUtils)1 AuditQuerySource (org.apache.inlong.manager.common.enums.AuditQuerySource)1 Preconditions (org.apache.inlong.manager.common.util.Preconditions)1 AuditEntityMapper (org.apache.inlong.manager.dao.mapper.AuditEntityMapper)1 ServiceBaseTest (org.apache.inlong.manager.service.ServiceBaseTest)1 AuditService (org.apache.inlong.manager.service.core.AuditService)1 ElasticsearchApi (org.apache.inlong.manager.service.thirdparty.elasticsearch.ElasticsearchApi)1 SearchRequest (org.elasticsearch.action.search.SearchRequest)1 SearchResponse (org.elasticsearch.action.search.SearchResponse)1 BoolQueryBuilder (org.elasticsearch.index.query.BoolQueryBuilder)1 QueryBuilders.termQuery (org.elasticsearch.index.query.QueryBuilders.termQuery)1