Examples with ESBulkRequest com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest used on opensource projects

Search in sources :

Example 1 with ESBulkRequest

use of com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest in project canal by alibaba.

the class ESEtlService method executeSqlImport.

protected boolean executeSqlImport(DataSource ds, String sql, List<Object> values, AdapterConfig.AdapterMapping adapterMapping, AtomicLong impCount, List<String> errMsg) {
    try {
        ESMapping mapping = (ESMapping) adapterMapping;
        Util.sqlRS(ds, sql, values, rs -> {
            int count = 0;
            try {
                ESBulkRequest esBulkRequest = this.esConnection.new ES7xBulkRequest();
                long batchBegin = System.currentTimeMillis();
                while (rs.next()) {
                    Map<String, Object> esFieldData = new LinkedHashMap<>();
                    Object idVal = null;
                    for (FieldItem fieldItem : mapping.getSchemaItem().getSelectFields().values()) {
                        String fieldName = fieldItem.getFieldName();
                        if (mapping.getSkips().contains(fieldName)) {
                            continue;
                        }
                        // 如果是主键字段则不插入
                        if (fieldItem.getFieldName().equals(mapping.get_id())) {
                            idVal = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
                        } else {
                            Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
                            esFieldData.put(Util.cleanColumn(fieldName), val);
                        }
                    }
                    if (!mapping.getRelations().isEmpty()) {
                        mapping.getRelations().forEach((relationField, relationMapping) -> {
                            Map<String, Object> relations = new HashMap<>();
                            relations.put("name", relationMapping.getName());
                            if (StringUtils.isNotEmpty(relationMapping.getParent())) {
                                FieldItem parentFieldItem = mapping.getSchemaItem().getSelectFields().get(relationMapping.getParent());
                                Object parentVal;
                                try {
                                    parentVal = esTemplate.getValFromRS(mapping, rs, parentFieldItem.getFieldName(), parentFieldItem.getFieldName());
                                } catch (SQLException e) {
                                    throw new RuntimeException(e);
                                }
                                if (parentVal != null) {
                                    relations.put("parent", parentVal.toString());
                                    esFieldData.put("$parent_routing", parentVal.toString());
                                }
                            }
                            esFieldData.put(Util.cleanColumn(relationField), relations);
                        });
                    }
                    if (idVal != null) {
                        String parentVal = (String) esFieldData.remove("$parent_routing");
                        if (mapping.isUpsert()) {
                            ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest(mapping.get_index(), idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
                            if (StringUtils.isNotEmpty(parentVal)) {
                                esUpdateRequest.setRouting(parentVal);
                            }
                            esBulkRequest.add(esUpdateRequest);
                        } else {
                            ESIndexRequest esIndexRequest = this.esConnection.new ES7xIndexRequest(mapping.get_index(), idVal.toString()).setSource(esFieldData);
                            if (StringUtils.isNotEmpty(parentVal)) {
                                esIndexRequest.setRouting(parentVal);
                            }
                            esBulkRequest.add(esIndexRequest);
                        }
                    } else {
                        idVal = esFieldData.get(mapping.getPk());
                        ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index()).setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal)).size(10000);
                        SearchResponse response = esSearchRequest.getResponse();
                        for (SearchHit hit : response.getHits()) {
                            ESUpdateRequest esUpdateRequest = this.esConnection.new ES7xUpdateRequest(mapping.get_index(), hit.getId()).setDoc(esFieldData);
                            esBulkRequest.add(esUpdateRequest);
                        }
                    }
                    if (esBulkRequest.numberOfActions() % mapping.getCommitBatch() == 0 && esBulkRequest.numberOfActions() > 0) {
                        long esBatchBegin = System.currentTimeMillis();
                        ESBulkResponse rp = esBulkRequest.bulk();
                        if (rp.hasFailures()) {
                            rp.processFailBulkResponse("全量数据 etl 异常 ");
                        }
                        if (logger.isTraceEnabled()) {
                            logger.trace("全量数据批量导入批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}", (System.currentTimeMillis() - batchBegin), (System.currentTimeMillis() - esBatchBegin), esBulkRequest.numberOfActions(), mapping.get_index());
                        }
                        batchBegin = System.currentTimeMillis();
                        esBulkRequest.resetBulk();
                    }
                    count++;
                    impCount.incrementAndGet();
                }
                if (esBulkRequest.numberOfActions() > 0) {
                    long esBatchBegin = System.currentTimeMillis();
                    ESBulkResponse rp = esBulkRequest.bulk();
                    if (rp.hasFailures()) {
                        rp.processFailBulkResponse("全量数据 etl 异常 ");
                    }
                    if (logger.isTraceEnabled()) {
                        logger.trace("全量数据批量导入最后批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}", (System.currentTimeMillis() - batchBegin), (System.currentTimeMillis() - esBatchBegin), esBulkRequest.numberOfActions(), mapping.get_index());
                    }
                }
            } catch (Exception e) {
                logger.error(e.getMessage(), e);
                errMsg.add(mapping.get_index() + " etl failed! ==>" + e.getMessage());
                throw new RuntimeException(e);
            }
            return count;
        });
        return true;
    } catch (Exception e) {
        logger.error(e.getMessage(), e);
        return false;
    }
}
Also used : SearchHit(org.elasticsearch.search.SearchHit) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) SQLException(java.sql.SQLException) ESSearchRequest(com.alibaba.otter.canal.client.adapter.es7x.support.ESConnection.ESSearchRequest) SQLException(java.sql.SQLException) LinkedHashMap(java.util.LinkedHashMap) SearchResponse(org.elasticsearch.action.search.SearchResponse) ESUpdateRequest(com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESUpdateRequest) ESMapping(com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping) ESIndexRequest(com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESIndexRequest) ESBulkRequest(com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest) FieldItem(com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem) ESBulkResponse(com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESBulkResponse)

Example 2 with ESBulkRequest

use of com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest in project canal by alibaba.

the class ESEtlService method executeSqlImport.

protected boolean executeSqlImport(DataSource ds, String sql, List<Object> values, AdapterConfig.AdapterMapping adapterMapping, AtomicLong impCount, List<String> errMsg) {
    try {
        ESMapping mapping = (ESMapping) adapterMapping;
        Util.sqlRS(ds, sql, values, rs -> {
            int count = 0;
            try {
                ESBulkRequest esBulkRequest = this.esConnection.new ES6xBulkRequest();
                long batchBegin = System.currentTimeMillis();
                while (rs.next()) {
                    Map<String, Object> esFieldData = new LinkedHashMap<>();
                    Object idVal = null;
                    for (FieldItem fieldItem : mapping.getSchemaItem().getSelectFields().values()) {
                        String fieldName = fieldItem.getFieldName();
                        if (mapping.getSkips().contains(fieldName)) {
                            continue;
                        }
                        // 如果是主键字段则不插入
                        if (fieldItem.getFieldName().equals(mapping.get_id())) {
                            idVal = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
                        } else {
                            Object val = esTemplate.getValFromRS(mapping, rs, fieldName, fieldName);
                            esFieldData.put(Util.cleanColumn(fieldName), val);
                        }
                    }
                    if (!mapping.getRelations().isEmpty()) {
                        mapping.getRelations().forEach((relationField, relationMapping) -> {
                            Map<String, Object> relations = new HashMap<>();
                            relations.put("name", relationMapping.getName());
                            if (StringUtils.isNotEmpty(relationMapping.getParent())) {
                                FieldItem parentFieldItem = mapping.getSchemaItem().getSelectFields().get(relationMapping.getParent());
                                Object parentVal;
                                try {
                                    parentVal = esTemplate.getValFromRS(mapping, rs, parentFieldItem.getFieldName(), parentFieldItem.getFieldName());
                                } catch (SQLException e) {
                                    throw new RuntimeException(e);
                                }
                                if (parentVal != null) {
                                    relations.put("parent", parentVal.toString());
                                    esFieldData.put("$parent_routing", parentVal.toString());
                                }
                            }
                            esFieldData.put(Util.cleanColumn(relationField), relations);
                        });
                    }
                    if (idVal != null) {
                        String parentVal = (String) esFieldData.remove("$parent_routing");
                        if (mapping.isUpsert()) {
                            ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(), mapping.get_type(), idVal.toString()).setDoc(esFieldData).setDocAsUpsert(true);
                            if (StringUtils.isNotEmpty(parentVal)) {
                                esUpdateRequest.setRouting(parentVal);
                            }
                            esBulkRequest.add(esUpdateRequest);
                        } else {
                            ESIndexRequest esIndexRequest = this.esConnection.new ES6xIndexRequest(mapping.get_index(), mapping.get_type(), idVal.toString()).setSource(esFieldData);
                            if (StringUtils.isNotEmpty(parentVal)) {
                                esIndexRequest.setRouting(parentVal);
                            }
                            esBulkRequest.add(esIndexRequest);
                        }
                    } else {
                        idVal = esFieldData.get(mapping.getPk());
                        ESSearchRequest esSearchRequest = this.esConnection.new ESSearchRequest(mapping.get_index(), mapping.get_type()).setQuery(QueryBuilders.termQuery(mapping.getPk(), idVal)).size(10000);
                        SearchResponse response = esSearchRequest.getResponse();
                        for (SearchHit hit : response.getHits()) {
                            ESUpdateRequest esUpdateRequest = this.esConnection.new ES6xUpdateRequest(mapping.get_index(), mapping.get_type(), hit.getId()).setDoc(esFieldData);
                            esBulkRequest.add(esUpdateRequest);
                        }
                    }
                    if (esBulkRequest.numberOfActions() % mapping.getCommitBatch() == 0 && esBulkRequest.numberOfActions() > 0) {
                        long esBatchBegin = System.currentTimeMillis();
                        ESBulkResponse rp = esBulkRequest.bulk();
                        if (rp.hasFailures()) {
                            rp.processFailBulkResponse("全量数据 etl 异常 ");
                        }
                        if (logger.isTraceEnabled()) {
                            logger.trace("全量数据批量导入批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}", (System.currentTimeMillis() - batchBegin), (System.currentTimeMillis() - esBatchBegin), esBulkRequest.numberOfActions(), mapping.get_index());
                        }
                        batchBegin = System.currentTimeMillis();
                        esBulkRequest.resetBulk();
                    }
                    count++;
                    impCount.incrementAndGet();
                }
                if (esBulkRequest.numberOfActions() > 0) {
                    long esBatchBegin = System.currentTimeMillis();
                    ESBulkResponse rp = esBulkRequest.bulk();
                    if (rp.hasFailures()) {
                        rp.processFailBulkResponse("全量数据 etl 异常 ");
                    }
                    if (logger.isTraceEnabled()) {
                        logger.trace("全量数据批量导入最后批次耗时: {}, es执行时间: {}, 批次大小: {}, index; {}", (System.currentTimeMillis() - batchBegin), (System.currentTimeMillis() - esBatchBegin), esBulkRequest.numberOfActions(), mapping.get_index());
                    }
                }
            } catch (Exception e) {
                logger.error(e.getMessage(), e);
                errMsg.add(mapping.get_index() + " etl failed! ==>" + e.getMessage());
                throw new RuntimeException(e);
            }
            return count;
        });
        return true;
    } catch (Exception e) {
        logger.error(e.getMessage(), e);
        return false;
    }
}
Also used : SearchHit(org.elasticsearch.search.SearchHit) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) SQLException(java.sql.SQLException) ESSearchRequest(com.alibaba.otter.canal.client.adapter.es6x.support.ESConnection.ESSearchRequest) SQLException(java.sql.SQLException) LinkedHashMap(java.util.LinkedHashMap) SearchResponse(org.elasticsearch.action.search.SearchResponse) ESUpdateRequest(com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESUpdateRequest) ESMapping(com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping) ESIndexRequest(com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESIndexRequest) ESBulkRequest(com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest) FieldItem(com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem) ESBulkResponse(com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESBulkResponse)

Aggregations

ESMapping (com.alibaba.otter.canal.client.adapter.es.core.config.ESSyncConfig.ESMapping)2 FieldItem (com.alibaba.otter.canal.client.adapter.es.core.config.SchemaItem.FieldItem)2 ESBulkRequest (com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest)2 ESBulkResponse (com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESBulkResponse)2 ESIndexRequest (com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESIndexRequest)2 ESUpdateRequest (com.alibaba.otter.canal.client.adapter.es.core.support.ESBulkRequest.ESUpdateRequest)2 SQLException (java.sql.SQLException)2 HashMap (java.util.HashMap)2 LinkedHashMap (java.util.LinkedHashMap)2 SearchResponse (org.elasticsearch.action.search.SearchResponse)2 SearchHit (org.elasticsearch.search.SearchHit)2 ESSearchRequest (com.alibaba.otter.canal.client.adapter.es6x.support.ESConnection.ESSearchRequest)1 ESSearchRequest (com.alibaba.otter.canal.client.adapter.es7x.support.ESConnection.ESSearchRequest)1
About Me
UseOf

Java Tutorials :

Simple Java RabbitMQ Example with Spring
Simple Springboot JPA Eclipeslink Example
Simple SpringBoot JPA Hibernate Example
Jackson JSON @JsonIgnore and @JsonIgnoreProperties Examples
Java Infinite recursion (StackOverflowError) Jackson solutions
Simple Java Jackson Serialize Objects to JSON and convert them back To Object
ElasticSearch 5 - Upload files using Java API in binary field Example
Java JAXB marshall unmarshall Examples from String and Stream
Java 8 forEach List and Map examples
ElasticSearch 5 Java API SearchRequestBuilder examples
Java ElasticSearch 5 examples with node index and mapping - running local
Convert String to int or Integer Java 8
Java 9 in action - JShell - Ubuntu
Java - removing invalid characters from a file name
Hello world!? or Hello Tutorial