Examples with WriterResult com.alicloud.openservices.tablestore.writer.WriterResult used on opensource projects

Example 1 with WriterResult

use of com.alicloud.openservices.tablestore.writer.WriterResult in project canal by alibaba.

the class TablestoreAdapter method sync.

@Override
public void sync(List<Dml> dmls) {
    if (dmls == null || dmls.isEmpty()) {
        return;
    }
    try {
        Set<TableStoreWriter> writerSet = new HashSet<>();
        List<Future<WriterResult>> futureList = new ArrayList<>();
        for (Dml dml : dmls) {
            String destination = StringUtils.trimToEmpty(dml.getDestination());
            String groupId = StringUtils.trimToEmpty(dml.getGroupId());
            String database = dml.getDatabase();
            String table = dml.getTable();
            String key;
            if (envProperties != null && !"tcp".equalsIgnoreCase(envProperties.getProperty("canal.conf.mode"))) {
                key = destination + "-" + groupId + "_" + database + "-" + table;
            } else {
                key = destination + "_" + database + "-" + table;
            }
            Map<String, MappingConfig> configMap = mappingConfigCache.get(key);
            if (configMap == null) {
                // 可能有dml中涉及到的表并没有出现在配置中，说明此类dml并不需要同步
                continue;
            }
            Map<String, TableStoreWriter> writerMap = writerCache.get(key);
            for (Map.Entry<String, MappingConfig> entry : configMap.entrySet()) {
                TableStoreWriter w = writerMap.get(entry.getKey());
                // 拿到所有future用于判定失败的记录
                Future<WriterResult> futureTemp = tablestoreSyncService.sync(entry.getValue(), dml, w);
                if (futureTemp != null) {
                    writerSet.add(w);
                    futureList.add(futureTemp);
                }
            }
        }
        if (writerSet.isEmpty()) {
            return;
        }
        writerSet.forEach(e -> e.flush());
        List<WriterResult.RowChangeStatus> totalFailedRows = new ArrayList<>();
        for (Future<WriterResult> future : futureList) {
            try {
                WriterResult result = future.get();
                List<WriterResult.RowChangeStatus> failedRows = result.getFailedRows();
                if (!CollectionUtils.isEmpty(failedRows)) {
                    totalFailedRows.addAll(failedRows);
                }
            } catch (InterruptedException e) {
                logger.info("InterruptedException", e);
                Thread.currentThread().interrupt();
            } catch (ExecutionException e) {
                throw new RuntimeException(e);
            }
        }
        if (!CollectionUtils.isEmpty(totalFailedRows)) {
            // 认为有失败的请求
            List<String> msgs = totalFailedRows.stream().map(e -> buildErrorMsgForFailedRowChange(e)).collect(Collectors.toList());
            throw new RuntimeException("Failed rows:" + org.springframework.util.StringUtils.collectionToDelimitedString(msgs, ",", "[", "]"));
        }
    } catch (Exception e) {
        throw e;
    }
}

Example 2 with WriterResult

use of com.alicloud.openservices.tablestore.writer.WriterResult in project canal by alibaba.

the class TablestoreEtlService method executeSqlImport.

@Override
protected boolean executeSqlImport(DataSource srcDS, String sql, List<Object> values, AdapterConfig.AdapterMapping mapping, AtomicLong impCount, List<String> errMsg) {
    try {
        MappingConfig.DbMapping dbMapping = (MappingConfig.DbMapping) mapping;
        Map<String, String> columnsMap = dbMapping.getTargetColumnsParsed();
        Util.sqlRS(srcDS, sql, values, rs -> {
            int idx = 0;
            List<Future<WriterResult>> futureList = new ArrayList<>();
            while (true) {
                try {
                    if (!rs.next())
                        break;
                } catch (SQLException throwables) {
                    logger.error("Error while get data from srcDs", throwables);
                    break;
                }
                Dml dml = getDMLByRs(columnsMap, rs);
                List<RowChange> rowChanges = syncService.getRowChanges(dml, config);
                if (CollectionUtils.isEmpty(rowChanges)) {
                    return null;
                }
                Future<WriterResult> future = writer.addRowChangeWithFuture(rowChanges);
                if (future != null) {
                    futureList.add(future);
                }
            }
            writer.flush();
            for (Future<WriterResult> future : futureList) {
                try {
                    WriterResult result = future.get();
                    if (result != null && result.isAllSucceed()) {
                        impCount.incrementAndGet();
                        idx++;
                    } else if (result != null && !result.isAllSucceed()) {
                        List<WriterResult.RowChangeStatus> totalFailedRows = result.getFailedRows();
                        List<String> msgs = totalFailedRows.stream().map(e -> TablestoreAdapter.buildErrorMsgForFailedRowChange(e)).collect(Collectors.toList());
                        logger.error("Failed rows when ETL:" + org.springframework.util.StringUtils.collectionToDelimitedString(msgs, ",", "[", "]"));
                    }
                } catch (InterruptedException e) {
                    logger.info("InterruptedException", e);
                    errMsg.add(e.getMessage());
                    Thread.currentThread().interrupt();
                } catch (ExecutionException e) {
                    errMsg.add(e.getMessage());
                    throw new RuntimeException(e);
                }
            }
            return idx;
        });
        return true;
    } catch (Exception e) {
        logger.error(e.getMessage(), e);
        return false;
    }
}