Examples with RowChange com.alicloud.openservices.tablestore.model.RowChange used on opensource projects

Example 1 with RowChange

use of com.alicloud.openservices.tablestore.model.RowChange in project canal by alibaba.

the class TablestoreEtlService method executeSqlImport.

@Override
protected boolean executeSqlImport(DataSource srcDS, String sql, List<Object> values, AdapterConfig.AdapterMapping mapping, AtomicLong impCount, List<String> errMsg) {
    try {
        MappingConfig.DbMapping dbMapping = (MappingConfig.DbMapping) mapping;
        Map<String, String> columnsMap = dbMapping.getTargetColumnsParsed();
        Util.sqlRS(srcDS, sql, values, rs -> {
            int idx = 0;
            List<Future<WriterResult>> futureList = new ArrayList<>();
            while (true) {
                try {
                    if (!rs.next())
                        break;
                } catch (SQLException throwables) {
                    logger.error("Error while get data from srcDs", throwables);
                    break;
                }
                Dml dml = getDMLByRs(columnsMap, rs);
                List<RowChange> rowChanges = syncService.getRowChanges(dml, config);
                if (CollectionUtils.isEmpty(rowChanges)) {
                    return null;
                }
                Future<WriterResult> future = writer.addRowChangeWithFuture(rowChanges);
                if (future != null) {
                    futureList.add(future);
                }
            }
            writer.flush();
            for (Future<WriterResult> future : futureList) {
                try {
                    WriterResult result = future.get();
                    if (result != null && result.isAllSucceed()) {
                        impCount.incrementAndGet();
                        idx++;
                    } else if (result != null && !result.isAllSucceed()) {
                        List<WriterResult.RowChangeStatus> totalFailedRows = result.getFailedRows();
                        List<String> msgs = totalFailedRows.stream().map(e -> TablestoreAdapter.buildErrorMsgForFailedRowChange(e)).collect(Collectors.toList());
                        logger.error("Failed rows when ETL:" + org.springframework.util.StringUtils.collectionToDelimitedString(msgs, ",", "[", "]"));
                    }
                } catch (InterruptedException e) {
                    logger.info("InterruptedException", e);
                    errMsg.add(e.getMessage());
                    Thread.currentThread().interrupt();
                } catch (ExecutionException e) {
                    errMsg.add(e.getMessage());
                    throw new RuntimeException(e);
                }
            }
            return idx;
        });
        return true;
    } catch (Exception e) {
        logger.error(e.getMessage(), e);
        return false;
    }
}