Examples with BatchExecutor com.alibaba.otter.canal.client.adapter.phoenix.support.BatchExecutor used on opensource projects

Example 1 with BatchExecutor

use of com.alibaba.otter.canal.client.adapter.phoenix.support.BatchExecutor in project canal by alibaba.

the class PhoenixSyncService method sync.

/**
 * 批量同步回调
 *
 * @param dmls     批量 DML
 * @param function 回调方法
 */
private void sync(List<Dml> dmls, Function<Dml, Boolean> function) {
    try {
        boolean toExecute = false;
        for (Dml dml : dmls) {
            if (!toExecute) {
                toExecute = function.apply(dml);
            } else {
                function.apply(dml);
            }
        }
        if (toExecute) {
            List<Future<Boolean>> futures = new ArrayList<>();
            for (int i = 0; i < threads; i++) {
                int j = i;
                if (dmlsPartition[j].isEmpty()) {
                    // bypass
                    continue;
                }
                futures.add(executorThreads[i].submit(() -> {
                    try {
                        dmlsPartition[j].forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml));
                        // 相对于RDB同步 少了  dmlsPartition[j].clear();
                        // 在 try catch中获取异常后再次执行一次batchExecutors[j].commit()
                        batchExecutors[j].commit();
                        return true;
                    } catch (Throwable e) {
                        batchExecutors[j].rollback();
                        if (!e.getClass().getName().endsWith("ColumnNotFoundException") && !e.getClass().getName().endsWith("TableNotFoundException")) {
                            throw new RuntimeException(e);
                        }
                        logger.info("table or column not found: " + e.getMessage());
                        boolean synced = false;
                        for (SyncItem syncItem : dmlsPartition[j]) {
                            if (PhoenixEtlService.syncSchema(batchExecutors[j].getConn(), syncItem.config)) {
                                synced = true;
                            }
                        }
                        if (!synced) {
                            throw new RuntimeException(e);
                        }
                        dmlsPartition[j].forEach(syncItem -> sync(batchExecutors[j], syncItem.config, syncItem.singleDml));
                        try {
                            batchExecutors[j].commit();
                            return true;
                        } catch (Throwable e1) {
                            batchExecutors[j].rollback();
                            throw new RuntimeException(e1);
                        }
                    } finally {
                        dmlsPartition[j].clear();
                    }
                }));
            }
            futures.forEach(future -> {
                try {
                    future.get();
                } catch (ExecutionException | InterruptedException e) {
                    throw new RuntimeException(e);
                }
            });
        }
    } finally {
        for (BatchExecutor batchExecutor : batchExecutors) {
            if (batchExecutor != null) {
                batchExecutor.close();
            }
        }
    }
}