Examples with OperationResponse org.apache.kudu.client.OperationResponse used on opensource projects

Example 1 with OperationResponse

use of org.apache.kudu.client.OperationResponse in project canal by alibaba.

the class KuduTemplate method upsert.

/**
 * 更新/插入字段
 *
 * @param tableName
 * @param dataList
 * @throws KuduException
 */
public void upsert(String tableName, List<Map<String, Object>> dataList) throws KuduException {
    this.checkClient();
    KuduTable kuduTable = kuduClient.openTable(tableName);
    KuduSession session = kuduClient.newSession();
    try {
        session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
        session.setMutationBufferSpace(OPERATION_BATCH);
        // 获取元数据结构
        Map<String, Type> metaMap = new HashMap<>();
        Schema schema = kuduTable.getSchema();
        for (ColumnSchema columnSchema : schema.getColumns()) {
            String colName = columnSchema.getName().toLowerCase();
            Type type = columnSchema.getType();
            metaMap.put(colName, type);
        }
        int uncommit = 0;
        for (Map<String, Object> data : dataList) {
            Upsert upsert = kuduTable.newUpsert();
            PartialRow row = upsert.getRow();
            for (Map.Entry<String, Object> entry : data.entrySet()) {
                String name = entry.getKey().toLowerCase();
                Type type = metaMap.get(name);
                Object value = entry.getValue();
                // 填充行数据
                fillRow(row, name, value, type);
            }
            session.apply(upsert);
            // 对于手工提交, 需要buffer在未满的时候flush,这里采用了buffer一半时即提交
            uncommit = uncommit + 1;
            if (uncommit > OPERATION_BATCH / 3 * 2) {
                List<OperationResponse> update_option = session.flush();
                if (update_option.size() > 0) {
                    OperationResponse response = update_option.get(0);
                    if (response.hasRowError()) {
                        logger.error("update row fail table name is :{} ", tableName);
                        logger.error("update list is :{}", response.getRowError().getMessage());
                    }
                }
                uncommit = 0;
            }
        }
        List<OperationResponse> update_option = session.flush();
        if (update_option.size() > 0) {
            OperationResponse response = update_option.get(0);
            if (response.hasRowError()) {
                logger.error("update row fail table name is :{} ", tableName);
                logger.error("update list is :{}", response.getRowError().getMessage());
            }
        }
    } catch (KuduException e) {
        logger.error("error message is :{}", dataList.toString());
        throw e;
    } finally {
        if (!session.isClosed()) {
            session.close();
        }
    }
}

Example 2 with OperationResponse

use of org.apache.kudu.client.OperationResponse in project canal by alibaba.

the class KuduTemplate method insert.

/**
 * 插入数据
 *
 * @param tableName
 * @param dataList
 * @throws KuduException
 */
public void insert(String tableName, List<Map<String, Object>> dataList) throws KuduException {
    this.checkClient();
    // 打开表
    KuduTable kuduTable = kuduClient.openTable(tableName);
    // 创建写session,kudu必须通过session写入
    KuduSession session = kuduClient.newSession();
    try {
        // 采取Flush方式
        session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
        // 手动刷新
        session.setMutationBufferSpace(OPERATION_BATCH);
        // 获取元数据结构
        Map<String, Type> metaMap = new HashMap<>();
        Schema schema = kuduTable.getSchema();
        for (ColumnSchema columnSchema : schema.getColumns()) {
            String colName = columnSchema.getName().toLowerCase();
            Type type = columnSchema.getType();
            metaMap.put(colName, type);
        }
        int uncommit = 0;
        for (Map<String, Object> data : dataList) {
            Insert insert = kuduTable.newInsert();
            PartialRow row = insert.getRow();
            for (Map.Entry<String, Object> entry : data.entrySet()) {
                String name = entry.getKey().toLowerCase();
                Type type = metaMap.get(name);
                Object value = entry.getValue();
                // 填充行数据
                fillRow(row, name, value, type);
            }
            session.apply(insert);
            // 对于手工提交, 需要buffer在未满的时候flush,这里采用了buffer一半时即提交
            uncommit = uncommit + 1;
            if (uncommit > OPERATION_BATCH / 3 * 2) {
                List<OperationResponse> insert_option = session.flush();
                if (insert_option.size() > 0) {
                    OperationResponse response = insert_option.get(0);
                    if (response.hasRowError()) {
                        logger.error("insert row fail table name is :{} ", tableName);
                        logger.error("insert list is :{}", response.getRowError().getMessage());
                    }
                }
                uncommit = 0;
            }
        }
        List<OperationResponse> insert_option = session.flush();
        if (insert_option.size() > 0) {
            OperationResponse response = insert_option.get(0);
            if (response.hasRowError()) {
                logger.error("insert row fail table name is :{} ", tableName);
                logger.error("insert list is :{}", response.getRowError().getMessage());
            }
        }
    } catch (KuduException e) {
        logger.error("error message is :{}", dataList.toString());
        throw e;
    } finally {
        if (!session.isClosed()) {
            session.close();
        }
    }
}

Example 3 with OperationResponse

use of org.apache.kudu.client.OperationResponse in project gora by apache.

the class KuduStore method delete.

@Override
public boolean delete(K key) throws GoraException {
    try {
        Column pkc = kuduMapping.getPrimaryKey().get(0);
        Delete delete = table.newDelete();
        PartialRow row = delete.getRow();
        KuduClientUtils.addObjectRow(row, pkc, key);
        OperationResponse apply = session.apply(delete);
        return !apply.hasRowError();
    } catch (KuduException ex) {
        throw new GoraException(ex);
    }
}

Example 4 with OperationResponse

use of org.apache.kudu.client.OperationResponse in project apex-malhar by apache.

the class KuduInputOperatorCommons method addTestDataRows.

public void addTestDataRows(int numRowsInEachPartition) throws Exception {
    int intRowKeyStepsize = Integer.MAX_VALUE / SPLIT_COUNT_FOR_INT_ROW_KEY;
    int splitBoundaryForIntRowKey = intRowKeyStepsize;
    int[] inputrowkeyPartitionEntries = new int[SPLIT_COUNT_FOR_INT_ROW_KEY + 1];
    // setting the int keys that will fall in the range of all partitions
    for (int i = 0; i < SPLIT_COUNT_FOR_INT_ROW_KEY; i++) {
        // 3 to fall into the partition next to boundary
        inputrowkeyPartitionEntries[i] = splitBoundaryForIntRowKey + 3;
        splitBoundaryForIntRowKey += intRowKeyStepsize;
    }
    inputrowkeyPartitionEntries[SPLIT_COUNT_FOR_INT_ROW_KEY] = splitBoundaryForIntRowKey + 3;
    AbstractKuduPartitionScanner<UnitTestTablePojo, InputOperatorControlTuple> scannerForAddingRows = unitTestStepwiseScanInputOperator.getScanner();
    ApexKuduConnection aCurrentConnection = scannerForAddingRows.getConnectionPoolForThreads().get(0);
    KuduSession aSessionForInserts = aCurrentConnection.getKuduClient().newSession();
    KuduTable currentTable = aCurrentConnection.getKuduTable();
    // constant to allow for data landing on first partition for unit tests
    long seedValueForTimestampRowKey = 0L;
    for (int i = 0; i <= SPLIT_COUNT_FOR_INT_ROW_KEY; i++) {
        // range key iterator
        int intRowKeyBaseValue = inputrowkeyPartitionEntries[i] + i;
        for (int k = 0; k < 2; k++) {
            // hash key iterator . The table defines two hash partitions
            // to avoid spilling to another tablet
            long timestampRowKeyValue = seedValueForTimestampRowKey + k;
            // to avoid spilling to another tablet randomly
            String stringRowKeyValue = "" + timestampRowKeyValue + k;
            for (int y = 0; y < numRowsInEachPartition; y++) {
                Upsert aNewRow = currentTable.newUpsert();
                PartialRow rowValue = aNewRow.getRow();
                // Start assigning row keys below the current split boundary.
                rowValue.addInt("introwkey", intRowKeyBaseValue - y - 1);
                rowValue.addString("stringrowkey", stringRowKeyValue);
                rowValue.addLong("timestamprowkey", timestampRowKeyValue);
                rowValue.addLong("longdata", (seedValueForTimestampRowKey + y));
                rowValue.addString("stringdata", ("" + seedValueForTimestampRowKey + y));
                OperationResponse response = aSessionForInserts.apply(aNewRow);
            }
        }
    }
    List<OperationResponse> insertResponse = aSessionForInserts.flush();
    aSessionForInserts.close();
    // Sleep to allow for scans to complete
    Thread.sleep(2000);
}

Example 5 with OperationResponse

use of org.apache.kudu.client.OperationResponse in project canal by alibaba.

the class KuduTemplate method delete.

/**
 * 删除行
 *
 * @param tableName
 * @param dataList
 * @throws KuduException
 */
public void delete(String tableName, List<Map<String, Object>> dataList) throws KuduException {
    this.checkClient();
    KuduTable kuduTable = kuduClient.openTable(tableName);
    KuduSession session = kuduClient.newSession();
    try {
        session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
        session.setMutationBufferSpace(OPERATION_BATCH);
        // 获取元数据结构
        Map<String, Type> metaMap = new HashMap<>();
        Schema schema = kuduTable.getSchema();
        for (ColumnSchema columnSchema : schema.getColumns()) {
            String colName = columnSchema.getName().toLowerCase();
            Type type = columnSchema.getType();
            metaMap.put(colName, type);
        }
        int uncommit = 0;
        for (Map<String, Object> data : dataList) {
            Delete delete = kuduTable.newDelete();
            PartialRow row = delete.getRow();
            for (Map.Entry<String, Object> entry : data.entrySet()) {
                String name = entry.getKey().toLowerCase();
                Type type = metaMap.get(name);
                Object value = entry.getValue();
                // 填充行数据
                fillRow(row, name, value, type);
            }
            session.apply(delete);
            // 对于手工提交, 需要buffer在未满的时候flush,这里采用了buffer一半时即提交
            uncommit = uncommit + 1;
            if (uncommit > OPERATION_BATCH / 3 * 2) {
                List<OperationResponse> delete_option = session.flush();
                if (delete_option.size() > 0) {
                    OperationResponse response = delete_option.get(0);
                    if (response.hasRowError()) {
                        logger.error("delete row fail table name is :{} ", tableName);
                        logger.error("error list is :{}", response.getRowError().getMessage());
                    }
                }
                uncommit = 0;
            }
        }
        List<OperationResponse> delete_option = session.flush();
        if (delete_option.size() > 0) {
            OperationResponse response = delete_option.get(0);
            if (response.hasRowError()) {
                logger.error("delete row fail table name is :{} ", tableName);
                logger.error("error list is :{}", response.getRowError().getMessage());
            }
        }
    } catch (KuduException e) {
        logger.error("error message is :{}", dataList.toString());
        throw e;
    } finally {
        if (!session.isClosed()) {
            session.close();
        }
    }
}