Examples with Upsert org.apache.kudu.client.Upsert used on opensource projects

Example 1 with Upsert

use of org.apache.kudu.client.Upsert in project canal by alibaba.

the class KuduTemplate method upsert.

/**
 * 更新/插入字段
 *
 * @param tableName
 * @param dataList
 * @throws KuduException
 */
public void upsert(String tableName, List<Map<String, Object>> dataList) throws KuduException {
    this.checkClient();
    KuduTable kuduTable = kuduClient.openTable(tableName);
    KuduSession session = kuduClient.newSession();
    try {
        session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
        session.setMutationBufferSpace(OPERATION_BATCH);
        // 获取元数据结构
        Map<String, Type> metaMap = new HashMap<>();
        Schema schema = kuduTable.getSchema();
        for (ColumnSchema columnSchema : schema.getColumns()) {
            String colName = columnSchema.getName().toLowerCase();
            Type type = columnSchema.getType();
            metaMap.put(colName, type);
        }
        int uncommit = 0;
        for (Map<String, Object> data : dataList) {
            Upsert upsert = kuduTable.newUpsert();
            PartialRow row = upsert.getRow();
            for (Map.Entry<String, Object> entry : data.entrySet()) {
                String name = entry.getKey().toLowerCase();
                Type type = metaMap.get(name);
                Object value = entry.getValue();
                // 填充行数据
                fillRow(row, name, value, type);
            }
            session.apply(upsert);
            // 对于手工提交, 需要buffer在未满的时候flush,这里采用了buffer一半时即提交
            uncommit = uncommit + 1;
            if (uncommit > OPERATION_BATCH / 3 * 2) {
                List<OperationResponse> update_option = session.flush();
                if (update_option.size() > 0) {
                    OperationResponse response = update_option.get(0);
                    if (response.hasRowError()) {
                        logger.error("update row fail table name is :{} ", tableName);
                        logger.error("update list is :{}", response.getRowError().getMessage());
                    }
                }
                uncommit = 0;
            }
        }
        List<OperationResponse> update_option = session.flush();
        if (update_option.size() > 0) {
            OperationResponse response = update_option.get(0);
            if (response.hasRowError()) {
                logger.error("update row fail table name is :{} ", tableName);
                logger.error("update list is :{}", response.getRowError().getMessage());
            }
        }
    } catch (KuduException e) {
        logger.error("error message is :{}", dataList.toString());
        throw e;
    } finally {
        if (!session.isClosed()) {
            session.close();
        }
    }
}

Example 2 with Upsert

use of org.apache.kudu.client.Upsert in project gora by apache.

the class KuduStore method put.

@Override
public void put(K key, T obj) throws GoraException {
    try {
        if (obj.isDirty()) {
            Column pkc = kuduMapping.getPrimaryKey().get(0);
            Upsert upsert = table.newUpsert();
            PartialRow row = upsert.getRow();
            KuduClientUtils.addObjectRow(row, pkc, key);
            Schema schemaObj = obj.getSchema();
            List<Schema.Field> fields = schemaObj.getFields();
            for (Schema.Field field : fields) {
                Column mappedColumn = kuduMapping.getFields().get(field.name());
                if (mappedColumn != null) {
                    Object fieldValue = obj.get(field.pos());
                    if (fieldValue != null) {
                        Schema fieldSchema = field.schema();
                        Object serializedObj = serializeFieldValue(fieldSchema, fieldValue);
                        KuduClientUtils.addObjectRow(row, mappedColumn, serializedObj);
                    } else {
                        row.setNull(mappedColumn.getName());
                    }
                } else {
                    throw new GoraException("Unmapped field : " + field.name());
                }
            }
            session.apply(upsert);
        } else {
            LOG.info("Ignored putting object {} in the store as it is neither " + "new, neither dirty.", new Object[] { obj });
        }
    } catch (Exception e) {
        throw new GoraException(e);
    }
}

Example 3 with Upsert

use of org.apache.kudu.client.Upsert in project apex-malhar by apache.

the class KuduInputOperatorCommons method addTestDataRows.

public void addTestDataRows(int numRowsInEachPartition) throws Exception {
    int intRowKeyStepsize = Integer.MAX_VALUE / SPLIT_COUNT_FOR_INT_ROW_KEY;
    int splitBoundaryForIntRowKey = intRowKeyStepsize;
    int[] inputrowkeyPartitionEntries = new int[SPLIT_COUNT_FOR_INT_ROW_KEY + 1];
    // setting the int keys that will fall in the range of all partitions
    for (int i = 0; i < SPLIT_COUNT_FOR_INT_ROW_KEY; i++) {
        // 3 to fall into the partition next to boundary
        inputrowkeyPartitionEntries[i] = splitBoundaryForIntRowKey + 3;
        splitBoundaryForIntRowKey += intRowKeyStepsize;
    }
    inputrowkeyPartitionEntries[SPLIT_COUNT_FOR_INT_ROW_KEY] = splitBoundaryForIntRowKey + 3;
    AbstractKuduPartitionScanner<UnitTestTablePojo, InputOperatorControlTuple> scannerForAddingRows = unitTestStepwiseScanInputOperator.getScanner();
    ApexKuduConnection aCurrentConnection = scannerForAddingRows.getConnectionPoolForThreads().get(0);
    KuduSession aSessionForInserts = aCurrentConnection.getKuduClient().newSession();
    KuduTable currentTable = aCurrentConnection.getKuduTable();
    // constant to allow for data landing on first partition for unit tests
    long seedValueForTimestampRowKey = 0L;
    for (int i = 0; i <= SPLIT_COUNT_FOR_INT_ROW_KEY; i++) {
        // range key iterator
        int intRowKeyBaseValue = inputrowkeyPartitionEntries[i] + i;
        for (int k = 0; k < 2; k++) {
            // hash key iterator . The table defines two hash partitions
            // to avoid spilling to another tablet
            long timestampRowKeyValue = seedValueForTimestampRowKey + k;
            // to avoid spilling to another tablet randomly
            String stringRowKeyValue = "" + timestampRowKeyValue + k;
            for (int y = 0; y < numRowsInEachPartition; y++) {
                Upsert aNewRow = currentTable.newUpsert();
                PartialRow rowValue = aNewRow.getRow();
                // Start assigning row keys below the current split boundary.
                rowValue.addInt("introwkey", intRowKeyBaseValue - y - 1);
                rowValue.addString("stringrowkey", stringRowKeyValue);
                rowValue.addLong("timestamprowkey", timestampRowKeyValue);
                rowValue.addLong("longdata", (seedValueForTimestampRowKey + y));
                rowValue.addString("stringdata", ("" + seedValueForTimestampRowKey + y));
                OperationResponse response = aSessionForInserts.apply(aNewRow);
            }
        }
    }
    List<OperationResponse> insertResponse = aSessionForInserts.flush();
    aSessionForInserts.close();
    // Sleep to allow for scans to complete
    Thread.sleep(2000);
}

Example 4 with Upsert

use of org.apache.kudu.client.Upsert in project nifi by apache.

the class PutKudu method upsertRecordToKudu.

@Override
protected Upsert upsertRecordToKudu(KuduTable kuduTable, Record record, List<String> fieldNames) throws IllegalStateException, Exception {
    Upsert upsert = kuduTable.newUpsert();
    this.insert(kuduTable, upsert, record, fieldNames);
    return upsert;
}

Example 5 with Upsert

use of org.apache.kudu.client.Upsert in project presto by prestodb.

the class SchemaEmulationByTableNameConvention method createSchema.

@Override
public void createSchema(KuduClient client, String schemaName) {
    if (DEFAULT_SCHEMA.equals(schemaName)) {
        throw new SchemaAlreadyExistsException(schemaName);
    } else {
        try {
            KuduTable schemasTable = getSchemasTable(client);
            KuduSession session = client.newSession();
            try {
                Upsert upsert = schemasTable.newUpsert();
                upsert.getRow().addString(0, schemaName);
                session.apply(upsert);
            } finally {
                session.close();
            }
        } catch (KuduException e) {
            throw new PrestoException(GENERIC_INTERNAL_ERROR, e);
        }
    }
}