Examples with Insert org.apache.kudu.client.Insert used on opensource projects

Example 1 with Insert

use of org.apache.kudu.client.Insert in project nifi by apache.

the class PutKudu method insertRecordToKudu.

@Override
protected Insert insertRecordToKudu(KuduTable kuduTable, Record record, List<String> fieldNames) throws IllegalStateException, Exception {
    Insert insert = kuduTable.newInsert();
    this.insert(kuduTable, insert, record, fieldNames);
    return insert;
}

Example 2 with Insert

use of org.apache.kudu.client.Insert in project drill by axbaretto.

the class TestKuduConnect method createKuduTable.

public static void createKuduTable(String tableName, int tablets, int replicas, int rows) throws Exception {
    try (KuduClient client = new KuduClient.KuduClientBuilder(KUDU_MASTER).build()) {
        ListTablesResponse tables = client.getTablesList(tableName);
        if (!tables.getTablesList().isEmpty()) {
            client.deleteTable(tableName);
        }
        List<ColumnSchema> columns = new ArrayList<>(5);
        columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32).key(true).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("binary", Type.BINARY).nullable(false).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("boolean", Type.BOOL).nullable(true).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("float", Type.FLOAT).nullable(false).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("string", Type.STRING).nullable(true).build());
        Schema schema = new Schema(columns);
        CreateTableOptions builder = new CreateTableOptions();
        builder.setNumReplicas(replicas);
        builder.setRangePartitionColumns(Arrays.asList("key"));
        for (int i = 1; i < tablets; i++) {
            PartialRow splitRow = schema.newPartialRow();
            splitRow.addInt("key", i * 1000);
            builder.addSplitRow(splitRow);
        }
        client.createTable(tableName, schema, builder);
        KuduTable table = client.openTable(tableName);
        KuduSession session = client.newSession();
        session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC);
        for (int i = 0; i < rows; i++) {
            Insert insert = table.newInsert();
            PartialRow row = insert.getRow();
            row.addInt(0, i);
            row.addBinary(1, ("Row " + i).getBytes());
            row.addBoolean(2, i % 2 == 0);
            row.addFloat(3, i + 0.01f);
            row.addString(4, ("Row " + i));
            session.apply(insert);
        }
        List<String> projectColumns = new ArrayList<>(1);
        projectColumns.add("float");
        KuduScanner scanner = client.newScannerBuilder(table).setProjectedColumnNames(projectColumns).build();
        while (scanner.hasMoreRows()) {
            RowResultIterator results = scanner.nextRows();
            while (results.hasNext()) {
                RowResult result = results.next();
                System.out.println(result.toStringLongFormat());
            }
        }
    }
}

Example 3 with Insert

use of org.apache.kudu.client.Insert in project drill by apache.

the class TestKuduConnect method createKuduTable.

public static void createKuduTable(String tableName, int tablets, int replicas, int rows) throws Exception {
    try (KuduClient client = new KuduClient.KuduClientBuilder(KUDU_MASTER).build()) {
        ListTablesResponse tables = client.getTablesList(tableName);
        if (!tables.getTablesList().isEmpty()) {
            client.deleteTable(tableName);
        }
        List<ColumnSchema> columns = new ArrayList<>(5);
        columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32).key(true).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("binary", Type.BINARY).nullable(false).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("boolean", Type.BOOL).nullable(true).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("float", Type.FLOAT).nullable(false).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("string", Type.STRING).nullable(true).build());
        Schema schema = new Schema(columns);
        CreateTableOptions builder = new CreateTableOptions();
        builder.setNumReplicas(replicas);
        builder.setRangePartitionColumns(Arrays.asList("key"));
        for (int i = 1; i < tablets; i++) {
            PartialRow splitRow = schema.newPartialRow();
            splitRow.addInt("key", i * 1000);
            builder.addSplitRow(splitRow);
        }
        client.createTable(tableName, schema, builder);
        KuduTable table = client.openTable(tableName);
        KuduSession session = client.newSession();
        session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC);
        for (int i = 0; i < rows; i++) {
            Insert insert = table.newInsert();
            PartialRow row = insert.getRow();
            row.addInt(0, i);
            row.addBinary(1, ("Row " + i).getBytes());
            row.addBoolean(2, i % 2 == 0);
            row.addFloat(3, i + 0.01f);
            row.addString(4, ("Row " + i));
            session.apply(insert);
        }
        List<String> projectColumns = new ArrayList<>(1);
        projectColumns.add("float");
        KuduScanner scanner = client.newScannerBuilder(table).setProjectedColumnNames(projectColumns).build();
        while (scanner.hasMoreRows()) {
            RowResultIterator results = scanner.nextRows();
            while (results.hasNext()) {
                logger.debug(results.next().toString());
            }
        }
    }
}

Example 4 with Insert

use of org.apache.kudu.client.Insert in project hive by apache.

the class TestKuduInputFormat method testMultipleSplits.

@Test
public void testMultipleSplits() throws Exception {
    String tableName = "default.twoPartitionTable";
    Schema schema = new Schema(Arrays.asList(new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32).key(true).build(), new ColumnSchema.ColumnSchemaBuilder("string", Type.STRING).build()));
    CreateTableOptions options = new CreateTableOptions().addHashPartitions(Collections.singletonList("key"), 2);
    harness.getClient().createTable(tableName, schema, options);
    // Insert multiple test rows.
    KuduTable table = harness.getClient().openTable(tableName);
    KuduSession session = harness.getClient().newSession();
    Insert insert1 = table.newInsert();
    PartialRow row1 = insert1.getRow();
    row1.addInt("key", 1);
    row1.addString("string", "one");
    session.apply(insert1);
    Insert insert2 = table.newInsert();
    PartialRow row2 = insert2.getRow();
    row2.addInt("key", 2);
    row2.addString("string", "two");
    session.apply(insert2);
    session.close();
    KuduInputFormat input = new KuduInputFormat();
    JobConf jobConf = new JobConf(BASE_CONF);
    jobConf.set(KUDU_TABLE_NAME_KEY, tableName);
    jobConf.set(serdeConstants.LIST_COLUMNS, "key");
    InputSplit[] splits = input.getSplits(jobConf, 1);
    assertEquals(2, splits.length);
}

Example 5 with Insert

use of org.apache.kudu.client.Insert in project canal by alibaba.

the class KuduTemplate method insert.

/**
 * 插入数据
 *
 * @param tableName
 * @param dataList
 * @throws KuduException
 */
public void insert(String tableName, List<Map<String, Object>> dataList) throws KuduException {
    this.checkClient();
    // 打开表
    KuduTable kuduTable = kuduClient.openTable(tableName);
    // 创建写session,kudu必须通过session写入
    KuduSession session = kuduClient.newSession();
    try {
        // 采取Flush方式
        session.setFlushMode(SessionConfiguration.FlushMode.MANUAL_FLUSH);
        // 手动刷新
        session.setMutationBufferSpace(OPERATION_BATCH);
        // 获取元数据结构
        Map<String, Type> metaMap = new HashMap<>();
        Schema schema = kuduTable.getSchema();
        for (ColumnSchema columnSchema : schema.getColumns()) {
            String colName = columnSchema.getName().toLowerCase();
            Type type = columnSchema.getType();
            metaMap.put(colName, type);
        }
        int uncommit = 0;
        for (Map<String, Object> data : dataList) {
            Insert insert = kuduTable.newInsert();
            PartialRow row = insert.getRow();
            for (Map.Entry<String, Object> entry : data.entrySet()) {
                String name = entry.getKey().toLowerCase();
                Type type = metaMap.get(name);
                Object value = entry.getValue();
                // 填充行数据
                fillRow(row, name, value, type);
            }
            session.apply(insert);
            // 对于手工提交, 需要buffer在未满的时候flush,这里采用了buffer一半时即提交
            uncommit = uncommit + 1;
            if (uncommit > OPERATION_BATCH / 3 * 2) {
                List<OperationResponse> insert_option = session.flush();
                if (insert_option.size() > 0) {
                    OperationResponse response = insert_option.get(0);
                    if (response.hasRowError()) {
                        logger.error("insert row fail table name is :{} ", tableName);
                        logger.error("insert list is :{}", response.getRowError().getMessage());
                    }
                }
                uncommit = 0;
            }
        }
        List<OperationResponse> insert_option = session.flush();
        if (insert_option.size() > 0) {
            OperationResponse response = insert_option.get(0);
            if (response.hasRowError()) {
                logger.error("insert row fail table name is :{} ", tableName);
                logger.error("insert list is :{}", response.getRowError().getMessage());
            }
        }
    } catch (KuduException e) {
        logger.error("error message is :{}", dataList.toString());
        throw e;
    } finally {
        if (!session.isClosed()) {
            session.close();
        }
    }
}