Examples with KuduScanner org.apache.kudu.client.KuduScanner used on opensource projects

Example 1 with KuduScanner

use of org.apache.kudu.client.KuduScanner in project drill by axbaretto.

the class TestKuduConnect method createKuduTable.

public static void createKuduTable(String tableName, int tablets, int replicas, int rows) throws Exception {
    try (KuduClient client = new KuduClient.KuduClientBuilder(KUDU_MASTER).build()) {
        ListTablesResponse tables = client.getTablesList(tableName);
        if (!tables.getTablesList().isEmpty()) {
            client.deleteTable(tableName);
        }
        List<ColumnSchema> columns = new ArrayList<>(5);
        columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32).key(true).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("binary", Type.BINARY).nullable(false).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("boolean", Type.BOOL).nullable(true).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("float", Type.FLOAT).nullable(false).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("string", Type.STRING).nullable(true).build());
        Schema schema = new Schema(columns);
        CreateTableOptions builder = new CreateTableOptions();
        builder.setNumReplicas(replicas);
        builder.setRangePartitionColumns(Arrays.asList("key"));
        for (int i = 1; i < tablets; i++) {
            PartialRow splitRow = schema.newPartialRow();
            splitRow.addInt("key", i * 1000);
            builder.addSplitRow(splitRow);
        }
        client.createTable(tableName, schema, builder);
        KuduTable table = client.openTable(tableName);
        KuduSession session = client.newSession();
        session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC);
        for (int i = 0; i < rows; i++) {
            Insert insert = table.newInsert();
            PartialRow row = insert.getRow();
            row.addInt(0, i);
            row.addBinary(1, ("Row " + i).getBytes());
            row.addBoolean(2, i % 2 == 0);
            row.addFloat(3, i + 0.01f);
            row.addString(4, ("Row " + i));
            session.apply(insert);
        }
        List<String> projectColumns = new ArrayList<>(1);
        projectColumns.add("float");
        KuduScanner scanner = client.newScannerBuilder(table).setProjectedColumnNames(projectColumns).build();
        while (scanner.hasMoreRows()) {
            RowResultIterator results = scanner.nextRows();
            while (results.hasNext()) {
                RowResult result = results.next();
                System.out.println(result.toStringLongFormat());
            }
        }
    }
}

Example 2 with KuduScanner

use of org.apache.kudu.client.KuduScanner in project drill by apache.

the class TestKuduConnect method createKuduTable.

public static void createKuduTable(String tableName, int tablets, int replicas, int rows) throws Exception {
    try (KuduClient client = new KuduClient.KuduClientBuilder(KUDU_MASTER).build()) {
        ListTablesResponse tables = client.getTablesList(tableName);
        if (!tables.getTablesList().isEmpty()) {
            client.deleteTable(tableName);
        }
        List<ColumnSchema> columns = new ArrayList<>(5);
        columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32).key(true).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("binary", Type.BINARY).nullable(false).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("boolean", Type.BOOL).nullable(true).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("float", Type.FLOAT).nullable(false).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("string", Type.STRING).nullable(true).build());
        Schema schema = new Schema(columns);
        CreateTableOptions builder = new CreateTableOptions();
        builder.setNumReplicas(replicas);
        builder.setRangePartitionColumns(Arrays.asList("key"));
        for (int i = 1; i < tablets; i++) {
            PartialRow splitRow = schema.newPartialRow();
            splitRow.addInt("key", i * 1000);
            builder.addSplitRow(splitRow);
        }
        client.createTable(tableName, schema, builder);
        KuduTable table = client.openTable(tableName);
        KuduSession session = client.newSession();
        session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC);
        for (int i = 0; i < rows; i++) {
            Insert insert = table.newInsert();
            PartialRow row = insert.getRow();
            row.addInt(0, i);
            row.addBinary(1, ("Row " + i).getBytes());
            row.addBoolean(2, i % 2 == 0);
            row.addFloat(3, i + 0.01f);
            row.addString(4, ("Row " + i));
            session.apply(insert);
        }
        List<String> projectColumns = new ArrayList<>(1);
        projectColumns.add("float");
        KuduScanner scanner = client.newScannerBuilder(table).setProjectedColumnNames(projectColumns).build();
        while (scanner.hasMoreRows()) {
            RowResultIterator results = scanner.nextRows();
            while (results.hasNext()) {
                logger.debug(results.next().toString());
            }
        }
    }
}

Example 3 with KuduScanner

use of org.apache.kudu.client.KuduScanner in project presto by prestodb.

the class KuduRecordSet method cursor.

@Override
public RecordCursor cursor() {
    KuduScanner scanner = clientSession.createScanner(kuduSplit);
    if (!containsVirtualRowId) {
        return new KuduRecordCursor(scanner, getColumnTypes());
    } else {
        final int primaryKeyColumnCount = kuduSplit.getPrimaryKeyColumnCount();
        Map<Integer, Integer> fieldMapping = new HashMap<>();
        int index = primaryKeyColumnCount;
        for (int i = 0; i < columns.size(); i++) {
            KuduColumnHandle handle = (KuduColumnHandle) columns.get(i);
            if (!handle.isVirtualRowId()) {
                if (handle.getOrdinalPosition() < primaryKeyColumnCount) {
                    fieldMapping.put(i, handle.getOrdinalPosition());
                } else {
                    fieldMapping.put(i, index);
                    index++;
                }
            } else {
                fieldMapping.put(i, -1);
            }
        }
        KuduTable table = getTable();
        return new KuduRecordCursorWithVirtualRowId(scanner, table, getColumnTypes(), fieldMapping);
    }
}

Example 4 with KuduScanner

use of org.apache.kudu.client.KuduScanner in project hive by apache.

the class TestKuduOutputFormat method testGoodRow.

@Test
public void testGoodRow() throws Exception {
    KuduOutputFormat outputFormat = new KuduOutputFormat();
    KuduRecordWriter writer = (KuduRecordWriter) outputFormat.getHiveRecordWriter(new JobConf(BASE_CONF), null, null, false, TBL_PROPS, null);
    // Write a good row.
    try {
        PartialRow row = SCHEMA.newPartialRow();
        row.addByte("key", (byte) 1);
        row.addShort("int16", (short) 1);
        row.addInt("int32", 1);
        row.addLong("int64", 1L);
        row.addBoolean("bool", true);
        row.addFloat("float", 1.1f);
        row.addDouble("double", 1.1d);
        row.addString("string", "one");
        row.addBinary("binary", "one".getBytes(UTF_8));
        row.addTimestamp("timestamp", new Timestamp(NOW_MS));
        row.addDecimal("decimal", new BigDecimal("1.111"));
        row.setNull("null");
        // Not setting the "default" column.
        KuduWritable writable = new KuduWritable(row);
        writer.write(writable);
    } finally {
        writer.close(false);
    }
    // Verify the written row.
    KuduClient client = harness.getClient();
    KuduTable table = client.openTable(TABLE_NAME);
    KuduScanner scanner = client.newScannerBuilder(table).build();
    List<RowResult> results = new ArrayList<>();
    for (RowResult result : scanner) {
        results.add(result);
    }
    assertEquals(1, results.size());
    RowResult result = results.get(0);
    assertEquals((byte) 1, result.getByte(0));
    assertEquals((short) 1, result.getShort(1));
    assertEquals(1, result.getInt(2));
    assertEquals(1L, result.getLong(3));
    assertTrue(result.getBoolean(4));
    assertEquals(1.1f, result.getFloat(5), 0);
    assertEquals(1.1d, result.getDouble(6), 0);
    assertEquals("one", result.getString(7));
    assertEquals("one", new String(result.getBinaryCopy(8), UTF_8));
    assertEquals(NOW_MS, result.getTimestamp(9).getTime());
    assertEquals(new BigDecimal("1.111"), result.getDecimal(10));
    assertTrue(result.isNull(11));
    // default.
    assertEquals(1, result.getInt(12));
}

Example 5 with KuduScanner

use of org.apache.kudu.client.KuduScanner in project gora by apache.

the class KuduStore method deleteByQuery.

@Override
public long deleteByQuery(Query<K, T> query) throws GoraException {
    try {
        long count = 0;
        Column pkc = kuduMapping.getPrimaryKey().get(0);
        ColumnSchema column = table.getSchema().getColumn(pkc.getName());
        List<String> dbFields = new ArrayList<>();
        dbFields.add(pkc.getName());
        List<KuduPredicate> rangePredicates = KuduClientUtils.createRangePredicate(column, query.getStartKey(), query.getEndKey());
        rangePredicates.add(KuduPredicate.newIsNotNullPredicate(column));
        KuduScanner build = createScanner(rangePredicates, dbFields, query.getLimit());
        while (build.hasMoreRows()) {
            RowResultIterator nextRows = build.nextRows();
            for (RowResult it : nextRows) {
                count++;
                K key = (K) KuduClientUtils.getObjectRow(it, pkc);
                if (query.getFields() != null && query.getFields().length < kuduMapping.getFields().size()) {
                    Update updateOp = table.newUpdate();
                    PartialRow row = updateOp.getRow();
                    String[] avFields = getFieldsToQuery(query.getFields());
                    KuduClientUtils.addObjectRow(row, pkc, key);
                    for (String af : avFields) {
                        row.setNull(kuduMapping.getFields().get(af).getName());
                    }
                    session.apply(updateOp);
                } else {
                    delete(key);
                }
            }
        }
        build.close();
        return count;
    } catch (Exception e) {
        throw new GoraException(e);
    }
}