use of org.apache.kudu.client.RowResult in project drill by axbaretto.
the class TestKuduConnect method createKuduTable.
public static void createKuduTable(String tableName, int tablets, int replicas, int rows) throws Exception {
try (KuduClient client = new KuduClient.KuduClientBuilder(KUDU_MASTER).build()) {
ListTablesResponse tables = client.getTablesList(tableName);
if (!tables.getTablesList().isEmpty()) {
client.deleteTable(tableName);
}
List<ColumnSchema> columns = new ArrayList<>(5);
columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32).key(true).build());
columns.add(new ColumnSchema.ColumnSchemaBuilder("binary", Type.BINARY).nullable(false).build());
columns.add(new ColumnSchema.ColumnSchemaBuilder("boolean", Type.BOOL).nullable(true).build());
columns.add(new ColumnSchema.ColumnSchemaBuilder("float", Type.FLOAT).nullable(false).build());
columns.add(new ColumnSchema.ColumnSchemaBuilder("string", Type.STRING).nullable(true).build());
Schema schema = new Schema(columns);
CreateTableOptions builder = new CreateTableOptions();
builder.setNumReplicas(replicas);
builder.setRangePartitionColumns(Arrays.asList("key"));
for (int i = 1; i < tablets; i++) {
PartialRow splitRow = schema.newPartialRow();
splitRow.addInt("key", i * 1000);
builder.addSplitRow(splitRow);
}
client.createTable(tableName, schema, builder);
KuduTable table = client.openTable(tableName);
KuduSession session = client.newSession();
session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC);
for (int i = 0; i < rows; i++) {
Insert insert = table.newInsert();
PartialRow row = insert.getRow();
row.addInt(0, i);
row.addBinary(1, ("Row " + i).getBytes());
row.addBoolean(2, i % 2 == 0);
row.addFloat(3, i + 0.01f);
row.addString(4, ("Row " + i));
session.apply(insert);
}
List<String> projectColumns = new ArrayList<>(1);
projectColumns.add("float");
KuduScanner scanner = client.newScannerBuilder(table).setProjectedColumnNames(projectColumns).build();
while (scanner.hasMoreRows()) {
RowResultIterator results = scanner.nextRows();
while (results.hasNext()) {
RowResult result = results.next();
System.out.println(result.toStringLongFormat());
}
}
}
}
use of org.apache.kudu.client.RowResult in project hive by apache.
the class TestKuduInputFormat method testAllColumns.
@Test
public void testAllColumns() throws Exception {
KuduInputFormat input = new KuduInputFormat();
JobConf jobConf = new JobConf(BASE_CONF);
String columnsStr = SCHEMA.getColumns().stream().map(ColumnSchema::getName).collect(Collectors.joining(","));
jobConf.set(serdeConstants.LIST_COLUMNS, columnsStr);
InputSplit[] splits = input.getSplits(jobConf, 1);
assertEquals(1, splits.length);
KuduInputSplit split = (KuduInputSplit) splits[0];
KuduRecordReader reader = (KuduRecordReader) input.getRecordReader(split, jobConf, null);
assertTrue(reader.nextKeyValue());
RowResult value = reader.getCurrentValue().getRowResult();
verfiyRow(value);
assertFalse(reader.nextKeyValue());
}
use of org.apache.kudu.client.RowResult in project hive by apache.
the class TestKuduInputFormat method testProjection.
@Test
public void testProjection() throws Exception {
KuduInputFormat input = new KuduInputFormat();
JobConf jobConf = new JobConf(BASE_CONF);
jobConf.set(serdeConstants.LIST_COLUMNS, "bool,key");
InputSplit[] splits = input.getSplits(jobConf, 1);
assertEquals(1, splits.length);
KuduInputSplit split = (KuduInputSplit) splits[0];
KuduRecordReader reader = (KuduRecordReader) input.getRecordReader(split, jobConf, null);
assertTrue(reader.nextKeyValue());
RowResult value = reader.getCurrentValue().getRowResult();
assertEquals(2, value.getSchema().getColumnCount());
assertTrue(value.getBoolean(0));
assertEquals((byte) 1, value.getByte(1));
assertFalse(reader.nextKeyValue());
}
use of org.apache.kudu.client.RowResult in project hive by apache.
the class TestKuduOutputFormat method testGoodRow.
@Test
public void testGoodRow() throws Exception {
KuduOutputFormat outputFormat = new KuduOutputFormat();
KuduRecordWriter writer = (KuduRecordWriter) outputFormat.getHiveRecordWriter(new JobConf(BASE_CONF), null, null, false, TBL_PROPS, null);
// Write a good row.
try {
PartialRow row = SCHEMA.newPartialRow();
row.addByte("key", (byte) 1);
row.addShort("int16", (short) 1);
row.addInt("int32", 1);
row.addLong("int64", 1L);
row.addBoolean("bool", true);
row.addFloat("float", 1.1f);
row.addDouble("double", 1.1d);
row.addString("string", "one");
row.addBinary("binary", "one".getBytes(UTF_8));
row.addTimestamp("timestamp", new Timestamp(NOW_MS));
row.addDecimal("decimal", new BigDecimal("1.111"));
row.setNull("null");
// Not setting the "default" column.
KuduWritable writable = new KuduWritable(row);
writer.write(writable);
} finally {
writer.close(false);
}
// Verify the written row.
KuduClient client = harness.getClient();
KuduTable table = client.openTable(TABLE_NAME);
KuduScanner scanner = client.newScannerBuilder(table).build();
List<RowResult> results = new ArrayList<>();
for (RowResult result : scanner) {
results.add(result);
}
assertEquals(1, results.size());
RowResult result = results.get(0);
assertEquals((byte) 1, result.getByte(0));
assertEquals((short) 1, result.getShort(1));
assertEquals(1, result.getInt(2));
assertEquals(1L, result.getLong(3));
assertTrue(result.getBoolean(4));
assertEquals(1.1f, result.getFloat(5), 0);
assertEquals(1.1d, result.getDouble(6), 0);
assertEquals("one", result.getString(7));
assertEquals("one", new String(result.getBinaryCopy(8), UTF_8));
assertEquals(NOW_MS, result.getTimestamp(9).getTime());
assertEquals(new BigDecimal("1.111"), result.getDecimal(10));
assertTrue(result.isNull(11));
// default.
assertEquals(1, result.getInt(12));
}
use of org.apache.kudu.client.RowResult in project gora by apache.
the class KuduStore method deleteByQuery.
@Override
public long deleteByQuery(Query<K, T> query) throws GoraException {
try {
long count = 0;
Column pkc = kuduMapping.getPrimaryKey().get(0);
ColumnSchema column = table.getSchema().getColumn(pkc.getName());
List<String> dbFields = new ArrayList<>();
dbFields.add(pkc.getName());
List<KuduPredicate> rangePredicates = KuduClientUtils.createRangePredicate(column, query.getStartKey(), query.getEndKey());
rangePredicates.add(KuduPredicate.newIsNotNullPredicate(column));
KuduScanner build = createScanner(rangePredicates, dbFields, query.getLimit());
while (build.hasMoreRows()) {
RowResultIterator nextRows = build.nextRows();
for (RowResult it : nextRows) {
count++;
K key = (K) KuduClientUtils.getObjectRow(it, pkc);
if (query.getFields() != null && query.getFields().length < kuduMapping.getFields().size()) {
Update updateOp = table.newUpdate();
PartialRow row = updateOp.getRow();
String[] avFields = getFieldsToQuery(query.getFields());
KuduClientUtils.addObjectRow(row, pkc, key);
for (String af : avFields) {
row.setNull(kuduMapping.getFields().get(af).getName());
}
session.apply(updateOp);
} else {
delete(key);
}
}
}
build.close();
return count;
} catch (Exception e) {
throw new GoraException(e);
}
}
Aggregations