Examples with ColumnSchema org.apache.kudu.ColumnSchema used on opensource projects

Example 1 with ColumnSchema

use of org.apache.kudu.ColumnSchema in project YCSB by brianfrankcooper.

the class KuduYCSBClient method scan.

@Override
public Status scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
    try {
        KuduScanner.KuduScannerBuilder scannerBuilder = client.newScannerBuilder(kuduTable);
        List<String> querySchema;
        if (fields == null) {
            querySchema = COLUMN_NAMES;
        // No need to set the projected columns with the whole schema.
        } else {
            querySchema = new ArrayList<>(fields);
            scannerBuilder.setProjectedColumnNames(querySchema);
        }
        ColumnSchema column = schema.getColumnByIndex(0);
        KuduPredicate.ComparisonOp predicateOp = recordcount == 1 ? EQUAL : GREATER_EQUAL;
        KuduPredicate predicate = KuduPredicate.newComparisonPredicate(column, predicateOp, startkey);
        scannerBuilder.addPredicate(predicate);
        // currently noop
        scannerBuilder.limit(recordcount);
        KuduScanner scanner = scannerBuilder.build();
        while (scanner.hasMoreRows()) {
            RowResultIterator data = scanner.nextRows();
            addAllRowsToResult(data, recordcount, querySchema, result);
            if (recordcount == result.size()) {
                break;
            }
        }
        RowResultIterator closer = scanner.close();
        addAllRowsToResult(closer, recordcount, querySchema, result);
    } catch (TimeoutException te) {
        LOG.info("Waited too long for a scan operation with start key={}", startkey);
        return TIMEOUT;
    } catch (Exception e) {
        LOG.warn("Unexpected exception", e);
        return Status.ERROR;
    }
    return Status.OK;
}

Example 2 with ColumnSchema

use of org.apache.kudu.ColumnSchema in project drill by apache.

the class DrillKuduTable method getRowType.

@Override
public RelDataType getRowType(RelDataTypeFactory typeFactory) {
    List<String> names = Lists.newArrayList();
    List<RelDataType> types = Lists.newArrayList();
    for (ColumnSchema column : schema.getColumns()) {
        names.add(column.getName());
        RelDataType type = getSqlTypeFromKuduType(typeFactory, column.getType());
        type = typeFactory.createTypeWithNullability(type, column.isNullable());
        types.add(type);
    }
    return typeFactory.createStructType(types, names);
}

Example 3 with ColumnSchema

use of org.apache.kudu.ColumnSchema in project drill by apache.

the class TestKuduConnect method createKuduTable.

public static void createKuduTable(String tableName, int tablets, int replicas, int rows) throws Exception {
    try (KuduClient client = new KuduClient.KuduClientBuilder(KUDU_MASTER).build()) {
        ListTablesResponse tables = client.getTablesList(tableName);
        if (!tables.getTablesList().isEmpty()) {
            client.deleteTable(tableName);
        }
        List<ColumnSchema> columns = new ArrayList<>(5);
        columns.add(new ColumnSchema.ColumnSchemaBuilder("key", Type.INT32).key(true).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("binary", Type.BINARY).nullable(false).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("boolean", Type.BOOL).nullable(true).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("float", Type.FLOAT).nullable(false).build());
        columns.add(new ColumnSchema.ColumnSchemaBuilder("string", Type.STRING).nullable(true).build());
        Schema schema = new Schema(columns);
        CreateTableOptions builder = new CreateTableOptions();
        builder.setNumReplicas(replicas);
        builder.setRangePartitionColumns(Arrays.asList("key"));
        for (int i = 1; i < tablets; i++) {
            PartialRow splitRow = schema.newPartialRow();
            splitRow.addInt("key", i * 1000);
            builder.addSplitRow(splitRow);
        }
        client.createTable(tableName, schema, builder);
        KuduTable table = client.openTable(tableName);
        KuduSession session = client.newSession();
        session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_SYNC);
        for (int i = 0; i < rows; i++) {
            Insert insert = table.newInsert();
            PartialRow row = insert.getRow();
            row.addInt(0, i);
            row.addBinary(1, ("Row " + i).getBytes());
            row.addBoolean(2, i % 2 == 0);
            row.addFloat(3, i + 0.01f);
            row.addString(4, ("Row " + i));
            session.apply(insert);
        }
        List<String> projectColumns = new ArrayList<>(1);
        projectColumns.add("float");
        KuduScanner scanner = client.newScannerBuilder(table).setProjectedColumnNames(projectColumns).build();
        while (scanner.hasMoreRows()) {
            RowResultIterator results = scanner.nextRows();
            while (results.hasNext()) {
                RowResult result = results.next();
                System.out.println(result.toStringLongFormat());
            }
        }
    }
}

Example 4 with ColumnSchema

use of org.apache.kudu.ColumnSchema in project YCSB by brianfrankcooper.

the class KuduYCSBClient method initClient.

private static synchronized void initClient(String tableName, Properties prop) throws DBException {
    if (client != null) {
        return;
    }
    String masterAddresses = prop.getProperty(MASTER_ADDRESSES_OPT);
    if (masterAddresses == null) {
        masterAddresses = "localhost:7051";
    }
    int numTablets = getIntFromProp(prop, PRE_SPLIT_NUM_TABLETS_OPT, 4);
    if (numTablets > MAX_TABLETS) {
        throw new DBException(String.format("Specified number of tablets (%s) must be equal or below %s", numTablets, MAX_TABLETS));
    }
    int numReplicas = getIntFromProp(prop, TABLE_NUM_REPLICAS, 3);
    int blockSize = getIntFromProp(prop, BLOCK_SIZE_OPT, BLOCK_SIZE_DEFAULT);
    client = new KuduClient.KuduClientBuilder(masterAddresses).defaultSocketReadTimeoutMs(DEFAULT_SLEEP).defaultOperationTimeoutMs(DEFAULT_SLEEP).defaultAdminOperationTimeoutMs(DEFAULT_SLEEP).build();
    LOG.debug("Connecting to the masters at {}", masterAddresses);
    int fieldCount = getIntFromProp(prop, CoreWorkload.FIELD_COUNT_PROPERTY, Integer.parseInt(CoreWorkload.FIELD_COUNT_PROPERTY_DEFAULT));
    List<ColumnSchema> columns = new ArrayList<>(fieldCount + 1);
    ColumnSchema keyColumn = new ColumnSchema.ColumnSchemaBuilder(KEY, STRING).key(true).desiredBlockSize(blockSize).build();
    columns.add(keyColumn);
    COLUMN_NAMES.add(KEY);
    for (int i = 0; i < fieldCount; i++) {
        String name = "field" + i;
        COLUMN_NAMES.add(name);
        columns.add(new ColumnSchema.ColumnSchemaBuilder(name, STRING).desiredBlockSize(blockSize).build());
    }
    schema = new Schema(columns);
    CreateTableOptions builder = new CreateTableOptions();
    builder.setRangePartitionColumns(new ArrayList<String>());
    List<String> hashPartitionColumns = new ArrayList<>();
    hashPartitionColumns.add(KEY);
    builder.addHashPartitions(hashPartitionColumns, numTablets);
    builder.setNumReplicas(numReplicas);
    try {
        client.createTable(tableName, schema, builder);
    } catch (Exception e) {
        if (!e.getMessage().contains("already exists")) {
            throw new DBException("Couldn't create the table", e);
        }
    }
}

Example 5 with ColumnSchema

use of org.apache.kudu.ColumnSchema in project drill by apache.

the class KuduRecordReader method initCols.

private void initCols(Schema schema) throws SchemaChangeException {
    ImmutableList.Builder<ProjectedColumnInfo> pciBuilder = ImmutableList.builder();
    for (int i = 0; i < schema.getColumnCount(); i++) {
        ColumnSchema col = schema.getColumnByIndex(i);
        final String name = col.getName();
        final Type kuduType = col.getType();
        MinorType minorType = TYPES.get(kuduType);
        if (minorType == null) {
            logger.warn("Ignoring column that is unsupported.", UserException.unsupportedError().message("A column you queried has a data type that is not currently supported by the Kudu storage plugin. " + "The column's name was %s and its Kudu data type was %s. ", name, kuduType.toString()).addContext("column Name", name).addContext("plugin", "kudu").build(logger));
            continue;
        }
        MajorType majorType;
        if (col.isNullable()) {
            majorType = Types.optional(minorType);
        } else {
            majorType = Types.required(minorType);
        }
        MaterializedField field = MaterializedField.create(name, majorType);
        final Class<? extends ValueVector> clazz = (Class<? extends ValueVector>) TypeHelper.getValueVectorClass(minorType, majorType.getMode());
        ValueVector vector = output.addField(field, clazz);
        vector.allocateNew();
        ProjectedColumnInfo pci = new ProjectedColumnInfo();
        pci.vv = vector;
        pci.kuduColumn = col;
        pci.index = i;
        pciBuilder.add(pci);
    }
    projectedCols = pciBuilder.build();
}