Examples with CreateTableOptions org.apache.kudu.client.CreateTableOptions used on opensource projects

Example 11 with CreateTableOptions

use of org.apache.kudu.client.CreateTableOptions in project hive by apache.

the class TestKuduInputFormat method setUp.

@Before
public void setUp() throws Exception {
    // Set the base configuration values.
    BASE_CONF.set(KUDU_MASTER_ADDRS_KEY, harness.getMasterAddressesAsString());
    BASE_CONF.set(KUDU_TABLE_NAME_KEY, TABLE_NAME);
    BASE_CONF.set(FileInputFormat.INPUT_DIR, "dummy");
    // Create the test Kudu table.
    CreateTableOptions options = new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("key"));
    harness.getClient().createTable(TABLE_NAME, SCHEMA, options);
    // Insert a test row.
    KuduTable table = harness.getClient().openTable(TABLE_NAME);
    KuduSession session = harness.getClient().newSession();
    Insert insert = table.newInsert();
    PartialRow insertRow = insert.getRow();
    // Use KuduWritable, to populate the insert row.
    new KuduWritable(ROW).populateRow(insertRow);
    session.apply(insert);
    session.close();
}

Example 12 with CreateTableOptions

use of org.apache.kudu.client.CreateTableOptions in project hive by apache.

the class TestKuduSerDe method setUp.

@Before
public void setUp() throws Exception {
    // Set the base configuration values.
    BASE_CONF.set(KUDU_MASTER_ADDRS_KEY, harness.getMasterAddressesAsString());
    TBL_PROPS.setProperty(KUDU_TABLE_NAME_KEY, TABLE_NAME);
    // Create the test Kudu table.
    CreateTableOptions options = new CreateTableOptions().setRangePartitionColumns(ImmutableList.of("key"));
    harness.getClient().createTable(TABLE_NAME, SCHEMA, options);
}

Example 13 with CreateTableOptions

use of org.apache.kudu.client.CreateTableOptions in project apex-malhar by apache.

the class KuduClientTestCommons method createTestTable.

public static void createTestTable(String tableName, KuduClient client) throws Exception {
    List<String> rangeKeys = new ArrayList<>();
    rangeKeys.add("introwkey");
    List<String> hashPartitions = new ArrayList<>();
    hashPartitions.add("stringrowkey");
    hashPartitions.add("timestamprowkey");
    CreateTableOptions thisTableOptions = new CreateTableOptions().setNumReplicas(1).addHashPartitions(hashPartitions, HASH_BUCKETS_SIZE_FOR_ALL_HASH_COL).setRangePartitionColumns(rangeKeys);
    int stepsize = Integer.MAX_VALUE / SPLIT_COUNT_FOR_INT_ROW_KEY;
    int splitBoundary = stepsize;
    Schema schema = buildSchemaForUnitTestsTable();
    for (int i = 0; i < SPLIT_COUNT_FOR_INT_ROW_KEY; i++) {
        PartialRow splitRowBoundary = schema.newPartialRow();
        splitRowBoundary.addInt("introwkey", splitBoundary);
        thisTableOptions = thisTableOptions.addSplitRow(splitRowBoundary);
        splitBoundary += stepsize;
    }
    try {
        client.createTable(tableName, schema, thisTableOptions);
    } catch (KuduException e) {
        LOG.error("Error while creating table for unit tests " + e.getMessage(), e);
        throw e;
    }
}

Example 14 with CreateTableOptions

use of org.apache.kudu.client.CreateTableOptions in project presto by prestodb.

the class SchemaEmulationByTableNameConvention method createAndFillSchemasTable.

private void createAndFillSchemasTable(KuduClient client) throws KuduException {
    List<String> existingSchemaNames = listSchemaNamesFromTablets(client);
    ColumnSchema schemaColumnSchema = new ColumnSchema.ColumnSchemaBuilder("schema", Type.STRING).key(true).build();
    Schema schema = new Schema(ImmutableList.of(schemaColumnSchema));
    CreateTableOptions options = new CreateTableOptions();
    options.addHashPartitions(ImmutableList.of(schemaColumnSchema.getName()), 2);
    KuduTable schemasTable = client.createTable(rawSchemasTableName, schema, options);
    KuduSession session = client.newSession();
    try {
        session.setFlushMode(SessionConfiguration.FlushMode.AUTO_FLUSH_BACKGROUND);
        for (String schemaName : existingSchemaNames) {
            Insert insert = schemasTable.newInsert();
            insert.getRow().addString(0, schemaName);
            session.apply(insert);
        }
    } finally {
        session.close();
    }
}

Example 15 with CreateTableOptions

use of org.apache.kudu.client.CreateTableOptions in project gora by apache.

the class KuduStore method createSchema.

@Override
public void createSchema() throws GoraException {
    if (client == null) {
        throw new GoraException("Impossible to create the schema as no connection has been initiated.");
    }
    if (schemaExists()) {
        return;
    }
    try {
        List<ColumnSchema> columns = new ArrayList<>();
        List<String> keys = new ArrayList<>();
        for (Column primaryKey : kuduMapping.getPrimaryKey()) {
            columns.add(new ColumnSchema.ColumnSchemaBuilder(primaryKey.getName(), primaryKey.getDataType().getType()).key(true).build());
            keys.add(primaryKey.getName());
        }
        for (Map.Entry<String, Column> clt : kuduMapping.getFields().entrySet()) {
            Column aColumn = clt.getValue();
            ColumnSchema aColumnSch;
            ColumnSchema.ColumnSchemaBuilder aBaseColumn = new ColumnSchema.ColumnSchemaBuilder(aColumn.getName(), aColumn.getDataType().getType()).nullable(true);
            if (aColumn.getDataType().getType() == Type.DECIMAL) {
                aColumnSch = aBaseColumn.typeAttributes(DecimalUtil.typeAttributes(aColumn.getDataType().getPrecision(), aColumn.getDataType().getScale())).build();
            } else {
                aColumnSch = aBaseColumn.build();
            }
            columns.add(aColumnSch);
        }
        org.apache.kudu.Schema sch = new org.apache.kudu.Schema(columns);
        CreateTableOptions cto = new CreateTableOptions();
        cto.setNumReplicas(kuduMapping.getNumReplicas());
        if (kuduMapping.getHashBuckets() > 0) {
            cto.addHashPartitions(keys, kuduMapping.getHashBuckets());
        }
        if (!kuduMapping.getRangePartitions().isEmpty()) {
            cto.setRangePartitionColumns(keys);
            for (Map.Entry<String, String> range : kuduMapping.getRangePartitions()) {
                PartialRow lowerPar = sch.newPartialRow();
                PartialRow upperPar = sch.newPartialRow();
                for (String ky : keys) {
                    if (!range.getKey().isEmpty()) {
                        lowerPar.addString(ky, range.getKey());
                    }
                    if (!range.getValue().isEmpty()) {
                        upperPar.addString(ky, range.getValue());
                    }
                }
                cto.addRangePartition(lowerPar, upperPar);
            }
        }
        table = client.createTable(kuduMapping.getTableName(), sch, cto);
    } catch (KuduException ex) {
        throw new GoraException(ex);
    }
}