Search in sources :

Example 1 with InvalidFieldException

use of io.cdap.cdap.spi.data.InvalidFieldException in project cdap by caskdata.

the class NoSqlStructuredTable method convertFieldsToBytes.

/**
 * Convert the fields to a {@link Put} to write to table. The primary key must all be provided. The method will
 * add the table name as prefix to the row key.
 *
 * @param fields the fields to write
 * @return a PUT object
 * @throws InvalidFieldException if primary keys are missing or the column is not in schema
 */
private Put convertFieldsToBytes(Collection<Field<?>> fields) throws InvalidFieldException {
    Set<String> fieldNames = fields.stream().map(Field::getName).collect(Collectors.toSet());
    if (!fieldNames.containsAll(schema.getPrimaryKeys())) {
        throw new InvalidFieldException(schema.getTableId(), fields, String.format("Given fields %s does not contain all the " + "primary keys %s", fieldNames, schema.getPrimaryKeys()));
    }
    int numColumns = fields.size() - schema.getPrimaryKeys().size();
    // add the table name as the prefix
    MDSKey.Builder key = new MDSKey.Builder(keyPrefix);
    byte[][] columns = new byte[numColumns][];
    byte[][] values = new byte[numColumns][];
    int i = 0;
    for (Field<?> field : fields) {
        fieldValidator.validateField(field);
        if (schema.isPrimaryKeyColumn(field.getName())) {
            addKey(key, field, schema.getType(field.getName()));
        } else {
            if (schema.getType(field.getName()) == null) {
                throw new InvalidFieldException(schema.getTableId(), field.getName());
            }
            columns[i] = Bytes.toBytes(field.getName());
            values[i] = fieldToBytes(field);
            i++;
        }
    }
    Put put = new Put(key.build().getKey());
    for (int index = 0; index < columns.length; index++) {
        put.add(columns[index], values[index]);
    }
    return put;
}
Also used : MDSKey(io.cdap.cdap.data2.dataset2.lib.table.MDSKey) InvalidFieldException(io.cdap.cdap.spi.data.InvalidFieldException) Put(io.cdap.cdap.api.dataset.table.Put)

Example 2 with InvalidFieldException

use of io.cdap.cdap.spi.data.InvalidFieldException in project cdap by caskdata.

the class SpannerStructuredTable method scan.

@Override
public CloseableIterator<StructuredRow> scan(Field<?> index) throws InvalidFieldException {
    fieldValidator.validateField(index);
    if (!schema.isIndexColumn(index.getName())) {
        throw new InvalidFieldException(schema.getTableId(), index.getName(), "is not an indexed column");
    }
    KeySet keySet = KeySet.singleKey(createKey(Collections.singleton(index)));
    String indexName = SpannerStructuredTableAdmin.getIndexName(schema.getTableId(), index.getName());
    ResultSet resultSet = transactionContext.readUsingIndex(schema.getTableId().getName(), indexName, keySet, schema.getFieldNames());
    return new ResultSetIterator(schema, resultSet);
}
Also used : KeySet(com.google.cloud.spanner.KeySet) ResultSet(com.google.cloud.spanner.ResultSet) InvalidFieldException(io.cdap.cdap.spi.data.InvalidFieldException)

Example 3 with InvalidFieldException

use of io.cdap.cdap.spi.data.InvalidFieldException in project cdap by caskdata.

the class SpannerStructuredTable method update.

@Override
public void update(Collection<Field<?>> fields) throws InvalidFieldException {
    List<Field<?>> primaryKeyFields = new ArrayList<>();
    List<Field<?>> updateFields = new ArrayList<>();
    Set<String> fieldNames = new HashSet<>();
    for (Field<?> field : fields) {
        fieldValidator.validateField(field);
        if (schema.isPrimaryKeyColumn(field.getName())) {
            primaryKeyFields.add(field);
        } else {
            updateFields.add(field);
        }
        fieldNames.add(field.getName());
    }
    if (!fieldNames.containsAll(schema.getPrimaryKeys())) {
        throw new InvalidFieldException(schema.getTableId(), fields, String.format("Given fields %s do not contain all the " + "primary keys %s", fieldNames, schema.getPrimaryKeys()));
    }
    String sql = "UPDATE " + escapeName(schema.getTableId().getName()) + " SET " + updateFields.stream().map(this::fieldToParam).collect(Collectors.joining(", ")) + " WHERE " + primaryKeyFields.stream().map(this::fieldToParam).collect(Collectors.joining(" AND "));
    LOG.trace("Updating row: {}", sql);
    Statement statement = fields.stream().reduce(Statement.newBuilder(sql), (builder, field) -> builder.bind(field.getName()).to(getValue(field)), (builder1, builder2) -> builder1).build();
    transactionContext.executeUpdate(statement);
}
Also used : IntStream(java.util.stream.IntStream) StructuredRow(io.cdap.cdap.spi.data.StructuredRow) LoggerFactory(org.slf4j.LoggerFactory) FieldValidator(io.cdap.cdap.spi.data.table.field.FieldValidator) Fields(io.cdap.cdap.spi.data.table.field.Fields) HashMap(java.util.HashMap) Value(com.google.cloud.spanner.Value) Function(java.util.function.Function) KeyRange(com.google.cloud.spanner.KeyRange) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) LinkedHashMap(java.util.LinkedHashMap) ResultSet(com.google.cloud.spanner.ResultSet) Key(com.google.cloud.spanner.Key) Map(java.util.Map) Field(io.cdap.cdap.spi.data.table.field.Field) ByteArray(com.google.cloud.ByteArray) LinkedHashSet(java.util.LinkedHashSet) Nullable(javax.annotation.Nullable) TransactionContext(com.google.cloud.spanner.TransactionContext) InvalidFieldException(io.cdap.cdap.spi.data.InvalidFieldException) Logger(org.slf4j.Logger) Collection(java.util.Collection) Set(java.util.Set) FieldType(io.cdap.cdap.spi.data.table.field.FieldType) CloseableIterator(io.cdap.cdap.api.dataset.lib.CloseableIterator) Collectors(java.util.stream.Collectors) KeySet(com.google.cloud.spanner.KeySet) Statement(com.google.cloud.spanner.Statement) Objects(java.util.Objects) AbstractCloseableIterator(io.cdap.cdap.api.dataset.lib.AbstractCloseableIterator) List(java.util.List) Struct(com.google.cloud.spanner.Struct) StructuredTable(io.cdap.cdap.spi.data.StructuredTable) Range(io.cdap.cdap.spi.data.table.field.Range) Optional(java.util.Optional) StructuredTableSchema(io.cdap.cdap.spi.data.table.StructuredTableSchema) Collections(java.util.Collections) Field(io.cdap.cdap.spi.data.table.field.Field) Statement(com.google.cloud.spanner.Statement) ArrayList(java.util.ArrayList) InvalidFieldException(io.cdap.cdap.spi.data.InvalidFieldException) HashSet(java.util.HashSet) LinkedHashSet(java.util.LinkedHashSet)

Example 4 with InvalidFieldException

use of io.cdap.cdap.spi.data.InvalidFieldException in project cdap by caskdata.

the class SpannerStructuredTable method insert.

private void insert(Collection<Field<?>> fields) throws InvalidFieldException {
    List<Field<?>> insertFields = new ArrayList<>();
    for (Field<?> field : fields) {
        fieldValidator.validateField(field);
        insertFields.add(field);
    }
    String sql = "INSERT INTO " + escapeName(schema.getTableId().getName()) + " (" + insertFields.stream().map(Field::getName).map(this::escapeName).collect(Collectors.joining(",")) + ") VALUES (" + insertFields.stream().map(f -> "@" + f.getName()).collect(Collectors.joining(",")) + ")";
    LOG.trace("Inserting row: {}", sql);
    Statement statement = fields.stream().reduce(Statement.newBuilder(sql), (builder, field) -> builder.bind(field.getName()).to(getValue(field)), (builder1, builder2) -> builder1).build();
    transactionContext.executeUpdate(statement);
}
Also used : IntStream(java.util.stream.IntStream) StructuredRow(io.cdap.cdap.spi.data.StructuredRow) LoggerFactory(org.slf4j.LoggerFactory) FieldValidator(io.cdap.cdap.spi.data.table.field.FieldValidator) Fields(io.cdap.cdap.spi.data.table.field.Fields) HashMap(java.util.HashMap) Value(com.google.cloud.spanner.Value) Function(java.util.function.Function) KeyRange(com.google.cloud.spanner.KeyRange) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) LinkedHashMap(java.util.LinkedHashMap) ResultSet(com.google.cloud.spanner.ResultSet) Key(com.google.cloud.spanner.Key) Map(java.util.Map) Field(io.cdap.cdap.spi.data.table.field.Field) ByteArray(com.google.cloud.ByteArray) LinkedHashSet(java.util.LinkedHashSet) Nullable(javax.annotation.Nullable) TransactionContext(com.google.cloud.spanner.TransactionContext) InvalidFieldException(io.cdap.cdap.spi.data.InvalidFieldException) Logger(org.slf4j.Logger) Collection(java.util.Collection) Set(java.util.Set) FieldType(io.cdap.cdap.spi.data.table.field.FieldType) CloseableIterator(io.cdap.cdap.api.dataset.lib.CloseableIterator) Collectors(java.util.stream.Collectors) KeySet(com.google.cloud.spanner.KeySet) Statement(com.google.cloud.spanner.Statement) Objects(java.util.Objects) AbstractCloseableIterator(io.cdap.cdap.api.dataset.lib.AbstractCloseableIterator) List(java.util.List) Struct(com.google.cloud.spanner.Struct) StructuredTable(io.cdap.cdap.spi.data.StructuredTable) Range(io.cdap.cdap.spi.data.table.field.Range) Optional(java.util.Optional) StructuredTableSchema(io.cdap.cdap.spi.data.table.StructuredTableSchema) Collections(java.util.Collections) Field(io.cdap.cdap.spi.data.table.field.Field) Statement(com.google.cloud.spanner.Statement) ArrayList(java.util.ArrayList)

Example 5 with InvalidFieldException

use of io.cdap.cdap.spi.data.InvalidFieldException in project cdap by caskdata.

the class SpannerStructuredTable method delete.

@Override
public void delete(Collection<Field<?>> keys) throws InvalidFieldException {
    fieldValidator.validatePrimaryKeys(keys, false);
    String sql = "DELETE FROM " + escapeName(schema.getTableId().getName()) + " WHERE " + keys.stream().map(f -> escapeName(f.getName()) + " = @" + f.getName()).collect(Collectors.joining(" AND "));
    Statement statement = keys.stream().reduce(Statement.newBuilder(sql), (builder, field) -> builder.bind(field.getName()).to(getValue(field)), (builder1, builder2) -> builder1).build();
    transactionContext.executeUpdate(statement);
}
Also used : IntStream(java.util.stream.IntStream) StructuredRow(io.cdap.cdap.spi.data.StructuredRow) LoggerFactory(org.slf4j.LoggerFactory) FieldValidator(io.cdap.cdap.spi.data.table.field.FieldValidator) Fields(io.cdap.cdap.spi.data.table.field.Fields) HashMap(java.util.HashMap) Value(com.google.cloud.spanner.Value) Function(java.util.function.Function) KeyRange(com.google.cloud.spanner.KeyRange) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) LinkedHashMap(java.util.LinkedHashMap) ResultSet(com.google.cloud.spanner.ResultSet) Key(com.google.cloud.spanner.Key) Map(java.util.Map) Field(io.cdap.cdap.spi.data.table.field.Field) ByteArray(com.google.cloud.ByteArray) LinkedHashSet(java.util.LinkedHashSet) Nullable(javax.annotation.Nullable) TransactionContext(com.google.cloud.spanner.TransactionContext) InvalidFieldException(io.cdap.cdap.spi.data.InvalidFieldException) Logger(org.slf4j.Logger) Collection(java.util.Collection) Set(java.util.Set) FieldType(io.cdap.cdap.spi.data.table.field.FieldType) CloseableIterator(io.cdap.cdap.api.dataset.lib.CloseableIterator) Collectors(java.util.stream.Collectors) KeySet(com.google.cloud.spanner.KeySet) Statement(com.google.cloud.spanner.Statement) Objects(java.util.Objects) AbstractCloseableIterator(io.cdap.cdap.api.dataset.lib.AbstractCloseableIterator) List(java.util.List) Struct(com.google.cloud.spanner.Struct) StructuredTable(io.cdap.cdap.spi.data.StructuredTable) Range(io.cdap.cdap.spi.data.table.field.Range) Optional(java.util.Optional) StructuredTableSchema(io.cdap.cdap.spi.data.table.StructuredTableSchema) Collections(java.util.Collections) Statement(com.google.cloud.spanner.Statement)

Aggregations

InvalidFieldException (io.cdap.cdap.spi.data.InvalidFieldException)13 StructuredRow (io.cdap.cdap.spi.data.StructuredRow)7 StructuredTable (io.cdap.cdap.spi.data.StructuredTable)7 Field (io.cdap.cdap.spi.data.table.field.Field)7 FieldType (io.cdap.cdap.spi.data.table.field.FieldType)7 ArrayList (java.util.ArrayList)6 KeySet (com.google.cloud.spanner.KeySet)5 ResultSet (com.google.cloud.spanner.ResultSet)5 AbstractCloseableIterator (io.cdap.cdap.api.dataset.lib.AbstractCloseableIterator)5 CloseableIterator (io.cdap.cdap.api.dataset.lib.CloseableIterator)5 StructuredTableSchema (io.cdap.cdap.spi.data.table.StructuredTableSchema)5 FieldValidator (io.cdap.cdap.spi.data.table.field.FieldValidator)5 Range (io.cdap.cdap.spi.data.table.field.Range)5 IOException (java.io.IOException)5 Collection (java.util.Collection)5 Collections (java.util.Collections)5 HashSet (java.util.HashSet)5 List (java.util.List)5 Map (java.util.Map)5 Optional (java.util.Optional)5