use of com.google.spanner.v1.Mutation in project grpc-gcp-java by GoogleCloudPlatform.
the class BigtableIntegrationTest method getMutateRequest.
private static MutateRowRequest getMutateRequest(String val, int col, String rowKey) {
ByteString value = ByteString.copyFromUtf8(val);
Mutation mutation = Mutation.newBuilder().setSetCell(Mutation.SetCell.newBuilder().setFamilyName(FAMILY_NAME).setColumnQualifier(ByteString.copyFromUtf8(COLUMN_NAME + col)).setValue(value)).build();
MutateRowRequest request = MutateRowRequest.newBuilder().setTableName(TABLE_NAME).setRowKey(ByteString.copyFromUtf8(rowKey)).addMutations(mutation).build();
return request;
}
use of com.google.spanner.v1.Mutation in project pgadapter by GoogleCloudPlatform.
the class MutationWriter method buildMutationList.
/**
* Build mutation to add to mutations list with data contained within a CopyData payload
*/
public void buildMutationList(ConnectionHandler connectionHandler) throws Exception {
List<CSVRecord> records = parsePayloadData(this.payload.toByteArray());
for (CSVRecord record : records) {
// Check that the number of columns in a record matches the number of columns in the table
if (record.size() != this.tableColumns.keySet().size()) {
handleError(connectionHandler);
throw SpannerExceptionFactory.newSpannerException(ErrorCode.INVALID_ARGUMENT, "Invalid COPY data: Row length mismatched. Expected " + this.tableColumns.keySet().size() + " columns, but only found " + record.size());
}
WriteBuilder builder = Mutation.newInsertBuilder(this.tableName);
// Iterate through all table column to copy into
for (String columnName : this.tableColumns.keySet()) {
TypeCode columnType = this.tableColumns.get(columnName);
String recordValue = "";
try {
recordValue = record.get(columnName).trim();
switch(columnType) {
case STRING:
builder.set(columnName).to(recordValue);
break;
case BOOL:
builder.set(columnName).to(Boolean.parseBoolean(recordValue));
break;
case INT64:
builder.set(columnName).to(Long.parseLong(recordValue));
break;
case FLOAT64:
builder.set(columnName).to(Double.parseDouble(recordValue));
break;
case BYTES:
builder.set(columnName).to(Byte.parseByte(recordValue));
break;
case TIMESTAMP:
builder.set(columnName).to(com.google.cloud.Timestamp.parseTimestamp(recordValue));
break;
}
} catch (NumberFormatException | DateTimeParseException e) {
handleError(connectionHandler);
throw SpannerExceptionFactory.newSpannerException(ErrorCode.INVALID_ARGUMENT, "Invalid input syntax for type " + columnType.toString() + ":" + "\"" + recordValue + "\"");
} catch (IllegalArgumentException e) {
handleError(connectionHandler);
throw SpannerExceptionFactory.newSpannerException(ErrorCode.INVALID_ARGUMENT, "Invalid input syntax for column \"" + columnName + "\"");
} catch (Exception e) {
handleError(connectionHandler);
throw e;
}
}
// Add write builder to mutation list
this.mutations.add(builder.build());
// Increment the number of mutations being added
this.mutationCount += record.size();
// Increment the number of COPY rows by one
this.rowCount++;
}
if (!mutationCountIsWithinLimit()) {
handleError(connectionHandler);
throw SpannerExceptionFactory.newSpannerException(ErrorCode.FAILED_PRECONDITION, "Mutation count: " + mutationCount + " has exceeded the limit: " + MUTATION_LIMIT);
}
}
use of com.google.spanner.v1.Mutation in project SpinalTap by airbnb.
the class KafkaDestinationTest method createMutation.
private Mutation createMutation(MutationType type) {
Mapper<com.airbnb.spinaltap.Mutation<?>, ? extends TBase<?, ?>> thriftMutationMapper = ThriftMutationMapper.create("spinaltap");
Table table = new Table(0L, TABLE, DATABASE, null, ImmutableList.of(new ColumnMetadata("id", ColumnDataType.LONGLONG, true, 0)), ImmutableList.of("id"));
MysqlMutationMetadata metadata = new MysqlMutationMetadata(new DataSource(HOSTNAME, 0, SOURCE_NAME), new BinlogFilePos(), table, 0L, 0L, 0L, null, null, 0L, 0);
Row row = new Row(table, ImmutableMap.of("id", new Column(new ColumnMetadata("id", ColumnDataType.LONGLONG, true, 0), 1L)));
MysqlMutation mutation;
switch(type) {
case INSERT:
mutation = new MysqlInsertMutation(metadata, row);
break;
case UPDATE:
mutation = new MysqlUpdateMutation(metadata, row, row);
break;
case DELETE:
mutation = new MysqlDeleteMutation(metadata, row);
break;
default:
mutation = null;
}
return (Mutation) (thriftMutationMapper.map(mutation));
}
use of com.google.spanner.v1.Mutation in project SpinalTap by airbnb.
the class KafkaDestinationTest method KafkaDestination.
@SuppressWarnings("unchecked")
@Test
public void KafkaDestination() throws Exception {
createKafkaTopic(TOPIC);
KafkaProducerConfiguration configs = new KafkaProducerConfiguration(this.bootstrapServers());
KafkaDestination kafkaDestination = new KafkaDestination(null, configs, x -> x, metrics, 0L);
List<Mutation> messages = new ArrayList<>();
messages.add(createMutation(MutationType.INSERT));
messages.add(createMutation(MutationType.UPDATE));
messages.add(createMutation(MutationType.DELETE));
kafkaDestination.publish(messages);
Properties props = new Properties();
props.setProperty("bootstrap.servers", this.bootstrapServers());
props.setProperty("key.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer");
props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer");
KafkaConsumer<byte[], byte[]> kafkaConsumer = new KafkaConsumer<>(props);
kafkaConsumer.assign(Collections.singletonList(new TopicPartition(TOPIC, 0)));
kafkaConsumer.seekToBeginning(new TopicPartition(TOPIC, 0));
List<ConsumerRecords<byte[], byte[]>> records = new ArrayList<>();
ConsumerRecords<byte[], byte[]> record;
long startMs = current();
while (current() - startMs <= 10000L) {
record = kafkaConsumer.poll(1000L);
records.add(record);
if (records.size() == 3)
break;
}
Assert.assertEquals(records.size(), 3);
for (ConsumerRecords<byte[], byte[]> consumerRecords : records) {
for (ConsumerRecord<byte[], byte[]> consumerRecord : consumerRecords) {
com.airbnb.jitney.event.spinaltap.v1.Mutation mutation = getMutation(consumerRecord.value());
switch(mutation.getType()) {
case INSERT:
Assert.assertEquals(mutation, createMutation(MutationType.INSERT));
break;
case UPDATE:
Assert.assertEquals(mutation, createMutation(MutationType.UPDATE));
break;
case DELETE:
Assert.assertEquals(mutation, createMutation(MutationType.DELETE));
break;
}
}
}
kafkaDestination.close();
kafkaConsumer.close();
}
use of com.google.spanner.v1.Mutation in project SpinalTap by airbnb.
the class KafkaDestinationTest method getMutation.
private Mutation getMutation(byte[] payload) throws Exception {
Mutation mutation = new Mutation();
deserializer.get().deserialize(mutation, payload);
return mutation;
}
Aggregations