use of com.google.datastore.v1.Mutation in project beam by apache.
the class BigtableIOTest method testWritingFailsTableDoesNotExist.
/**
* Tests that when writing to a non-existent table, the write fails.
*/
@Test
public void testWritingFailsTableDoesNotExist() throws Exception {
final String table = "TEST-TABLE";
PCollection<KV<ByteString, Iterable<Mutation>>> emptyInput = p.apply(Create.empty(KvCoder.of(ByteStringCoder.of(), IterableCoder.of(ProtoCoder.of(Mutation.class)))));
// Exception will be thrown by write.validate() when writeToDynamic is applied.
thrown.expect(IllegalArgumentException.class);
thrown.expectMessage(String.format("Table %s does not exist", table));
emptyInput.apply("write", defaultWrite.withTableId(table));
p.run();
}
use of com.google.datastore.v1.Mutation in project beam by apache.
the class BigtableServiceImplTest method testWrite.
/**
* This test ensures that protobuf creation and interactions with {@link BulkMutation} work as
* expected.
*
* @throws IOException
* @throws InterruptedException
*/
@Test
public void testWrite() throws IOException, InterruptedException {
BigtableService.Writer underTest = new BigtableServiceImpl.BigtableWriterImpl(mockSession, TABLE_NAME);
Mutation mutation = Mutation.newBuilder().setSetCell(SetCell.newBuilder().setFamilyName("Family").build()).build();
ByteString key = ByteString.copyFromUtf8("key");
SettableFuture<MutateRowResponse> fakeResponse = SettableFuture.create();
when(mockBulkMutation.add(any(MutateRowsRequest.Entry.class))).thenReturn(fakeResponse);
underTest.writeRecord(KV.of(key, ImmutableList.of(mutation)));
Entry expected = MutateRowsRequest.Entry.newBuilder().setRowKey(key).addMutations(mutation).build();
verify(mockBulkMutation, times(1)).add(expected);
underTest.close();
verify(mockBulkMutation, times(1)).flush();
}
use of com.google.datastore.v1.Mutation in project YCSB by brianfrankcooper.
the class GoogleDatastoreClient method doSingleItemMutation.
private Status doSingleItemMutation(String table, String key, @Nullable Map<String, ByteIterator> values, MutationType mutationType) {
// First build the key.
Key.Builder datastoreKey = buildPrimaryKey(table, key);
// Build a commit request in non-transactional mode.
// Single item mutation to google datastore
// is always atomic and strongly consistent. Transaction is only necessary
// for multi-item mutation, or Read-modify-write operation.
CommitRequest.Builder commitRequest = CommitRequest.newBuilder();
commitRequest.setMode(Mode.NON_TRANSACTIONAL);
if (mutationType == MutationType.DELETE) {
commitRequest.addMutationsBuilder().setDelete(datastoreKey);
} else {
// If this is not for delete, build the entity.
Entity.Builder entityBuilder = Entity.newBuilder();
entityBuilder.setKey(datastoreKey);
for (Entry<String, ByteIterator> val : values.entrySet()) {
entityBuilder.getMutableProperties().put(val.getKey(), Value.newBuilder().setStringValue(val.getValue().toString()).setExcludeFromIndexes(skipIndex).build());
}
Entity entity = entityBuilder.build();
logger.debug("entity built as: " + entity.toString());
if (mutationType == MutationType.UPSERT) {
commitRequest.addMutationsBuilder().setUpsert(entity);
} else if (mutationType == MutationType.UPDATE) {
commitRequest.addMutationsBuilder().setUpdate(entity);
} else {
throw new RuntimeException("Impossible MutationType, code bug.");
}
}
try {
datastore.commit(commitRequest.build());
logger.debug("successfully committed.");
} catch (DatastoreException exception) {
// Catch all Datastore rpc errors.
// Log the exception, the name of the method called and the error code.
logger.error(String.format("Datastore Exception when committing (%s): %s %s", exception.getMessage(), exception.getMethodName(), exception.getCode()));
// will bubble up to the user as part of the YCSB Status "name".
return new Status("ERROR-" + exception.getCode(), exception.getMessage());
}
return Status.OK;
}
use of com.google.datastore.v1.Mutation in project SpinalTap by airbnb.
the class UpdateMutationMapper method map.
public Mutation map(MysqlUpdateMutation mutation) {
MysqlMutationMetadata metadata = mutation.getMetadata();
Mutation thriftMutation = new Mutation(MutationType.UPDATE, metadata.getTimestamp(), sourceId, metadata.getDataSource().getThriftDataSource(), createBinlogHeader(metadata, mutation.getType().getCode()), metadata.getTable().getThriftTable(), transformToEntity(mutation.getRow()));
thriftMutation.setPreviousEntity(transformToEntity(mutation.getPreviousRow()));
return thriftMutation;
}
use of com.google.datastore.v1.Mutation in project SpinalTap by airbnb.
the class KafkaDestinationTest method KafkaDestination.
@Test
public void KafkaDestination() throws Exception {
createKafkaTopic(TOPIC);
KafkaProducerConfiguration configs = new KafkaProducerConfiguration(this.bootstrapServers());
KafkaDestination kafkaDestination = new KafkaDestination(null, configs, null, null, 0L);
List<Mutation> messages = new ArrayList<>();
messages.add(createMutation(MutationType.INSERT));
messages.add(createMutation(MutationType.UPDATE));
messages.add(createMutation(MutationType.DELETE));
kafkaDestination.publish(messages);
Properties props = new Properties();
props.setProperty("bootstrap.servers", this.bootstrapServers());
props.setProperty("key.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer");
props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer");
KafkaConsumer<byte[], byte[]> kafkaConsumer = new KafkaConsumer<>(props);
kafkaConsumer.assign(Collections.singletonList(new TopicPartition(TOPIC, 0)));
kafkaConsumer.seekToBeginning(new TopicPartition(TOPIC, 0));
List<ConsumerRecords<byte[], byte[]>> records = new ArrayList<>();
ConsumerRecords<byte[], byte[]> record;
long startMs = current();
while (current() - startMs <= 10000L) {
record = kafkaConsumer.poll(1000L);
records.add(record);
if (records.size() == 3)
break;
}
Assert.assertEquals(records.size(), 3);
for (ConsumerRecords<byte[], byte[]> consumerRecords : records) {
for (ConsumerRecord<byte[], byte[]> consumerRecord : consumerRecords) {
com.airbnb.jitney.event.spinaltap.v1.Mutation mutation = getMutation(consumerRecord.value());
switch(mutation.getType()) {
case INSERT:
Assert.assertEquals(mutation, createMutation(MutationType.INSERT));
break;
case UPDATE:
Assert.assertEquals(mutation, createMutation(MutationType.UPDATE));
break;
case DELETE:
Assert.assertEquals(mutation, createMutation(MutationType.DELETE));
break;
}
}
}
kafkaDestination.close();
kafkaConsumer.close();
}
Aggregations