Examples with Mutation com.airbnb.jitney.event.spinaltap.v1.Mutation used on opensource projects

Example 21 with Mutation

use of com.airbnb.jitney.event.spinaltap.v1.Mutation in project SpinalTap by airbnb.

the class KafkaDestinationTest method getMutation.

private Mutation getMutation(byte[] payload) throws Exception {
    Mutation mutation = new Mutation();
    deserializer.get().deserialize(mutation, payload);
    return mutation;
}

Example 22 with Mutation

use of com.airbnb.jitney.event.spinaltap.v1.Mutation in project beam by apache.

the class DatastoreV1Test method testDeleteEntities.

/**
 * Test that entities with valid keys are transformed to delete mutations.
 */
@Test
public void testDeleteEntities() throws Exception {
    Key key = makeKey("bird", "finch").build();
    Entity entity = Entity.newBuilder().setKey(key).build();
    DeleteEntityFn deleteEntityFn = new DeleteEntityFn();
    Mutation expectedMutation = makeDelete(entity.getKey()).build();
    assertEquals(expectedMutation, deleteEntityFn.apply(entity));
}

Example 23 with Mutation

use of com.airbnb.jitney.event.spinaltap.v1.Mutation in project beam by apache.

the class DatastoreV1Test method testDatatoreWriterFnWithLargeEntities.

/**
 * Tests {@link DatastoreWriterFn} with large entities that need to be split into more batches.
 */
@Test
public void testDatatoreWriterFnWithLargeEntities() throws Exception {
    List<Mutation> mutations = new ArrayList<>();
    int entitySize = 0;
    for (int i = 0; i < 12; ++i) {
        Entity entity = Entity.newBuilder().setKey(makeKey("key" + i, i + 1)).putProperties("long", makeValue(new String(new char[900_000])).setExcludeFromIndexes(true).build()).build();
        // Take the size of any one entity.
        entitySize = entity.getSerializedSize();
        mutations.add(makeUpsert(entity).build());
    }
    DatastoreWriterFn datastoreWriter = new DatastoreWriterFn(StaticValueProvider.of(PROJECT_ID), null, mockDatastoreFactory, new FakeWriteBatcher());
    DoFnTester<Mutation, Void> doFnTester = DoFnTester.of(datastoreWriter);
    doFnTester.setCloningBehavior(CloningBehavior.DO_NOT_CLONE);
    doFnTester.processBundle(mutations);
    // This test is over-specific currently; it requires that we split the 12 entity writes into 3
    // requests, but we only need each CommitRequest to be less than 10MB in size.
    int entitiesPerRpc = DATASTORE_BATCH_UPDATE_BYTES_LIMIT / entitySize;
    int start = 0;
    while (start < mutations.size()) {
        int end = Math.min(mutations.size(), start + entitiesPerRpc);
        CommitRequest.Builder commitRequest = CommitRequest.newBuilder();
        commitRequest.setMode(CommitRequest.Mode.NON_TRANSACTIONAL);
        commitRequest.addAllMutations(mutations.subList(start, end));
        // Verify all the batch requests were made with the expected mutations.
        verify(mockDatastore).commit(commitRequest.build());
        start = end;
    }
}

Example 24 with Mutation

use of com.airbnb.jitney.event.spinaltap.v1.Mutation in project beam by apache.

the class BigtableIOTest method testWritingFailsTableDoesNotExist.

/**
 * Tests that when writing to a non-existent table, the write fails.
 */
@Test
public void testWritingFailsTableDoesNotExist() throws Exception {
    final String table = "TEST-TABLE";
    PCollection<KV<ByteString, Iterable<Mutation>>> emptyInput = p.apply(Create.empty(KvCoder.of(ByteStringCoder.of(), IterableCoder.of(ProtoCoder.of(Mutation.class)))));
    // Exception will be thrown by write.validate() when writeToDynamic is applied.
    thrown.expect(IllegalArgumentException.class);
    thrown.expectMessage(String.format("Table %s does not exist", table));
    emptyInput.apply("write", defaultWrite.withTableId(table));
    p.run();
}

Example 25 with Mutation

use of com.airbnb.jitney.event.spinaltap.v1.Mutation in project beam by apache.

the class BigtableServiceImplTest method testWrite.

/**
 * This test ensures that protobuf creation and interactions with {@link BulkMutation} work as
 * expected.
 *
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public void testWrite() throws IOException, InterruptedException {
    BigtableService.Writer underTest = new BigtableServiceImpl.BigtableWriterImpl(mockSession, TABLE_NAME);
    Mutation mutation = Mutation.newBuilder().setSetCell(SetCell.newBuilder().setFamilyName("Family").build()).build();
    ByteString key = ByteString.copyFromUtf8("key");
    SettableFuture<MutateRowResponse> fakeResponse = SettableFuture.create();
    when(mockBulkMutation.add(any(MutateRowsRequest.Entry.class))).thenReturn(fakeResponse);
    underTest.writeRecord(KV.of(key, ImmutableList.of(mutation)));
    Entry expected = MutateRowsRequest.Entry.newBuilder().setRowKey(key).addMutations(mutation).build();
    verify(mockBulkMutation, times(1)).add(expected);
    underTest.close();
    verify(mockBulkMutation, times(1)).flush();
}