Search in sources :

Example 41 with Row

use of com.google.api.ads.admanager.axis.v202111.Row in project beam by apache.

the class BigtableWriteIT method getTableData.

/**
 * Helper function to get a table's data.
 */
private List<KV<ByteString, ByteString>> getTableData(String tableName) throws IOException {
    // Add empty range to avoid TARGET_NOT_SET error
    RowRange range = RowRange.newBuilder().setStartKeyClosed(ByteString.EMPTY).setEndKeyOpen(ByteString.EMPTY).build();
    RowSet rowSet = RowSet.newBuilder().addRowRanges(range).build();
    ReadRowsRequest.Builder readRowsRequestBuilder = ReadRowsRequest.newBuilder().setTableName(tableName).setRows(rowSet);
    ResultScanner<Row> scanner = session.getDataClient().readRows(readRowsRequestBuilder.build());
    Row currentRow;
    List<KV<ByteString, ByteString>> tableData = new ArrayList<>();
    while ((currentRow = scanner.next()) != null) {
        ByteString key = currentRow.getKey();
        ByteString value = currentRow.getFamilies(0).getColumns(0).getCells(0).getValue();
        tableData.add(KV.of(key, value));
    }
    scanner.close();
    return tableData;
}
Also used : RowRange(com.google.bigtable.v2.RowRange) ByteString(com.google.protobuf.ByteString) RowSet(com.google.bigtable.v2.RowSet) ArrayList(java.util.ArrayList) ReadRowsRequest(com.google.bigtable.v2.ReadRowsRequest) Row(com.google.bigtable.v2.Row) KV(org.apache.beam.sdk.values.KV)

Example 42 with Row

use of com.google.api.ads.admanager.axis.v202111.Row in project beam by apache.

the class BigtableIOTest method testReadingWithRuntimeParameterizedKeyRange.

/**
 * Tests reading key ranges specified through a ValueProvider.
 */
@Test
public void testReadingWithRuntimeParameterizedKeyRange() throws Exception {
    final String table = "TEST-KEY-RANGE-TABLE";
    final int numRows = 1001;
    List<Row> testRows = makeTableData(table, numRows);
    ByteKey startKey = ByteKey.copyFrom("key000000100".getBytes(StandardCharsets.UTF_8));
    ByteKey endKey = ByteKey.copyFrom("key000000300".getBytes(StandardCharsets.UTF_8));
    service.setupSampleRowKeys(table, numRows / 10, "key000000100".length());
    final ByteKeyRange middleRange = ByteKeyRange.of(startKey, endKey);
    List<Row> middleRows = filterToRange(testRows, middleRange);
    runReadTest(defaultRead.withTableId(table).withKeyRanges(StaticValueProvider.of(Collections.singletonList(middleRange))), middleRows);
    assertThat(middleRows, allOf(hasSize(lessThan(numRows)), hasSize(greaterThan(0))));
}
Also used : ByteKey(org.apache.beam.sdk.io.range.ByteKey) ByteKeyRange(org.apache.beam.sdk.io.range.ByteKeyRange) ByteString(com.google.protobuf.ByteString) Row(com.google.bigtable.v2.Row) Test(org.junit.Test)

Example 43 with Row

use of com.google.api.ads.admanager.axis.v202111.Row in project beam by apache.

the class BigtableIOTest method testReading.

/**
 * Tests reading all rows from a table.
 */
@Test
public void testReading() throws Exception {
    final String table = "TEST-MANY-ROWS-TABLE";
    final int numRows = 1001;
    List<Row> testRows = makeTableData(table, numRows);
    service.setupSampleRowKeys(table, 3, 1000L);
    runReadTest(defaultRead.withTableId(table), testRows);
    logged.verifyInfo(String.format("Closing reader after reading %d records.", numRows / 3));
}
Also used : ByteString(com.google.protobuf.ByteString) Row(com.google.bigtable.v2.Row) Test(org.junit.Test)

Example 44 with Row

use of com.google.api.ads.admanager.axis.v202111.Row in project beam by apache.

the class BigtableIOTest method testReadingWithKeyRanges.

/**
 * Tests reading three key ranges with one read.
 */
@Test
public void testReadingWithKeyRanges() throws Exception {
    final String table = "TEST-KEY-RANGE-TABLE";
    final int numRows = 11;
    List<Row> testRows = makeTableData(table, numRows);
    ByteKey startKey1 = ByteKey.copyFrom("key000000001".getBytes(StandardCharsets.UTF_8));
    ByteKey endKey1 = ByteKey.copyFrom("key000000003".getBytes(StandardCharsets.UTF_8));
    ByteKey startKey2 = ByteKey.copyFrom("key000000004".getBytes(StandardCharsets.UTF_8));
    ByteKey endKey2 = ByteKey.copyFrom("key000000007".getBytes(StandardCharsets.UTF_8));
    ByteKey startKey3 = ByteKey.copyFrom("key000000008".getBytes(StandardCharsets.UTF_8));
    ByteKey endKey3 = ByteKey.copyFrom("key000000009".getBytes(StandardCharsets.UTF_8));
    service.setupSampleRowKeys(table, numRows / 10, "key000000001".length());
    final ByteKeyRange range1 = ByteKeyRange.of(startKey1, endKey1);
    final ByteKeyRange range2 = ByteKeyRange.of(startKey2, endKey2);
    final ByteKeyRange range3 = ByteKeyRange.of(startKey3, endKey3);
    List<ByteKeyRange> ranges = ImmutableList.of(range1, range2, range3);
    List<Row> rangeRows = filterToRanges(testRows, ranges);
    runReadTest(defaultRead.withTableId(table).withKeyRanges(ranges), rangeRows);
    // range rows should be non-trivial (non-zero,non-all).
    assertThat(rangeRows, allOf(hasSize(lessThan(numRows)), hasSize(greaterThan(0))));
}
Also used : ByteKey(org.apache.beam.sdk.io.range.ByteKey) ByteKeyRange(org.apache.beam.sdk.io.range.ByteKeyRange) ByteString(com.google.protobuf.ByteString) Row(com.google.bigtable.v2.Row) Test(org.junit.Test)

Example 45 with Row

use of com.google.api.ads.admanager.axis.v202111.Row in project beam by apache.

the class BigtableIOTest method makeTableData.

/**
 * Helper function to create a table and return the rows that it created.
 */
private static List<Row> makeTableData(String tableId, int numRows) {
    service.createTable(tableId);
    Map<ByteString, ByteString> testData = service.getTable(tableId);
    List<Row> testRows = new ArrayList<>(numRows);
    for (int i = 0; i < numRows; ++i) {
        ByteString key = ByteString.copyFromUtf8(String.format("key%09d", i));
        ByteString value = ByteString.copyFromUtf8(String.format("value%09d", i));
        testData.put(key, value);
        testRows.add(makeRow(key, value));
    }
    return testRows;
}
Also used : ByteString(com.google.protobuf.ByteString) ArrayList(java.util.ArrayList) Row(com.google.bigtable.v2.Row)

Aggregations

Test (org.junit.Test)36 Row (com.google.bigtable.v2.Row)16 ByteString (com.google.protobuf.ByteString)11 Function (com.google.common.base.Function)8 Row (com.google.api.ads.admanager.axis.v202108.Row)6 Row (com.google.api.ads.admanager.axis.v202111.Row)6 Row (com.google.api.ads.admanager.axis.v202202.Row)6 Family (com.google.bigtable.v2.Family)6 ByteKey (org.apache.beam.sdk.io.range.ByteKey)6 Row (com.google.api.ads.admanager.axis.v202105.Row)5 Row (com.google.api.ads.admanager.jaxws.v202105.Row)5 Row (com.google.api.ads.admanager.jaxws.v202108.Row)5 Row (com.google.api.ads.admanager.jaxws.v202111.Row)5 Row (com.google.api.ads.admanager.jaxws.v202202.Row)5 ResultSet (com.google.api.ads.admanager.axis.v202108.ResultSet)4 ResultSet (com.google.api.ads.admanager.axis.v202111.ResultSet)4 ResultSet (com.google.api.ads.admanager.axis.v202202.ResultSet)4 Column (com.google.bigtable.v2.Column)4 ReadRowsRequest (com.google.bigtable.v2.ReadRowsRequest)4 ResultSet (com.google.api.ads.admanager.jaxws.v202105.ResultSet)3