Examples with FakeBigQueryServices org.apache.beam.sdk.io.gcp.testing.FakeBigQueryServices used on opensource projects

Example 16 with FakeBigQueryServices

use of org.apache.beam.sdk.io.gcp.testing.FakeBigQueryServices in project beam by apache.

the class BigQueryIOReadTest method checkSetsProject.

private void checkSetsProject(String projectId) throws Exception {
    fakeDatasetService.createDataset(projectId, "dataset-id", "", "", null);
    String tableId = "sometable";
    TableReference tableReference = new TableReference().setProjectId(projectId).setDatasetId("dataset-id").setTableId(tableId);
    fakeDatasetService.createTable(new Table().setTableReference(tableReference).setSchema(new TableSchema().setFields(ImmutableList.of(new TableFieldSchema().setName("name").setType("STRING"), new TableFieldSchema().setName("number").setType("INTEGER")))));
    FakeBigQueryServices fakeBqServices = new FakeBigQueryServices().withJobService(new FakeJobService()).withDatasetService(fakeDatasetService);
    List<TableRow> expected = ImmutableList.of(new TableRow().set("name", "a").set("number", 1L), new TableRow().set("name", "b").set("number", 2L), new TableRow().set("name", "c").set("number", 3L), new TableRow().set("name", "d").set("number", 4L), new TableRow().set("name", "e").set("number", 5L), new TableRow().set("name", "f").set("number", 6L));
    fakeDatasetService.insertAll(tableReference, expected, null);
    TableReference tableRef = new TableReference().setDatasetId("dataset-id").setTableId(tableId);
    PCollection<KV<String, Long>> output = p.apply(BigQueryIO.read().from(tableRef).withTestServices(fakeBqServices)).apply(ParDo.of(new DoFn<TableRow, KV<String, Long>>() {

        @ProcessElement
        public void processElement(ProcessContext c) throws Exception {
            c.output(KV.of((String) c.element().get("name"), Long.valueOf((String) c.element().get("number"))));
        }
    }));
    PAssert.that(output).containsInAnyOrder(ImmutableList.of(KV.of("a", 1L), KV.of("b", 2L), KV.of("c", 3L), KV.of("d", 4L), KV.of("e", 5L), KV.of("f", 6L)));
    p.run();
}

Example 17 with FakeBigQueryServices

use of org.apache.beam.sdk.io.gcp.testing.FakeBigQueryServices in project beam by apache.

the class BigQueryIOReadTest method testReadTableWithSchema.

@Test
public void testReadTableWithSchema() throws IOException, InterruptedException {
    // setup
    Table someTable = new Table();
    someTable.setSchema(new TableSchema().setFields(ImmutableList.of(new TableFieldSchema().setName("name").setType("STRING"), new TableFieldSchema().setName("number").setType("INTEGER"))));
    someTable.setTableReference(new TableReference().setProjectId("non-executing-project").setDatasetId("schema_dataset").setTableId("schema_table"));
    someTable.setNumBytes(1024L * 1024L);
    FakeDatasetService fakeDatasetService = new FakeDatasetService();
    fakeDatasetService.createDataset("non-executing-project", "schema_dataset", "", "", null);
    fakeDatasetService.createTable(someTable);
    List<TableRow> records = Lists.newArrayList(new TableRow().set("name", "a").set("number", 1L), new TableRow().set("name", "b").set("number", 2L), new TableRow().set("name", "c").set("number", 3L));
    fakeDatasetService.insertAll(someTable.getTableReference(), records, null);
    FakeBigQueryServices fakeBqServices = new FakeBigQueryServices().withJobService(new FakeJobService()).withDatasetService(fakeDatasetService);
    // test
    BigQueryIO.TypedRead<TableRow> read = BigQueryIO.readTableRowsWithSchema().from("non-executing-project:schema_dataset.schema_table").withTestServices(fakeBqServices).withoutValidation();
    PCollection<TableRow> bqRows = p.apply(read);
    Schema expectedSchema = Schema.of(Schema.Field.of("name", Schema.FieldType.STRING).withNullable(true), Schema.Field.of("number", Schema.FieldType.INT64).withNullable(true));
    assertEquals(expectedSchema, bqRows.getSchema());
    PCollection<Row> output = bqRows.apply(Select.fieldNames("name", "number"));
    PAssert.that(output).containsInAnyOrder(ImmutableList.of(Row.withSchema(expectedSchema).addValues("a", 1L).build(), Row.withSchema(expectedSchema).addValues("b", 2L).build(), Row.withSchema(expectedSchema).addValues("c", 3L).build()));
    p.run();
}

Example 18 with FakeBigQueryServices

use of org.apache.beam.sdk.io.gcp.testing.FakeBigQueryServices in project beam by apache.

the class BigQueryIOReadTest method testReadFromTable.

private void testReadFromTable(boolean useTemplateCompatibility, boolean useReadTableRows) throws IOException, InterruptedException {
    Table sometable = new Table();
    sometable.setSchema(new TableSchema().setFields(ImmutableList.of(new TableFieldSchema().setName("name").setType("STRING"), new TableFieldSchema().setName("number").setType("INTEGER"))));
    sometable.setTableReference(new TableReference().setProjectId("non-executing-project").setDatasetId("somedataset").setTableId("sometable"));
    sometable.setNumBytes(1024L * 1024L);
    FakeDatasetService fakeDatasetService = new FakeDatasetService();
    fakeDatasetService.createDataset("non-executing-project", "somedataset", "", "", null);
    fakeDatasetService.createTable(sometable);
    List<TableRow> records = Lists.newArrayList(new TableRow().set("name", "a").set("number", 1L), new TableRow().set("name", "b").set("number", 2L), new TableRow().set("name", "c").set("number", 3L));
    fakeDatasetService.insertAll(sometable.getTableReference(), records, null);
    FakeBigQueryServices fakeBqServices = new FakeBigQueryServices().withJobService(new FakeJobService()).withDatasetService(fakeDatasetService);
    PTransform<PBegin, PCollection<TableRow>> readTransform;
    if (useReadTableRows) {
        BigQueryIO.Read read = BigQueryIO.read().from("non-executing-project:somedataset.sometable").withTestServices(fakeBqServices).withoutValidation();
        readTransform = useTemplateCompatibility ? read.withTemplateCompatibility() : read;
    } else {
        BigQueryIO.TypedRead<TableRow> read = BigQueryIO.readTableRows().from("non-executing-project:somedataset.sometable").withTestServices(fakeBqServices).withoutValidation();
        readTransform = useTemplateCompatibility ? read.withTemplateCompatibility() : read;
    }
    PCollection<KV<String, Long>> output = p.apply(readTransform).apply(ParDo.of(new DoFn<TableRow, KV<String, Long>>() {

        @ProcessElement
        public void processElement(ProcessContext c) throws Exception {
            c.output(KV.of((String) c.element().get("name"), Long.valueOf((String) c.element().get("number"))));
        }
    }));
    PAssert.that(output).containsInAnyOrder(ImmutableList.of(KV.of("a", 1L), KV.of("b", 2L), KV.of("c", 3L)));
    p.run();
}

Example 19 with FakeBigQueryServices

use of org.apache.beam.sdk.io.gcp.testing.FakeBigQueryServices in project beam by apache.

the class BigQueryIOStorageReadTest method testReadFromBigQueryIOArrow.

@Test
public void testReadFromBigQueryIOArrow() throws Exception {
    fakeDatasetService.createDataset("foo.com:project", "dataset", "", "", null);
    TableReference tableRef = BigQueryHelpers.parseTableSpec("foo.com:project:dataset.table");
    Table table = new Table().setTableReference(tableRef).setNumBytes(10L).setSchema(TABLE_SCHEMA);
    fakeDatasetService.createTable(table);
    CreateReadSessionRequest expectedCreateReadSessionRequest = CreateReadSessionRequest.newBuilder().setParent("projects/project-id").setReadSession(ReadSession.newBuilder().setTable("projects/foo.com:project/datasets/dataset/tables/table").setDataFormat(DataFormat.ARROW)).setMaxStreamCount(10).build();
    ReadSession readSession = ReadSession.newBuilder().setName("readSessionName").setArrowSchema(ArrowSchema.newBuilder().setSerializedSchema(serializeArrowSchema(ARROW_SCHEMA)).build()).addStreams(ReadStream.newBuilder().setName("streamName")).setDataFormat(DataFormat.ARROW).build();
    ReadRowsRequest expectedReadRowsRequest = ReadRowsRequest.newBuilder().setReadStream("streamName").build();
    List<String> names = Arrays.asList("A", "B", "C", "D");
    List<Long> values = Arrays.asList(1L, 2L, 3L, 4L);
    List<ReadRowsResponse> readRowsResponses = Lists.newArrayList(createResponseArrow(ARROW_SCHEMA, names.subList(0, 2), values.subList(0, 2), 0.0, 0.50), createResponseArrow(ARROW_SCHEMA, names.subList(2, 4), values.subList(2, 4), 0.5, 0.75));
    StorageClient fakeStorageClient = mock(StorageClient.class, withSettings().serializable());
    when(fakeStorageClient.createReadSession(expectedCreateReadSessionRequest)).thenReturn(readSession);
    when(fakeStorageClient.readRows(expectedReadRowsRequest, "")).thenReturn(new FakeBigQueryServerStream<>(readRowsResponses));
    PCollection<KV<String, Long>> output = p.apply(BigQueryIO.read(new ParseKeyValue()).from("foo.com:project:dataset.table").withMethod(Method.DIRECT_READ).withFormat(DataFormat.ARROW).withTestServices(new FakeBigQueryServices().withDatasetService(fakeDatasetService).withStorageClient(fakeStorageClient)));
    PAssert.that(output).containsInAnyOrder(ImmutableList.of(KV.of("A", 1L), KV.of("B", 2L), KV.of("C", 3L), KV.of("D", 4L)));
    p.run();
}

Example 20 with FakeBigQueryServices

use of org.apache.beam.sdk.io.gcp.testing.FakeBigQueryServices in project beam by apache.

the class BigQueryIOStorageReadTest method testFractionConsumedWithSplit.

@Test
public void testFractionConsumedWithSplit() throws Exception {
    ReadSession readSession = ReadSession.newBuilder().setName("readSession").setAvroSchema(AvroSchema.newBuilder().setSchema(AVRO_SCHEMA_STRING)).build();
    ReadRowsRequest expectedRequest = ReadRowsRequest.newBuilder().setReadStream("parentStream").build();
    List<GenericRecord> records = Lists.newArrayList(createRecord("A", 1, AVRO_SCHEMA), createRecord("B", 2, AVRO_SCHEMA), createRecord("C", 3, AVRO_SCHEMA), createRecord("D", 4, AVRO_SCHEMA), createRecord("E", 5, AVRO_SCHEMA), createRecord("F", 6, AVRO_SCHEMA), createRecord("G", 7, AVRO_SCHEMA));
    List<ReadRowsResponse> parentResponses = Lists.newArrayList(createResponse(AVRO_SCHEMA, records.subList(0, 2), 0.000, 0.250), createResponse(AVRO_SCHEMA, records.subList(2, 4), 0.300, 0.500), createResponse(AVRO_SCHEMA, records.subList(4, 7), 0.800, 0.875));
    StorageClient fakeStorageClient = mock(StorageClient.class);
    when(fakeStorageClient.readRows(expectedRequest, "")).thenReturn(new FakeBigQueryServerStream<>(parentResponses));
    when(fakeStorageClient.splitReadStream(SplitReadStreamRequest.newBuilder().setName("parentStream").setFraction(0.5f).build())).thenReturn(SplitReadStreamResponse.newBuilder().setPrimaryStream(ReadStream.newBuilder().setName("primaryStream")).setRemainderStream(ReadStream.newBuilder().setName("remainderStream")).build());
    List<ReadRowsResponse> primaryResponses = Lists.newArrayList(createResponse(AVRO_SCHEMA, records.subList(1, 3), 0.25, 0.75), createResponse(AVRO_SCHEMA, records.subList(3, 4), 0.8, 1.0));
    when(fakeStorageClient.readRows(ReadRowsRequest.newBuilder().setReadStream("primaryStream").setOffset(1).build(), "")).thenReturn(new FakeBigQueryServerStream<>(primaryResponses));
    BigQueryStorageStreamSource<TableRow> streamSource = BigQueryStorageStreamSource.create(readSession, ReadStream.newBuilder().setName("parentStream").build(), TABLE_SCHEMA, new TableRowParser(), TableRowJsonCoder.of(), new FakeBigQueryServices().withStorageClient(fakeStorageClient));
    BoundedReader<TableRow> reader = streamSource.createReader(options);
    // Before call to BoundedReader#start, fraction consumed must be zero.
    assertEquals(0.0, reader.getFractionConsumed(), DELTA);
    // Reads A.
    assertTrue(reader.start());
    assertEquals(0.125, reader.getFractionConsumed(), DELTA);
    reader.splitAtFraction(0.5);
    assertEquals(0.125, reader.getFractionConsumed(), DELTA);
    // Reads B.
    assertTrue(reader.advance());
    assertEquals(0.5, reader.getFractionConsumed(), DELTA);
    // Reads C.
    assertTrue(reader.advance());
    assertEquals(0.75, reader.getFractionConsumed(), DELTA);
    // Reads D.
    assertTrue(reader.advance());
    assertEquals(1.0, reader.getFractionConsumed(), DELTA);
    assertFalse(reader.advance());
    assertEquals(1.0, reader.getFractionConsumed(), DELTA);
}