Examples with Schema com.google.pubsub.v1.Schema used on opensource projects

Example 61 with Schema

use of com.google.pubsub.v1.Schema in project DataflowTemplates by GoogleCloudPlatform.

the class PubSubChangeconsumerTest method testBasicRecordAndFilteredRecordInput.

@Test
public void testBasicRecordAndFilteredRecordInput() throws InterruptedException {
    DataCatalogSchemaManager dataCatalogMock = Mockito.mock(DataCatalogSchemaManager.class);
    Publisher pubsubMock = Mockito.mock(Publisher.class);
    Mockito.when(dataCatalogMock.updateSchemaForTable(Mockito.anyString(), Mockito.any(org.apache.beam.sdk.schemas.Schema.class))).thenReturn(Entry.newBuilder().build());
    Mockito.when(pubsubMock.publish(Mockito.any())).thenReturn(Mockito.mock(ApiFuture.class));
    PubSubChangeConsumer changeConsumer = new PubSubChangeConsumer(Sets.newHashSet("mainstance.cdcForDataflow.team_metadata", "table2"), dataCatalogMock, (input1, input2) -> pubsubMock);
    Schema keySchema = SchemaBuilder.struct().field("team", Schema.STRING_SCHEMA).build();
    Struct key = new Struct(keySchema).put("team", "team_PXHU");
    Schema valueAfterSchema = SchemaBuilder.struct().field("team", Schema.STRING_SCHEMA).field("city", Schema.STRING_SCHEMA).field("country", Schema.STRING_SCHEMA).field("year_founded", Schema.INT32_SCHEMA).field("some_timestamp", Schema.INT64_SCHEMA).build();
    Schema valueSchema = SchemaBuilder.struct().field("after", valueAfterSchema).field("op", Schema.STRING_SCHEMA).field("ts_ms", Schema.INT64_SCHEMA).build();
    Struct value = new Struct(valueSchema).put("op", "c").put("ts_ms", 1569287580660L).put("after", new Struct(valueAfterSchema).put("team", "team_PXHU").put("city", "Mexico").put("country", "Mexico as well").put("year_founded", 1916).put("some_timestamp", 123456579L));
    String topicName = "mainstance.cdcForDataflow.team_metadata";
    // We are going to pass two records to be checked, but only one of them belongs to a table that
    // is whitelisted, therefore, only one will be published to pubsub.
    List<SourceRecord> recordBatch = ImmutableList.of(new SourceRecord(ImmutableMap.of("server", "mainstance"), ImmutableMap.of("file", "mysql-bin.000023", "pos", 110489, "gtids", "36797132-a366-11e9-ac33-42010a800456:1-6407169", "row", 1, "snapshot", true), topicName, keySchema, key, valueSchema, value), new SourceRecord(ImmutableMap.of("server", "mainstance"), ImmutableMap.of("file", "mysql-bin.000023", "pos", 110490, "gtids", "36797132-a366-11e9-ac33-42010a800456:1-6407169", "row", 1, "snapshot", true), // A topic that was NOT whitelisted
    "NOTWHITELISTEDTOPIC!", keySchema, key, valueSchema, value));
    RecordCommitter mockCommitter = Mockito.mock(RecordCommitter.class);
    changeConsumer.handleBatch(recordBatch, mockCommitter);
    Mockito.verify(pubsubMock, Mockito.times(1)).publish(Mockito.any(PubsubMessage.class));
    Mockito.verify(mockCommitter).markProcessed(recordBatch.get(0));
    Mockito.verify(mockCommitter).markBatchFinished();
}

Example 62 with Schema

use of com.google.pubsub.v1.Schema in project DataflowTemplates by GoogleCloudPlatform.

the class SchemaUtils method fromBeamField.

private static ColumnSchema fromBeamField(org.apache.beam.sdk.schemas.Schema.Field beamField) {
    ColumnSchema.Builder columnBuilder = ColumnSchema.newBuilder();
    if (beamField.getType().getNullable()) {
        columnBuilder.setMode("NULLABLE");
    } else {
        columnBuilder.setMode("REQUIRED");
    }
    if (beamField.getType().getTypeName() == TypeName.ROW) {
        String columnType = "STRUCT";
        Schema subSchema = fromBeamSchema(beamField.getType().getRowSchema());
        return columnBuilder.setColumn(beamField.getName()).setType(columnType).addAllSubcolumns(subSchema.getColumnsList()).build();
    } else if (LOGICAL_FIELD_TYPES.inverse().containsKey(beamField.getType())) {
        String columnType = LOGICAL_FIELD_TYPES.inverse().get(beamField.getType());
        return columnBuilder.setColumn(beamField.getName()).setType(columnType).build();
    } else {
        String columnType = FIELD_TYPE_NAMES.inverse().get(beamField.getType().getTypeName());
        // TODO(pabloem): Include other characteristics of the field (e.g. required/nullable/etc).
        return columnBuilder.setColumn(beamField.getName()).setType(columnType).build();
    }
}

Example 63 with Schema

use of com.google.pubsub.v1.Schema in project java-pubsub by googleapis.

the class SchemaServiceClientTest method createSchemaExceptionTest2.

@Test
public void createSchemaExceptionTest2() throws Exception {
    StatusRuntimeException exception = new StatusRuntimeException(io.grpc.Status.INVALID_ARGUMENT);
    mockSchemaService.addException(exception);
    try {
        String parent = "parent-995424086";
        Schema schema = Schema.newBuilder().build();
        String schemaId = "schemaId-697673060";
        client.createSchema(parent, schema, schemaId);
        Assert.fail("No exception raised");
    } catch (InvalidArgumentException e) {
    // Expected exception.
    }
}

Example 64 with Schema

use of com.google.pubsub.v1.Schema in project java-pubsub by googleapis.

the class SchemaServiceClientTest method validateSchemaExceptionTest2.

@Test
public void validateSchemaExceptionTest2() throws Exception {
    StatusRuntimeException exception = new StatusRuntimeException(io.grpc.Status.INVALID_ARGUMENT);
    mockSchemaService.addException(exception);
    try {
        String parent = "parent-995424086";
        Schema schema = Schema.newBuilder().build();
        client.validateSchema(parent, schema);
        Assert.fail("No exception raised");
    } catch (InvalidArgumentException e) {
    // Expected exception.
    }
}

Example 65 with Schema

use of com.google.pubsub.v1.Schema in project java-pubsub by googleapis.

the class SchemaServiceClientTest method createSchemaTest2.

@Test
public void createSchemaTest2() throws Exception {
    Schema expectedResponse = Schema.newBuilder().setName(SchemaName.of("[PROJECT]", "[SCHEMA]").toString()).setDefinition("definition-1014418093").build();
    mockSchemaService.addResponse(expectedResponse);
    String parent = "parent-995424086";
    Schema schema = Schema.newBuilder().build();
    String schemaId = "schemaId-697673060";
    Schema actualResponse = client.createSchema(parent, schema, schemaId);
    Assert.assertEquals(expectedResponse, actualResponse);
    List<AbstractMessage> actualRequests = mockSchemaService.getRequests();
    Assert.assertEquals(1, actualRequests.size());
    CreateSchemaRequest actualRequest = ((CreateSchemaRequest) actualRequests.get(0));
    Assert.assertEquals(parent, actualRequest.getParent());
    Assert.assertEquals(schema, actualRequest.getSchema());
    Assert.assertEquals(schemaId, actualRequest.getSchemaId());
    Assert.assertTrue(channelProvider.isHeaderSent(ApiClientHeaderProvider.getDefaultApiClientHeaderKey(), GaxGrpcProperties.getDefaultApiClientHeaderPattern()));
}