Examples with Schema com.google.pubsub.v1.Schema used on opensource projects

Example 76 with Schema

use of com.google.pubsub.v1.Schema in project java-pubsub by googleapis.

the class SubscribeWithAvroSchemaExample method subscribeWithAvroSchemaExample.

public static void subscribeWithAvroSchemaExample(String projectId, String subscriptionId) {
    ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.of(projectId, subscriptionId);
    // Prepare a reader for the encoded Avro records.
    SpecificDatumReader<State> reader = new SpecificDatumReader<>(State.getClassSchema());
    // Instantiate an asynchronous message receiver.
    MessageReceiver receiver = (PubsubMessage message, AckReplyConsumer consumer) -> {
        ByteString data = message.getData();
        // Get the schema encoding type.
        String encoding = message.getAttributesMap().get("googclient_schemaencoding");
        // Send the message data to a byte[] input stream.
        InputStream inputStream = new ByteArrayInputStream(data.toByteArray());
        Decoder decoder = null;
        // based on the schema encoding type.
        block: try {
            switch(encoding) {
                case "BINARY":
                    decoder = DecoderFactory.get().directBinaryDecoder(inputStream, /*reuse=*/
                    null);
                    System.out.println("Receiving a binary-encoded message:");
                    break;
                case "JSON":
                    decoder = DecoderFactory.get().jsonDecoder(State.getClassSchema(), inputStream);
                    System.out.println("Receiving a JSON-encoded message:");
                    break;
                default:
                    break block;
            }
            // Obtain an object of the generated Avro class using the decoder.
            State state = reader.read(null, decoder);
            System.out.println(state.getName() + " is abbreviated as " + state.getPostAbbr());
        } catch (IOException e) {
            System.err.println(e);
        }
        // Ack the message.
        consumer.ack();
    };
    Subscriber subscriber = null;
    try {
        subscriber = Subscriber.newBuilder(subscriptionName, receiver).build();
        subscriber.startAsync().awaitRunning();
        System.out.printf("Listening for messages on %s:\n", subscriptionName.toString());
        subscriber.awaitTerminated(30, TimeUnit.SECONDS);
    } catch (TimeoutException timeoutException) {
        subscriber.stopAsync();
    }
}

Example 77 with Schema

use of com.google.pubsub.v1.Schema in project java-pubsub by googleapis.

the class CreateTopicWithSchemaExample method main.

public static void main(String... args) throws Exception {
    // TODO(developer): Replace these variables before running the sample.
    String projectId = "your-project-id";
    String topicId = "your-topic-id";
    // Use an existing schema.
    String schemaId = "your-schema-id";
    // Choose either BINARY or JSON message serialization in this topic.
    Encoding encoding = Encoding.BINARY;
    createTopicWithSchemaExample(projectId, topicId, schemaId, encoding);
}

Example 78 with Schema

use of com.google.pubsub.v1.Schema in project java-pubsub by googleapis.

the class DeleteSchemaExample method deleteSchemaExample.

public static void deleteSchemaExample(String projectId, String schemaId) throws IOException {
    SchemaName schemaName = SchemaName.of(projectId, schemaId);
    try (SchemaServiceClient schemaServiceClient = SchemaServiceClient.create()) {
        schemaServiceClient.deleteSchema(schemaName);
        System.out.println("Deleted a schema:" + schemaName);
    } catch (NotFoundException e) {
        System.out.println(schemaName + "not found.");
    }
}

Example 79 with Schema

use of com.google.pubsub.v1.Schema in project java-pubsub by googleapis.

the class GetSchemaExample method getSchemaExample.

public static void getSchemaExample(String projectId, String schemaId) throws IOException {
    SchemaName schemaName = SchemaName.of(projectId, schemaId);
    try (SchemaServiceClient schemaServiceClient = SchemaServiceClient.create()) {
        Schema schema = schemaServiceClient.getSchema(schemaName);
        System.out.println("Got a schema:\n" + schema);
    } catch (NotFoundException e) {
        System.out.println(schemaName + "not found.");
    }
}

Example 80 with Schema

use of com.google.pubsub.v1.Schema in project java-docs-samples by GoogleCloudPlatform.

the class StreamingBeamSql method main.

public static void main(final String[] args) {
    Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
    options.setStreaming(true);
    var project = options.as(GcpOptions.class).getProject();
    var subscription = ProjectSubscriptionName.of(project, options.getInputSubscription()).toString();
    var schema = Schema.builder().addStringField("url").addDoubleField("page_score").addDateTimeField("processing_time").build();
    var pipeline = Pipeline.create(options);
    pipeline.apply("Read messages from Pub/Sub", PubsubIO.readStrings().fromSubscription(subscription)).apply("Parse JSON into SQL rows", MapElements.into(TypeDescriptor.of(Row.class)).via(message -> {
        // This is a good place to add error handling.
        // The first transform should act as a validation layer to make sure
        // that any data coming to the processing pipeline must be valid.
        // See `MapElements.MapWithFailures` for more details.
        LOG.info("message: {}", message);
        var msg = GSON.fromJson(message, PageReviewMessage.class);
        return Row.withSchema(schema).addValues(// row url
        msg.url, // row page_score
        msg.review.equals("positive") ? 1.0 : 0.0, // row processing_time
        new Instant()).build();
    })).setRowSchema(// make sure to set the row schema for the PCollection
    schema).apply("Add processing time", WithTimestamps.of((row) -> row.getDateTime("processing_time").toInstant())).apply("Fixed-size windows", Window.into(FixedWindows.of(Duration.standardMinutes(1)))).apply("Run Beam SQL query", SqlTransform.query("SELECT " + "  url, " + "  COUNT(page_score) AS num_reviews, " + "  AVG(page_score) AS score, " + "  MIN(processing_time) AS first_date, " + "  MAX(processing_time) AS last_date " + "FROM PCOLLECTION " + "GROUP BY url")).apply("Convert to BigQuery TableRow", MapElements.into(TypeDescriptor.of(TableRow.class)).via(row -> {
        LOG.info("rating summary: {} {} ({} reviews)", row.getDouble("score"), row.getString("url"), row.getInt64("num_reviews"));
        return new TableRow().set("url", row.getString("url")).set("num_reviews", row.getInt64("num_reviews")).set("score", row.getDouble("score")).set("first_date", row.getDateTime("first_date").toInstant().toString()).set("last_date", row.getDateTime("last_date").toInstant().toString());
    })).apply("Write to BigQuery", BigQueryIO.writeTableRows().to(options.getOutputTable()).withSchema(new TableSchema().setFields(Arrays.asList(// https://cloud.google.com/bigquery/docs/reference/standard-sql/data-types
    new TableFieldSchema().setName("url").setType("STRING"), new TableFieldSchema().setName("num_reviews").setType("INTEGER"), new TableFieldSchema().setName("score").setType("FLOAT64"), new TableFieldSchema().setName("first_date").setType("TIMESTAMP"), new TableFieldSchema().setName("last_date").setType("TIMESTAMP")))).withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED).withWriteDisposition(WriteDisposition.WRITE_APPEND));
    // For a Dataflow Flex Template, do NOT waitUntilFinish().
    pipeline.run();
}