Examples with MAIN_TAG org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageToRow.MAIN_TAG used on opensource projects

Example 1 with MAIN_TAG

use of org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageToRow.MAIN_TAG in project beam by apache.

the class PubsubMessageToRowTest method testSendsFlatRowInvalidToDLQ.

@Test
public void testSendsFlatRowInvalidToDLQ() {
    Schema messageSchema = Schema.builder().addDateTimeField("event_timestamp").addInt32Field("id").addStringField("name").build();
    PCollectionTuple outputs = pipeline.apply("create", Create.timestamped(message(1, map("attr1", "val1"), "{ \"invalid1\" : \"sdfsd\" }"), message(2, map("attr2", "val2"), "{ \"invalid2"), message(3, map("attr", "val"), "{ \"id\" : 3, \"name\" : \"foo\" }"), message(4, map("bttr", "vbl"), "{ \"name\" : \"baz\", \"id\" : 5 }"))).apply("convert", PubsubMessageToRow.builder().messageSchema(messageSchema).useDlq(true).useFlatSchema(true).serializerProvider(JSON_SERIALIZER_PROVIDER).build());
    PCollection<Row> rows = outputs.get(MAIN_TAG);
    PCollection<PubsubMessage> dlqMessages = outputs.get(DLQ_TAG);
    PAssert.that(dlqMessages).satisfies(messages -> {
        assertEquals(2, size(messages));
        assertEquals(ImmutableSet.of(map("attr1", "val1"), map("attr2", "val2")), convertToSet(messages, PubsubMessage::getAttributeMap));
        assertEquals(ImmutableSet.of("{ \"invalid1\" : \"sdfsd\" }", "{ \"invalid2"), convertToSet(messages, m -> new String(m.getPayload(), UTF_8)));
        return null;
    });
    PAssert.that(rows).containsInAnyOrder(Row.withSchema(messageSchema).addValues(ts(3), /* map("attr", "val"), */
    3, "foo").build(), Row.withSchema(messageSchema).addValues(ts(4), /* map("bttr", "vbl"), */
    5, "baz").build());
    pipeline.run();
}

Example 2 with MAIN_TAG

use of org.apache.beam.sdk.io.gcp.pubsub.PubsubMessageToRow.MAIN_TAG in project beam by apache.

the class PubsubMessageToRowTest method testSendsInvalidToDLQ.

@Test
public void testSendsInvalidToDLQ() {
    Schema payloadSchema = Schema.builder().addInt32Field("id").addStringField("name").build();
    Schema messageSchema = Schema.builder().addDateTimeField("event_timestamp").addMapField("attributes", FieldType.STRING, FieldType.STRING).addRowField("payload", payloadSchema).build();
    PCollectionTuple outputs = pipeline.apply("create", Create.timestamped(message(1, map("attr1", "val1"), "{ \"invalid1\" : \"sdfsd\" }"), message(2, map("attr2", "val2"), "{ \"invalid2"), message(3, map("attr", "val"), "{ \"id\" : 3, \"name\" : \"foo\" }"), message(4, map("bttr", "vbl"), "{ \"name\" : \"baz\", \"id\" : 5 }"))).apply("convert", PubsubMessageToRow.builder().messageSchema(messageSchema).useDlq(true).useFlatSchema(false).serializerProvider(JSON_SERIALIZER_PROVIDER).build());
    PCollection<Row> rows = outputs.get(MAIN_TAG);
    PCollection<PubsubMessage> dlqMessages = outputs.get(DLQ_TAG);
    PAssert.that(dlqMessages).satisfies(messages -> {
        assertEquals(2, size(messages));
        assertEquals(ImmutableSet.of(map("attr1", "val1"), map("attr2", "val2")), convertToSet(messages, PubsubMessage::getAttributeMap));
        assertEquals(ImmutableSet.of("{ \"invalid1\" : \"sdfsd\" }", "{ \"invalid2"), convertToSet(messages, m -> new String(m.getPayload(), UTF_8)));
        return null;
    });
    PAssert.that(rows).containsInAnyOrder(Row.withSchema(messageSchema).addValues(ts(3), map("attr", "val"), row(payloadSchema, 3, "foo")).build(), Row.withSchema(messageSchema).addValues(ts(4), map("bttr", "vbl"), row(payloadSchema, 5, "baz")).build());
    pipeline.run();
}