Examples with GenericObject org.apache.pulsar.client.api.schema.GenericObject used on opensource projects

Example 1 with GenericObject

use of org.apache.pulsar.client.api.schema.GenericObject in project pulsar by apache.

the class CmdConsume method interpretMessage.

/**
 * Interprets the message to create a string representation.
 *
 * @param message
 *            The message to interpret
 * @param displayHex
 *            Whether to display BytesMessages in hexdump style, ignored for simple text messages
 * @return String representation of the message
 */
private String interpretMessage(Message<?> message, boolean displayHex) throws IOException {
    StringBuilder sb = new StringBuilder();
    String properties = Arrays.toString(message.getProperties().entrySet().toArray());
    String data;
    Object value = message.getValue();
    if (value == null) {
        data = "null";
    } else if (value instanceof byte[]) {
        byte[] msgData = (byte[]) value;
        data = interpretByteArray(displayHex, msgData);
    } else if (value instanceof GenericObject) {
        Map<String, Object> asMap = genericObjectToMap((GenericObject) value, displayHex);
        data = asMap.toString();
    } else if (value instanceof ByteBuffer) {
        data = new String(getBytes((ByteBuffer) value));
    } else {
        data = value.toString();
    }
    String key = null;
    if (message.hasKey()) {
        key = message.getKey();
    }
    sb.append("key:[").append(key).append("], ");
    if (!properties.isEmpty()) {
        sb.append("properties:").append(properties).append(", ");
    }
    sb.append("content:").append(data);
    return sb.toString();
}

Example 2 with GenericObject

use of org.apache.pulsar.client.api.schema.GenericObject in project pulsar by apache.

the class ElasticSearchClientTests method testBulkRetry.

@Test
public void testBulkRetry() throws Exception {
    try (ElasticToxiproxiContainer toxiproxy = new ElasticToxiproxiContainer(container, network)) {
        toxiproxy.start();
        final String index = "indexbulktest-" + UUID.randomUUID();
        ElasticSearchConfig config = new ElasticSearchConfig().setElasticSearchUrl("http://" + toxiproxy.getHttpHostAddress()).setIndexName(index).setBulkEnabled(true).setMaxRetries(1000).setBulkActions(2).setRetryBackoffInMs(100).setBulkFlushIntervalInMs(-1);
        try (ElasticSearchClient client = new ElasticSearchClient(config)) {
            try {
                assertTrue(client.createIndexIfNeeded(index));
                MockRecord<GenericObject> mockRecord = new MockRecord<>();
                client.bulkIndex(mockRecord, Pair.of("1", "{\"a\":1}"));
                client.bulkIndex(mockRecord, Pair.of("2", "{\"a\":2}"));
                assertEquals(mockRecord.acked, 2);
                assertEquals(mockRecord.failed, 0);
                assertEquals(client.totalHits(index), 2);
                log.info("starting the toxic");
                toxiproxy.getProxy().setConnectionCut(false);
                toxiproxy.getProxy().toxics().latency("elasticpause", ToxicDirection.DOWNSTREAM, 15000);
                toxiproxy.removeToxicAfterDelay("elasticpause", 15000);
                client.bulkIndex(mockRecord, Pair.of("3", "{\"a\":3}"));
                assertEquals(mockRecord.acked, 2);
                assertEquals(mockRecord.failed, 0);
                assertEquals(client.totalHits(index), 2);
                client.flush();
                assertEquals(mockRecord.acked, 3);
                assertEquals(mockRecord.failed, 0);
                assertEquals(client.totalHits(index), 3);
            } finally {
                client.delete(index);
            }
        }
    }
}

Example 3 with GenericObject

use of org.apache.pulsar.client.api.schema.GenericObject in project pulsar by apache.

the class ElasticSearchExtractTests method testKeyValueGenericRecord.

@Test(dataProvider = "schemaType")
public void testKeyValueGenericRecord(SchemaType schemaType) throws Exception {
    RecordSchemaBuilder keySchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("key");
    keySchemaBuilder.field("a").type(SchemaType.STRING).optional().defaultValue(null);
    keySchemaBuilder.field("b").type(SchemaType.INT32).optional().defaultValue(null);
    GenericSchema<GenericRecord> keySchema = Schema.generic(keySchemaBuilder.build(schemaType));
    GenericRecord keyGenericRecord = keySchema.newRecordBuilder().set("a", "1").set("b", 1).build();
    RecordSchemaBuilder valueSchemaBuilder = org.apache.pulsar.client.api.schema.SchemaBuilder.record("value");
    valueSchemaBuilder.field("c").type(SchemaType.STRING).optional().defaultValue(null);
    valueSchemaBuilder.field("d").type(SchemaType.INT32).optional().defaultValue(null);
    RecordSchemaBuilder udtSchemaBuilder = SchemaBuilder.record("type1");
    udtSchemaBuilder.field("a").type(SchemaType.STRING).optional().defaultValue(null);
    udtSchemaBuilder.field("b").type(SchemaType.BOOLEAN).optional().defaultValue(null);
    udtSchemaBuilder.field("d").type(SchemaType.DOUBLE).optional().defaultValue(null);
    udtSchemaBuilder.field("f").type(SchemaType.FLOAT).optional().defaultValue(null);
    udtSchemaBuilder.field("i").type(SchemaType.INT32).optional().defaultValue(null);
    udtSchemaBuilder.field("l").type(SchemaType.INT64).optional().defaultValue(null);
    GenericSchema<GenericRecord> udtGenericSchema = Schema.generic(udtSchemaBuilder.build(schemaType));
    valueSchemaBuilder.field("e", udtGenericSchema).type(schemaType).optional().defaultValue(null);
    GenericSchema<GenericRecord> valueSchema = Schema.generic(valueSchemaBuilder.build(schemaType));
    GenericRecord valueGenericRecord = valueSchema.newRecordBuilder().set("c", "1").set("d", 1).set("e", udtGenericSchema.newRecordBuilder().set("a", "a").set("b", true).set("d", 1.0).set("f", 1.0f).set("i", 1).set("l", 10L).build()).build();
    Schema<KeyValue<GenericRecord, GenericRecord>> keyValueSchema = Schema.KeyValue(keySchema, valueSchema, KeyValueEncodingType.INLINE);
    KeyValue<GenericRecord, GenericRecord> keyValue = new KeyValue<>(keyGenericRecord, valueGenericRecord);
    GenericObject genericObject = new GenericObject() {

        @Override
        public SchemaType getSchemaType() {
            return SchemaType.KEY_VALUE;
        }

        @Override
        public Object getNativeObject() {
            return keyValue;
        }
    };
    Record<GenericObject> genericObjectRecord = new Record<GenericObject>() {

        @Override
        public Optional<String> getTopicName() {
            return Optional.of("data-ks1.table1");
        }

        @Override
        public org.apache.pulsar.client.api.Schema getSchema() {
            return keyValueSchema;
        }

        @Override
        public GenericObject getValue() {
            return genericObject;
        }
    };
    ElasticSearchSink elasticSearchSink = new ElasticSearchSink();
    elasticSearchSink.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", "schemaEnable", "true", "keyIgnore", "false"), null);
    Pair<String, String> pair = elasticSearchSink.extractIdAndDocument(genericObjectRecord);
    assertEquals(pair.getLeft(), "[\"1\",1]");
    assertEquals(pair.getRight(), "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}");
    ElasticSearchSink elasticSearchSink2 = new ElasticSearchSink();
    elasticSearchSink2.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", "schemaEnable", "true"), null);
    Pair<String, String> pair2 = elasticSearchSink2.extractIdAndDocument(genericObjectRecord);
    assertNull(pair2.getLeft());
    assertEquals(pair2.getRight(), "{\"c\":\"1\",\"d\":1,\"e\":{\"a\":\"a\",\"b\":true,\"d\":1.0,\"f\":1.0,\"i\":1,\"l\":10}}");
    // test null value
    ElasticSearchSink elasticSearchSink3 = new ElasticSearchSink();
    elasticSearchSink3.open(ImmutableMap.of("elasticSearchUrl", "http://localhost:9200", "schemaEnable", "true", "keyIgnore", "false"), null);
    Pair<String, String> pair3 = elasticSearchSink.extractIdAndDocument(new Record<GenericObject>() {

        @Override
        public Optional<String> getTopicName() {
            return Optional.of("data-ks1.table1");
        }

        @Override
        public org.apache.pulsar.client.api.Schema getSchema() {
            return keyValueSchema;
        }

        @Override
        public GenericObject getValue() {
            return new GenericObject() {

                @Override
                public SchemaType getSchemaType() {
                    return SchemaType.KEY_VALUE;
                }

                @Override
                public Object getNativeObject() {
                    return new KeyValue<>(keyGenericRecord, null);
                }
            };
        }
    });
    assertEquals(pair3.getLeft(), "[\"1\",1]");
    assertNull(pair3.getRight());
}

Example 4 with GenericObject

use of org.apache.pulsar.client.api.schema.GenericObject in project pulsar by apache.

the class ElasticSearchSink method extractIdAndDocument.

/**
 * Extract ES _id and _source using the Schema if available.
 *
 * @param record
 * @return A pair for _id and _source
 */
public Pair<String, String> extractIdAndDocument(Record<GenericObject> record) throws JsonProcessingException {
    if (elasticSearchConfig.isSchemaEnable()) {
        Object key = null;
        GenericObject value = null;
        Schema<?> keySchema = null;
        Schema<?> valueSchema = null;
        if (record.getSchema() != null && record.getSchema() instanceof KeyValueSchema) {
            KeyValueSchema<GenericObject, GenericObject> keyValueSchema = (KeyValueSchema) record.getSchema();
            keySchema = keyValueSchema.getKeySchema();
            valueSchema = keyValueSchema.getValueSchema();
            KeyValue<GenericObject, GenericObject> keyValue = (KeyValue<GenericObject, GenericObject>) record.getValue().getNativeObject();
            key = keyValue.getKey();
            value = keyValue.getValue();
        } else {
            key = record.getKey().orElse(null);
            valueSchema = record.getSchema();
            value = record.getValue();
        }
        String id = null;
        if (!elasticSearchConfig.isKeyIgnore() && key != null && keySchema != null) {
            id = stringifyKey(keySchema, key);
        }
        String doc = null;
        if (value != null) {
            if (valueSchema != null) {
                doc = stringifyValue(valueSchema, value);
            } else {
                if (value.getNativeObject() instanceof byte[]) {
                    // for BWC with the ES-Sink
                    doc = new String((byte[]) value.getNativeObject(), StandardCharsets.UTF_8);
                } else {
                    doc = value.getNativeObject().toString();
                }
            }
        }
        if (doc != null && primaryFields != null) {
            try {
                // extract the PK from the JSON document
                JsonNode jsonNode = objectMapper.readTree(doc);
                id = stringifyKey(jsonNode, primaryFields);
            } catch (JsonProcessingException e) {
                log.error("Failed to read JSON", e);
                throw e;
            }
        }
        if (log.isDebugEnabled()) {
            SchemaType schemaType = null;
            if (record.getSchema() != null && record.getSchema().getSchemaInfo() != null) {
                schemaType = record.getSchema().getSchemaInfo().getType();
            }
            log.debug("recordType={} schemaType={} id={} doc={}", record.getClass().getName(), schemaType, id, doc);
        }
        return Pair.of(id, doc);
    } else {
        return Pair.of(null, new String(record.getMessage().orElseThrow(() -> new IllegalArgumentException("Record does not carry message information")).getData(), StandardCharsets.UTF_8));
    }
}

Example 5 with GenericObject

use of org.apache.pulsar.client.api.schema.GenericObject in project pulsar by apache.

the class KafkaConnectSinkTest method smokeTest.

@Test
public void smokeTest() throws Exception {
    KafkaConnectSink sink = new KafkaConnectSink();
    sink.open(props, context);
    final GenericRecord rec = getGenericRecord("value", Schema.STRING);
    Message msg = mock(MessageImpl.class);
    when(msg.getValue()).thenReturn(rec);
    when(msg.getMessageId()).thenReturn(new MessageIdImpl(1, 0, 0));
    final AtomicInteger status = new AtomicInteger(0);
    Record<GenericObject> record = PulsarRecord.<String>builder().topicName("fake-topic").message(msg).ackFunction(status::incrementAndGet).failFunction(status::decrementAndGet).schema(Schema.STRING).build();
    sink.write(record);
    sink.flush();
    assertEquals(status.get(), 1);
    sink.close();
    List<String> lines = Files.readAllLines(file, StandardCharsets.US_ASCII);
    assertEquals(lines.get(0), "value");
}