Examples with Field org.apache.pulsar.client.api.schema.Field used on opensource projects

Example 16 with Field

use of org.apache.pulsar.client.api.schema.Field in project pulsar by yahoo.

the class InfluxDBGenericRecordSink method buildPoint.

@Override
protected Point buildPoint(Record<GenericRecord> message) throws Exception {
    Map<String, String> tags;
    Map<String, Object> fields = Maps.newHashMap();
    GenericRecord record = message.getValue();
    Object measurementField = getFiled(record, "measurement");
    if (null == measurementField) {
        throw new SchemaSerializationException("measurement is a required field.");
    }
    String measurement = measurementField.toString();
    // Looking for tags
    Object tagsField = getFiled(record, "tags");
    if (null == tagsField) {
        tags = ImmutableMap.of();
    } else if (Map.class.isAssignableFrom(tagsField.getClass())) {
        tags = ((Map<Object, Object>) tagsField).entrySet().stream().collect(Collectors.toMap(entry -> entry.getKey().toString(), entry -> entry.getValue().toString()));
    } else {
        // Field 'tags' that is not of Map type will be ignored
        tags = ImmutableMap.of();
    }
    // Just insert the current time millis
    long timestamp = System.currentTimeMillis();
    for (Field field : record.getFields()) {
        String fieldName = field.getName();
        if (FIELDS_TO_SKIP.contains(fieldName)) {
            continue;
        }
        Object fieldValue = record.getField(field);
        if (null != fieldValue) {
            fields.put(fieldName, fieldValue);
        }
    }
    Point.Builder builder = Point.measurement(measurement).time(timestamp, TimeUnit.MILLISECONDS).tag(tags).fields(fields);
    return builder.build();
}

Example 17 with Field

use of org.apache.pulsar.client.api.schema.Field in project pulsar by yahoo.

the class GenericJsonRecord method getField.

@Override
public Object getField(String fieldName) {
    JsonNode fn = jn.get(fieldName);
    if (fn == null) {
        return null;
    }
    if (fn.isContainerNode()) {
        AtomicInteger idx = new AtomicInteger(0);
        List<Field> fields = Lists.newArrayList(fn.fieldNames()).stream().map(f -> new Field(f, idx.getAndIncrement())).collect(Collectors.toList());
        return new GenericJsonRecord(schemaVersion, fields, fn, schemaInfo);
    } else if (fn.isBoolean()) {
        return fn.asBoolean();
    } else if (fn.isFloatingPointNumber()) {
        return fn.asDouble();
    } else if (fn.isBigInteger()) {
        if (fn.canConvertToLong()) {
            return fn.asLong();
        } else {
            return fn.asText();
        }
    } else if (fn.isNumber()) {
        return fn.numberValue();
    } else if (fn.isBinary()) {
        try {
            return fn.binaryValue();
        } catch (IOException e) {
            return fn.asText();
        }
    } else if (isBinaryValue(fieldName)) {
        try {
            return fn.binaryValue();
        } catch (IOException e) {
            return fn.asText();
        }
    } else if (fn.isNull()) {
        return null;
    } else {
        return fn.asText();
    }
}

Example 18 with Field

use of org.apache.pulsar.client.api.schema.Field in project cdc-apache-cassandra by datastax.

the class PulsarCassandraSourceTests method testSchema.

// docker exec -it pulsar cat /pulsar/logs/functions/public/default/cassandra-source-ks1-table3/cassandra-source-ks1-table3-0.log
public void testSchema(String ksName, Class<? extends Converter> keyConverter, Class<? extends Converter> valueConverter) throws InterruptedException, IOException {
    try {
        try (CqlSession cqlSession = cassandraContainer1.getCqlSession()) {
            cqlSession.execute("CREATE KEYSPACE IF NOT EXISTS " + ksName + " WITH replication = {'class':'SimpleStrategy','replication_factor':'1'};");
            cqlSession.execute("CREATE TYPE IF NOT EXISTS " + ksName + ".zudt (" + "ztext text, zascii ascii, zboolean boolean, zblob blob, ztimestamp timestamp, ztime time, zdate date, zuuid uuid, ztimeuuid timeuuid, " + "ztinyint tinyint, zsmallint smallint, zint int, zbigint bigint, zvarint varint, zdecimal decimal, zduration duration, zdouble double, " + "zfloat float, zinet4 inet, zinet6 inet, zlist frozen<list<text>>, zset frozen<set<int>>, zmap frozen<map<text, double>>" + ");");
            UserDefinedType zudt = cqlSession.getMetadata().getKeyspace(ksName).flatMap(ks -> ks.getUserDefinedType("zudt")).orElseThrow(() -> new IllegalArgumentException("Missing UDT zudt definition"));
            UdtValue zudtValue = zudt.newValue(dataSpecMap.get("text").cqlValue, dataSpecMap.get("ascii").cqlValue, dataSpecMap.get("boolean").cqlValue, dataSpecMap.get("blob").cqlValue, dataSpecMap.get("timestamp").cqlValue, dataSpecMap.get("time").cqlValue, dataSpecMap.get("date").cqlValue, dataSpecMap.get("uuid").cqlValue, dataSpecMap.get("timeuuid").cqlValue, dataSpecMap.get("tinyint").cqlValue, dataSpecMap.get("smallint").cqlValue, dataSpecMap.get("int").cqlValue, dataSpecMap.get("bigint").cqlValue, dataSpecMap.get("varint").cqlValue, dataSpecMap.get("decimal").cqlValue, dataSpecMap.get("duration").cqlValue, dataSpecMap.get("double").cqlValue, dataSpecMap.get("float").cqlValue, dataSpecMap.get("inet4").cqlValue, dataSpecMap.get("inet6").cqlValue, dataSpecMap.get("list").cqlValue, dataSpecMap.get("set").cqlValue, dataSpecMap.get("map").cqlValue);
            cqlSession.execute("CREATE TABLE IF NOT EXISTS " + ksName + ".table3 (" + "xtext text, xascii ascii, xboolean boolean, xblob blob, xtimestamp timestamp, xtime time, xdate date, xuuid uuid, xtimeuuid timeuuid, xtinyint tinyint, xsmallint smallint, xint int, xbigint bigint, xvarint varint, xdecimal decimal, xdouble double, xfloat float, xinet4 inet, xinet6 inet, " + "ytext text, yascii ascii, yboolean boolean, yblob blob, ytimestamp timestamp, ytime time, ydate date, yuuid uuid, ytimeuuid timeuuid, ytinyint tinyint, ysmallint smallint, yint int, ybigint bigint, yvarint varint, ydecimal decimal, ydouble double, yfloat float, yinet4 inet, yinet6 inet, yduration duration, yudt zudt, ylist list<text>, yset set<int>, ymap map<text, double>, ylistofmap list<frozen<map<text,double>>>, ysetofudt set<frozen<zudt>>," + "primary key (xtext, xascii, xboolean, xblob, xtimestamp, xtime, xdate, xuuid, xtimeuuid, xtinyint, xsmallint, xint, xbigint, xvarint, xdecimal, xdouble, xfloat, xinet4, xinet6)) " + "WITH CLUSTERING ORDER BY (xascii ASC, xboolean DESC, xblob ASC, xtimestamp DESC, xtime DESC, xdate ASC, xuuid DESC, xtimeuuid ASC, xtinyint DESC, xsmallint ASC, xint DESC, xbigint ASC, xvarint DESC, xdecimal ASC, xdouble DESC, xfloat ASC, xinet4 ASC, xinet6 DESC) AND cdc=true");
            cqlSession.execute("INSERT INTO " + ksName + ".table3 (" + "xtext, xascii, xboolean, xblob, xtimestamp, xtime, xdate, xuuid, xtimeuuid, xtinyint, xsmallint, xint, xbigint, xvarint, xdecimal, xdouble, xfloat, xinet4, xinet6, " + "ytext, yascii, yboolean, yblob, ytimestamp, ytime, ydate, yuuid, ytimeuuid, ytinyint, ysmallint, yint, ybigint, yvarint, ydecimal, ydouble, yfloat, yinet4, yinet6, yduration, yudt, ylist, yset, ymap, ylistofmap, ysetofudt" + ") VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, ?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, ?,?, ?,?,?,?,?)", dataSpecMap.get("text").cqlValue, dataSpecMap.get("ascii").cqlValue, dataSpecMap.get("boolean").cqlValue, dataSpecMap.get("blob").cqlValue, dataSpecMap.get("timestamp").cqlValue, dataSpecMap.get("time").cqlValue, dataSpecMap.get("date").cqlValue, dataSpecMap.get("uuid").cqlValue, dataSpecMap.get("timeuuid").cqlValue, dataSpecMap.get("tinyint").cqlValue, dataSpecMap.get("smallint").cqlValue, dataSpecMap.get("int").cqlValue, dataSpecMap.get("bigint").cqlValue, dataSpecMap.get("varint").cqlValue, dataSpecMap.get("decimal").cqlValue, dataSpecMap.get("double").cqlValue, dataSpecMap.get("float").cqlValue, dataSpecMap.get("inet4").cqlValue, dataSpecMap.get("inet6").cqlValue, dataSpecMap.get("text").cqlValue, dataSpecMap.get("ascii").cqlValue, dataSpecMap.get("boolean").cqlValue, dataSpecMap.get("blob").cqlValue, dataSpecMap.get("timestamp").cqlValue, dataSpecMap.get("time").cqlValue, dataSpecMap.get("date").cqlValue, dataSpecMap.get("uuid").cqlValue, dataSpecMap.get("timeuuid").cqlValue, dataSpecMap.get("tinyint").cqlValue, dataSpecMap.get("smallint").cqlValue, dataSpecMap.get("int").cqlValue, dataSpecMap.get("bigint").cqlValue, dataSpecMap.get("varint").cqlValue, dataSpecMap.get("decimal").cqlValue, dataSpecMap.get("double").cqlValue, dataSpecMap.get("float").cqlValue, dataSpecMap.get("inet4").cqlValue, dataSpecMap.get("inet6").cqlValue, dataSpecMap.get("duration").cqlValue, zudtValue, dataSpecMap.get("list").cqlValue, dataSpecMap.get("set").cqlValue, dataSpecMap.get("map").cqlValue, dataSpecMap.get("listofmap").cqlValue, ImmutableSet.of(zudtValue, zudtValue));
        }
        deployConnector(ksName, "table3", keyConverter, valueConverter);
        try (PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsarContainer.getPulsarBrokerUrl()).build()) {
            try (Consumer<GenericRecord> consumer = pulsarClient.newConsumer(org.apache.pulsar.client.api.Schema.AUTO_CONSUME()).topic(String.format(Locale.ROOT, "data-%s.table3", ksName)).subscriptionName("sub1").subscriptionType(SubscriptionType.Key_Shared).subscriptionMode(SubscriptionMode.Durable).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
                int mutationTable3Count = 0;
                Message<GenericRecord> msg;
                while ((msg = consumer.receive(120, TimeUnit.SECONDS)) != null && mutationTable3Count < 1) {
                    GenericObject genericObject = msg.getValue();
                    mutationTable3Count++;
                    assertEquals(SchemaType.KEY_VALUE, genericObject.getSchemaType());
                    KeyValue<GenericRecord, GenericRecord> kv = (KeyValue<GenericRecord, GenericRecord>) genericObject.getNativeObject();
                    GenericRecord key = kv.getKey();
                    GenericRecord value = kv.getValue();
                    // check primary key fields
                    Map<String, Object> keyMap = genericRecordToMap(key);
                    for (Field field : key.getFields()) {
                        assertField(field.getName(), keyMap.get(field.getName()));
                    }
                    // check regular columns.
                    Map<String, Object> valueMap = genericRecordToMap(value);
                    for (Field field : value.getFields()) {
                        assertField(field.getName(), valueMap.get(field.getName()));
                    }
                    consumer.acknowledge(msg);
                }
                assertEquals(1, mutationTable3Count);
            }
        }
    } finally {
        dumpFunctionLogs("cassandra-source-" + ksName + "-table3");
        undeployConnector(ksName, "table3");
    }
}

Example 19 with Field

use of org.apache.pulsar.client.api.schema.Field in project cdc-apache-cassandra by datastax.

the class PulsarSingleNodeTests method testSchema.

@Test
@SuppressWarnings("unchecked")
public void testSchema() throws IOException, InterruptedException {
    final String pulsarServiceUrl = "pulsar://pulsar:" + pulsarContainer.BROKER_PORT;
    try (CassandraContainer<?> cassandraContainer1 = createCassandraContainer(1, pulsarServiceUrl, testNetwork)) {
        cassandraContainer1.start();
        try (CqlSession cqlSession = cassandraContainer1.getCqlSession()) {
            cqlSession.execute("CREATE KEYSPACE IF NOT EXISTS ks2 WITH replication = {'class':'SimpleStrategy','replication_factor':'1'};");
            cqlSession.execute("CREATE TABLE IF NOT EXISTS ks2.table1 (" + "xtext text, xascii ascii, xboolean boolean, xblob blob, xtimestamp timestamp, xtime time, xdate date, xuuid uuid, xtimeuuid timeuuid, xtinyint tinyint, xsmallint smallint, xint int, xbigint bigint, xvarint varint, xdecimal decimal, xdouble double, xfloat float, xinet4 inet, xinet6 inet, " + "primary key (xtext, xascii, xboolean, xblob, xtimestamp, xtime, xdate, xuuid, xtimeuuid, xtinyint, xsmallint, xint, xbigint, xvarint, xdecimal, xdouble, xfloat, xinet4, xinet6)) " + "WITH CLUSTERING ORDER BY (xascii ASC, xboolean DESC, xblob ASC, xtimestamp DESC, xtime DESC, xdate ASC, xuuid DESC, xtimeuuid ASC, xtinyint DESC, xsmallint ASC, xint DESC, xbigint ASC, xvarint DESC, xdecimal ASC, xdouble DESC, xfloat ASC, xinet4 ASC, xinet6 DESC) AND cdc=true");
            cqlSession.execute("INSERT INTO ks2.table1 (xtext, xascii, xboolean, xblob, xtimestamp, xtime, xdate, xuuid, xtimeuuid, xtinyint, xsmallint, xint, xbigint, xvarint, xdecimal, xdouble, xfloat, xinet4, xinet6) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", dataSpecMap.get("text").cqlValue, dataSpecMap.get("ascii").cqlValue, dataSpecMap.get("boolean").cqlValue, dataSpecMap.get("blob").cqlValue, dataSpecMap.get("timestamp").cqlValue, dataSpecMap.get("time").cqlValue, dataSpecMap.get("date").cqlValue, dataSpecMap.get("uuid").cqlValue, dataSpecMap.get("timeuuid").cqlValue, dataSpecMap.get("tinyint").cqlValue, dataSpecMap.get("smallint").cqlValue, dataSpecMap.get("int").cqlValue, dataSpecMap.get("bigint").cqlValue, dataSpecMap.get("varint").cqlValue, dataSpecMap.get("decimal").cqlValue, dataSpecMap.get("double").cqlValue, dataSpecMap.get("float").cqlValue, dataSpecMap.get("inet4").cqlValue, dataSpecMap.get("inet6").cqlValue);
        }
        drain(cassandraContainer1);
        try (PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(pulsarContainer.getPulsarBrokerUrl()).build();
            Consumer<GenericRecord> consumer = pulsarClient.newConsumer(Schema.AUTO_CONSUME()).topic("events-ks2.table1").subscriptionName("sub1").subscriptionType(SubscriptionType.Key_Shared).subscriptionMode(SubscriptionMode.Durable).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe()) {
            Message<GenericRecord> msg = consumer.receive(60, TimeUnit.SECONDS);
            Assert.assertNotNull("Expecting one message, check the agent log", msg);
            GenericRecord gr = msg.getValue();
            KeyValue<GenericRecord, GenericRecord> kv = (KeyValue<GenericRecord, GenericRecord>) gr.getNativeObject();
            GenericRecord key = kv.getKey();
            System.out.println("Consumer Record: topicName=" + msg.getTopicName() + " key=" + AgentTestUtil.genericRecordToString(key));
            Map<String, Object> keyMap = AgentTestUtil.genericRecordToMap(key);
            for (Field field : key.getFields()) {
                String vKey = field.getName().substring(1);
                Assert.assertTrue("Unknown field " + vKey, dataSpecMap.containsKey(vKey));
                if (keyMap.get(field.getName()) instanceof GenericRecord) {
                    AgentTestUtil.assertGenericRecords(vKey, (GenericRecord) keyMap.get(field.getName()), dataSpecMap);
                } else {
                    Assert.assertEquals("Wrong value for PK field " + field.getName(), dataSpecMap.get(vKey).avroValue, keyMap.get(field.getName()));
                }
            }
            consumer.acknowledgeAsync(msg);
        }
    }
}

Example 20 with Field

use of org.apache.pulsar.client.api.schema.Field in project incubator-pulsar by apache.

the class MultiVersionGenericJsonReader method loadReader.

@Override
protected SchemaReader<GenericRecord> loadReader(BytesSchemaVersion schemaVersion) {
    SchemaInfo schemaInfo = getSchemaInfoByVersion(schemaVersion.get());
    if (schemaInfo != null) {
        LOG.info("Load schema reader for version({}), schema is : {}", SchemaUtils.getStringSchemaVersion(schemaVersion.get()), schemaInfo.getSchemaDefinition());
        Schema readerSchema;
        if (useProvidedSchemaAsReaderSchema) {
            readerSchema = this.readerSchema;
        } else {
            readerSchema = parseAvroSchema(schemaInfo.getSchemaDefinition());
        }
        return new GenericJsonReader(schemaVersion.get(), readerSchema.getFields().stream().map(f -> new Field(f.name(), f.pos())).collect(Collectors.toList()), schemaInfo);
    } else {
        LOG.warn("No schema found for version({}), use latest schema : {}", SchemaUtils.getStringSchemaVersion(schemaVersion.get()), this.readerSchema);
        return providerSchemaReader;
    }
}