Examples with Field org.apache.kafka.connect.data.Field used on opensource projects

Example 76 with Field

use of org.apache.kafka.connect.data.Field in project debezium by debezium.

the class KeyValueStore method sourceFor.

protected static Struct sourceFor(SourceRecord record) {
    Struct envelope = (Struct) record.value();
    Field field = envelope.schema().field(Envelope.FieldName.SOURCE);
    if (field != null) {
        return envelope.getStruct(field.name());
    }
    return null;
}

Example 77 with Field

use of org.apache.kafka.connect.data.Field in project debezium by debezium.

the class KeyValueStore method valueFor.

protected static Struct valueFor(SourceRecord record) {
    Struct envelope = (Struct) record.value();
    Field afterField = envelope.schema().field(Envelope.FieldName.AFTER);
    if (afterField != null) {
        return envelope.getStruct(afterField.name());
    }
    return null;
}

Example 78 with Field

use of org.apache.kafka.connect.data.Field in project debezium by debezium.

the class VerifyRecord method assertEquals.

@SuppressWarnings("unchecked")
protected static void assertEquals(Schema schema, Object o1, Object o2, String keyOrValue, String field, Predicate<String> ignoreFields, Map<String, RecordValueComparator> comparatorsByName, Map<String, RecordValueComparator> comparatorsBySchemaName) {
    if (o1 == o2)
        return;
    if (o1 == null) {
        if (o2 == null)
            return;
        fail(nameOf(keyOrValue, field) + " was null but expected " + SchemaUtil.asString(o2));
    } else if (o2 == null) {
        fail("expecting a null " + nameOf(keyOrValue, field) + " but found " + SchemaUtil.asString(o1));
    }
    // See if there is a custom comparator for this field ...
    String pathToField = keyOrValue.toUpperCase() + "/" + field;
    RecordValueComparator comparator = comparatorsByName.get(pathToField);
    if (comparator != null) {
        comparator.assertEquals(nameOf(keyOrValue, field), o1, o2);
        return;
    }
    // See if there is a custom comparator for this schema type ...
    String schemaName = schemaName(schema);
    if (schemaName != null) {
        comparator = comparatorsBySchemaName.get(schemaName);
        if (comparator != null) {
            comparator.assertEquals(nameOf(keyOrValue, field), o1, o2);
        }
    }
    if (o1 instanceof ByteBuffer) {
        o1 = ((ByteBuffer) o1).array();
    }
    if (o2 instanceof ByteBuffer) {
        o2 = ((ByteBuffer) o2).array();
    }
    if (o2 instanceof byte[]) {
        if (!(o1 instanceof byte[])) {
            fail("expecting " + nameOf(keyOrValue, field) + " to be byte[] but found " + o1.getClass().toString());
        }
        if (!Arrays.equals((byte[]) o1, (byte[]) o2)) {
            fail("byte[] at " + nameOf(keyOrValue, field) + " is " + o1 + " but was expected to be " + o2);
        }
    } else if (o2 instanceof Object[]) {
        if (!(o1 instanceof Object[])) {
            fail("expecting " + nameOf(keyOrValue, field) + " to be Object[] but was " + o1.getClass().toString());
        }
        if (!deepEquals((Object[]) o1, (Object[]) o2)) {
            fail("Object[] at " + nameOf(keyOrValue, field) + " is " + o1 + " but was expected to be " + o2);
        }
    } else if (o2 instanceof Map) {
        if (!(o1 instanceof Map)) {
            fail("expecting " + nameOf(keyOrValue, field) + " to be Map<String,?> but was " + o1.getClass().toString());
        }
        Map<String, Object> m1 = (Map<String, Object>) o1;
        Map<String, Object> m2 = (Map<String, Object>) o2;
        if (!m1.keySet().equals(m2.keySet())) {
            fail("Map at " + nameOf(keyOrValue, field) + " has entry keys " + m1.keySet() + " but expected " + m2.keySet());
        }
        for (Map.Entry<String, Object> entry : m1.entrySet()) {
            String key = entry.getKey();
            String fieldName = field.isEmpty() ? key : field + "/" + key;
            String predicate = keyOrValue.toUpperCase() + "/" + fieldName;
            if (ignoreFields != null && ignoreFields.test(predicate)) {
                continue;
            }
            Object v1 = entry.getValue();
            Object v2 = m2.get(key);
            assertEquals(null, v1, v2, keyOrValue, fieldName(field, key), ignoreFields, comparatorsByName, comparatorsBySchemaName);
        }
    } else if (o2 instanceof Collection) {
        if (!(o1 instanceof Collection)) {
            fail("expecting " + nameOf(keyOrValue, field) + " to be Collection<?> but was " + o1.getClass().toString());
        }
        Collection<Object> m1 = (Collection<Object>) o1;
        Collection<Object> m2 = (Collection<Object>) o2;
        if (m1.size() != m2.size()) {
            fail("Collection at " + nameOf(keyOrValue, field) + " has " + SchemaUtil.asString(m1) + " but expected " + SchemaUtil.asString(m2));
        }
        Iterator<?> iter1 = m1.iterator();
        Iterator<?> iter2 = m2.iterator();
        int index = 0;
        while (iter1.hasNext() && iter2.hasNext()) {
            assertEquals(null, iter1.next(), iter2.next(), keyOrValue, field + "[" + (index++) + "]", ignoreFields, comparatorsByName, comparatorsBySchemaName);
        }
    } else if (o2 instanceof Struct) {
        if (!(o1 instanceof Struct)) {
            fail("expecting " + nameOf(keyOrValue, field) + " to be Struct but was " + o1.getClass().toString());
        }
        // Unfortunately, the Struct.equals() method has a bug in that it is not using Arrays.deepEquals(...) to
        // compare values in two Struct objects. The result is that the equals only works if the values of the
        // first level Struct are non arrays; otherwise, the array values are compared using == and that obviously
        // does not work for non-primitive values.
        Struct struct1 = (Struct) o1;
        Struct struct2 = (Struct) o2;
        if (!Objects.equals(struct1.schema(), struct2.schema())) {
            fail("Schema at " + nameOf(keyOrValue, field) + " is " + SchemaUtil.asString(struct1.schema()) + " but expected " + SchemaUtil.asString(struct2.schema()));
        }
        for (Field f : struct1.schema().fields()) {
            String fieldName = fieldName(field, f.name());
            String predicate = keyOrValue.toUpperCase() + "/" + fieldName;
            if (ignoreFields != null && ignoreFields.test(predicate)) {
                continue;
            }
            Object value1 = struct1.get(f);
            Object value2 = struct2.get(f);
            assertEquals(f.schema(), value1, value2, keyOrValue, fieldName, ignoreFields, comparatorsByName, comparatorsBySchemaName);
        }
        return;
    } else if (o2 instanceof Double || o2 instanceof Float || o2 instanceof BigDecimal) {
        // Value should be within 1%
        double expectedNumericValue = ((Number) o2).doubleValue();
        double actualNumericValue = ((Number) o1).doubleValue();
        String desc = "found " + nameOf(keyOrValue, field) + " is " + o1 + " but expected " + o2;
        assertThat(actualNumericValue).as(desc).isEqualTo(expectedNumericValue, Delta.delta(0.01d * expectedNumericValue));
    } else if (o2 instanceof Integer || o2 instanceof Long || o2 instanceof Short) {
        long expectedNumericValue = ((Number) o2).longValue();
        long actualNumericValue = ((Number) o1).longValue();
        String desc = "found " + nameOf(keyOrValue, field) + " is " + o1 + " but expected " + o2;
        assertThat(actualNumericValue).as(desc).isEqualTo(expectedNumericValue);
    } else if (o2 instanceof Boolean) {
        boolean expectedValue = ((Boolean) o2).booleanValue();
        boolean actualValue = ((Boolean) o1).booleanValue();
        String desc = "found " + nameOf(keyOrValue, field) + " is " + o1 + " but expected " + o2;
        assertThat(actualValue).as(desc).isEqualTo(expectedValue);
    } else if (ZonedTimestamp.SCHEMA_NAME.equals(schemaName)) {
        // the actual value (produced by the connectors) should always be properly formatted
        String actualValueString = o1.toString();
        ZonedDateTime actualValue = ZonedDateTime.parse(o1.toString(), ZonedTimestamp.FORMATTER);
        String expectedValueString = o2.toString();
        ZonedDateTime expectedValue;
        try {
            // first try a standard offset format which contains the TZ information
            expectedValue = ZonedDateTime.parse(expectedValueString, ZonedTimestamp.FORMATTER);
        } catch (DateTimeParseException e) {
            // then try a local format using the system default offset
            LocalDateTime localDateTime = LocalDateTime.parse(expectedValueString);
            expectedValue = ZonedDateTime.of(localDateTime, ZoneId.systemDefault());
        }
        assertThat(actualValue.toInstant()).as(actualValueString).isEqualTo(expectedValue.toInstant()).as(expectedValueString);
    } else {
        assertThat(o1).isEqualTo(o2);
    }
}

Example 79 with Field

use of org.apache.kafka.connect.data.Field in project debezium by debezium.

the class TableSchemaBuilder method createKeyGenerator.

/**
 * Creates the function that produces a Kafka Connect key object for a row of data.
 *
 * @param schema the Kafka Connect schema for the key; may be null if there is no known schema, in which case the generator
 *            will be null
 * @param columnSetName the name for the set of columns, used in error messages; may not be null
 * @param columns the column definitions for the table that defines the row; may not be null
 * @return the key-generating function, or null if there is no key schema
 */
protected Function<Object[], Object> createKeyGenerator(Schema schema, TableId columnSetName, List<Column> columns) {
    if (schema != null) {
        int[] recordIndexes = indexesForColumns(columns);
        Field[] fields = fieldsForColumns(schema, columns);
        int numFields = recordIndexes.length;
        ValueConverter[] converters = convertersForColumns(schema, columnSetName, columns, null, null);
        return (row) -> {
            Struct result = new Struct(schema);
            for (int i = 0; i != numFields; ++i) {
                Object value = row[recordIndexes[i]];
                ValueConverter converter = converters[i];
                if (converter != null) {
                    value = value == null ? value : converter.convert(value);
                    try {
                        result.put(fields[i], value);
                    } catch (DataException e) {
                        Column col = columns.get(i);
                        LOGGER.error("Failed to properly convert key value for '{}.{}' of type {} for row {}:", columnSetName, col.name(), col.typeName(), row, e);
                    }
                }
            }
            return result;
        };
    }
    return null;
}

Example 80 with Field

use of org.apache.kafka.connect.data.Field in project debezium by debezium.

the class TableSchemaBuilder method createValueGenerator.

/**
 * Creates the function that produces a Kafka Connect value object for a row of data.
 *
 * @param schema the Kafka Connect schema for the value; may be null if there is no known schema, in which case the generator
 *            will be null
 * @param tableId the table identifier; may not be null
 * @param columns the column definitions for the table that defines the row; may not be null
 * @param filter the filter that specifies whether columns in the table should be included; may be null if all columns
 *            are to be included
 * @param mappers the mapping functions for columns; may be null if none of the columns are to be mapped to different values
 * @return the value-generating function, or null if there is no value schema
 */
protected Function<Object[], Struct> createValueGenerator(Schema schema, TableId tableId, List<Column> columns, Predicate<ColumnId> filter, ColumnMappers mappers) {
    if (schema != null) {
        int[] recordIndexes = indexesForColumns(columns);
        Field[] fields = fieldsForColumns(schema, columns);
        int numFields = recordIndexes.length;
        ValueConverter[] converters = convertersForColumns(schema, tableId, columns, filter, mappers);
        return (row) -> {
            Struct result = new Struct(schema);
            for (int i = 0; i != numFields; ++i) {
                Object value = row[recordIndexes[i]];
                ValueConverter converter = converters[i];
                if (converter != null) {
                    try {
                        value = converter.convert(value);
                        result.put(fields[i], value);
                    } catch (DataException | IllegalArgumentException e) {
                        Column col = columns.get(i);
                        LOGGER.error("Failed to properly convert data value for '{}.{}' of type {} for row {}:", tableId, col.name(), col.typeName(), row, e);
                    } catch (final Exception e) {
                        Column col = columns.get(i);
                        LOGGER.error("Failed to properly convert data value for '{}.{}' of type {} for row {}:", tableId, col.name(), col.typeName(), row, e);
                    }
                }
            }
            return result;
        };
    }
    return null;
}