Examples with Field org.apache.kafka.connect.data.Field used on opensource projects

Example 1 with Field

use of org.apache.kafka.connect.data.Field in project kafka by apache.

the class MaskFieldTest method withSchema.

@Test
public void withSchema() {
    Schema schema = SchemaBuilder.struct().field("magic", Schema.INT32_SCHEMA).field("bool", Schema.BOOLEAN_SCHEMA).field("byte", Schema.INT8_SCHEMA).field("short", Schema.INT16_SCHEMA).field("int", Schema.INT32_SCHEMA).field("long", Schema.INT64_SCHEMA).field("float", Schema.FLOAT32_SCHEMA).field("double", Schema.FLOAT64_SCHEMA).field("string", Schema.STRING_SCHEMA).field("date", org.apache.kafka.connect.data.Date.SCHEMA).field("time", Time.SCHEMA).field("timestamp", Timestamp.SCHEMA).field("decimal", Decimal.schema(0)).field("array", SchemaBuilder.array(Schema.INT32_SCHEMA)).field("map", SchemaBuilder.map(Schema.STRING_SCHEMA, Schema.STRING_SCHEMA)).build();
    final Struct value = new Struct(schema);
    value.put("magic", 42);
    value.put("bool", true);
    value.put("byte", (byte) 42);
    value.put("short", (short) 42);
    value.put("int", 42);
    value.put("long", 42L);
    value.put("float", 42f);
    value.put("double", 42d);
    value.put("string", "hmm");
    value.put("date", new Date());
    value.put("time", new Date());
    value.put("timestamp", new Date());
    value.put("decimal", new BigDecimal(42));
    value.put("array", Arrays.asList(1, 2, 3));
    value.put("map", Collections.singletonMap("what", "what"));
    final List<String> maskFields = new ArrayList<>(schema.fields().size());
    for (Field field : schema.fields()) {
        if (!field.name().equals("magic")) {
            maskFields.add(field.name());
        }
    }
    final Struct updatedValue = (Struct) transform(maskFields).apply(record(schema, value)).value();
    assertEquals(42, updatedValue.get("magic"));
    assertEquals(false, updatedValue.get("bool"));
    assertEquals((byte) 0, updatedValue.get("byte"));
    assertEquals((short) 0, updatedValue.get("short"));
    assertEquals(0, updatedValue.get("int"));
    assertEquals(0L, updatedValue.get("long"));
    assertEquals(0f, updatedValue.get("float"));
    assertEquals(0d, updatedValue.get("double"));
    assertEquals("", updatedValue.get("string"));
    assertEquals(new Date(0), updatedValue.get("date"));
    assertEquals(new Date(0), updatedValue.get("time"));
    assertEquals(new Date(0), updatedValue.get("timestamp"));
    assertEquals(BigDecimal.ZERO, updatedValue.get("decimal"));
    assertEquals(Collections.emptyList(), updatedValue.get("array"));
    assertEquals(Collections.emptyMap(), updatedValue.get("map"));
}

Example 2 with Field

use of org.apache.kafka.connect.data.Field in project connect-utils by jcustenborder.

the class MarkdownFormatter method toMarkdown.

public static String toMarkdown(Schema schema) {
    Preconditions.checkNotNull(schema, "schema cannot be null.");
    StringBuilder builder = new StringBuilder();
    builder.append("## ");
    if (Strings.isNullOrEmpty(schema.name())) {
        builder.append(schema.type());
    } else {
        builder.append(schema.name());
    }
    if (!Strings.isNullOrEmpty(schema.doc())) {
        builder.append("\n\n");
        builder.append(schema.doc());
    }
    if (Schema.Type.STRUCT == schema.type()) {
        List<List<String>> rows = new ArrayList<>();
        rows.add(ImmutableList.of("Name", "Optional", "Schema", "Default Value", "Documentation"));
        for (Field field : schema.fields()) {
            Schema fieldSchema = field.schema();
            List<String> row = ImmutableList.of(field.name(), String.valueOf(fieldSchema.isOptional()), schema(fieldSchema), fieldSchema.defaultValue() == null ? "" : fieldSchema.defaultValue().toString(), Strings.isNullOrEmpty(fieldSchema.doc()) ? "" : fieldSchema.doc());
            rows.add(row);
        }
        builder.append("\n\n");
        builder.append(markdownTable(rows));
    } else if (Schema.Type.MAP == schema.type()) {
    } else {
    }
    return builder.toString();
}

Example 3 with Field

use of org.apache.kafka.connect.data.Field in project connect-utils by jcustenborder.

the class Parser method parseJsonNode.

public Object parseJsonNode(Schema schema, JsonNode input) {
    checkSchemaAndInput(schema, input);
    if (null == input || input.isNull()) {
        return null;
    }
    log.trace("parseJsonNode() - schema.type() = {}", schema.type());
    Object result;
    if (Schema.Type.STRUCT == schema.type()) {
        Struct struct = new Struct(schema);
        Preconditions.checkState(input.isObject(), "struct schemas require a ObjectNode to be supplied for input.");
        log.trace("parseJsonNode() - Processing as struct.");
        final Set<String> processedFields = Sets.newHashSetWithExpectedSize(schema.fields().size());
        for (Field field : schema.fields()) {
            log.trace("parseJsonNode() - Processing field '{}:{}'", schema.name(), field.name());
            JsonNode fieldInput = input.findValue(field.name());
            try {
                Object convertedValue = parseJsonNode(field.schema(), fieldInput);
                struct.put(field, convertedValue);
            } catch (Exception ex) {
                throw new DataException(String.format("Exception thrown while processing %s:%s", schema.name(), field.name()), ex);
            }
            processedFields.add(field.name());
        }
        if (log.isTraceEnabled()) {
            final Set<String> jsonFieldNames = Sets.newLinkedHashSet(ImmutableList.copyOf(input.fieldNames()));
            Sets.SetView<String> difference = Sets.difference(jsonFieldNames, processedFields);
            if (!difference.isEmpty()) {
                log.trace("parseJsonNode() - Unprocessed fields for {}:\n{}", schema.name(), Joiner.on('\n').join(difference));
            }
        }
        result = struct;
    } else if (Schema.Type.ARRAY == schema.type()) {
        Preconditions.checkState(input.isArray(), "array schemas require a ArrayNode to be supplied for input.");
        log.trace("parseJsonNode() - Processing as array.");
        List<Object> array = new ArrayList<>();
        Iterator<JsonNode> arrayIterator = input.iterator();
        int index = 0;
        while (arrayIterator.hasNext()) {
            log.trace("parseJsonNode() - Processing index {}", index);
            JsonNode arrayInput = arrayIterator.next();
            try {
                Object arrayResult = parseJsonNode(schema.valueSchema(), arrayInput);
                array.add(arrayResult);
            } catch (Exception ex) {
                throw new DataException(String.format("Exception thrown while processing index %s", index), ex);
            }
            index++;
        }
        result = array;
    } else if (Schema.Type.MAP == schema.type()) {
        Preconditions.checkState(input.isObject(), "map schemas require a ObjectNode to be supplied for input.");
        log.trace("parseJsonNode() - Processing as map.");
        Map<Object, Object> map = new LinkedHashMap<>();
        Iterator<String> fieldNameIterator = input.fieldNames();
        while (fieldNameIterator.hasNext()) {
            final String fieldName = fieldNameIterator.next();
            final JsonNode fieldInput = input.findValue(fieldName);
            log.trace("parseJsonNode() - Processing key. Key='{}'", fieldName);
            final Object mapKey;
            try {
                mapKey = parseString(schema.keySchema(), fieldName);
            } catch (Exception ex) {
                throw new DataException(String.format("Exception thrown while parsing key. Key='%s'", fieldName), ex);
            }
            log.trace("parseJsonNode() - Processing value. Key='{}'", fieldName);
            final Object mapValue;
            try {
                mapValue = parseJsonNode(schema.keySchema(), fieldInput);
            } catch (Exception ex) {
                throw new DataException(String.format("Exception thrown while parsing value. Key='%s'", fieldName), ex);
            }
            map.put(mapKey, mapValue);
        }
        result = map;
    } else {
        TypeParser parser = findParser(schema);
        try {
            result = parser.parseJsonNode(input, schema);
        } catch (Exception ex) {
            String message = String.format("Could not parse '%s' to %s", input, parser.expectedClass().getSimpleName());
            throw new DataException(message, ex);
        }
    }
    return result;
}

Example 4 with Field

use of org.apache.kafka.connect.data.Field in project connect-utils by jcustenborder.

the class AssertSchema method assertSchema.

public static void assertSchema(final Schema expected, final Schema actual, String message) {
    final String prefix = Strings.isNullOrEmpty(message) ? "" : message + ": ";
    if (null == expected) {
        assertNull(actual, prefix + "actual should not be null.");
        return;
    }
    assertNotNull(expected, prefix + "expected schema should not be null.");
    assertNotNull(actual, prefix + "actual schema should not be null.");
    assertEquals(expected.name(), actual.name(), prefix + "schema.name() should match.");
    assertEquals(expected.type(), actual.type(), prefix + "schema.type() should match.");
    assertEquals(expected.defaultValue(), actual.defaultValue(), prefix + "schema.defaultValue() should match.");
    assertEquals(expected.isOptional(), actual.isOptional(), prefix + "schema.isOptional() should match.");
    assertEquals(expected.doc(), actual.doc(), prefix + "schema.doc() should match.");
    assertEquals(expected.version(), actual.version(), prefix + "schema.version() should match.");
    assertMap(expected.parameters(), actual.parameters(), prefix + "schema.parameters() should match.");
    if (null != expected.defaultValue()) {
        assertNotNull(actual.defaultValue(), "actual.defaultValue() should not be null.");
        Class<?> expectedType = null;
        switch(expected.type()) {
            case INT8:
                expectedType = Byte.class;
                break;
            case INT16:
                expectedType = Short.class;
                break;
            case INT32:
                expectedType = Integer.class;
                break;
            case INT64:
                expectedType = Long.class;
                break;
            case FLOAT32:
                expectedType = Float.class;
                break;
            case FLOAT64:
                expectedType = Float.class;
                break;
            default:
                break;
        }
        if (null != expectedType) {
            assertTrue(actual.defaultValue().getClass().isAssignableFrom(expectedType), String.format("actual.defaultValue() should be a %s", expectedType.getSimpleName()));
        }
    }
    switch(expected.type()) {
        case ARRAY:
            assertSchema(expected.valueSchema(), actual.valueSchema(), message + "valueSchema does not match.");
            break;
        case MAP:
            assertSchema(expected.keySchema(), actual.keySchema(), message + "keySchema does not match.");
            assertSchema(expected.valueSchema(), actual.valueSchema(), message + "valueSchema does not match.");
            break;
        case STRUCT:
            List<Field> expectedFields = expected.fields();
            List<Field> actualFields = actual.fields();
            assertEquals(expectedFields.size(), actualFields.size(), prefix + "Number of fields do not match.");
            for (int i = 0; i < expectedFields.size(); i++) {
                Field expectedField = expectedFields.get(i);
                Field actualField = actualFields.get(i);
                assertField(expectedField, actualField, "index " + i);
            }
            break;
    }
}

Example 5 with Field

use of org.apache.kafka.connect.data.Field in project connect-utils by jcustenborder.

the class AssertStruct method assertStruct.

public static void assertStruct(final Struct expected, final Struct actual, String message) {
    String prefix = Strings.isNullOrEmpty(message) ? "" : message + ": ";
    if (null == expected) {
        assertNull(actual, prefix + "actual should be null.");
        return;
    }
    assertSchema(expected.schema(), actual.schema(), "schema does not match.");
    final Set<String> logicalTypes = ImmutableSet.of(Timestamp.LOGICAL_NAME, Date.LOGICAL_NAME, Time.LOGICAL_NAME, Decimal.LOGICAL_NAME);
    for (Field field : expected.schema().fields()) {
        log.trace("assertStruct() - testing field '{}'", field.name());
        final Object expectedValue = expected.get(field.name());
        final Object actualValue = actual.get(field.name());
        if (!Strings.isNullOrEmpty(field.schema().name()) && logicalTypes.contains(field.schema().name())) {
            assertEquals(expectedValue, actualValue, prefix + field.name() + " does not match.");
        } else {
            switch(field.schema().type()) {
                case ARRAY:
                    assertTrue(null == expectedValue || expectedValue instanceof List);
                    assertTrue(null == actualValue || actualValue instanceof List);
                    List<Object> expectedArray = (List<Object>) expectedValue;
                    List<Object> actualArray = (List<Object>) actualValue;
                    assertEquals(expectedArray, actualArray, prefix + field.name() + " does not match.");
                    break;
                case MAP:
                    assertTrue(null == expectedValue || expectedValue instanceof Map);
                    assertTrue(null == actualValue || actualValue instanceof Map);
                    Map<Object, Object> expectedMap = (Map<Object, Object>) expectedValue;
                    Map<Object, Object> actualMap = (Map<Object, Object>) actualValue;
                    assertEquals(expectedMap, actualMap, prefix + field.name() + " does not match.");
                    break;
                case STRUCT:
                    assertTrue(null == expectedValue || expectedValue instanceof Struct);
                    assertTrue(null == actualValue || actualValue instanceof Struct);
                    Struct expectedStruct = (Struct) expectedValue;
                    Struct actualStruct = (Struct) actualValue;
                    assertStruct(expectedStruct, actualStruct, prefix + field.name() + " does not match.");
                    break;
                case BYTES:
                    assertTrue(null == expectedValue || expectedValue instanceof byte[]);
                    assertTrue(null == actualValue || actualValue instanceof byte[]);
                    byte[] expectedByteArray = (byte[]) expectedValue;
                    byte[] actualByteArray = (byte[]) actualValue;
                    assertEquals(null == expectedByteArray ? "" : BaseEncoding.base32Hex().encode(expectedByteArray).toString(), null == actualByteArray ? "" : BaseEncoding.base32Hex().encode(actualByteArray).toString(), prefix + field.name() + " does not match.");
                    break;
                default:
                    assertEquals(expectedValue, actualValue, prefix + field.name() + " does not match.");
                    break;
            }
        }
    }
}