Examples with Transformation org.apache.kafka.connect.transforms.Transformation used on opensource projects

Example 1 with Transformation

use of org.apache.kafka.connect.transforms.Transformation in project connect-utils by jcustenborder.

the class BaseDocumentationTest method before.

@BeforeEach
public void before() throws MalformedURLException {
    log.info("before() - Configuring reflections to use package '{}'", packages());
    if (null == this.reflections) {
        this.reflections = new Reflections(new ConfigurationBuilder().setUrls(ClasspathHelper.forJavaClassPath()).forPackages(packages()));
    }
    this.packages = ImmutableSet.copyOf(packages());
    List<Class<? extends Transformation>> transformClasses = list(Transformation.class);
    List<Class<? extends SourceConnector>> sourceConnectorClasses = list(SourceConnector.class);
    List<Class<? extends SinkConnector>> sinkConnectorClasses = list(SinkConnector.class);
    this.pluginTemplate = PluginTemplate.from(sourceConnectorClasses, sinkConnectorClasses, transformClasses);
}

Example 2 with Transformation

use of org.apache.kafka.connect.transforms.Transformation in project apache-kafka-on-k8s by banzaicloud.

the class AbstractHerderTest method testConfigValidationTransformsExtendResults.

@Test()
public void testConfigValidationTransformsExtendResults() {
    AbstractHerder herder = createConfigValidationHerder(TestSourceConnector.class);
    // 2 transform aliases defined -> 2 plugin lookups
    Set<PluginDesc<Transformation>> transformations = new HashSet<>();
    transformations.add(new PluginDesc<Transformation>(SampleTransformation.class, "1.0", classLoader));
    EasyMock.expect(plugins.transformations()).andReturn(transformations).times(2);
    replayAll();
    // Define 2 transformations. One has a class defined and so can get embedded configs, the other is missing
    // class info that should generate an error.
    Map<String, String> config = new HashMap<>();
    config.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, TestSourceConnector.class.getName());
    config.put(ConnectorConfig.NAME_CONFIG, "connector-name");
    config.put(ConnectorConfig.TRANSFORMS_CONFIG, "xformA,xformB");
    config.put(ConnectorConfig.TRANSFORMS_CONFIG + ".xformA.type", SampleTransformation.class.getName());
    // connector required config
    config.put("required", "value");
    ConfigInfos result = herder.validateConnectorConfig(config);
    assertEquals(herder.connectorTypeForClass(config.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG)), ConnectorType.SOURCE);
    // We expect there to be errors due to the missing name and .... Note that these assertions depend heavily on
    // the config fields for SourceConnectorConfig, but we expect these to change rarely.
    assertEquals(TestSourceConnector.class.getName(), result.name());
    // Each transform also gets its own group
    List<String> expectedGroups = Arrays.asList(ConnectorConfig.COMMON_GROUP, ConnectorConfig.TRANSFORMS_GROUP, "Transforms: xformA", "Transforms: xformB");
    assertEquals(expectedGroups, result.groups());
    assertEquals(2, result.errorCount());
    // Base connector config has 7 fields, connector's configs add 2, 2 type fields from the transforms, and
    // 1 from the valid transformation's config
    assertEquals(12, result.values().size());
    // Should get 2 type fields from the transforms, first adds its own config since it has a valid class
    assertEquals("transforms.xformA.type", result.values().get(7).configValue().name());
    assertTrue(result.values().get(7).configValue().errors().isEmpty());
    assertEquals("transforms.xformA.subconfig", result.values().get(8).configValue().name());
    assertEquals("transforms.xformB.type", result.values().get(9).configValue().name());
    assertFalse(result.values().get(9).configValue().errors().isEmpty());
    verifyAll();
}

Example 3 with Transformation

use of org.apache.kafka.connect.transforms.Transformation in project kafka by apache.

the class ConnectorConfig method transformations.

/**
 * Returns the initialized list of {@link Transformation} which are specified in {@link #TRANSFORMS_CONFIG}.
 */
public <R extends ConnectRecord<R>> List<Transformation<R>> transformations() {
    final List<String> transformAliases = getList(TRANSFORMS_CONFIG);
    final List<Transformation<R>> transformations = new ArrayList<>(transformAliases.size());
    for (String alias : transformAliases) {
        final String prefix = TRANSFORMS_CONFIG + "." + alias + ".";
        try {
            @SuppressWarnings("unchecked") final Transformation<R> transformation = Utils.newInstance(getClass(prefix + "type"), Transformation.class);
            Map<String, Object> configs = originalsWithPrefix(prefix);
            Object predicateAlias = configs.remove(PredicatedTransformation.PREDICATE_CONFIG);
            Object negate = configs.remove(PredicatedTransformation.NEGATE_CONFIG);
            transformation.configure(configs);
            if (predicateAlias != null) {
                String predicatePrefix = PREDICATES_PREFIX + predicateAlias + ".";
                @SuppressWarnings("unchecked") Predicate<R> predicate = Utils.newInstance(getClass(predicatePrefix + "type"), Predicate.class);
                predicate.configure(originalsWithPrefix(predicatePrefix));
                transformations.add(new PredicatedTransformation<>(predicate, negate == null ? false : Boolean.parseBoolean(negate.toString()), transformation));
            } else {
                transformations.add(transformation);
            }
        } catch (Exception e) {
            throw new ConnectException(e);
        }
    }
    return transformations;
}

Example 4 with Transformation

use of org.apache.kafka.connect.transforms.Transformation in project kafka by apache.

the class TopicCreationTest method topicCreationWithTwoGroupsAndTwoTransformations.

@Test
public void topicCreationWithTwoGroupsAndTwoTransformations() {
    short fooReplicas = 3;
    int partitions = 5;
    int barPartitions = 1;
    sourceProps = defaultConnectorPropsWithTopicCreation();
    sourceProps.put(TOPIC_CREATION_GROUPS_CONFIG, String.join(",", FOO_GROUP, BAR_GROUP));
    sourceProps.put(DEFAULT_TOPIC_CREATION_PREFIX + PARTITIONS_CONFIG, String.valueOf(partitions));
    // Setting here but they should be ignored for the default group
    sourceProps.put(TOPIC_CREATION_PREFIX + FOO_GROUP + "." + INCLUDE_REGEX_CONFIG, FOO_TOPIC);
    sourceProps.put(TOPIC_CREATION_PREFIX + FOO_GROUP + "." + REPLICATION_FACTOR_CONFIG, String.valueOf(fooReplicas));
    sourceProps.put(TOPIC_CREATION_PREFIX + BAR_GROUP + "." + INCLUDE_REGEX_CONFIG, BAR_REGEX);
    sourceProps.put(TOPIC_CREATION_PREFIX + BAR_GROUP + "." + PARTITIONS_CONFIG, String.valueOf(barPartitions));
    String castName = "cast";
    String castType = "int8";
    sourceProps.put("transforms." + castName + ".type", Cast.Value.class.getName());
    sourceProps.put("transforms." + castName + ".spec", castType);
    String regexRouterName = "regex";
    sourceProps.put("transforms." + regexRouterName + ".type", RegexRouter.class.getName());
    sourceProps.put("transforms." + regexRouterName + ".regex", "(.*)");
    sourceProps.put("transforms." + regexRouterName + ".replacement", "prefix-$1");
    sourceProps.put("transforms", String.join(",", castName, regexRouterName));
    Map<String, String> fooTopicProps = new HashMap<>();
    fooTopicProps.put(RETENTION_MS_CONFIG, String.valueOf(TimeUnit.DAYS.toMillis(30)));
    fooTopicProps.forEach((k, v) -> sourceProps.put(TOPIC_CREATION_PREFIX + FOO_GROUP + "." + k, v));
    Map<String, String> barTopicProps = new HashMap<>();
    barTopicProps.put(CLEANUP_POLICY_CONFIG, CLEANUP_POLICY_COMPACT);
    barTopicProps.forEach((k, v) -> sourceProps.put(TOPIC_CREATION_PREFIX + BAR_GROUP + "." + k, v));
    // verify config creation
    sourceConfig = new SourceConnectorConfig(MOCK_PLUGINS, sourceProps, true);
    assertTrue(sourceConfig.usesTopicCreation());
    assertEquals(DEFAULT_REPLICATION_FACTOR, (short) sourceConfig.topicCreationReplicationFactor(DEFAULT_TOPIC_CREATION_GROUP));
    assertEquals(partitions, (int) sourceConfig.topicCreationPartitions(DEFAULT_TOPIC_CREATION_GROUP));
    assertThat(sourceConfig.topicCreationInclude(DEFAULT_TOPIC_CREATION_GROUP), is(Collections.singletonList(".*")));
    assertThat(sourceConfig.topicCreationExclude(DEFAULT_TOPIC_CREATION_GROUP), is(Collections.emptyList()));
    assertThat(sourceConfig.topicCreationOtherConfigs(DEFAULT_TOPIC_CREATION_GROUP), is(Collections.emptyMap()));
    // verify topic creation group is instantiated correctly
    Map<String, TopicCreationGroup> groups = TopicCreationGroup.configuredGroups(sourceConfig);
    assertEquals(3, groups.size());
    assertThat(groups.keySet(), hasItems(DEFAULT_TOPIC_CREATION_GROUP, FOO_GROUP, BAR_GROUP));
    // verify topic creation
    TopicCreation topicCreation = TopicCreation.newTopicCreation(workerConfig, groups);
    TopicCreationGroup defaultGroup = topicCreation.defaultTopicGroup();
    // Default group will match all topics besides empty string
    assertTrue(defaultGroup.matches(" "));
    assertTrue(defaultGroup.matches(FOO_TOPIC));
    assertTrue(defaultGroup.matches(BAR_TOPIC));
    assertEquals(DEFAULT_TOPIC_CREATION_GROUP, defaultGroup.name());
    TopicCreationGroup fooGroup = groups.get(FOO_GROUP);
    assertFalse(fooGroup.matches(" "));
    assertTrue(fooGroup.matches(FOO_TOPIC));
    assertFalse(fooGroup.matches(BAR_TOPIC));
    assertEquals(FOO_GROUP, fooGroup.name());
    TopicCreationGroup barGroup = groups.get(BAR_GROUP);
    assertTrue(barGroup.matches(BAR_TOPIC));
    assertFalse(barGroup.matches(FOO_TOPIC));
    assertEquals(BAR_GROUP, barGroup.name());
    assertTrue(topicCreation.isTopicCreationEnabled());
    assertTrue(topicCreation.isTopicCreationRequired(FOO_TOPIC));
    assertTrue(topicCreation.isTopicCreationRequired(BAR_TOPIC));
    assertEquals(2, topicCreation.topicGroups().size());
    assertThat(topicCreation.topicGroups().keySet(), hasItems(FOO_GROUP, BAR_GROUP));
    assertEquals(fooGroup, topicCreation.findFirstGroup(FOO_TOPIC));
    assertEquals(barGroup, topicCreation.findFirstGroup(BAR_TOPIC));
    topicCreation.addTopic(FOO_TOPIC);
    topicCreation.addTopic(BAR_TOPIC);
    assertFalse(topicCreation.isTopicCreationRequired(FOO_TOPIC));
    assertFalse(topicCreation.isTopicCreationRequired(BAR_TOPIC));
    // verify new topic properties
    String otherTopic = "any-other-topic";
    NewTopic defaultTopicSpec = topicCreation.findFirstGroup(otherTopic).newTopic(otherTopic);
    assertEquals(otherTopic, defaultTopicSpec.name());
    assertEquals(DEFAULT_REPLICATION_FACTOR, defaultTopicSpec.replicationFactor());
    assertEquals(partitions, defaultTopicSpec.numPartitions());
    assertThat(defaultTopicSpec.configs(), is(Collections.emptyMap()));
    NewTopic fooTopicSpec = topicCreation.findFirstGroup(FOO_TOPIC).newTopic(FOO_TOPIC);
    assertEquals(FOO_TOPIC, fooTopicSpec.name());
    assertEquals(fooReplicas, fooTopicSpec.replicationFactor());
    assertEquals(partitions, fooTopicSpec.numPartitions());
    assertThat(fooTopicSpec.configs(), is(fooTopicProps));
    NewTopic barTopicSpec = topicCreation.findFirstGroup(BAR_TOPIC).newTopic(BAR_TOPIC);
    assertEquals(BAR_TOPIC, barTopicSpec.name());
    assertEquals(DEFAULT_REPLICATION_FACTOR, barTopicSpec.replicationFactor());
    assertEquals(barPartitions, barTopicSpec.numPartitions());
    assertThat(barTopicSpec.configs(), is(barTopicProps));
    List<Transformation<SourceRecord>> transformations = sourceConfig.transformations();
    assertEquals(2, transformations.size());
    Cast<SourceRecord> castXForm = (Cast<SourceRecord>) transformations.get(0);
    SourceRecord transformed = castXForm.apply(new SourceRecord(null, null, "topic", 0, null, null, Schema.INT8_SCHEMA, 42));
    assertEquals(Schema.Type.INT8, transformed.valueSchema().type());
    assertEquals((byte) 42, transformed.value());
    RegexRouter<SourceRecord> regexRouterXForm = (RegexRouter<SourceRecord>) transformations.get(1);
    transformed = regexRouterXForm.apply(new SourceRecord(null, null, "topic", 0, null, null, Schema.INT8_SCHEMA, 42));
    assertEquals("prefix-topic", transformed.topic());
}

Example 5 with Transformation

use of org.apache.kafka.connect.transforms.Transformation in project connect-utils by jcustenborder.

the class TemplateInput method fromTransformation.

public static TemplateInput fromTransformation(Class<? extends Transformation> transform) throws IllegalAccessException, InstantiationException {
    final TemplateInput result = new TemplateInput();
    Transformation sourceConnector = transform.newInstance();
    ConfigDef config = sourceConnector.config();
    assertNotNull(config, "config() cannot return a null.");
    populateTemplate(transform, result, config);
    return result;
}