use of org.apache.kafka.connect.transforms.Transformation in project apache-kafka-on-k8s by banzaicloud.
the class ConnectorConfig method transformations.
/**
* Returns the initialized list of {@link Transformation} which are specified in {@link #TRANSFORMS_CONFIG}.
*/
public <R extends ConnectRecord<R>> List<Transformation<R>> transformations() {
final List<String> transformAliases = getList(TRANSFORMS_CONFIG);
final List<Transformation<R>> transformations = new ArrayList<>(transformAliases.size());
for (String alias : transformAliases) {
final String prefix = TRANSFORMS_CONFIG + "." + alias + ".";
final Transformation<R> transformation;
try {
transformation = getClass(prefix + "type").asSubclass(Transformation.class).newInstance();
} catch (Exception e) {
throw new ConnectException(e);
}
transformation.configure(originalsWithPrefix(prefix));
transformations.add(transformation);
}
return transformations;
}
use of org.apache.kafka.connect.transforms.Transformation in project kafka by apache.
the class AbstractHerderTest method testConfigValidationTransformsExtendResults.
@Test()
public void testConfigValidationTransformsExtendResults() {
AbstractHerder herder = createConfigValidationHerder(TestSourceConnector.class, noneConnectorClientConfigOverridePolicy);
// 2 transform aliases defined -> 2 plugin lookups
Set<PluginDesc<Transformation<?>>> transformations = new HashSet<>();
transformations.add(transformationPluginDesc());
EasyMock.expect(plugins.transformations()).andReturn(transformations).times(2);
replayAll();
// Define 2 transformations. One has a class defined and so can get embedded configs, the other is missing
// class info that should generate an error.
Map<String, String> config = new HashMap<>();
config.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, TestSourceConnector.class.getName());
config.put(ConnectorConfig.NAME_CONFIG, "connector-name");
config.put(ConnectorConfig.TRANSFORMS_CONFIG, "xformA,xformB");
config.put(ConnectorConfig.TRANSFORMS_CONFIG + ".xformA.type", SampleTransformation.class.getName());
// connector required config
config.put("required", "value");
ConfigInfos result = herder.validateConnectorConfig(config, false);
assertEquals(herder.connectorTypeForClass(config.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG)), ConnectorType.SOURCE);
// We expect there to be errors due to the missing name and .... Note that these assertions depend heavily on
// the config fields for SourceConnectorConfig, but we expect these to change rarely.
assertEquals(TestSourceConnector.class.getName(), result.name());
// Each transform also gets its own group
List<String> expectedGroups = Arrays.asList(ConnectorConfig.COMMON_GROUP, ConnectorConfig.TRANSFORMS_GROUP, ConnectorConfig.PREDICATES_GROUP, ConnectorConfig.ERROR_GROUP, SourceConnectorConfig.TOPIC_CREATION_GROUP, "Transforms: xformA", "Transforms: xformB");
assertEquals(expectedGroups, result.groups());
assertEquals(2, result.errorCount());
Map<String, ConfigInfo> infos = result.values().stream().collect(Collectors.toMap(info -> info.configKey().name(), Function.identity()));
assertEquals(22, infos.size());
// Should get 2 type fields from the transforms, first adds its own config since it has a valid class
assertEquals("transforms.xformA.type", infos.get("transforms.xformA.type").configValue().name());
assertTrue(infos.get("transforms.xformA.type").configValue().errors().isEmpty());
assertEquals("transforms.xformA.subconfig", infos.get("transforms.xformA.subconfig").configValue().name());
assertEquals("transforms.xformB.type", infos.get("transforms.xformB.type").configValue().name());
assertFalse(infos.get("transforms.xformB.type").configValue().errors().isEmpty());
verifyAll();
}
use of org.apache.kafka.connect.transforms.Transformation in project kafka by apache.
the class AbstractHerderTest method testConfigValidationPredicatesExtendResults.
@Test()
public void testConfigValidationPredicatesExtendResults() {
AbstractHerder herder = createConfigValidationHerder(TestSourceConnector.class, noneConnectorClientConfigOverridePolicy);
// 2 transform aliases defined -> 2 plugin lookups
Set<PluginDesc<Transformation<?>>> transformations = new HashSet<>();
transformations.add(transformationPluginDesc());
EasyMock.expect(plugins.transformations()).andReturn(transformations).times(1);
Set<PluginDesc<Predicate<?>>> predicates = new HashSet<>();
predicates.add(predicatePluginDesc());
EasyMock.expect(plugins.predicates()).andReturn(predicates).times(2);
replayAll();
// Define 2 transformations. One has a class defined and so can get embedded configs, the other is missing
// class info that should generate an error.
Map<String, String> config = new HashMap<>();
config.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, TestSourceConnector.class.getName());
config.put(ConnectorConfig.NAME_CONFIG, "connector-name");
config.put(ConnectorConfig.TRANSFORMS_CONFIG, "xformA");
config.put(ConnectorConfig.TRANSFORMS_CONFIG + ".xformA.type", SampleTransformation.class.getName());
config.put(ConnectorConfig.TRANSFORMS_CONFIG + ".xformA.predicate", "predX");
config.put(ConnectorConfig.PREDICATES_CONFIG, "predX,predY");
config.put(ConnectorConfig.PREDICATES_CONFIG + ".predX.type", SamplePredicate.class.getName());
// connector required config
config.put("required", "value");
ConfigInfos result = herder.validateConnectorConfig(config, false);
assertEquals(herder.connectorTypeForClass(config.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG)), ConnectorType.SOURCE);
// We expect there to be errors due to the missing name and .... Note that these assertions depend heavily on
// the config fields for SourceConnectorConfig, but we expect these to change rarely.
assertEquals(TestSourceConnector.class.getName(), result.name());
// Each transform also gets its own group
List<String> expectedGroups = Arrays.asList(ConnectorConfig.COMMON_GROUP, ConnectorConfig.TRANSFORMS_GROUP, ConnectorConfig.PREDICATES_GROUP, ConnectorConfig.ERROR_GROUP, SourceConnectorConfig.TOPIC_CREATION_GROUP, "Transforms: xformA", "Predicates: predX", "Predicates: predY");
assertEquals(expectedGroups, result.groups());
assertEquals(2, result.errorCount());
Map<String, ConfigInfo> infos = result.values().stream().collect(Collectors.toMap(info -> info.configKey().name(), Function.identity()));
assertEquals(24, infos.size());
// Should get 2 type fields from the transforms, first adds its own config since it has a valid class
assertEquals("transforms.xformA.type", infos.get("transforms.xformA.type").configValue().name());
assertTrue(infos.get("transforms.xformA.type").configValue().errors().isEmpty());
assertEquals("transforms.xformA.subconfig", infos.get("transforms.xformA.subconfig").configValue().name());
assertEquals("transforms.xformA.predicate", infos.get("transforms.xformA.predicate").configValue().name());
assertTrue(infos.get("transforms.xformA.predicate").configValue().errors().isEmpty());
assertEquals("transforms.xformA.negate", infos.get("transforms.xformA.negate").configValue().name());
assertTrue(infos.get("transforms.xformA.negate").configValue().errors().isEmpty());
assertEquals("predicates.predX.type", infos.get("predicates.predX.type").configValue().name());
assertEquals("predicates.predX.predconfig", infos.get("predicates.predX.predconfig").configValue().name());
assertEquals("predicates.predY.type", infos.get("predicates.predY.type").configValue().name());
assertFalse(infos.get("predicates.predY.type").configValue().errors().isEmpty());
verifyAll();
}
use of org.apache.kafka.connect.transforms.Transformation in project kafka by apache.
the class ConnectorConfig method enrich.
/**
* Returns an enriched {@link ConfigDef} building upon the {@code ConfigDef}, using the current configuration specified in {@code props} as an input.
* <p>
* {@code requireFullConfig} specifies whether required config values that are missing should cause an exception to be thrown.
*/
@SuppressWarnings({ "rawtypes", "unchecked" })
public static ConfigDef enrich(Plugins plugins, ConfigDef baseConfigDef, Map<String, String> props, boolean requireFullConfig) {
ConfigDef newDef = new ConfigDef(baseConfigDef);
new EnrichablePlugin<Transformation<?>>("Transformation", TRANSFORMS_CONFIG, TRANSFORMS_GROUP, (Class) Transformation.class, props, requireFullConfig) {
@SuppressWarnings("rawtypes")
@Override
protected Set<PluginDesc<Transformation<?>>> plugins() {
return plugins.transformations();
}
@Override
protected ConfigDef initialConfigDef() {
// All Transformations get these config parameters implicitly
return super.initialConfigDef().define(PredicatedTransformation.PREDICATE_CONFIG, Type.STRING, "", Importance.MEDIUM, "The alias of a predicate used to determine whether to apply this transformation.").define(PredicatedTransformation.NEGATE_CONFIG, Type.BOOLEAN, false, Importance.MEDIUM, "Whether the configured predicate should be negated.");
}
@Override
protected Stream<Map.Entry<String, ConfigDef.ConfigKey>> configDefsForClass(String typeConfig) {
return super.configDefsForClass(typeConfig).filter(entry -> {
// The implicit parameters mask any from the transformer with the same name
if (PredicatedTransformation.PREDICATE_CONFIG.equals(entry.getKey()) || PredicatedTransformation.NEGATE_CONFIG.equals(entry.getKey())) {
log.warn("Transformer config {} is masked by implicit config of that name", entry.getKey());
return false;
} else {
return true;
}
});
}
@Override
protected ConfigDef config(Transformation<?> transformation) {
return transformation.config();
}
@Override
protected void validateProps(String prefix) {
String prefixedNegate = prefix + PredicatedTransformation.NEGATE_CONFIG;
String prefixedPredicate = prefix + PredicatedTransformation.PREDICATE_CONFIG;
if (props.containsKey(prefixedNegate) && !props.containsKey(prefixedPredicate)) {
throw new ConfigException("Config '" + prefixedNegate + "' was provided " + "but there is no config '" + prefixedPredicate + "' defining a predicate to be negated.");
}
}
}.enrich(newDef);
new EnrichablePlugin<Predicate<?>>("Predicate", PREDICATES_CONFIG, PREDICATES_GROUP, (Class) Predicate.class, props, requireFullConfig) {
@Override
protected Set<PluginDesc<Predicate<?>>> plugins() {
return plugins.predicates();
}
@Override
protected ConfigDef config(Predicate<?> predicate) {
return predicate.config();
}
}.enrich(newDef);
return newDef;
}
use of org.apache.kafka.connect.transforms.Transformation in project kafka by apache.
the class TopicCreationTest method testTopicCreationWithSingleTransformation.
@Test
public void testTopicCreationWithSingleTransformation() {
sourceProps = defaultConnectorPropsWithTopicCreation();
sourceProps.put(TOPIC_CREATION_GROUPS_CONFIG, String.join(",", FOO_GROUP, BAR_GROUP));
String xformName = "example";
String castType = "int8";
sourceProps.put("transforms", xformName);
sourceProps.put("transforms." + xformName + ".type", Cast.Value.class.getName());
sourceProps.put("transforms." + xformName + ".spec", castType);
sourceConfig = new SourceConnectorConfig(MOCK_PLUGINS, sourceProps, true);
Map<String, TopicCreationGroup> groups = TopicCreationGroup.configuredGroups(sourceConfig);
TopicCreation topicCreation = TopicCreation.newTopicCreation(workerConfig, groups);
assertTrue(topicCreation.isTopicCreationEnabled());
assertTrue(topicCreation.isTopicCreationRequired(FOO_TOPIC));
assertThat(topicCreation.defaultTopicGroup(), is(groups.get(DEFAULT_TOPIC_CREATION_GROUP)));
assertEquals(2, topicCreation.topicGroups().size());
assertThat(topicCreation.topicGroups().keySet(), hasItems(FOO_GROUP, BAR_GROUP));
assertEquals(topicCreation.defaultTopicGroup(), topicCreation.findFirstGroup(FOO_TOPIC));
topicCreation.addTopic(FOO_TOPIC);
assertFalse(topicCreation.isTopicCreationRequired(FOO_TOPIC));
List<Transformation<SourceRecord>> transformations = sourceConfig.transformations();
assertEquals(1, transformations.size());
Cast<SourceRecord> xform = (Cast<SourceRecord>) transformations.get(0);
SourceRecord transformed = xform.apply(new SourceRecord(null, null, "topic", 0, null, null, Schema.INT8_SCHEMA, 42));
assertEquals(Schema.Type.INT8, transformed.valueSchema().type());
assertEquals((byte) 42, transformed.value());
}
Aggregations