Examples with PluginDesc org.apache.kafka.connect.runtime.isolation.PluginDesc used on opensource projects

Example 1 with PluginDesc

use of org.apache.kafka.connect.runtime.isolation.PluginDesc in project apache-kafka-on-k8s by banzaicloud.

the class AbstractHerderTest method testConfigValidationTransformsExtendResults.

@Test()
public void testConfigValidationTransformsExtendResults() {
    AbstractHerder herder = createConfigValidationHerder(TestSourceConnector.class);
    // 2 transform aliases defined -> 2 plugin lookups
    Set<PluginDesc<Transformation>> transformations = new HashSet<>();
    transformations.add(new PluginDesc<Transformation>(SampleTransformation.class, "1.0", classLoader));
    EasyMock.expect(plugins.transformations()).andReturn(transformations).times(2);
    replayAll();
    // Define 2 transformations. One has a class defined and so can get embedded configs, the other is missing
    // class info that should generate an error.
    Map<String, String> config = new HashMap<>();
    config.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, TestSourceConnector.class.getName());
    config.put(ConnectorConfig.NAME_CONFIG, "connector-name");
    config.put(ConnectorConfig.TRANSFORMS_CONFIG, "xformA,xformB");
    config.put(ConnectorConfig.TRANSFORMS_CONFIG + ".xformA.type", SampleTransformation.class.getName());
    // connector required config
    config.put("required", "value");
    ConfigInfos result = herder.validateConnectorConfig(config);
    assertEquals(herder.connectorTypeForClass(config.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG)), ConnectorType.SOURCE);
    // We expect there to be errors due to the missing name and .... Note that these assertions depend heavily on
    // the config fields for SourceConnectorConfig, but we expect these to change rarely.
    assertEquals(TestSourceConnector.class.getName(), result.name());
    // Each transform also gets its own group
    List<String> expectedGroups = Arrays.asList(ConnectorConfig.COMMON_GROUP, ConnectorConfig.TRANSFORMS_GROUP, "Transforms: xformA", "Transforms: xformB");
    assertEquals(expectedGroups, result.groups());
    assertEquals(2, result.errorCount());
    // Base connector config has 7 fields, connector's configs add 2, 2 type fields from the transforms, and
    // 1 from the valid transformation's config
    assertEquals(12, result.values().size());
    // Should get 2 type fields from the transforms, first adds its own config since it has a valid class
    assertEquals("transforms.xformA.type", result.values().get(7).configValue().name());
    assertTrue(result.values().get(7).configValue().errors().isEmpty());
    assertEquals("transforms.xformA.subconfig", result.values().get(8).configValue().name());
    assertEquals("transforms.xformB.type", result.values().get(9).configValue().name());
    assertFalse(result.values().get(9).configValue().errors().isEmpty());
    verifyAll();
}

Example 2 with PluginDesc

use of org.apache.kafka.connect.runtime.isolation.PluginDesc in project kafka by apache.

the class AbstractHerderTest method testConfigValidationTransformsExtendResults.

@Test()
public void testConfigValidationTransformsExtendResults() {
    AbstractHerder herder = createConfigValidationHerder(TestSourceConnector.class, noneConnectorClientConfigOverridePolicy);
    // 2 transform aliases defined -> 2 plugin lookups
    Set<PluginDesc<Transformation<?>>> transformations = new HashSet<>();
    transformations.add(transformationPluginDesc());
    EasyMock.expect(plugins.transformations()).andReturn(transformations).times(2);
    replayAll();
    // Define 2 transformations. One has a class defined and so can get embedded configs, the other is missing
    // class info that should generate an error.
    Map<String, String> config = new HashMap<>();
    config.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, TestSourceConnector.class.getName());
    config.put(ConnectorConfig.NAME_CONFIG, "connector-name");
    config.put(ConnectorConfig.TRANSFORMS_CONFIG, "xformA,xformB");
    config.put(ConnectorConfig.TRANSFORMS_CONFIG + ".xformA.type", SampleTransformation.class.getName());
    // connector required config
    config.put("required", "value");
    ConfigInfos result = herder.validateConnectorConfig(config, false);
    assertEquals(herder.connectorTypeForClass(config.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG)), ConnectorType.SOURCE);
    // We expect there to be errors due to the missing name and .... Note that these assertions depend heavily on
    // the config fields for SourceConnectorConfig, but we expect these to change rarely.
    assertEquals(TestSourceConnector.class.getName(), result.name());
    // Each transform also gets its own group
    List<String> expectedGroups = Arrays.asList(ConnectorConfig.COMMON_GROUP, ConnectorConfig.TRANSFORMS_GROUP, ConnectorConfig.PREDICATES_GROUP, ConnectorConfig.ERROR_GROUP, SourceConnectorConfig.TOPIC_CREATION_GROUP, "Transforms: xformA", "Transforms: xformB");
    assertEquals(expectedGroups, result.groups());
    assertEquals(2, result.errorCount());
    Map<String, ConfigInfo> infos = result.values().stream().collect(Collectors.toMap(info -> info.configKey().name(), Function.identity()));
    assertEquals(22, infos.size());
    // Should get 2 type fields from the transforms, first adds its own config since it has a valid class
    assertEquals("transforms.xformA.type", infos.get("transforms.xformA.type").configValue().name());
    assertTrue(infos.get("transforms.xformA.type").configValue().errors().isEmpty());
    assertEquals("transforms.xformA.subconfig", infos.get("transforms.xformA.subconfig").configValue().name());
    assertEquals("transforms.xformB.type", infos.get("transforms.xformB.type").configValue().name());
    assertFalse(infos.get("transforms.xformB.type").configValue().errors().isEmpty());
    verifyAll();
}

Example 3 with PluginDesc

use of org.apache.kafka.connect.runtime.isolation.PluginDesc in project kafka by apache.

the class AbstractHerderTest method testConfigValidationPredicatesExtendResults.

@Test()
public void testConfigValidationPredicatesExtendResults() {
    AbstractHerder herder = createConfigValidationHerder(TestSourceConnector.class, noneConnectorClientConfigOverridePolicy);
    // 2 transform aliases defined -> 2 plugin lookups
    Set<PluginDesc<Transformation<?>>> transformations = new HashSet<>();
    transformations.add(transformationPluginDesc());
    EasyMock.expect(plugins.transformations()).andReturn(transformations).times(1);
    Set<PluginDesc<Predicate<?>>> predicates = new HashSet<>();
    predicates.add(predicatePluginDesc());
    EasyMock.expect(plugins.predicates()).andReturn(predicates).times(2);
    replayAll();
    // Define 2 transformations. One has a class defined and so can get embedded configs, the other is missing
    // class info that should generate an error.
    Map<String, String> config = new HashMap<>();
    config.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, TestSourceConnector.class.getName());
    config.put(ConnectorConfig.NAME_CONFIG, "connector-name");
    config.put(ConnectorConfig.TRANSFORMS_CONFIG, "xformA");
    config.put(ConnectorConfig.TRANSFORMS_CONFIG + ".xformA.type", SampleTransformation.class.getName());
    config.put(ConnectorConfig.TRANSFORMS_CONFIG + ".xformA.predicate", "predX");
    config.put(ConnectorConfig.PREDICATES_CONFIG, "predX,predY");
    config.put(ConnectorConfig.PREDICATES_CONFIG + ".predX.type", SamplePredicate.class.getName());
    // connector required config
    config.put("required", "value");
    ConfigInfos result = herder.validateConnectorConfig(config, false);
    assertEquals(herder.connectorTypeForClass(config.get(ConnectorConfig.CONNECTOR_CLASS_CONFIG)), ConnectorType.SOURCE);
    // We expect there to be errors due to the missing name and .... Note that these assertions depend heavily on
    // the config fields for SourceConnectorConfig, but we expect these to change rarely.
    assertEquals(TestSourceConnector.class.getName(), result.name());
    // Each transform also gets its own group
    List<String> expectedGroups = Arrays.asList(ConnectorConfig.COMMON_GROUP, ConnectorConfig.TRANSFORMS_GROUP, ConnectorConfig.PREDICATES_GROUP, ConnectorConfig.ERROR_GROUP, SourceConnectorConfig.TOPIC_CREATION_GROUP, "Transforms: xformA", "Predicates: predX", "Predicates: predY");
    assertEquals(expectedGroups, result.groups());
    assertEquals(2, result.errorCount());
    Map<String, ConfigInfo> infos = result.values().stream().collect(Collectors.toMap(info -> info.configKey().name(), Function.identity()));
    assertEquals(24, infos.size());
    // Should get 2 type fields from the transforms, first adds its own config since it has a valid class
    assertEquals("transforms.xformA.type", infos.get("transforms.xformA.type").configValue().name());
    assertTrue(infos.get("transforms.xformA.type").configValue().errors().isEmpty());
    assertEquals("transforms.xformA.subconfig", infos.get("transforms.xformA.subconfig").configValue().name());
    assertEquals("transforms.xformA.predicate", infos.get("transforms.xformA.predicate").configValue().name());
    assertTrue(infos.get("transforms.xformA.predicate").configValue().errors().isEmpty());
    assertEquals("transforms.xformA.negate", infos.get("transforms.xformA.negate").configValue().name());
    assertTrue(infos.get("transforms.xformA.negate").configValue().errors().isEmpty());
    assertEquals("predicates.predX.type", infos.get("predicates.predX.type").configValue().name());
    assertEquals("predicates.predX.predconfig", infos.get("predicates.predX.predconfig").configValue().name());
    assertEquals("predicates.predY.type", infos.get("predicates.predY.type").configValue().name());
    assertFalse(infos.get("predicates.predY.type").configValue().errors().isEmpty());
    verifyAll();
}

Example 4 with PluginDesc

use of org.apache.kafka.connect.runtime.isolation.PluginDesc in project kafka by apache.

the class ConnectorConfig method enrich.

/**
 * Returns an enriched {@link ConfigDef} building upon the {@code ConfigDef}, using the current configuration specified in {@code props} as an input.
 * <p>
 * {@code requireFullConfig} specifies whether required config values that are missing should cause an exception to be thrown.
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
public static ConfigDef enrich(Plugins plugins, ConfigDef baseConfigDef, Map<String, String> props, boolean requireFullConfig) {
    ConfigDef newDef = new ConfigDef(baseConfigDef);
    new EnrichablePlugin<Transformation<?>>("Transformation", TRANSFORMS_CONFIG, TRANSFORMS_GROUP, (Class) Transformation.class, props, requireFullConfig) {

        @SuppressWarnings("rawtypes")
        @Override
        protected Set<PluginDesc<Transformation<?>>> plugins() {
            return plugins.transformations();
        }

        @Override
        protected ConfigDef initialConfigDef() {
            // All Transformations get these config parameters implicitly
            return super.initialConfigDef().define(PredicatedTransformation.PREDICATE_CONFIG, Type.STRING, "", Importance.MEDIUM, "The alias of a predicate used to determine whether to apply this transformation.").define(PredicatedTransformation.NEGATE_CONFIG, Type.BOOLEAN, false, Importance.MEDIUM, "Whether the configured predicate should be negated.");
        }

        @Override
        protected Stream<Map.Entry<String, ConfigDef.ConfigKey>> configDefsForClass(String typeConfig) {
            return super.configDefsForClass(typeConfig).filter(entry -> {
                // The implicit parameters mask any from the transformer with the same name
                if (PredicatedTransformation.PREDICATE_CONFIG.equals(entry.getKey()) || PredicatedTransformation.NEGATE_CONFIG.equals(entry.getKey())) {
                    log.warn("Transformer config {} is masked by implicit config of that name", entry.getKey());
                    return false;
                } else {
                    return true;
                }
            });
        }

        @Override
        protected ConfigDef config(Transformation<?> transformation) {
            return transformation.config();
        }

        @Override
        protected void validateProps(String prefix) {
            String prefixedNegate = prefix + PredicatedTransformation.NEGATE_CONFIG;
            String prefixedPredicate = prefix + PredicatedTransformation.PREDICATE_CONFIG;
            if (props.containsKey(prefixedNegate) && !props.containsKey(prefixedPredicate)) {
                throw new ConfigException("Config '" + prefixedNegate + "' was provided " + "but there is no config '" + prefixedPredicate + "' defining a predicate to be negated.");
            }
        }
    }.enrich(newDef);
    new EnrichablePlugin<Predicate<?>>("Predicate", PREDICATES_CONFIG, PREDICATES_GROUP, (Class) Predicate.class, props, requireFullConfig) {

        @Override
        protected Set<PluginDesc<Predicate<?>>> plugins() {
            return plugins.predicates();
        }

        @Override
        protected ConfigDef config(Predicate<?> predicate) {
            return predicate.config();
        }
    }.enrich(newDef);
    return newDef;
}