Search in sources :

Example 1 with MacroEvaluator

use of io.cdap.cdap.api.macro.MacroEvaluator in project cdap by caskdata.

the class PipelinePhase method registerPlugins.

/**
 * Registers all the plugin to the given pluginConfigurer by calling {@link PluginConfigurer#usePluginClass(String,
 * String, String, PluginProperties, PluginSelector)}
 *
 * @param pluginConfigurer the {@link PluginConfigurer} to which the plugins in this {@link PipelinePhase} needs to be
 * registered
 * @param runtimeConfigurer the runtime configurer to provide runtime arguments to resolve macro better, null
 *                          if this is the initial deploy
 * @param namespace namespace the app is deployed
 */
public void registerPlugins(PluginConfigurer pluginConfigurer, @Nullable RuntimeConfigurer runtimeConfigurer, String namespace) {
    MacroParserOptions options = MacroParserOptions.builder().skipInvalidMacros().setEscaping(false).setFunctionWhitelist(ConnectionMacroEvaluator.FUNCTION_NAME).build();
    MacroEvaluator runtimeEvaluator = null;
    if (runtimeConfigurer != null) {
        Map<String, MacroEvaluator> evaluators = Collections.singletonMap(ConnectionMacroEvaluator.FUNCTION_NAME, new ConnectionMacroEvaluator(namespace, runtimeConfigurer));
        runtimeEvaluator = new DefaultMacroEvaluator(new BasicArguments(runtimeConfigurer.getRuntimeArguments()), evaluators, Collections.singleton(ConnectionMacroEvaluator.FUNCTION_NAME));
    }
    for (StageSpec stageSpec : stagesByName.values()) {
        // we don't need to register connectors only source, sink and transform plugins
        if (stageSpec.getPluginType().equals(Constants.Connector.PLUGIN_TYPE)) {
            continue;
        }
        PluginSpec pluginSpec = stageSpec.getPlugin();
        ArtifactVersion version = pluginSpec.getArtifact().getVersion();
        ArtifactSelector artifactSelector = new ArtifactSelector(pluginSpec.getArtifact().getScope(), pluginSpec.getArtifact().getName(), new ArtifactVersionRange(version, true, version, true));
        Map<String, String> prop = pluginSpec.getProperties();
        pluginConfigurer.usePluginClass(pluginSpec.getType(), pluginSpec.getName(), stageSpec.getName(), PluginProperties.builder().addAll(runtimeConfigurer == null ? prop : pluginConfigurer.evaluateMacros(prop, runtimeEvaluator, options)).build(), artifactSelector);
    }
}
Also used : MacroEvaluator(io.cdap.cdap.api.macro.MacroEvaluator) ArtifactVersionRange(io.cdap.cdap.api.artifact.ArtifactVersionRange) MacroParserOptions(io.cdap.cdap.api.macro.MacroParserOptions) PluginSpec(io.cdap.cdap.etl.proto.v2.spec.PluginSpec) ArtifactVersion(io.cdap.cdap.api.artifact.ArtifactVersion) StageSpec(io.cdap.cdap.etl.proto.v2.spec.StageSpec)

Example 2 with MacroEvaluator

use of io.cdap.cdap.api.macro.MacroEvaluator in project cdap by caskdata.

the class MacroParserTest method testSkipInvalidMacrosInMacroFunctions.

@Test
public void testSkipInvalidMacrosInMacroFunctions() {
    MacroEvaluator evaluator = new TestMacroEvaluator(Collections.emptyMap(), Collections.emptyMap(), true);
    MacroParser parser = new MacroParser(evaluator, MacroParserOptions.builder().skipInvalidMacros().build());
    // test macros in the macro function still get skipped correctly
    Assert.assertEquals("${test(${k1})}", parser.parse("${test(${k1})}"));
    Assert.assertEquals("${test(${k1${k2}})}", parser.parse("${test(${k1${k2}})}"));
    Assert.assertEquals("${test(${k1})}${t(${t1})}", parser.parse("${test(${k1})}${t(${t1})}"));
}
Also used : MacroEvaluator(io.cdap.cdap.api.macro.MacroEvaluator) Test(org.junit.Test)

Example 3 with MacroEvaluator

use of io.cdap.cdap.api.macro.MacroEvaluator in project cdap by caskdata.

the class MacroParserTest method assertSubstitution.

private static void assertSubstitution(String macro, String expected, Map<String, String> propertySubstitutions, Map<String, String> macroFunctionSubstitutions) {
    MacroEvaluator macroEvaluator = new TestMacroEvaluator(propertySubstitutions, macroFunctionSubstitutions);
    MacroParser macroParser = new MacroParser(macroEvaluator);
    Assert.assertEquals(expected, macroParser.parse(macro));
}
Also used : MacroEvaluator(io.cdap.cdap.api.macro.MacroEvaluator)

Example 4 with MacroEvaluator

use of io.cdap.cdap.api.macro.MacroEvaluator in project cdap by caskdata.

the class MacroParserTest method testDisableFunctions.

@Test
public void testDisableFunctions() {
    MacroEvaluator evaluator = new TestMacroEvaluator(Collections.emptyMap(), Collections.emptyMap());
    MacroParser parser = new MacroParser(evaluator, MacroParserOptions.builder().disableFunctions().build());
    Assert.assertEquals("${test(key)}", parser.parse("${test(key)}"));
}
Also used : MacroEvaluator(io.cdap.cdap.api.macro.MacroEvaluator) Test(org.junit.Test)

Example 5 with MacroEvaluator

use of io.cdap.cdap.api.macro.MacroEvaluator in project cdap by caskdata.

the class MacroParserTest method testFunctionWhitelist.

@Test
public void testFunctionWhitelist() {
    MacroEvaluator evaluator = new TestMacroEvaluator(Collections.emptyMap(), ImmutableMap.of("key", "val"));
    MacroParser parser = new MacroParser(evaluator, MacroParserOptions.builder().setFunctionWhitelist("t").build());
    // $t(key) should get evaluated, but $test(key) should get skipped.
    Assert.assertEquals("${test(key)}val", parser.parse("${test(key)}${t(key)}"));
    Assert.assertEquals("val${test(key)}", parser.parse("${t(key)}${test(key)}"));
    Assert.assertEquals("${test(val)}", parser.parse("${test(${t(key)})}"));
}
Also used : MacroEvaluator(io.cdap.cdap.api.macro.MacroEvaluator) Test(org.junit.Test)

Aggregations

MacroEvaluator (io.cdap.cdap.api.macro.MacroEvaluator)32 DefaultMacroEvaluator (io.cdap.cdap.etl.common.DefaultMacroEvaluator)19 BasicArguments (io.cdap.cdap.etl.common.BasicArguments)13 PipelineRuntime (io.cdap.cdap.etl.common.PipelineRuntime)10 StageSpec (io.cdap.cdap.etl.proto.v2.spec.StageSpec)10 HashMap (java.util.HashMap)10 Map (java.util.Map)10 Test (org.junit.Test)8 PluginContext (io.cdap.cdap.api.plugin.PluginContext)7 MacroParserOptions (io.cdap.cdap.api.macro.MacroParserOptions)6 BatchPhaseSpec (io.cdap.cdap.etl.batch.BatchPhaseSpec)6 SparkPipelinePluginContext (io.cdap.cdap.etl.spark.plugin.SparkPipelinePluginContext)6 PipelinePluginContext (io.cdap.cdap.etl.common.plugin.PipelinePluginContext)5 OAuthMacroEvaluator (io.cdap.cdap.etl.common.OAuthMacroEvaluator)4 SecureStoreMacroEvaluator (io.cdap.cdap.etl.common.SecureStoreMacroEvaluator)4 SparkPipelineRuntime (io.cdap.cdap.etl.spark.SparkPipelineRuntime)4 ImmutableMap (com.google.common.collect.ImmutableMap)3 Gson (com.google.gson.Gson)3 TxRunnable (io.cdap.cdap.api.TxRunnable)3 DatasetContext (io.cdap.cdap.api.data.DatasetContext)3