Search in sources :

Example 11 with PluginPropertyField

use of io.cdap.cdap.api.plugin.PluginPropertyField in project cdap by caskdata.

the class ArtifactStoreTest method testGetPlugins.

@Test
public void testGetPlugins() throws Exception {
    ArtifactRange parentArtifacts = new ArtifactRange(NamespaceId.DEFAULT.getNamespace(), "parent", new ArtifactVersion("1.0.0"), new ArtifactVersion("2.0.0"));
    // we have 2 plugins of type A and 2 plugins of type B
    PluginClass pluginA1 = PluginClass.builder().setName("p1").setType("A").setDescription("desc").setClassName("c.p1").setConfigFieldName("cfg").setProperties(ImmutableMap.of("threshold", new PluginPropertyField("thresh", "description", "double", true, false), "retry", new PluginPropertyField("retries", "description", "int", false, false))).build();
    PluginClass pluginA2 = PluginClass.builder().setName("p2").setType("A").setDescription("desc").setClassName("c.p2").setConfigFieldName("conf").setProperties(ImmutableMap.of("stream", new PluginPropertyField("stream", "description", "string", true, false))).build();
    PluginClass pluginB1 = PluginClass.builder().setName("p1").setType("B").setDescription("desc").setClassName("c.p1").setConfigFieldName("cfg").setProperties(ImmutableMap.of("createIfNotExist", new PluginPropertyField("createIfNotExist", "desc", "boolean", false, false))).build();
    PluginClass pluginB2 = PluginClass.builder().setName("p2").setType("B").setDescription("desc").setClassName("c.p2").setConfigFieldName("stuff").setProperties(ImmutableMap.of("numer", new PluginPropertyField("numerator", "description", "double", true, false), "denom", new PluginPropertyField("denominator", "description", "double", true, false))).build();
    // add artifacts
    // not interested in artifact contents for this test, using some dummy value
    String contents = "0";
    // write parent
    Id.Artifact parentArtifactId = Id.Artifact.from(Id.Namespace.DEFAULT, "parent", "1.0.0");
    ArtifactMeta parentMeta = new ArtifactMeta(ArtifactClasses.builder().build());
    writeArtifact(parentArtifactId, parentMeta, contents);
    // artifact artifactX-1.0.0 contains plugin A1
    Id.Artifact artifactXv100 = Id.Artifact.from(Id.Namespace.DEFAULT, "artifactX", "1.0.0");
    ArtifactMeta metaXv100 = new ArtifactMeta(ArtifactClasses.builder().addPlugin(pluginA1).build(), ImmutableSet.of(parentArtifacts));
    writeArtifact(artifactXv100, metaXv100, contents);
    ArtifactDescriptor artifactXv100Info = artifactStore.getArtifact(artifactXv100).getDescriptor();
    // artifact artifactX-1.1.0 contains plugin A1
    Id.Artifact artifactXv110 = Id.Artifact.from(Id.Namespace.DEFAULT, "artifactX", "1.1.0");
    ArtifactMeta metaXv110 = new ArtifactMeta(ArtifactClasses.builder().addPlugin(pluginA1).build(), ImmutableSet.of(parentArtifacts));
    writeArtifact(artifactXv110, metaXv110, contents);
    ArtifactDescriptor artifactXv110Info = artifactStore.getArtifact(artifactXv110).getDescriptor();
    // artifact artifactX-2.0.0 contains plugins A1 and A2
    Id.Artifact artifactXv200 = Id.Artifact.from(Id.Namespace.DEFAULT, "artifactX", "2.0.0");
    ArtifactMeta metaXv200 = new ArtifactMeta(ArtifactClasses.builder().addPlugins(pluginA1, pluginA2).build(), ImmutableSet.of(parentArtifacts));
    writeArtifact(artifactXv200, metaXv200, contents);
    ArtifactDescriptor artifactXv200Info = artifactStore.getArtifact(artifactXv200).getDescriptor();
    // artifact artifactY-1.0.0 contains plugin B1
    Id.Artifact artifactYv100 = Id.Artifact.from(Id.Namespace.DEFAULT, "artifactY", "1.0.0");
    ArtifactMeta metaYv100 = new ArtifactMeta(ArtifactClasses.builder().addPlugin(pluginB1).build(), ImmutableSet.of(parentArtifacts));
    writeArtifact(artifactYv100, metaYv100, contents);
    ArtifactDescriptor artifactYv100Info = artifactStore.getArtifact(artifactYv100).getDescriptor();
    // artifact artifactY-2.0.0 contains plugin B2
    Id.Artifact artifactYv200 = Id.Artifact.from(Id.Namespace.DEFAULT, "artifactY", "2.0.0");
    ArtifactMeta metaYv200 = new ArtifactMeta(ArtifactClasses.builder().addPlugin(pluginB2).build(), ImmutableSet.of(parentArtifacts));
    writeArtifact(artifactYv200, metaYv200, contents);
    ArtifactDescriptor artifactYv200Info = artifactStore.getArtifact(artifactYv200).getDescriptor();
    // artifact artifactZ-1.0.0 contains plugins A1 and B1
    Id.Artifact artifactZv100 = Id.Artifact.from(Id.Namespace.DEFAULT, "artifactZ", "1.0.0");
    ArtifactMeta metaZv100 = new ArtifactMeta(ArtifactClasses.builder().addPlugins(pluginA1, pluginB1).build(), ImmutableSet.of(parentArtifacts));
    writeArtifact(artifactZv100, metaZv100, contents);
    ArtifactDescriptor artifactZv100Info = artifactStore.getArtifact(artifactZv100).getDescriptor();
    // artifact artifactZ-2.0.0 contains plugins A1, A2, B1, and B2
    Id.Artifact artifactZv200 = Id.Artifact.from(Id.Namespace.DEFAULT, "artifactZ", "2.0.0");
    ArtifactMeta metaZv200 = new ArtifactMeta(ArtifactClasses.builder().addPlugins(pluginA1, pluginA2, pluginB1, pluginB2).build(), ImmutableSet.of(parentArtifacts));
    writeArtifact(artifactZv200, metaZv200, contents);
    ArtifactDescriptor artifactZv200Info = artifactStore.getArtifact(artifactZv200).getDescriptor();
    // test getting all plugins in the namespace
    Map<ArtifactDescriptor, Set<PluginClass>> expected = Maps.newHashMap();
    expected.put(artifactXv100Info, ImmutableSet.of(pluginA1));
    expected.put(artifactXv110Info, ImmutableSet.of(pluginA1));
    expected.put(artifactXv200Info, ImmutableSet.of(pluginA1, pluginA2));
    expected.put(artifactYv100Info, ImmutableSet.of(pluginB1));
    expected.put(artifactYv200Info, ImmutableSet.of(pluginB2));
    expected.put(artifactZv100Info, ImmutableSet.of(pluginA1, pluginB1));
    expected.put(artifactZv200Info, ImmutableSet.of(pluginA1, pluginA2, pluginB1, pluginB2));
    Map<ArtifactDescriptor, Set<PluginClass>> actual = artifactStore.getPluginClasses(NamespaceId.DEFAULT, parentArtifactId);
    Assert.assertEquals(expected, actual);
    // test getting all plugins by namespace and type
    // get all of type A
    expected = Maps.newHashMap();
    expected.put(artifactXv100Info, ImmutableSet.of(pluginA1));
    expected.put(artifactXv110Info, ImmutableSet.of(pluginA1));
    expected.put(artifactXv200Info, ImmutableSet.of(pluginA1, pluginA2));
    expected.put(artifactZv100Info, ImmutableSet.of(pluginA1));
    expected.put(artifactZv200Info, ImmutableSet.of(pluginA1, pluginA2));
    actual = artifactStore.getPluginClasses(NamespaceId.DEFAULT, parentArtifactId, "A");
    Assert.assertEquals(expected, actual);
    // get all of type B
    expected = Maps.newHashMap();
    expected.put(artifactYv100Info, ImmutableSet.of(pluginB1));
    expected.put(artifactYv200Info, ImmutableSet.of(pluginB2));
    expected.put(artifactZv100Info, ImmutableSet.of(pluginB1));
    expected.put(artifactZv200Info, ImmutableSet.of(pluginB1, pluginB2));
    actual = artifactStore.getPluginClasses(NamespaceId.DEFAULT, parentArtifactId, "B");
    Assert.assertEquals(expected, actual);
    // test getting plugins by namespace, type, and name
    // get all of type A and name p1
    Map<ArtifactDescriptor, PluginClass> expectedMap = Maps.newHashMap();
    expectedMap.put(artifactXv100Info, pluginA1);
    expectedMap.put(artifactXv110Info, pluginA1);
    expectedMap.put(artifactXv200Info, pluginA1);
    expectedMap.put(artifactZv100Info, pluginA1);
    expectedMap.put(artifactZv200Info, pluginA1);
    Map<ArtifactDescriptor, PluginClass> actualMap = artifactStore.getPluginClasses(NamespaceId.DEFAULT, parentArtifactId, "A", "p1", null, Integer.MAX_VALUE, ArtifactSortOrder.UNORDERED);
    Assert.assertEquals(expectedMap, actualMap);
    // test get limited number
    actualMap = artifactStore.getPluginClasses(NamespaceId.DEFAULT, parentArtifactId, "A", "p1", null, 1, ArtifactSortOrder.UNORDERED);
    Assert.assertEquals(ImmutableMap.of(artifactXv100Info, pluginA1), actualMap);
    // test get DESC order
    actualMap = new TreeMap<>(artifactStore.getPluginClasses(NamespaceId.DEFAULT, parentArtifactId, "A", "p1", null, Integer.MAX_VALUE, ArtifactSortOrder.DESC));
    Assert.assertEquals(expectedMap, new TreeMap<>(actualMap).descendingMap());
    // test Predicate
    Predicate<ArtifactId> predicate = input -> {
        try {
            return input.getParent().equals(NamespaceId.DEFAULT) && input.getArtifact().equals("artifactX") && ArtifactVersionRange.parse("[1.0.0, 1.1.0)").versionIsInRange(new ArtifactVersion(input.getVersion()));
        } catch (InvalidArtifactRangeException e) {
            return false;
        }
    };
    expectedMap = Maps.newHashMap();
    expectedMap.put(artifactXv100Info, pluginA1);
    actualMap = artifactStore.getPluginClasses(NamespaceId.DEFAULT, parentArtifactId, "A", "p1", predicate, Integer.MAX_VALUE, ArtifactSortOrder.UNORDERED);
    Assert.assertEquals(expectedMap, actualMap);
    // test limit and order combined
    actualMap = artifactStore.getPluginClasses(NamespaceId.DEFAULT, parentArtifactId, "A", "p1", null, 1, ArtifactSortOrder.DESC);
    Assert.assertEquals(ImmutableMap.of(artifactZv200Info, pluginA1), actualMap);
    // test limit, order, predicate combined
    actualMap = artifactStore.getPluginClasses(NamespaceId.DEFAULT, parentArtifactId, "A", "p1", predicate, 1, ArtifactSortOrder.DESC);
    Assert.assertEquals(ImmutableMap.of(artifactXv100Info, pluginA1), actualMap);
    // get all of type A and name p2
    expectedMap = Maps.newHashMap();
    expectedMap.put(artifactXv200Info, pluginA2);
    expectedMap.put(artifactZv200Info, pluginA2);
    actualMap = artifactStore.getPluginClasses(NamespaceId.DEFAULT, parentArtifactId, "A", "p2", null, Integer.MAX_VALUE, ArtifactSortOrder.UNORDERED);
    Assert.assertEquals(expectedMap, actualMap);
    // get all of type B and name p1
    expectedMap = Maps.newHashMap();
    expectedMap.put(artifactYv100Info, pluginB1);
    expectedMap.put(artifactZv100Info, pluginB1);
    expectedMap.put(artifactZv200Info, pluginB1);
    actualMap = artifactStore.getPluginClasses(NamespaceId.DEFAULT, parentArtifactId, "B", "p1", null, Integer.MAX_VALUE, ArtifactSortOrder.UNORDERED);
    Assert.assertEquals(expectedMap, actualMap);
    // get all of type B and name p2
    expectedMap = Maps.newHashMap();
    expectedMap.put(artifactYv200Info, pluginB2);
    expectedMap.put(artifactZv200Info, pluginB2);
    actualMap = artifactStore.getPluginClasses(NamespaceId.DEFAULT, parentArtifactId, "B", "p2", null, Integer.MAX_VALUE, ArtifactSortOrder.UNORDERED);
    Assert.assertEquals(expectedMap, actualMap);
}
Also used : Arrays(java.util.Arrays) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) ArtifactClasses(io.cdap.cdap.api.artifact.ArtifactClasses) Bytes(io.cdap.cdap.api.common.Bytes) Ids(io.cdap.cdap.proto.id.Ids) Future(java.util.concurrent.Future) CharStreams(com.google.common.io.CharStreams) ArtifactSortOrder(io.cdap.cdap.proto.artifact.ArtifactSortOrder) After(org.junit.After) Map(java.util.Map) ClassRule(org.junit.ClassRule) CyclicBarrier(java.util.concurrent.CyclicBarrier) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) Table(io.cdap.cdap.api.dataset.table.Table) Set(java.util.Set) ApplicationClass(io.cdap.cdap.api.artifact.ApplicationClass) Category(org.junit.experimental.categories.Category) Collectors(java.util.stream.Collectors) PluginClass(io.cdap.cdap.api.plugin.PluginClass) Executors(java.util.concurrent.Executors) Id(io.cdap.cdap.common.id.Id) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) ArtifactScope(io.cdap.cdap.api.artifact.ArtifactScope) ArtifactAlreadyExistsException(io.cdap.cdap.common.ArtifactAlreadyExistsException) SortedMap(java.util.SortedMap) Location(org.apache.twill.filesystem.Location) ArtifactRange(io.cdap.cdap.api.artifact.ArtifactRange) InvalidArtifactRangeException(io.cdap.cdap.api.artifact.InvalidArtifactRangeException) InspectionApp(io.cdap.cdap.internal.app.runtime.artifact.app.inspection.InspectionApp) DefaultImpersonator(io.cdap.cdap.security.impersonation.DefaultImpersonator) ArrayList(java.util.ArrayList) PluginNotExistsException(io.cdap.cdap.internal.app.runtime.plugin.PluginNotExistsException) SlowTests(io.cdap.cdap.test.SlowTests) SQLException(java.sql.SQLException) Lists(com.google.common.collect.Lists) ArtifactVersionRange(io.cdap.cdap.api.artifact.ArtifactVersionRange) ImmutableList(com.google.common.collect.ImmutableList) ReflectionSchemaGenerator(io.cdap.cdap.internal.io.ReflectionSchemaGenerator) ArtifactId(io.cdap.cdap.proto.id.ArtifactId) ExecutorService(java.util.concurrent.ExecutorService) KeyValueTable(io.cdap.cdap.api.dataset.lib.KeyValueTable) Charsets(com.google.common.base.Charsets) Cube(io.cdap.cdap.api.dataset.lib.cube.Cube) Files(java.nio.file.Files) Test(org.junit.Test) IOException(java.io.IOException) BrokenBarrierException(java.util.concurrent.BrokenBarrierException) Maps(com.google.common.collect.Maps) InputStreamReader(java.io.InputStreamReader) File(java.io.File) EntityImpersonator(io.cdap.cdap.security.impersonation.EntityImpersonator) Requirements(io.cdap.cdap.api.plugin.Requirements) ArtifactNotFoundException(io.cdap.cdap.common.ArtifactNotFoundException) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) TreeMap(java.util.TreeMap) ArtifactVersion(io.cdap.cdap.api.artifact.ArtifactVersion) PluginPropertyField(io.cdap.cdap.api.plugin.PluginPropertyField) AllProgramsApp(io.cdap.cdap.AllProgramsApp) Assert(org.junit.Assert) Collections(java.util.Collections) TemporaryFolder(org.junit.rules.TemporaryFolder) ImmutableSet(com.google.common.collect.ImmutableSet) Set(java.util.Set) ArtifactId(io.cdap.cdap.proto.id.ArtifactId) ArtifactRange(io.cdap.cdap.api.artifact.ArtifactRange) InvalidArtifactRangeException(io.cdap.cdap.api.artifact.InvalidArtifactRangeException) TreeMap(java.util.TreeMap) PluginPropertyField(io.cdap.cdap.api.plugin.PluginPropertyField) ArtifactVersion(io.cdap.cdap.api.artifact.ArtifactVersion) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Id(io.cdap.cdap.common.id.Id) ArtifactId(io.cdap.cdap.proto.id.ArtifactId) PluginClass(io.cdap.cdap.api.plugin.PluginClass) Test(org.junit.Test)

Example 12 with PluginPropertyField

use of io.cdap.cdap.api.plugin.PluginPropertyField in project cdap by caskdata.

the class PluginInstantiator method getFieldsWithMacro.

private Set<String> getFieldsWithMacro(Plugin plugin) {
    // TODO: cleanup after endpoint to get plugin details is merged (#6089)
    Set<String> macroFields = new HashSet<>();
    Map<String, PluginPropertyField> pluginPropertyFieldMap = plugin.getPluginClass().getProperties();
    TrackingMacroEvaluator trackingMacroEvaluator = new TrackingMacroEvaluator();
    for (Map.Entry<String, PluginPropertyField> pluginEntry : pluginPropertyFieldMap.entrySet()) {
        PluginPropertyField pluginField = pluginEntry.getValue();
        if (pluginEntry.getValue() != null && pluginField.isMacroSupported()) {
            String macroValue = plugin.getProperties().getProperties().get(pluginEntry.getKey());
            if (macroValue != null) {
                MacroParser macroParser = new MacroParser(trackingMacroEvaluator, MacroParserOptions.builder().setEscaping(pluginField.isMacroEscapingEnabled()).build());
                macroParser.parse(macroValue);
                if (trackingMacroEvaluator.hasMacro()) {
                    if (!pluginField.getChildren().isEmpty()) {
                        macroFields.addAll(pluginField.getChildren());
                    }
                    macroFields.add(pluginEntry.getKey());
                    trackingMacroEvaluator.reset();
                }
            }
        }
    }
    return macroFields;
}
Also used : PluginPropertyField(io.cdap.cdap.api.plugin.PluginPropertyField) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap) HashSet(java.util.HashSet)

Example 13 with PluginPropertyField

use of io.cdap.cdap.api.plugin.PluginPropertyField in project cdap by caskdata.

the class MockSQLEngineWithStageSettings method getPluginClass.

private static PluginClass getPluginClass() {
    Map<String, PluginPropertyField> properties = new HashMap<>();
    properties.put("name", new PluginPropertyField("name", "", "string", true, false));
    properties.put("inputDirName", new PluginPropertyField("inputDirName", "", "string", true, false));
    properties.put("outputDirName", new PluginPropertyField("outputDirName", "", "string", true, false));
    properties.put("outputSchema", new PluginPropertyField("outputSchema", "", "string", true, false));
    properties.put("includedStages", new PluginPropertyField("includedStages", "", "string", true, false));
    properties.put("excludedStages", new PluginPropertyField("excludedStages", "", "string", true, false));
    return new PluginClass(BatchSQLEngine.PLUGIN_TYPE, NAME, "", MockSQLEngineWithStageSettings.class.getName(), "config", properties);
}
Also used : HashMap(java.util.HashMap) PluginClass(io.cdap.cdap.api.plugin.PluginClass) PluginPropertyField(io.cdap.cdap.api.plugin.PluginPropertyField)

Example 14 with PluginPropertyField

use of io.cdap.cdap.api.plugin.PluginPropertyField in project cdap by caskdata.

the class NodeStatesAction method getPluginClass.

private static PluginClass getPluginClass() {
    Map<String, PluginPropertyField> properties = new HashMap<>();
    properties.put("tableName", new PluginPropertyField("tableName", "", "string", true, false));
    return PluginClass.builder().setName(NodeStatesAction.NAME).setType(PostAction.PLUGIN_TYPE).setDescription("").setClassName(NodeStatesAction.class.getName()).setProperties(properties).setConfigFieldName("conf").build();
}
Also used : HashMap(java.util.HashMap) PluginPropertyField(io.cdap.cdap.api.plugin.PluginPropertyField)

Example 15 with PluginPropertyField

use of io.cdap.cdap.api.plugin.PluginPropertyField in project cdap by caskdata.

the class MockRuntimeDatasetSource method getPluginClass.

private static PluginClass getPluginClass() {
    Map<String, PluginPropertyField> properties = new HashMap<>();
    properties.put("tableName", new PluginPropertyField("tableName", "", "string", true, false));
    properties.put("runtimeDatasetName", new PluginPropertyField("runtimeDatasetName", "", "string", true, true));
    return PluginClass.builder().setName("MockRuntime").setType(BatchSource.PLUGIN_TYPE).setDescription("").setClassName(MockRuntimeDatasetSource.class.getName()).setProperties(properties).setConfigFieldName("config").build();
}
Also used : HashMap(java.util.HashMap) PluginPropertyField(io.cdap.cdap.api.plugin.PluginPropertyField)

Aggregations

PluginPropertyField (io.cdap.cdap.api.plugin.PluginPropertyField)54 HashMap (java.util.HashMap)43 PluginClass (io.cdap.cdap.api.plugin.PluginClass)11 Test (org.junit.Test)8 ArtifactRange (io.cdap.cdap.api.artifact.ArtifactRange)6 ArtifactVersion (io.cdap.cdap.api.artifact.ArtifactVersion)6 Id (io.cdap.cdap.common.id.Id)6 File (java.io.File)5 ArtifactClasses (io.cdap.cdap.api.artifact.ArtifactClasses)4 ArtifactId (io.cdap.cdap.proto.id.ArtifactId)4 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)4 Location (org.apache.twill.filesystem.Location)4 ImmutableMap (com.google.common.collect.ImmutableMap)3 ImmutableSet (com.google.common.collect.ImmutableSet)3 ApplicationClass (io.cdap.cdap.api.artifact.ApplicationClass)3 ArtifactId (io.cdap.cdap.api.artifact.ArtifactId)3 Requirements (io.cdap.cdap.api.plugin.Requirements)3 ArtifactNotFoundException (io.cdap.cdap.common.ArtifactNotFoundException)3 ReflectionSchemaGenerator (io.cdap.cdap.internal.io.ReflectionSchemaGenerator)3 ArrayList (java.util.ArrayList)3