Examples with ArtifactId co.cask.cdap.api.artifact.ArtifactId used on opensource projects

Example 6 with ArtifactId

use of co.cask.cdap.api.artifact.ArtifactId in project cdap by caskdata.

the class ApplicationLifecycleService method updateApp.

/**
   * Update an existing application. An application's configuration and artifact version can be updated.
   *
   * @param appId the id of the application to update
   * @param appRequest the request to update the application, including new config and artifact
   * @param programTerminator a program terminator that will stop programs that are removed when updating an app.
   *                          For example, if an update removes a flow, the terminator defines how to stop that flow.
   * @return information about the deployed application
   * @throws ApplicationNotFoundException if the specified application does not exist
   * @throws ArtifactNotFoundException if the requested artifact does not exist
   * @throws InvalidArtifactException if the specified artifact is invalid. For example, if the artifact name changed,
   *                                  if the version is an invalid version, or the artifact contains no app classes
   * @throws Exception if there was an exception during the deployment pipeline. This exception will often wrap
   *                   the actual exception
   */
public ApplicationWithPrograms updateApp(ApplicationId appId, AppRequest appRequest, ProgramTerminator programTerminator) throws Exception {
    // check that app exists
    ApplicationSpecification currentSpec = store.getApplication(appId);
    if (currentSpec == null) {
        throw new ApplicationNotFoundException(appId);
    }
    // App exists. Check if the current user has admin privileges on it before updating. The user's write privileges on
    // the namespace will get enforced in the deployApp method.
    authorizationEnforcer.enforce(appId, authenticationContext.getPrincipal(), Action.ADMIN);
    ArtifactId currentArtifact = currentSpec.getArtifactId();
    // if no artifact is given, use the current one.
    ArtifactId newArtifactId = currentArtifact;
    // otherwise, check requested artifact is valid and use it
    ArtifactSummary requestedArtifact = appRequest.getArtifact();
    if (requestedArtifact != null) {
        // cannot change artifact name, only artifact version.
        if (!currentArtifact.getName().equals(requestedArtifact.getName())) {
            throw new InvalidArtifactException(String.format(" Only artifact version updates are allowed. Cannot change from artifact '%s' to '%s'.", currentArtifact.getName(), requestedArtifact.getName()));
        }
        if (!currentArtifact.getScope().equals(requestedArtifact.getScope())) {
            throw new InvalidArtifactException("Only artifact version updates are allowed. " + "Cannot change from a non-system artifact to a system artifact or vice versa.");
        }
        // check requested artifact version is valid
        ArtifactVersion requestedVersion = new ArtifactVersion(requestedArtifact.getVersion());
        if (requestedVersion.getVersion() == null) {
            throw new InvalidArtifactException(String.format("Requested artifact version '%s' is invalid", requestedArtifact.getVersion()));
        }
        newArtifactId = new ArtifactId(currentArtifact.getName(), requestedVersion, currentArtifact.getScope());
    }
    // ownerAdmin.getImpersonationPrincipal will give the owner which will be impersonated for the application
    // irrespective of the version
    SecurityUtil.verifyOwnerPrincipal(appId, appRequest.getOwnerPrincipal(), ownerAdmin);
    Object requestedConfigObj = appRequest.getConfig();
    // if config is null, use the previous config. Shouldn't use a static GSON since the request Config object can
    // be a user class, otherwise there will be ClassLoader leakage.
    String requestedConfigStr = requestedConfigObj == null ? currentSpec.getConfiguration() : new Gson().toJson(requestedConfigObj);
    Id.Artifact artifactId = Artifacts.toArtifactId(appId.getParent(), newArtifactId).toId();
    return deployApp(appId.getParent(), appId.getApplication(), null, artifactId, requestedConfigStr, programTerminator, ownerAdmin.getOwner(appId), appRequest.canUpdateSchedules());
}

Example 7 with ArtifactId

use of co.cask.cdap.api.artifact.ArtifactId in project cdap by caskdata.

the class ArtifactRepositoryTest method getFile.

private static File getFile() throws Exception {
    File pluginDir = DirUtils.createTempDir(tmpDir);
    // Create the plugin jar. There should be two plugins there (TestPlugin and TestPlugin2).
    Manifest manifest = createManifest(ManifestFields.EXPORT_PACKAGE, TestPlugin.class.getPackage().getName());
    File jarFile = createPluginJar(TestPlugin.class, new File(tmpDir, "myPlugin-1.0.jar"), manifest);
    // add the artifact
    Set<ArtifactRange> parents = ImmutableSet.of(new ArtifactRange(APP_ARTIFACT_ID.getNamespace().getId(), APP_ARTIFACT_ID.getName(), new ArtifactVersion("1.0.0"), new ArtifactVersion("2.0.0")));
    Id.Artifact artifactId = Id.Artifact.from(Id.Namespace.DEFAULT, "myPlugin", "1.0");
    artifactRepository.addArtifact(artifactId, jarFile, parents);
    return pluginDir;
}

Example 8 with ArtifactId

use of co.cask.cdap.api.artifact.ArtifactId in project cdap by caskdata.

the class ArtifactSelectorTest method testSelection.

@Test
public void testSelection() {
    SortedMap<ArtifactId, PluginClass> plugins = new TreeMap<>();
    // doesn't matter what this is, since we only select on artifact id.
    PluginClass pluginClass = new PluginClass("type", "name", "desc", "com.company.class", "field", ImmutableMap.<String, PluginPropertyField>of());
    // put every combination of abc or def as name, 1.0.0 or 2.0.0 as version, and system or user as scope
    plugins.put(new ArtifactId("abc", new ArtifactVersion("1.0.0"), ArtifactScope.SYSTEM), pluginClass);
    plugins.put(new ArtifactId("abc", new ArtifactVersion("2.0.0"), ArtifactScope.SYSTEM), pluginClass);
    plugins.put(new ArtifactId("abc", new ArtifactVersion("1.0.0"), ArtifactScope.USER), pluginClass);
    plugins.put(new ArtifactId("abc", new ArtifactVersion("2.0.0"), ArtifactScope.USER), pluginClass);
    plugins.put(new ArtifactId("def", new ArtifactVersion("1.0.0"), ArtifactScope.SYSTEM), pluginClass);
    plugins.put(new ArtifactId("def", new ArtifactVersion("2.0.0"), ArtifactScope.SYSTEM), pluginClass);
    plugins.put(new ArtifactId("def", new ArtifactVersion("1.0.0"), ArtifactScope.USER), pluginClass);
    plugins.put(new ArtifactId("def", new ArtifactVersion("2.0.0"), ArtifactScope.USER), pluginClass);
    // test scope only
    ArtifactSelector selector = new ArtifactSelector("type", "name", ArtifactScope.SYSTEM, null, null);
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("2.0.0"), ArtifactScope.SYSTEM), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", ArtifactScope.USER, null, null);
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("2.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    // test name only
    selector = new ArtifactSelector("type", "name", null, "abc", null);
    Assert.assertEquals(new ArtifactId("abc", new ArtifactVersion("2.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", null, "def", null);
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("2.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    try {
        selector = new ArtifactSelector("type", "name", null, "xyz", null);
        selector.select(plugins);
    } catch (Exception e) {
    // expected
    }
    // test version only
    selector = new ArtifactSelector("type", "name", null, null, new ArtifactVersionRange(new ArtifactVersion("1.0.0"), true, new ArtifactVersion("1.0.0"), true));
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("1.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", null, null, new ArtifactVersionRange(new ArtifactVersion("2.0.0"), true, new ArtifactVersion("2.0.0"), true));
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("2.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    try {
        selector = new ArtifactSelector("type", "name", null, null, new ArtifactVersionRange(new ArtifactVersion("3.0.0"), true, new ArtifactVersion("3.0.0"), true));
        selector.select(plugins);
    } catch (Exception e) {
    // expected
    }
    // test range only
    selector = new ArtifactSelector("type", "name", null, null, new ArtifactVersionRange(new ArtifactVersion("1.0.0-SNAPSHOT"), true, new ArtifactVersion("2.0.0"), false));
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("1.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", null, null, new ArtifactVersionRange(new ArtifactVersion("1.0.0-SNAPSHOT"), true, new ArtifactVersion("2.0.0"), true));
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("2.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    try {
        selector = new ArtifactSelector("type", "name", null, null, new ArtifactVersionRange(new ArtifactVersion("2.0.0"), false, new ArtifactVersion("3.0.0"), true));
        selector.select(plugins);
    } catch (Exception e) {
    // expected
    }
    // test name + version
    selector = new ArtifactSelector("type", "name", null, "abc", new ArtifactVersionRange(new ArtifactVersion("1.0.0"), true, new ArtifactVersion("1.0.0"), true));
    Assert.assertEquals(new ArtifactId("abc", new ArtifactVersion("1.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", null, "abc", new ArtifactVersionRange(new ArtifactVersion("2.0.0"), true, new ArtifactVersion("2.0.0"), true));
    Assert.assertEquals(new ArtifactId("abc", new ArtifactVersion("2.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", null, "def", new ArtifactVersionRange(new ArtifactVersion("1.0.0"), true, new ArtifactVersion("1.0.0"), true));
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("1.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", null, "def", new ArtifactVersionRange(new ArtifactVersion("2.0.0"), true, new ArtifactVersion("2.0.0"), true));
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("2.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    try {
        selector = new ArtifactSelector("type", "name", null, "xyz", new ArtifactVersionRange(new ArtifactVersion("1.0.0"), true, new ArtifactVersion("1.0.0"), true));
        selector.select(plugins);
    } catch (Exception e) {
    // expected
    }
    try {
        selector = new ArtifactSelector("type", "name", null, "abc", new ArtifactVersionRange(new ArtifactVersion("3.0.0"), true, new ArtifactVersion("3.0.0"), true));
        selector.select(plugins);
    } catch (Exception e) {
    // expected
    }
    // test name + scope
    selector = new ArtifactSelector("type", "name", ArtifactScope.SYSTEM, "abc", null);
    Assert.assertEquals(new ArtifactId("abc", new ArtifactVersion("2.0.0"), ArtifactScope.SYSTEM), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", ArtifactScope.SYSTEM, "def", null);
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("2.0.0"), ArtifactScope.SYSTEM), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", ArtifactScope.USER, "abc", null);
    Assert.assertEquals(new ArtifactId("abc", new ArtifactVersion("2.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", ArtifactScope.USER, "def", null);
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("2.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    try {
        selector = new ArtifactSelector("type", "name", ArtifactScope.SYSTEM, "xyz", null);
        selector.select(plugins);
    } catch (Exception e) {
    // expected
    }
    // test version + scope
    selector = new ArtifactSelector("type", "name", ArtifactScope.SYSTEM, null, new ArtifactVersionRange(new ArtifactVersion("1.0.0"), true, new ArtifactVersion("1.0.0"), true));
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("1.0.0"), ArtifactScope.SYSTEM), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", ArtifactScope.SYSTEM, null, new ArtifactVersionRange(new ArtifactVersion("2.0.0"), true, new ArtifactVersion("2.0.0"), true));
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("2.0.0"), ArtifactScope.SYSTEM), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", ArtifactScope.USER, null, new ArtifactVersionRange(new ArtifactVersion("1.0.0"), true, new ArtifactVersion("1.0.0"), true));
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("1.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", ArtifactScope.USER, null, new ArtifactVersionRange(new ArtifactVersion("2.0.0"), true, new ArtifactVersion("2.0.0"), true));
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("2.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    try {
        selector = new ArtifactSelector("type", "name", ArtifactScope.SYSTEM, null, new ArtifactVersionRange(new ArtifactVersion("3.0.0"), true, new ArtifactVersion("3.0.0"), true));
        selector.select(plugins);
    } catch (Exception e) {
    // expected
    }
    // test name + range
    selector = new ArtifactSelector("type", "name", null, "abc", new ArtifactVersionRange(new ArtifactVersion("1.0.0-SNAPSHOT"), true, new ArtifactVersion("2.0.0"), false));
    Assert.assertEquals(new ArtifactId("abc", new ArtifactVersion("1.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", null, "abc", new ArtifactVersionRange(new ArtifactVersion("1.0.0-SNAPSHOT"), true, new ArtifactVersion("2.0.0"), true));
    Assert.assertEquals(new ArtifactId("abc", new ArtifactVersion("2.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    try {
        selector = new ArtifactSelector("type", "name", null, "def", new ArtifactVersionRange(new ArtifactVersion("2.0.0"), false, new ArtifactVersion("3.0.0"), true));
        selector.select(plugins);
    } catch (Exception e) {
    // expected
    }
    // test scope + range
    selector = new ArtifactSelector("type", "name", ArtifactScope.SYSTEM, null, new ArtifactVersionRange(new ArtifactVersion("1.0.0-SNAPSHOT"), true, new ArtifactVersion("2.0.0"), false));
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("1.0.0"), ArtifactScope.SYSTEM), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", ArtifactScope.USER, null, new ArtifactVersionRange(new ArtifactVersion("1.0.0-SNAPSHOT"), true, new ArtifactVersion("2.0.0"), false));
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("1.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", ArtifactScope.SYSTEM, null, new ArtifactVersionRange(new ArtifactVersion("1.0.0-SNAPSHOT"), true, new ArtifactVersion("2.0.0"), true));
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("2.0.0"), ArtifactScope.SYSTEM), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", ArtifactScope.USER, null, new ArtifactVersionRange(new ArtifactVersion("1.0.0-SNAPSHOT"), true, new ArtifactVersion("2.0.0"), true));
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("2.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    try {
        selector = new ArtifactSelector("type", "name", ArtifactScope.SYSTEM, null, new ArtifactVersionRange(new ArtifactVersion("2.0.0"), false, new ArtifactVersion("3.0.0"), true));
        selector.select(plugins);
    } catch (Exception e) {
    // expected
    }
    // test name + version + scope
    selector = new ArtifactSelector("type", "name", ArtifactScope.USER, "def", new ArtifactVersionRange(new ArtifactVersion("2.0.0"), true, new ArtifactVersion("2.0.0"), true));
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("2.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    try {
        selector = new ArtifactSelector("type", "name", ArtifactScope.USER, "xyz", new ArtifactVersionRange(new ArtifactVersion("1.0.0"), true, new ArtifactVersion("1.0.0"), true));
        selector.select(plugins);
    } catch (Exception e) {
    // expected
    }
    // test name + scope + range
    selector = new ArtifactSelector("type", "name", ArtifactScope.SYSTEM, "abc", new ArtifactVersionRange(new ArtifactVersion("1.0.0-SNAPSHOT"), true, new ArtifactVersion("2.0.0"), false));
    Assert.assertEquals(new ArtifactId("abc", new ArtifactVersion("1.0.0"), ArtifactScope.SYSTEM), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", ArtifactScope.USER, "abc", new ArtifactVersionRange(new ArtifactVersion("1.0.0-SNAPSHOT"), true, new ArtifactVersion("2.0.0"), true));
    Assert.assertEquals(new ArtifactId("abc", new ArtifactVersion("2.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", ArtifactScope.SYSTEM, "def", new ArtifactVersionRange(new ArtifactVersion("1.0.0-SNAPSHOT"), true, new ArtifactVersion("2.0.0"), false));
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("1.0.0"), ArtifactScope.SYSTEM), selector.select(plugins).getKey());
    selector = new ArtifactSelector("type", "name", ArtifactScope.USER, "def", new ArtifactVersionRange(new ArtifactVersion("1.0.0-SNAPSHOT"), true, new ArtifactVersion("2.0.0"), true));
    Assert.assertEquals(new ArtifactId("def", new ArtifactVersion("2.0.0"), ArtifactScope.USER), selector.select(plugins).getKey());
    try {
        selector = new ArtifactSelector("type", "name", ArtifactScope.SYSTEM, "abc", new ArtifactVersionRange(new ArtifactVersion("2.0.0"), false, new ArtifactVersion("3.0.0"), true));
        selector.select(plugins);
    } catch (Exception e) {
    // expected
    }
}

Example 9 with ArtifactId

use of co.cask.cdap.api.artifact.ArtifactId in project cdap by caskdata.

the class PipelinePlannerTest method testGeneratePlan.

@Test
public void testGeneratePlan() {
    /*
             |--- n2(r) ----------|
             |                    |                                    |-- n10
        n1 --|--- n3(r) --- n5 ---|--- n6 --- n7(r) --- n8 --- n9(r) --|
             |                    |                                    |-- n11
             |--- n4(r) ----------|
     */
    // create the spec for this pipeline
    ArtifactId artifactId = new ArtifactId("dummy", new ArtifactVersion("1.0.0"), ArtifactScope.SYSTEM);
    Map<String, String> empty = ImmutableMap.of();
    PluginSpec nodePlugin = new PluginSpec(NODE, "mock", empty, artifactId);
    PluginSpec reducePlugin = new PluginSpec(AGGREGATOR, "mock", empty, artifactId);
    Schema schema = Schema.recordOf("stuff", Schema.Field.of("x", Schema.of(Schema.Type.INT)));
    Set<StageSpec> stageSpecs = ImmutableSet.of(StageSpec.builder("n1", nodePlugin).setOutputSchema(schema).addOutputs("n2", "n3", "n4").build(), StageSpec.builder("n2", reducePlugin).addInputSchema("n1", schema).setOutputSchema(schema).addInputs("n1").addOutputs("n6").build(), StageSpec.builder("n3", reducePlugin).addInputSchema("n1", schema).setOutputSchema(schema).addInputs("n1").addOutputs("n5").build(), StageSpec.builder("n4", reducePlugin).addInputSchema("n1", schema).setOutputSchema(schema).addInputs("n1").addOutputs("n6").build(), StageSpec.builder("n5", nodePlugin).addInputSchema("n3", schema).setOutputSchema(schema).addInputs("n3").addOutputs("n6").build(), StageSpec.builder("n6", nodePlugin).addInputSchemas(ImmutableMap.of("n2", schema, "n5", schema, "n4", schema)).setOutputSchema(schema).addInputs("n2", "n5", "n4").addOutputs("n7").build(), StageSpec.builder("n7", reducePlugin).addInputSchema("n6", schema).setOutputSchema(schema).addInputs("n6").addOutputs("n8").build(), StageSpec.builder("n8", nodePlugin).addInputSchema("n7", schema).setOutputSchema(schema).addInputs("n7").addOutputs("n9").build(), StageSpec.builder("n9", reducePlugin).addInputSchema("n8", schema).setOutputSchema(schema).addInputs("n8").addOutputs("n10", "n11").build(), StageSpec.builder("n10", nodePlugin).addInputSchema("n9", schema).addInputs("n9").build(), StageSpec.builder("n11", nodePlugin).addInputSchema("n9", schema).addInputs("n9").build());
    Set<Connection> connections = ImmutableSet.of(new Connection("n1", "n2"), new Connection("n1", "n3"), new Connection("n1", "n4"), new Connection("n2", "n6"), new Connection("n3", "n5"), new Connection("n4", "n6"), new Connection("n5", "n6"), new Connection("n6", "n7"), new Connection("n7", "n8"), new Connection("n8", "n9"), new Connection("n9", "n10"), new Connection("n9", "n11"));
    Set<String> pluginTypes = ImmutableSet.of(NODE, AGGREGATOR, Constants.CONNECTOR_TYPE);
    Set<String> reduceTypes = ImmutableSet.of(AGGREGATOR);
    Set<String> emptySet = ImmutableSet.of();
    PipelinePlanner planner = new PipelinePlanner(pluginTypes, reduceTypes, emptySet, emptySet);
    PipelineSpec pipelineSpec = PipelineSpec.builder().addStages(stageSpecs).addConnections(connections).build();
    Map<String, PipelinePhase> phases = new HashMap<>();
    /*
             |--- n2.connector
             |
        n1 --|--- n3.connector
             |
             |--- n4.connector
     */
    PipelinePhase phase1 = PipelinePhase.builder(pluginTypes).addStage(StageInfo.builder("n1", NODE).addOutputs("n2", "n3", "n4").setOutputSchema(schema).build()).addStage(StageInfo.builder("n2.connector", Constants.CONNECTOR_TYPE).build()).addStage(StageInfo.builder("n3.connector", Constants.CONNECTOR_TYPE).build()).addStage(StageInfo.builder("n4.connector", Constants.CONNECTOR_TYPE).build()).addConnections("n1", ImmutableSet.of("n2.connector", "n3.connector", "n4.connector")).build();
    String phase1Name = getPhaseName("n1", "n2.connector", "n3.connector", "n4.connector");
    phases.put(phase1Name, phase1);
    /*
        phase2:
        n2.connector --- n2(r) --- n6 --- n7.connector
     */
    PipelinePhase phase2 = PipelinePhase.builder(pluginTypes).addStage(StageInfo.builder("n2", AGGREGATOR).addInputs("n1").addInputSchema("n1", schema).addOutputs("n6").setOutputSchema(schema).build()).addStage(StageInfo.builder("n6", NODE).addInputs("n2", "n4", "n5").addInputSchema("n2", schema).addInputSchema("n4", schema).addInputSchema("n5", schema).addOutputs("n7").setOutputSchema(schema).build()).addStage(StageInfo.builder("n2.connector", Constants.CONNECTOR_TYPE).build()).addStage(StageInfo.builder("n7.connector", Constants.CONNECTOR_TYPE).build()).addConnection("n2.connector", "n2").addConnection("n2", "n6").addConnection("n6", "n7.connector").build();
    String phase2Name = getPhaseName("n2.connector", "n7.connector");
    phases.put(phase2Name, phase2);
    /*
        phase3:
        n3.connector --- n3(r) --- n5 --- n6 --- n7.connector
     */
    PipelinePhase phase3 = PipelinePhase.builder(pluginTypes).addStage(StageInfo.builder("n5", NODE).addInputs("n3").addInputSchema("n3", schema).addOutputs("n6").setOutputSchema(schema).build()).addStage(StageInfo.builder("n6", NODE).addInputs("n2", "n4", "n5").addInputSchema("n2", schema).addInputSchema("n4", schema).addInputSchema("n5", schema).addOutputs("n7").setOutputSchema(schema).build()).addStage(StageInfo.builder("n3", AGGREGATOR).addInputs("n1").addInputSchema("n1", schema).addOutputs("n5").setOutputSchema(schema).build()).addStage(StageInfo.builder("n3.connector", Constants.CONNECTOR_TYPE).build()).addStage(StageInfo.builder("n7.connector", Constants.CONNECTOR_TYPE).build()).addConnection("n3.connector", "n3").addConnection("n3", "n5").addConnection("n5", "n6").addConnection("n6", "n7.connector").build();
    String phase3Name = getPhaseName("n3.connector", "n7.connector");
    phases.put(phase3Name, phase3);
    /*
        phase4:
        n4.connector --- n4(r) --- n6 --- n7.connector
     */
    PipelinePhase phase4 = PipelinePhase.builder(pluginTypes).addStage(StageInfo.builder("n4", AGGREGATOR).addInputs("n1").addInputSchema("n1", schema).addOutputs("n6").setOutputSchema(schema).build()).addStage(StageInfo.builder("n6", NODE).addInputs("n2", "n4", "n5").addInputSchema("n2", schema).addInputSchema("n4", schema).addInputSchema("n5", schema).addOutputs("n7").setOutputSchema(schema).build()).addStage(StageInfo.builder("n4.connector", Constants.CONNECTOR_TYPE).build()).addStage(StageInfo.builder("n7.connector", Constants.CONNECTOR_TYPE).build()).addConnection("n4.connector", "n4").addConnection("n4", "n6").addConnection("n6", "n7.connector").build();
    String phase4Name = getPhaseName("n4.connector", "n7.connector");
    phases.put(phase4Name, phase4);
    /*
        phase5:
        n7.connector --- n7(r) --- n8 --- n9.connector
     */
    PipelinePhase phase5 = PipelinePhase.builder(pluginTypes).addStage(StageInfo.builder("n8", NODE).addInputs("n7").addInputSchema("n7", schema).addOutputs("n9").setOutputSchema(schema).build()).addStage(StageInfo.builder("n7", AGGREGATOR).addInputs("n6").addInputSchema("n6", schema).addOutputs("n8").setOutputSchema(schema).build()).addStage(StageInfo.builder("n7.connector", Constants.CONNECTOR_TYPE).build()).addStage(StageInfo.builder("n9.connector", Constants.CONNECTOR_TYPE).build()).addConnection("n7.connector", "n7").addConnection("n7", "n8").addConnection("n8", "n9.connector").build();
    String phase5Name = getPhaseName("n7.connector", "n9.connector");
    phases.put(phase5Name, phase5);
    /*
        phase6:
                                 |-- n10
        n9.connector --- n9(r) --|
                                 |-- n11
     */
    PipelinePhase phase6 = PipelinePhase.builder(pluginTypes).addStage(StageInfo.builder("n10", NODE).addInputs("n9").addInputSchema("n9", schema).build()).addStage(StageInfo.builder("n11", NODE).addInputs("n9").addInputSchema("n9", schema).build()).addStage(StageInfo.builder("n9", AGGREGATOR).addInputs("n8").addInputSchema("n8", schema).addOutputs("n10", "n11").setOutputSchema(schema).build()).addStage(StageInfo.builder("n9.connector", Constants.CONNECTOR_TYPE).build()).addConnection("n9.connector", "n9").addConnection("n9", "n10").addConnection("n9", "n11").build();
    String phase6Name = getPhaseName("n9.connector", "n10", "n11");
    phases.put(phase6Name, phase6);
    Set<Connection> phaseConnections = new HashSet<>();
    phaseConnections.add(new Connection(phase1Name, phase2Name));
    phaseConnections.add(new Connection(phase1Name, phase3Name));
    phaseConnections.add(new Connection(phase1Name, phase4Name));
    phaseConnections.add(new Connection(phase2Name, phase5Name));
    phaseConnections.add(new Connection(phase3Name, phase5Name));
    phaseConnections.add(new Connection(phase4Name, phase5Name));
    phaseConnections.add(new Connection(phase5Name, phase6Name));
    PipelinePlan expected = new PipelinePlan(phases, phaseConnections);
    PipelinePlan actual = planner.plan(pipelineSpec);
    Assert.assertEquals(expected, actual);
}

Example 10 with ArtifactId

use of co.cask.cdap.api.artifact.ArtifactId in project cdap by caskdata.

the class PipelineSpecGeneratorTest method setupTests.

@BeforeClass
public static void setupTests() {
    // populate some mock plugins.
    MockPluginConfigurer pluginConfigurer = new MockPluginConfigurer();
    Set<ArtifactId> artifactIds = ImmutableSet.of(ARTIFACT_ID);
    pluginConfigurer.addMockPlugin(BatchSource.PLUGIN_TYPE, "mocksource", MockPlugin.builder().setOutputSchema(SCHEMA_A).build(), artifactIds);
    pluginConfigurer.addMockPlugin(Transform.PLUGIN_TYPE, "mockA", MockPlugin.builder().setOutputSchema(SCHEMA_A).setErrorSchema(SCHEMA_B).build(), artifactIds);
    pluginConfigurer.addMockPlugin(Transform.PLUGIN_TYPE, "mockB", MockPlugin.builder().setOutputSchema(SCHEMA_B).build(), artifactIds);
    pluginConfigurer.addMockPlugin(BatchSink.PLUGIN_TYPE, "mocksink", MockPlugin.builder().build(), artifactIds);
    pluginConfigurer.addMockPlugin(Action.PLUGIN_TYPE, "mockaction", MockPlugin.builder().build(), artifactIds);
    pluginConfigurer.addMockPlugin(BatchJoiner.PLUGIN_TYPE, "mockjoiner", MockPlugin.builder().build(), artifactIds);
    pluginConfigurer.addMockPlugin(ErrorTransform.PLUGIN_TYPE, "mockerror", MockPlugin.builder().build(), artifactIds);
    specGenerator = new BatchPipelineSpecGenerator(pluginConfigurer, ImmutableSet.of(BatchSource.PLUGIN_TYPE), ImmutableSet.of(BatchSink.PLUGIN_TYPE), FileSet.class, DatasetProperties.EMPTY, Engine.MAPREDUCE);
}