Examples with ArtifactConfig co.cask.cdap.common.conf.ArtifactConfig used on opensource projects

Example 1 with ArtifactConfig

use of co.cask.cdap.common.conf.ArtifactConfig in project cdap by caskdata.

the class DefaultArtifactRepository method addSystemArtifacts.

@Override
public void addSystemArtifacts() throws Exception {
    // scan the directory for artifact .jar files and config files for those artifacts
    Map<Id.Artifact, SystemArtifactInfo> systemArtifacts = new HashMap<>();
    for (File systemArtifactDir : systemArtifactDirs) {
        for (File jarFile : DirUtils.listFiles(systemArtifactDir, "jar")) {
            // parse id from filename
            Id.Artifact artifactId;
            try {
                artifactId = Id.Artifact.parse(Id.Namespace.SYSTEM, jarFile.getName());
            } catch (IllegalArgumentException e) {
                LOG.warn(String.format("Skipping system artifact '%s' because the name is invalid: ", e.getMessage()));
                continue;
            }
            // check for a corresponding .json config file
            String artifactFileName = jarFile.getName();
            String configFileName = artifactFileName.substring(0, artifactFileName.length() - ".jar".length()) + ".json";
            File configFile = new File(systemArtifactDir, configFileName);
            try {
                // read and parse the config file if it exists. Otherwise use an empty config with the artifact filename
                ArtifactConfig artifactConfig = configFile.isFile() ? configReader.read(artifactId.getNamespace(), configFile) : new ArtifactConfig();
                validateParentSet(artifactId, artifactConfig.getParents());
                validatePluginSet(artifactConfig.getPlugins());
                systemArtifacts.put(artifactId, new SystemArtifactInfo(artifactId, jarFile, artifactConfig));
            } catch (InvalidArtifactException e) {
                LOG.warn(String.format("Could not add system artifact '%s' because it is invalid.", artifactFileName), e);
            }
        }
    }
    // child -> parents
    Multimap<Id.Artifact, Id.Artifact> childToParents = HashMultimap.create();
    // parent -> children
    Multimap<Id.Artifact, Id.Artifact> parentToChildren = HashMultimap.create();
    Set<Id.Artifact> remainingArtifacts = new HashSet<>();
    // build mapping from child to parents and from parents to children
    for (SystemArtifactInfo child : systemArtifacts.values()) {
        Id.Artifact childId = child.getArtifactId();
        remainingArtifacts.add(childId);
        for (SystemArtifactInfo potentialParent : systemArtifacts.values()) {
            Id.Artifact potentialParentId = potentialParent.getArtifactId();
            // skip if we're looking at ourselves
            if (childId.equals(potentialParentId)) {
                continue;
            }
            if (child.getConfig().hasParent(potentialParentId)) {
                childToParents.put(childId, potentialParentId);
                parentToChildren.put(potentialParentId, childId);
            }
        }
    }
    // loop until there is no change
    boolean nochange = false;
    while (!remainingArtifacts.isEmpty() && !nochange) {
        // add all artifacts that don't have any more parents
        Set<Id.Artifact> addedArtifacts = new HashSet<>();
        for (Id.Artifact remainingArtifact : remainingArtifacts) {
            if (!childToParents.containsKey(remainingArtifact)) {
                addSystemArtifact(systemArtifacts.get(remainingArtifact));
                addedArtifacts.add(remainingArtifact);
                for (Id.Artifact child : parentToChildren.get(remainingArtifact)) {
                    childToParents.remove(child, remainingArtifact);
                }
            }
        }
        remainingArtifacts.removeAll(addedArtifacts);
        nochange = addedArtifacts.isEmpty();
    }
    if (!remainingArtifacts.isEmpty()) {
        LOG.warn("Unable to add system artifacts {} due to cyclic dependencies", Joiner.on(",").join(remainingArtifacts));
    }
}

Example 2 with ArtifactConfig

use of co.cask.cdap.common.conf.ArtifactConfig in project cdap by caskdata.

the class LoadArtifactCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    File artifactFile = resolver.resolvePathToFile(arguments.get(ArgumentName.LOCAL_FILE_PATH.toString()));
    String name = arguments.getOptional(ArgumentName.ARTIFACT_NAME.toString());
    String version = arguments.getOptional(ArgumentName.ARTIFACT_VERSION.toString());
    ArtifactId artifactId;
    if (name == null && version != null) {
        throw new IllegalArgumentException("If a version is specified, name must also be specified.");
    } else if (name != null && version == null) {
        throw new IllegalArgumentException("If a name is specified, a version must also be specified.");
    } else if (name == null) {
        artifactId = new ArtifactId(cliConfig.getCurrentNamespace().getNamespace(), artifactFile.getName());
    } else {
        artifactId = cliConfig.getCurrentNamespace().artifact(name, version);
    }
    String configPath = arguments.getOptional(ArgumentName.ARTIFACT_CONFIG_FILE.toString());
    NamespaceId namespace = artifactId.getParent();
    if (configPath == null) {
        artifactClient.add(namespace, artifactId.getEntityName(), Files.newInputStreamSupplier(artifactFile), artifactId.getVersion());
    } else {
        File configFile = resolver.resolvePathToFile(configPath);
        ArtifactConfig artifactConfig = configReader.read(Id.Namespace.fromEntityId(namespace), configFile);
        artifactClient.add(namespace, artifactId.getEntityName(), Files.newInputStreamSupplier(artifactFile), artifactId.getVersion(), artifactConfig.getParents(), artifactConfig.getPlugins());
        Map<String, String> properties = artifactConfig.getProperties();
        if (properties != null && !properties.isEmpty()) {
            artifactClient.writeProperties(artifactId, properties);
        }
    }
    output.printf("Successfully added artifact with name '%s'\n", artifactId.getEntityName());
}

Example 3 with ArtifactConfig

use of co.cask.cdap.common.conf.ArtifactConfig in project cdap by caskdata.

the class ArtifactRepository method addSystemArtifacts.

/**
   * Scan all files in the local system artifact directory, looking for jar files and adding them as system artifacts.
   * If the artifact already exists it will not be added again unless it is a snapshot version.
   *
   * @throws IOException if there was some IO error adding the system artifacts
   */
public void addSystemArtifacts() throws Exception {
    // to add system artifacts, users should have write privileges on the system namespace
    Principal principal = authenticationContext.getPrincipal();
    authorizationEnforcer.enforce(NamespaceId.SYSTEM, principal, Action.WRITE);
    // scan the directory for artifact .jar files and config files for those artifacts
    List<SystemArtifactInfo> systemArtifacts = new ArrayList<>();
    for (File systemArtifactDir : systemArtifactDirs) {
        for (File jarFile : DirUtils.listFiles(systemArtifactDir, "jar")) {
            // parse id from filename
            Id.Artifact artifactId;
            try {
                artifactId = Id.Artifact.parse(Id.Namespace.SYSTEM, jarFile.getName());
            } catch (IllegalArgumentException e) {
                LOG.warn(String.format("Skipping system artifact '%s' because the name is invalid: ", e.getMessage()));
                continue;
            }
            // first revoke any orphane privileges
            co.cask.cdap.proto.id.ArtifactId artifact = artifactId.toEntityId();
            privilegesManager.revoke(artifact);
            // then grant all on the artifact
            privilegesManager.grant(artifact, principal, EnumSet.allOf(Action.class));
            // check for a corresponding .json config file
            String artifactFileName = jarFile.getName();
            String configFileName = artifactFileName.substring(0, artifactFileName.length() - ".jar".length()) + ".json";
            File configFile = new File(systemArtifactDir, configFileName);
            try {
                // read and parse the config file if it exists. Otherwise use an empty config with the artifact filename
                ArtifactConfig artifactConfig = configFile.isFile() ? configReader.read(artifactId.getNamespace(), configFile) : new ArtifactConfig();
                validateParentSet(artifactId, artifactConfig.getParents());
                validatePluginSet(artifactConfig.getPlugins());
                systemArtifacts.add(new SystemArtifactInfo(artifactId, jarFile, artifactConfig));
            } catch (InvalidArtifactException e) {
                LOG.warn(String.format("Could not add system artifact '%s' because it is invalid.", artifactFileName), e);
                // since adding artifact failed, revoke privileges, since they may be orphane now
                privilegesManager.revoke(artifact);
            }
        }
    }
    // taking advantage of the fact that we only have 1 level of dependencies
    // so we can add all the parents first, then we know its safe to add everything else
    // add all parents
    Set<Id.Artifact> parents = new HashSet<>();
    for (SystemArtifactInfo child : systemArtifacts) {
        Id.Artifact childId = child.getArtifactId();
        for (SystemArtifactInfo potentialParent : systemArtifacts) {
            Id.Artifact potentialParentId = potentialParent.getArtifactId();
            // skip if we're looking at ourselves
            if (childId.equals(potentialParentId)) {
                continue;
            }
            if (child.getConfig().hasParent(potentialParentId)) {
                parents.add(potentialParentId);
            }
        }
    }
    // add all parents first
    for (SystemArtifactInfo systemArtifact : systemArtifacts) {
        if (parents.contains(systemArtifact.getArtifactId())) {
            addSystemArtifact(systemArtifact);
        }
    }
    // add children next
    for (SystemArtifactInfo systemArtifact : systemArtifacts) {
        if (!parents.contains(systemArtifact.getArtifactId())) {
            addSystemArtifact(systemArtifact);
        }
    }
}

Example 4 with ArtifactConfig

use of co.cask.cdap.common.conf.ArtifactConfig in project cdap by caskdata.

the class ArtifactRepositoryTest method testAddSystemArtifacts.

@Test
public void testAddSystemArtifacts() throws Exception {
    Id.Artifact systemAppArtifactId = Id.Artifact.from(Id.Namespace.SYSTEM, "PluginTest", "1.0.0");
    File systemAppJar = createAppJar(PluginTestApp.class, new File(systemArtifactsDir1, "PluginTest-1.0.0.jar"), createManifest(ManifestFields.EXPORT_PACKAGE, PluginTestRunnable.class.getPackage().getName()));
    // write plugins jar
    Id.Artifact pluginArtifactId1 = Id.Artifact.from(Id.Namespace.SYSTEM, "APlugin", "1.0.0");
    Manifest manifest = createManifest(ManifestFields.EXPORT_PACKAGE, TestPlugin.class.getPackage().getName());
    File pluginJar1 = createPluginJar(TestPlugin.class, new File(systemArtifactsDir1, "APlugin-1.0.0.jar"), manifest);
    // write plugins config file
    Map<String, PluginPropertyField> emptyMap = Collections.emptyMap();
    Set<PluginClass> manuallyAddedPlugins1 = ImmutableSet.of(new PluginClass("typeA", "manual1", "desc", "co.cask.classname", null, emptyMap), new PluginClass("typeB", "manual2", "desc", "co.cask.otherclassname", null, emptyMap));
    File pluginConfigFile = new File(systemArtifactsDir1, "APlugin-1.0.0.json");
    ArtifactConfig pluginConfig1 = new ArtifactConfig(ImmutableSet.of(new ArtifactRange(NamespaceId.SYSTEM.getNamespace(), "PluginTest", new ArtifactVersion("0.9.0"), new ArtifactVersion("2.0.0"))), // add a dummy plugin to test explicit addition of plugins through the config file
    manuallyAddedPlugins1, ImmutableMap.of("k1", "v1", "k2", "v2"));
    try (BufferedWriter writer = Files.newWriter(pluginConfigFile, Charsets.UTF_8)) {
        writer.write(pluginConfig1.toString());
    }
    // write another plugins jar to a different directory, to test that plugins will get picked up from both directories
    Id.Artifact pluginArtifactId2 = Id.Artifact.from(Id.Namespace.SYSTEM, "BPlugin", "1.0.0");
    manifest = createManifest(ManifestFields.EXPORT_PACKAGE, TestPlugin.class.getPackage().getName());
    File pluginJar2 = createPluginJar(TestPlugin.class, new File(systemArtifactsDir2, "BPlugin-1.0.0.jar"), manifest);
    // write plugins config file
    Set<PluginClass> manuallyAddedPlugins2 = ImmutableSet.of(new PluginClass("typeA", "manual1", "desc", "co.notcask.classname", null, emptyMap), new PluginClass("typeB", "manual2", "desc", "co.notcask.otherclassname", null, emptyMap));
    pluginConfigFile = new File(systemArtifactsDir2, "BPlugin-1.0.0.json");
    ArtifactConfig pluginConfig2 = new ArtifactConfig(ImmutableSet.of(new ArtifactRange(NamespaceId.SYSTEM.getNamespace(), "PluginTest", new ArtifactVersion("0.9.0"), new ArtifactVersion("2.0.0"))), manuallyAddedPlugins2, ImmutableMap.of("k3", "v3"));
    try (BufferedWriter writer = Files.newWriter(pluginConfigFile, Charsets.UTF_8)) {
        writer.write(pluginConfig2.toString());
    }
    artifactRepository.addSystemArtifacts();
    Assert.assertTrue(systemAppJar.delete());
    Assert.assertTrue(pluginJar1.delete());
    Assert.assertTrue(pluginJar2.delete());
    try {
        // check app artifact added correctly
        ArtifactDetail appArtifactDetail = artifactRepository.getArtifact(systemAppArtifactId);
        Map<ArtifactDescriptor, Set<PluginClass>> plugins = artifactRepository.getPlugins(NamespaceId.DEFAULT, systemAppArtifactId);
        Assert.assertEquals(2, plugins.size());
        Set<PluginClass> pluginClasses = plugins.values().iterator().next();
        Set<String> pluginNames = Sets.newHashSet();
        for (PluginClass pluginClass : pluginClasses) {
            pluginNames.add(pluginClass.getName());
        }
        Assert.assertEquals(Sets.newHashSet("manual1", "manual2", "TestPlugin", "TestPlugin2"), pluginNames);
        Assert.assertEquals(systemAppArtifactId.getName(), appArtifactDetail.getDescriptor().getArtifactId().getName());
        Assert.assertEquals(systemAppArtifactId.getVersion(), appArtifactDetail.getDescriptor().getArtifactId().getVersion());
        // check plugin artifact added correctly
        ArtifactDetail pluginArtifactDetail = artifactRepository.getArtifact(pluginArtifactId1);
        Assert.assertEquals(pluginArtifactId1.getName(), pluginArtifactDetail.getDescriptor().getArtifactId().getName());
        Assert.assertEquals(pluginArtifactId1.getVersion(), pluginArtifactDetail.getDescriptor().getArtifactId().getVersion());
        // check manually added plugins are there
        Assert.assertTrue(pluginArtifactDetail.getMeta().getClasses().getPlugins().containsAll(manuallyAddedPlugins1));
        // check properties are there
        Assert.assertEquals(pluginConfig1.getProperties(), pluginArtifactDetail.getMeta().getProperties());
        // check other plugin artifact added correctly
        pluginArtifactDetail = artifactRepository.getArtifact(pluginArtifactId2);
        Assert.assertEquals(pluginArtifactId2.getName(), pluginArtifactDetail.getDescriptor().getArtifactId().getName());
        Assert.assertEquals(pluginArtifactId2.getVersion(), pluginArtifactDetail.getDescriptor().getArtifactId().getVersion());
        // check manually added plugins are there
        Assert.assertTrue(pluginArtifactDetail.getMeta().getClasses().getPlugins().containsAll(manuallyAddedPlugins2));
        // check properties are there
        Assert.assertEquals(pluginConfig2.getProperties(), pluginArtifactDetail.getMeta().getProperties());
    } finally {
        artifactRepository.clear(NamespaceId.SYSTEM);
    }
}