Examples with ArtifactConfig io.cdap.cdap.common.conf.ArtifactConfig used on opensource projects

Example 1 with ArtifactConfig

use of io.cdap.cdap.common.conf.ArtifactConfig in project cdap by caskdata.

the class DefaultArtifactRepository method addSystemArtifacts.

@Override
public void addSystemArtifacts() throws Exception {
    // scan the directory for artifact .jar files and config files for those artifacts
    Map<Id.Artifact, SystemArtifactInfo> systemArtifacts = new HashMap<>();
    for (File systemArtifactDir : systemArtifactDirs) {
        for (File jarFile : DirUtils.listFiles(systemArtifactDir, "jar")) {
            // parse id from filename
            Id.Artifact artifactId;
            try {
                artifactId = Id.Artifact.parse(Id.Namespace.SYSTEM, jarFile.getName());
            } catch (IllegalArgumentException e) {
                LOG.warn(String.format("Skipping system artifact '%s' because the name is invalid: ", e.getMessage()));
                continue;
            }
            // check for a corresponding .json config file
            String artifactFileName = jarFile.getName();
            String configFileName = artifactFileName.substring(0, artifactFileName.length() - ".jar".length()) + ".json";
            File configFile = new File(systemArtifactDir, configFileName);
            try {
                // read and parse the config file if it exists. Otherwise use an empty config with the artifact filename
                ArtifactConfig artifactConfig = configFile.isFile() ? configReader.read(artifactId.getNamespace(), configFile) : new ArtifactConfig();
                validateParentSet(artifactId, artifactConfig.getParents());
                validatePluginSet(artifactConfig.getPlugins());
                systemArtifacts.put(artifactId, new SystemArtifactInfo(artifactId, jarFile, artifactConfig));
            } catch (InvalidArtifactException e) {
                LOG.warn(String.format("Could not add system artifact '%s' because it is invalid.", artifactFileName), e);
            }
        }
    }
    // child -> parents
    Multimap<Id.Artifact, Id.Artifact> childToParents = HashMultimap.create();
    // parent -> children
    Multimap<Id.Artifact, Id.Artifact> parentToChildren = HashMultimap.create();
    Set<Id.Artifact> remainingArtifacts = new HashSet<>();
    // build mapping from child to parents and from parents to children
    for (SystemArtifactInfo child : systemArtifacts.values()) {
        Id.Artifact childId = child.getArtifactId();
        remainingArtifacts.add(childId);
        for (SystemArtifactInfo potentialParent : systemArtifacts.values()) {
            Id.Artifact potentialParentId = potentialParent.getArtifactId();
            // skip if we're looking at ourselves
            if (childId.equals(potentialParentId)) {
                continue;
            }
            if (child.getConfig().hasParent(potentialParentId)) {
                childToParents.put(childId, potentialParentId);
                parentToChildren.put(potentialParentId, childId);
            }
        }
    }
    if (!remainingArtifacts.isEmpty()) {
        ExecutorService executorService = Executors.newFixedThreadPool(Math.min(maxArtifactLoadParallelism, remainingArtifacts.size()), Threads.createDaemonThreadFactory("system-artifact-loader-%d"));
        try {
            // loop until there is no change
            boolean artifactsAdded = true;
            while (!remainingArtifacts.isEmpty() && artifactsAdded) {
                artifactsAdded = loadSystemArtifacts(executorService, systemArtifacts, remainingArtifacts, parentToChildren, childToParents);
            }
        } finally {
            executorService.shutdownNow();
        }
        if (!remainingArtifacts.isEmpty()) {
            LOG.warn("Unable to add system artifacts {} due to cyclic dependencies", Joiner.on(",").join(remainingArtifacts));
        }
    }
}

Example 2 with ArtifactConfig

use of io.cdap.cdap.common.conf.ArtifactConfig in project cdap by caskdata.

the class ArtifactRepositoryTest method testAddSystemArtifacts.

@Test
public void testAddSystemArtifacts() throws Exception {
    Id.Artifact systemAppArtifactId = Id.Artifact.from(Id.Namespace.SYSTEM, "PluginTest", "1.0.0");
    File systemAppJar = createAppJar(PluginTestApp.class, new File(systemArtifactsDir1, "PluginTest-1.0.0.jar"), createManifest(ManifestFields.EXPORT_PACKAGE, PluginTestRunnable.class.getPackage().getName()));
    // write plugins jar
    Id.Artifact pluginArtifactId1 = Id.Artifact.from(Id.Namespace.SYSTEM, "APlugin", "1.0.0");
    Manifest manifest = createManifest(ManifestFields.EXPORT_PACKAGE, TestPlugin.class.getPackage().getName());
    File pluginJar1 = createPluginJar(TestPlugin.class, new File(systemArtifactsDir1, "APlugin-1.0.0.jar"), manifest);
    // write plugins config file
    Map<String, PluginPropertyField> emptyMap = Collections.emptyMap();
    Set<PluginClass> manuallyAddedPlugins1 = ImmutableSet.of(PluginClass.builder().setName("manual1").setType("typeA").setDescription("desc").setClassName(TestPlugin.class.getName()).setProperties(emptyMap).build(), PluginClass.builder().setName("manual2").setType("typeB").setDescription("desc").setClassName(TestPlugin.class.getName()).setProperties(emptyMap).build());
    File pluginConfigFile = new File(systemArtifactsDir1, "APlugin-1.0.0.json");
    ArtifactConfig pluginConfig1 = new ArtifactConfig(ImmutableSet.of(new ArtifactRange(NamespaceId.SYSTEM.getNamespace(), "PluginTest", new ArtifactVersion("0.9.0"), new ArtifactVersion("2.0.0"))), // add a dummy plugin to test explicit addition of plugins through the config file
    manuallyAddedPlugins1, ImmutableMap.of("k1", "v1", "k2", "v2"));
    try (BufferedWriter writer = Files.newWriter(pluginConfigFile, Charsets.UTF_8)) {
        writer.write(pluginConfig1.toString());
    }
    // write another plugins jar to a different directory, to test that plugins will get picked up from both directories
    Id.Artifact pluginArtifactId2 = Id.Artifact.from(Id.Namespace.SYSTEM, "BPlugin", "1.0.0");
    manifest = createManifest(ManifestFields.EXPORT_PACKAGE, TestPlugin.class.getPackage().getName());
    File pluginJar2 = createPluginJar(TestPlugin.class, new File(systemArtifactsDir2, "BPlugin-1.0.0.jar"), manifest);
    // write plugins config file
    Set<PluginClass> manuallyAddedPlugins2 = ImmutableSet.of(PluginClass.builder().setName("manual1").setType("typeA").setDescription("desc").setClassName(TestPlugin.class.getName()).setProperties(emptyMap).build(), PluginClass.builder().setName("manual2").setType("typeB").setDescription("desc").setClassName(TestPlugin.class.getName()).setProperties(emptyMap).build());
    pluginConfigFile = new File(systemArtifactsDir2, "BPlugin-1.0.0.json");
    ArtifactConfig pluginConfig2 = new ArtifactConfig(ImmutableSet.of(new ArtifactRange(NamespaceId.SYSTEM.getNamespace(), "PluginTest", new ArtifactVersion("0.9.0"), new ArtifactVersion("2.0.0"))), manuallyAddedPlugins2, ImmutableMap.of("k3", "v3"));
    try (BufferedWriter writer = Files.newWriter(pluginConfigFile, Charsets.UTF_8)) {
        writer.write(pluginConfig2.toString());
    }
    artifactRepository.addSystemArtifacts();
    Assert.assertTrue(systemAppJar.delete());
    Assert.assertTrue(pluginJar1.delete());
    Assert.assertTrue(pluginJar2.delete());
    try {
        // check app artifact added correctly
        ArtifactDetail appArtifactDetail = artifactRepository.getArtifact(systemAppArtifactId);
        Map<ArtifactDescriptor, Set<PluginClass>> plugins = artifactRepository.getPlugins(NamespaceId.DEFAULT, systemAppArtifactId);
        Assert.assertEquals(2, plugins.size());
        Set<PluginClass> pluginClasses = plugins.values().iterator().next();
        Set<String> pluginNames = Sets.newHashSet();
        for (PluginClass pluginClass : pluginClasses) {
            pluginNames.add(pluginClass.getName());
        }
        Assert.assertEquals(Sets.newHashSet("manual1", "manual2", "TestPlugin", "TestPlugin2"), pluginNames);
        Assert.assertEquals(systemAppArtifactId.getName(), appArtifactDetail.getDescriptor().getArtifactId().getName());
        Assert.assertEquals(systemAppArtifactId.getVersion(), appArtifactDetail.getDescriptor().getArtifactId().getVersion());
        // check plugin artifact added correctly
        ArtifactDetail pluginArtifactDetail = artifactRepository.getArtifact(pluginArtifactId1);
        Assert.assertEquals(pluginArtifactId1.getName(), pluginArtifactDetail.getDescriptor().getArtifactId().getName());
        Assert.assertEquals(pluginArtifactId1.getVersion(), pluginArtifactDetail.getDescriptor().getArtifactId().getVersion());
        // check manually added plugins are there
        Assert.assertTrue(pluginArtifactDetail.getMeta().getClasses().getPlugins().containsAll(manuallyAddedPlugins1));
        // check properties are there
        Assert.assertEquals(pluginConfig1.getProperties(), pluginArtifactDetail.getMeta().getProperties());
        // check other plugin artifact added correctly
        pluginArtifactDetail = artifactRepository.getArtifact(pluginArtifactId2);
        Assert.assertEquals(pluginArtifactId2.getName(), pluginArtifactDetail.getDescriptor().getArtifactId().getName());
        Assert.assertEquals(pluginArtifactId2.getVersion(), pluginArtifactDetail.getDescriptor().getArtifactId().getVersion());
        // check manually added plugins are there
        Assert.assertTrue(pluginArtifactDetail.getMeta().getClasses().getPlugins().containsAll(manuallyAddedPlugins2));
        // check properties are there
        Assert.assertEquals(pluginConfig2.getProperties(), pluginArtifactDetail.getMeta().getProperties());
    } finally {
        artifactRepository.clear(NamespaceId.SYSTEM);
    }
}

Example 3 with ArtifactConfig

use of io.cdap.cdap.common.conf.ArtifactConfig in project cdap by caskdata.

the class LoadArtifactCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    File artifactFile = resolver.resolvePathToFile(arguments.get(ArgumentName.LOCAL_FILE_PATH.toString()));
    String name = arguments.getOptional(ArgumentName.ARTIFACT_NAME.toString());
    String version = arguments.getOptional(ArgumentName.ARTIFACT_VERSION.toString());
    ArtifactId artifactId;
    if (name == null && version != null) {
        throw new IllegalArgumentException("If a version is specified, name must also be specified.");
    } else if (name != null && version == null) {
        throw new IllegalArgumentException("If a name is specified, a version must also be specified.");
    } else if (name == null) {
        artifactId = new ArtifactId(cliConfig.getCurrentNamespace().getNamespace(), artifactFile.getName());
    } else {
        artifactId = cliConfig.getCurrentNamespace().artifact(name, version);
    }
    String configPath = arguments.getOptional(ArgumentName.ARTIFACT_CONFIG_FILE.toString());
    NamespaceId namespace = artifactId.getParent();
    if (configPath == null) {
        artifactClient.add(namespace, artifactId.getEntityName(), () -> new FileInputStream(artifactFile), artifactId.getVersion());
    } else {
        File configFile = resolver.resolvePathToFile(configPath);
        ArtifactConfig artifactConfig = configReader.read(Id.Namespace.fromEntityId(namespace), configFile);
        artifactClient.add(namespace, artifactId.getEntityName(), () -> new FileInputStream(artifactFile), artifactId.getVersion(), artifactConfig.getParents(), artifactConfig.getPlugins());
        Map<String, String> properties = artifactConfig.getProperties();
        if (properties != null && !properties.isEmpty()) {
            artifactClient.writeProperties(artifactId, properties);
        }
    }
    output.printf("Successfully added artifact with name '%s'\n", artifactId.getEntityName());
}