Examples with ProducerConsumerExecutor org.jfrog.build.extractor.producerConsumer.ProducerConsumerExecutor used on opensource projects

Example 1 with ProducerConsumerExecutor

use of org.jfrog.build.extractor.producerConsumer.ProducerConsumerExecutor in project build-info by JFrogDev.

the class NpmBuildInfoExtractor method populateDependenciesMap.

/**
 * Populate the dependencies map for the specified scope by:
 * 1. Create npm dependency tree from root node of 'npm ls' command tree. Populate each node with name, version and scope.
 * 2. For each dependency, retrieve sha1 and md5 from Artifactory. Use the producer-consumer mechanism to parallelize it.
 */
private void populateDependenciesMap(Map<String, Dependency> dependencies, Map<String, Dependency> previousBuildDependencies, JsonNode npmDependencyTree, NpmScope scope, Path workingDir) throws Exception {
    // Set of packages that could not be found in Artifactory.
    Set<NpmPackageInfo> badPackages = Collections.synchronizedSet(new HashSet<>());
    DefaultMutableTreeNode rootNode = NpmDependencyTree.createDependencyTree(npmDependencyTree, scope, workingDir);
    try (ArtifactoryManager artifactoryManager = artifactoryManagerBuilder.build()) {
        // Create producer Runnable.
        ProducerRunnableBase[] producerRunnable = new ProducerRunnableBase[] { new NpmExtractorProducer(rootNode) };
        // Create consumer Runnables.
        ConsumerRunnableBase[] consumerRunnables = new ConsumerRunnableBase[] { new NpmExtractorConsumer(artifactoryManager, dependencies, previousBuildDependencies, badPackages), new NpmExtractorConsumer(artifactoryManager, dependencies, previousBuildDependencies, badPackages), new NpmExtractorConsumer(artifactoryManager, dependencies, previousBuildDependencies, badPackages) };
        // Create the deployment executor.
        ProducerConsumerExecutor deploymentExecutor = new ProducerConsumerExecutor(logger, producerRunnable, consumerRunnables, CONNECTION_POOL_SIZE);
        deploymentExecutor.start();
        if (!badPackages.isEmpty()) {
            logger.info((Arrays.toString(badPackages.toArray())));
            logger.info("The npm dependencies above could not be found in Artifactory and therefore are not included in the build-info. " + "Make sure the dependencies are available in Artifactory for this build. " + "Deleting the local cache will force populating Artifactory with these dependencies.");
        }
    }
}

Example 2 with ProducerConsumerExecutor

use of org.jfrog.build.extractor.producerConsumer.ProducerConsumerExecutor in project build-info by JFrogDev.

the class SpecsHelper method uploadArtifactsBySpec.

/**
 * Upload artifacts according to a given spec, return a list describing the deployed items.
 *
 * @param uploadSpec                The required spec represented as String
 * @param numberOfThreads           Number of concurrent threads to use for handling uploads
 * @param workspace                 File object that represents the workspace
 * @param buildProperties           Upload properties
 * @param artifactoryManagerBuilder ArtifactoryManagerBuilder which will build the ArtifactoryManager per the number of passed threads number to perform the actual upload
 * @return Set of DeployDetails that was calculated from the given params
 * @throws IOException Thrown if any error occurs while reading the file, calculating the
 *                     checksums or in case of any file system exception
 */
public List<Artifact> uploadArtifactsBySpec(String uploadSpec, int numberOfThreads, File workspace, Multimap<String, String> buildProperties, ArtifactoryManagerBuilder artifactoryManagerBuilder) throws Exception {
    FileSpec fileSpec = FileSpec.fromString(uploadSpec);
    FileSpecsValidation.validateUploadFileSpec(fileSpec, this.log);
    try (ArtifactoryManager artifactoryManager = artifactoryManagerBuilder.build()) {
        // Create producer Runnable
        ProducerRunnableBase[] producerRunnable = new ProducerRunnableBase[] { new SpecDeploymentProducer(fileSpec, workspace, buildProperties) };
        // Create consumer Runnables
        ConsumerRunnableBase[] consumerRunnables = new ConsumerRunnableBase[numberOfThreads];
        for (int i = 0; i < numberOfThreads; i++) {
            consumerRunnables[i] = new SpecDeploymentConsumer(artifactoryManager);
        }
        // Create the deployment executor
        ProducerConsumerExecutor deploymentExecutor = new ProducerConsumerExecutor(log, producerRunnable, consumerRunnables, CONNECTION_POOL_SIZE);
        deploymentExecutor.start();
        Set<DeployDetails> deployedArtifacts = ((SpecDeploymentProducer) producerRunnable[0]).getDeployedArtifacts();
        return convertDeployDetailsToArtifacts(deployedArtifacts);
    }
}