Search in sources :

Example 6 with MutableDependencyGraph

use of com.synopsys.integration.bdio.graph.MutableDependencyGraph in project hub-detect by blackducksoftware.

the class NpmCliParser method convertNpmJsonFileToCodeLocation.

NpmParseResult convertNpmJsonFileToCodeLocation(final String sourcePath, final String npmLsOutput) {
    final JsonObject npmJson = new JsonParser().parse(npmLsOutput).getAsJsonObject();
    final MutableDependencyGraph graph = new MutableMapDependencyGraph();
    final JsonElement projectNameElement = npmJson.getAsJsonPrimitive(JSON_NAME);
    final JsonElement projectVersionElement = npmJson.getAsJsonPrimitive(JSON_VERSION);
    String projectName = null;
    String projectVersion = null;
    if (projectNameElement != null) {
        projectName = projectNameElement.getAsString();
    }
    if (projectVersionElement != null) {
        projectVersion = projectVersionElement.getAsString();
    }
    populateChildren(graph, null, npmJson.getAsJsonObject(JSON_DEPENDENCIES), true);
    final ExternalId externalId = externalIdFactory.createNameVersionExternalId(Forge.NPM, projectName, projectVersion);
    final DetectCodeLocation codeLocation = new DetectCodeLocation.Builder(DetectCodeLocationType.NPM, sourcePath, externalId, graph).build();
    return new NpmParseResult(projectName, projectVersion, codeLocation);
}
Also used : MutableDependencyGraph(com.synopsys.integration.bdio.graph.MutableDependencyGraph) JsonElement(com.google.gson.JsonElement) DetectCodeLocation(com.blackducksoftware.integration.hub.detect.workflow.codelocation.DetectCodeLocation) ExternalId(com.synopsys.integration.bdio.model.externalid.ExternalId) JsonObject(com.google.gson.JsonObject) MutableMapDependencyGraph(com.synopsys.integration.bdio.graph.MutableMapDependencyGraph) JsonParser(com.google.gson.JsonParser)

Example 7 with MutableDependencyGraph

use of com.synopsys.integration.bdio.graph.MutableDependencyGraph in project hub-detect by blackducksoftware.

the class NpmLockfileParser method parse.

public NpmParseResult parse(final String sourcePath, final Optional<String> packageJsonText, final String lockFileText, final boolean includeDevDependencies) {
    final MutableDependencyGraph dependencyGraph = new MutableMapDependencyGraph();
    logger.info("Parsing lock file text: ");
    logger.debug(lockFileText);
    Optional<PackageJson> packageJson = Optional.empty();
    if (packageJsonText.isPresent()) {
        logger.debug(packageJsonText.get());
        packageJson = Optional.of(gson.fromJson(packageJsonText.get(), PackageJson.class));
    }
    final PackageLock packageLock = gson.fromJson(lockFileText, PackageLock.class);
    logger.debug(lockFileText);
    logger.info("Processing project.");
    if (packageLock.dependencies != null) {
        logger.info(String.format("Found %d dependencies.", packageLock.dependencies.size()));
        // Convert to our custom format
        NpmDependencyConverter dependencyConverter = new NpmDependencyConverter(externalIdFactory);
        NpmDependency rootDependency = dependencyConverter.convertLockFile(packageLock, packageJson);
        traverse(rootDependency, dependencyGraph, true, includeDevDependencies);
    } else {
        logger.info("Lock file did not have a 'dependencies' section.");
    }
    logger.info("Finished processing.");
    final ExternalId projectId = externalIdFactory.createNameVersionExternalId(Forge.NPM, packageLock.name, packageLock.version);
    final DetectCodeLocation codeLocation = new DetectCodeLocation.Builder(DetectCodeLocationType.NPM, sourcePath, projectId, dependencyGraph).build();
    return new NpmParseResult(packageLock.name, packageLock.version, codeLocation);
}
Also used : MutableDependencyGraph(com.synopsys.integration.bdio.graph.MutableDependencyGraph) PackageLock(com.blackducksoftware.integration.hub.detect.detector.npm.model.PackageLock) NpmDependency(com.blackducksoftware.integration.hub.detect.detector.npm.model.NpmDependency) DetectCodeLocation(com.blackducksoftware.integration.hub.detect.workflow.codelocation.DetectCodeLocation) ExternalId(com.synopsys.integration.bdio.model.externalid.ExternalId) MutableMapDependencyGraph(com.synopsys.integration.bdio.graph.MutableMapDependencyGraph) PackageJson(com.blackducksoftware.integration.hub.detect.detector.npm.model.PackageJson)

Example 8 with MutableDependencyGraph

use of com.synopsys.integration.bdio.graph.MutableDependencyGraph in project hub-detect by blackducksoftware.

the class GradleReportParser method parseDependencies.

public Optional<DetectCodeLocation> parseDependencies(final File codeLocationFile) {
    DetectCodeLocation codeLocation = null;
    String projectSourcePath = "";
    String projectGroup = "";
    String projectName = "";
    String projectVersionName = "";
    boolean processingMetaData = false;
    final MutableDependencyGraph graph = new MutableMapDependencyGraph();
    final DependencyHistory history = new DependencyHistory();
    try (FileInputStream dependenciesInputStream = new FileInputStream(codeLocationFile);
        BufferedReader reader = new BufferedReader(new InputStreamReader(dependenciesInputStream, StandardCharsets.UTF_8))) {
        while (reader.ready()) {
            final String line = reader.readLine();
            /**
             * The meta data section will be at the end of the file after all of the "gradle dependencies" output
             */
            if (line.startsWith(DETECT_META_DATA_HEADER)) {
                processingMetaData = true;
                continue;
            }
            if (line.startsWith(DETECT_META_DATA_FOOTER)) {
                processingMetaData = false;
                continue;
            }
            if (processingMetaData) {
                if (line.startsWith(PROJECT_PATH_PREFIX)) {
                    projectSourcePath = line.substring(PROJECT_PATH_PREFIX.length()).trim();
                } else if (line.startsWith(PROJECT_GROUP_PREFIX)) {
                    projectGroup = line.substring(PROJECT_GROUP_PREFIX.length()).trim();
                } else if (line.startsWith(PROJECT_NAME_PREFIX)) {
                    projectName = line.substring(PROJECT_NAME_PREFIX.length()).trim();
                } else if (line.startsWith(PROJECT_VERSION_PREFIX)) {
                    projectVersionName = line.substring(PROJECT_VERSION_PREFIX.length()).trim();
                }
                continue;
            }
            if (StringUtils.isBlank(line)) {
                history.clear();
                gradleReportConfigurationParser = new GradleReportConfigurationParser();
                continue;
            }
            final Dependency dependency = gradleReportConfigurationParser.parseDependency(externalIdFactory, line);
            if (dependency == null) {
                continue;
            }
            final int lineTreeLevel = gradleReportConfigurationParser.getTreeLevel();
            try {
                history.clearDependenciesDeeperThan(lineTreeLevel);
            } catch (final IllegalStateException e) {
                logger.warn(String.format("Problem parsing line '%s': %s", line, e.getMessage()));
            }
            if (history.isEmpty()) {
                graph.addChildToRoot(dependency);
            } else {
                graph.addChildWithParents(dependency, history.getLastDependency());
            }
            history.add(dependency);
        }
        final ExternalId id = externalIdFactory.createMavenExternalId(projectGroup, projectName, projectVersionName);
        codeLocation = new DetectCodeLocation.Builder(DetectCodeLocationType.GRADLE, projectSourcePath, id, graph).build();
    } catch (final IOException e) {
        codeLocation = null;
    }
    return Optional.ofNullable(codeLocation);
}
Also used : MutableDependencyGraph(com.synopsys.integration.bdio.graph.MutableDependencyGraph) InputStreamReader(java.io.InputStreamReader) ExternalId(com.synopsys.integration.bdio.model.externalid.ExternalId) MutableMapDependencyGraph(com.synopsys.integration.bdio.graph.MutableMapDependencyGraph) DependencyHistory(com.blackducksoftware.integration.hub.detect.util.DependencyHistory) Dependency(com.synopsys.integration.bdio.model.dependency.Dependency) IOException(java.io.IOException) FileInputStream(java.io.FileInputStream) DetectCodeLocation(com.blackducksoftware.integration.hub.detect.workflow.codelocation.DetectCodeLocation) BufferedReader(java.io.BufferedReader)

Example 9 with MutableDependencyGraph

use of com.synopsys.integration.bdio.graph.MutableDependencyGraph in project hub-detect by blackducksoftware.

the class NugetDependencyNodeBuilder method createDependencyGraph.

public DependencyGraph createDependencyGraph(final List<NugetPackageId> packageDependencies) {
    final MutableDependencyGraph graph = new MutableMapDependencyGraph();
    if (packageSets != null) {
        for (final NugetPackageSet packageSet : packageSets) {
            if (packageSet.dependencies != null) {
                for (final NugetPackageId id : packageSet.dependencies) {
                    if (packageSet.packageId != null) {
                        graph.addParentWithChild(convertPackageId(packageSet.packageId), convertPackageId(id));
                    }
                }
            }
        }
    }
    packageDependencies.forEach(it -> {
        graph.addChildToRoot(convertPackageId(it));
    });
    return graph;
}
Also used : MutableDependencyGraph(com.synopsys.integration.bdio.graph.MutableDependencyGraph) MutableMapDependencyGraph(com.synopsys.integration.bdio.graph.MutableMapDependencyGraph) NugetPackageSet(com.blackducksoftware.integration.hub.detect.detector.nuget.model.NugetPackageSet) NugetPackageId(com.blackducksoftware.integration.hub.detect.detector.nuget.model.NugetPackageId)

Example 10 with MutableDependencyGraph

use of com.synopsys.integration.bdio.graph.MutableDependencyGraph in project hub-detect by blackducksoftware.

the class NugetInspectorExtractor method extract.

public Extraction extract(final File targetDirectory, File outputDirectory, NugetInspector inspector, final ExtractionId extractionId) {
    try {
        final List<String> options = new ArrayList<>(Arrays.asList("--target_path=" + targetDirectory.toString(), "--output_directory=" + outputDirectory.getCanonicalPath(), "--ignore_failure=" + detectConfiguration.getBooleanProperty(DetectProperty.DETECT_NUGET_IGNORE_FAILURE, PropertyAuthority.None)));
        final String nugetExcludedModules = detectConfiguration.getProperty(DetectProperty.DETECT_NUGET_EXCLUDED_MODULES, PropertyAuthority.None);
        if (StringUtils.isNotBlank(nugetExcludedModules)) {
            options.add("--excluded_modules=" + nugetExcludedModules);
        }
        final String nugetIncludedModules = detectConfiguration.getProperty(DetectProperty.DETECT_NUGET_INCLUDED_MODULES, PropertyAuthority.None);
        if (StringUtils.isNotBlank(nugetIncludedModules)) {
            options.add("--included_modules=" + nugetIncludedModules);
        }
        final String[] nugetPackagesRepo = detectConfiguration.getStringArrayProperty(DetectProperty.DETECT_NUGET_PACKAGES_REPO_URL, PropertyAuthority.None);
        if (nugetPackagesRepo.length > 0) {
            final String packagesRepos = Arrays.asList(nugetPackagesRepo).stream().collect(Collectors.joining(","));
            options.add("--packages_repo_url=" + packagesRepos);
        }
        final String nugetConfigPath = detectConfiguration.getProperty(DetectProperty.DETECT_NUGET_CONFIG_PATH, PropertyAuthority.None);
        if (StringUtils.isNotBlank(nugetConfigPath)) {
            options.add("--nuget_config_path=" + nugetConfigPath);
        }
        if (logger.isTraceEnabled()) {
            options.add("-v");
        }
        final ExecutableOutput executableOutput = inspector.execute(targetDirectory, options);
        if (executableOutput.getReturnCode() != 0) {
            return new Extraction.Builder().failure(String.format("Executing command '%s' returned a non-zero exit code %s", String.join(" ", options), executableOutput.getReturnCode())).build();
        }
        final List<File> dependencyNodeFiles = detectFileFinder.findFiles(outputDirectory, INSPECTOR_OUTPUT_PATTERN);
        final List<NugetParseResult> parseResults = new ArrayList<>();
        for (final File dependencyNodeFile : dependencyNodeFiles) {
            final NugetParseResult result = nugetInspectorPackager.createDetectCodeLocation(dependencyNodeFile);
            parseResults.add(result);
        }
        final List<DetectCodeLocation> codeLocations = parseResults.stream().flatMap(it -> it.codeLocations.stream()).collect(Collectors.toList());
        if (codeLocations.size() <= 0) {
            logger.warn("Unable to extract any dependencies from nuget");
        }
        final Map<String, DetectCodeLocation> codeLocationsBySource = new HashMap<>();
        final DependencyGraphCombiner combiner = new DependencyGraphCombiner();
        codeLocations.stream().forEach(codeLocation -> {
            final String sourcePathKey = codeLocation.getSourcePath().toLowerCase();
            if (codeLocationsBySource.containsKey(sourcePathKey)) {
                logger.info("Multiple project code locations were generated for: " + targetDirectory.toString());
                logger.info("This most likely means the same project exists in multiple solutions.");
                logger.info("The code location's dependencies will be combined, in the future they will exist seperately for each solution.");
                final DetectCodeLocation destination = codeLocationsBySource.get(sourcePathKey);
                combiner.addGraphAsChildrenToRoot((MutableDependencyGraph) destination.getDependencyGraph(), codeLocation.getDependencyGraph());
            } else {
                codeLocationsBySource.put(sourcePathKey, codeLocation);
            }
        });
        final List<DetectCodeLocation> uniqueCodeLocations = codeLocationsBySource.values().stream().collect(Collectors.toList());
        final Extraction.Builder builder = new Extraction.Builder().success(uniqueCodeLocations);
        final Optional<NugetParseResult> project = parseResults.stream().filter(it -> StringUtils.isNotBlank(it.projectName)).findFirst();
        if (project.isPresent()) {
            builder.projectName(project.get().projectName);
            builder.projectVersion(project.get().projectVersion);
        }
        return builder.build();
    } catch (final Exception e) {
        return new Extraction.Builder().exception(e).build();
    }
}
Also used : Arrays(java.util.Arrays) Logger(org.slf4j.Logger) MutableDependencyGraph(com.synopsys.integration.bdio.graph.MutableDependencyGraph) Extraction(com.blackducksoftware.integration.hub.detect.workflow.extraction.Extraction) LoggerFactory(org.slf4j.LoggerFactory) HashMap(java.util.HashMap) Collectors(java.util.stream.Collectors) StringUtils(org.apache.commons.lang3.StringUtils) File(java.io.File) DetectConfiguration(com.blackducksoftware.integration.hub.detect.configuration.DetectConfiguration) ArrayList(java.util.ArrayList) DependencyGraphCombiner(com.synopsys.integration.bdio.graph.DependencyGraphCombiner) List(java.util.List) ExtractionId(com.blackducksoftware.integration.hub.detect.detector.ExtractionId) ExecutableOutput(com.blackducksoftware.integration.hub.detect.util.executable.ExecutableOutput) PropertyAuthority(com.blackducksoftware.integration.hub.detect.configuration.PropertyAuthority) Map(java.util.Map) NugetInspector(com.blackducksoftware.integration.hub.detect.detector.nuget.inspector.NugetInspector) Optional(java.util.Optional) DetectFileFinder(com.blackducksoftware.integration.hub.detect.workflow.file.DetectFileFinder) DetectCodeLocation(com.blackducksoftware.integration.hub.detect.workflow.codelocation.DetectCodeLocation) DetectProperty(com.blackducksoftware.integration.hub.detect.configuration.DetectProperty) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) ExecutableOutput(com.blackducksoftware.integration.hub.detect.util.executable.ExecutableOutput) DetectCodeLocation(com.blackducksoftware.integration.hub.detect.workflow.codelocation.DetectCodeLocation) Extraction(com.blackducksoftware.integration.hub.detect.workflow.extraction.Extraction) File(java.io.File) DependencyGraphCombiner(com.synopsys.integration.bdio.graph.DependencyGraphCombiner)

Aggregations

MutableDependencyGraph (com.synopsys.integration.bdio.graph.MutableDependencyGraph)19 MutableMapDependencyGraph (com.synopsys.integration.bdio.graph.MutableMapDependencyGraph)16 Dependency (com.synopsys.integration.bdio.model.dependency.Dependency)11 ExternalId (com.synopsys.integration.bdio.model.externalid.ExternalId)10 DetectCodeLocation (com.blackducksoftware.integration.hub.detect.workflow.codelocation.DetectCodeLocation)8 DependencyHistory (com.blackducksoftware.integration.hub.detect.util.DependencyHistory)4 DependencyGraphCombiner (com.synopsys.integration.bdio.graph.DependencyGraphCombiner)3 ArrayList (java.util.ArrayList)3 List (java.util.List)3 Collectors (java.util.stream.Collectors)3 StringUtils (org.apache.commons.lang3.StringUtils)3 Logger (org.slf4j.Logger)3 LoggerFactory (org.slf4j.LoggerFactory)3 DetectConfiguration (com.blackducksoftware.integration.hub.detect.configuration.DetectConfiguration)2 DetectProperty (com.blackducksoftware.integration.hub.detect.configuration.DetectProperty)2 PropertyAuthority (com.blackducksoftware.integration.hub.detect.configuration.PropertyAuthority)2 DependencyGraph (com.synopsys.integration.bdio.graph.DependencyGraph)2 NameVersion (com.synopsys.integration.util.NameVersion)2 File (java.io.File)2 Arrays (java.util.Arrays)2