use of com.blackducksoftware.integration.hub.detect.workflow.extraction.Extraction in project hub-detect by blackducksoftware.
the class GradleInspectorExtractor method extract.
public Extraction extract(final File directory, final String gradleExe, final String gradleInspector, final File outputDirectory) {
try {
String gradleCommand = detectConfiguration.getProperty(DetectProperty.DETECT_GRADLE_BUILD_COMMAND, PropertyAuthority.None);
final List<String> arguments = new ArrayList<>();
if (StringUtils.isNotBlank(gradleCommand)) {
gradleCommand = gradleCommand.replaceAll("dependencies", "").trim();
Arrays.stream(gradleCommand.split(" ")).filter(StringUtils::isNotBlank).forEach(arguments::add);
}
arguments.add("dependencies");
arguments.add(String.format("--init-script=%s", gradleInspector));
arguments.add(String.format("-DGRADLEEXTRACTIONDIR=%s", outputDirectory.getCanonicalPath()));
arguments.add("--info");
final Executable executable = new Executable(directory, gradleExe, arguments);
final ExecutableOutput output = executableRunner.execute(executable);
if (output.getReturnCode() == 0) {
final File rootProjectMetadataFile = detectFileFinder.findFile(outputDirectory, "rootProjectMetadata.txt");
final List<File> codeLocationFiles = detectFileFinder.findFiles(outputDirectory, "*_dependencyGraph.txt");
final List<DetectCodeLocation> codeLocations = new ArrayList<>();
String projectName = null;
String projectVersion = null;
if (codeLocationFiles != null) {
codeLocationFiles.stream().map(codeLocationFile -> gradleReportParser.parseDependencies(codeLocationFile)).filter(Optional::isPresent).map(Optional::get).forEach(codeLocations::add);
if (rootProjectMetadataFile != null) {
final Optional<NameVersion> projectNameVersion = gradleReportParser.parseRootProjectNameVersion(rootProjectMetadataFile);
if (projectNameVersion.isPresent()) {
projectName = projectNameVersion.get().getName();
projectVersion = projectNameVersion.get().getVersion();
}
} else {
logger.warn("Gradle inspector did not create a meta data report so no project version information was found.");
}
}
return new Extraction.Builder().success(codeLocations).projectName(projectName).projectVersion(projectVersion).build();
} else {
return new Extraction.Builder().failure("The gradle inspector returned a non-zero exit code: " + output.getReturnCode()).build();
}
} catch (final Exception e) {
return new Extraction.Builder().exception(e).build();
}
}
use of com.blackducksoftware.integration.hub.detect.workflow.extraction.Extraction in project hub-detect by blackducksoftware.
the class NugetInspectorExtractor method extract.
public Extraction extract(final File targetDirectory, File outputDirectory, NugetInspector inspector, final ExtractionId extractionId) {
try {
final List<String> options = new ArrayList<>(Arrays.asList("--target_path=" + targetDirectory.toString(), "--output_directory=" + outputDirectory.getCanonicalPath(), "--ignore_failure=" + detectConfiguration.getBooleanProperty(DetectProperty.DETECT_NUGET_IGNORE_FAILURE, PropertyAuthority.None)));
final String nugetExcludedModules = detectConfiguration.getProperty(DetectProperty.DETECT_NUGET_EXCLUDED_MODULES, PropertyAuthority.None);
if (StringUtils.isNotBlank(nugetExcludedModules)) {
options.add("--excluded_modules=" + nugetExcludedModules);
}
final String nugetIncludedModules = detectConfiguration.getProperty(DetectProperty.DETECT_NUGET_INCLUDED_MODULES, PropertyAuthority.None);
if (StringUtils.isNotBlank(nugetIncludedModules)) {
options.add("--included_modules=" + nugetIncludedModules);
}
final String[] nugetPackagesRepo = detectConfiguration.getStringArrayProperty(DetectProperty.DETECT_NUGET_PACKAGES_REPO_URL, PropertyAuthority.None);
if (nugetPackagesRepo.length > 0) {
final String packagesRepos = Arrays.asList(nugetPackagesRepo).stream().collect(Collectors.joining(","));
options.add("--packages_repo_url=" + packagesRepos);
}
final String nugetConfigPath = detectConfiguration.getProperty(DetectProperty.DETECT_NUGET_CONFIG_PATH, PropertyAuthority.None);
if (StringUtils.isNotBlank(nugetConfigPath)) {
options.add("--nuget_config_path=" + nugetConfigPath);
}
if (logger.isTraceEnabled()) {
options.add("-v");
}
final ExecutableOutput executableOutput = inspector.execute(targetDirectory, options);
if (executableOutput.getReturnCode() != 0) {
return new Extraction.Builder().failure(String.format("Executing command '%s' returned a non-zero exit code %s", String.join(" ", options), executableOutput.getReturnCode())).build();
}
final List<File> dependencyNodeFiles = detectFileFinder.findFiles(outputDirectory, INSPECTOR_OUTPUT_PATTERN);
final List<NugetParseResult> parseResults = new ArrayList<>();
for (final File dependencyNodeFile : dependencyNodeFiles) {
final NugetParseResult result = nugetInspectorPackager.createDetectCodeLocation(dependencyNodeFile);
parseResults.add(result);
}
final List<DetectCodeLocation> codeLocations = parseResults.stream().flatMap(it -> it.codeLocations.stream()).collect(Collectors.toList());
if (codeLocations.size() <= 0) {
logger.warn("Unable to extract any dependencies from nuget");
}
final Map<String, DetectCodeLocation> codeLocationsBySource = new HashMap<>();
final DependencyGraphCombiner combiner = new DependencyGraphCombiner();
codeLocations.stream().forEach(codeLocation -> {
final String sourcePathKey = codeLocation.getSourcePath().toLowerCase();
if (codeLocationsBySource.containsKey(sourcePathKey)) {
logger.info("Multiple project code locations were generated for: " + targetDirectory.toString());
logger.info("This most likely means the same project exists in multiple solutions.");
logger.info("The code location's dependencies will be combined, in the future they will exist seperately for each solution.");
final DetectCodeLocation destination = codeLocationsBySource.get(sourcePathKey);
combiner.addGraphAsChildrenToRoot((MutableDependencyGraph) destination.getDependencyGraph(), codeLocation.getDependencyGraph());
} else {
codeLocationsBySource.put(sourcePathKey, codeLocation);
}
});
final List<DetectCodeLocation> uniqueCodeLocations = codeLocationsBySource.values().stream().collect(Collectors.toList());
final Extraction.Builder builder = new Extraction.Builder().success(uniqueCodeLocations);
final Optional<NugetParseResult> project = parseResults.stream().filter(it -> StringUtils.isNotBlank(it.projectName)).findFirst();
if (project.isPresent()) {
builder.projectName(project.get().projectName);
builder.projectVersion(project.get().projectVersion);
}
return builder.build();
} catch (final Exception e) {
return new Extraction.Builder().exception(e).build();
}
}
use of com.blackducksoftware.integration.hub.detect.workflow.extraction.Extraction in project hub-detect by blackducksoftware.
the class ClangExtractor method extract.
public Extraction extract(final ClangLinuxPackageManager pkgMgr, final File givenDir, final int depth, final ExtractionId extractionId, final File jsonCompilationDatabaseFile) {
try {
logger.info(String.format("Analyzing %s", jsonCompilationDatabaseFile.getAbsolutePath()));
final File rootDir = fileFinder.findContainingDir(givenDir, depth);
final File outputDirectory = directoryManager.getExtractionOutputDirectory(extractionId);
logger.debug(String.format("extract() called; compileCommandsJsonFilePath: %s", jsonCompilationDatabaseFile.getAbsolutePath()));
final Set<File> unManagedDependencyFiles = ConcurrentHashMap.newKeySet(64);
final List<CompileCommand> compileCommands = CompileCommandsJsonFile.parseJsonCompilationDatabaseFile(gson, jsonCompilationDatabaseFile);
final List<Dependency> bdioComponents = compileCommands.parallelStream().flatMap(compileCommandToDependencyFilePathsConverter(outputDirectory)).collect(Collectors.toSet()).parallelStream().filter(StringUtils::isNotBlank).map(File::new).filter(fileIsNewPredicate()).flatMap(dependencyFileToLinuxPackagesConverter(rootDir, unManagedDependencyFiles, pkgMgr)).collect(Collectors.toSet()).parallelStream().flatMap(linuxPackageToBdioComponentsConverter(pkgMgr)).collect(Collectors.toList());
final DetectCodeLocation detectCodeLocation = codeLocationAssembler.generateCodeLocation(pkgMgr.getDefaultForge(), rootDir, bdioComponents);
logSummary(bdioComponents, unManagedDependencyFiles);
return new Extraction.Builder().success(detectCodeLocation).build();
} catch (final Exception e) {
return new Extraction.Builder().exception(e).build();
}
}
use of com.blackducksoftware.integration.hub.detect.workflow.extraction.Extraction in project hub-detect by blackducksoftware.
the class RebarExtractor method extract.
public Extraction extract(final File directory, final File rebarExe) {
try {
final List<DetectCodeLocation> codeLocations = new ArrayList<>();
final Map<String, String> envVars = new HashMap<>();
envVars.put("REBAR_COLOR", "none");
final List<String> arguments = new ArrayList<>();
arguments.add("tree");
final Executable rebar3TreeExe = new Executable(directory, envVars, rebarExe.toString(), arguments);
final List<String> output = executableRunner.execute(rebar3TreeExe).getStandardOutputAsList();
final RebarParseResult parseResult = rebarTreeParser.parseRebarTreeOutput(output, directory.toString());
codeLocations.add(parseResult.getCodeLocation());
return new Extraction.Builder().success(codeLocations).projectName(parseResult.getProjectName()).projectVersion(parseResult.getProjectVersion()).build();
} catch (final Exception e) {
return new Extraction.Builder().exception(e).build();
}
}
use of com.blackducksoftware.integration.hub.detect.workflow.extraction.Extraction in project hub-detect by blackducksoftware.
the class MavenCliExtractor method extract.
public Extraction extract(final File directory, final String mavenExe) {
try {
String mavenCommand = detectConfiguration.getProperty(DetectProperty.DETECT_MAVEN_BUILD_COMMAND, PropertyAuthority.None);
if (StringUtils.isNotBlank(mavenCommand)) {
mavenCommand = mavenCommand.replace("dependency:tree", "");
if (StringUtils.isNotBlank(mavenCommand)) {
mavenCommand = mavenCommand.trim();
}
}
final List<String> arguments = new ArrayList<>();
if (StringUtils.isNotBlank(mavenCommand)) {
arguments.addAll(Arrays.asList(mavenCommand.split(" ")));
}
arguments.add("dependency:tree");
final Executable mvnExecutable = new Executable(directory, mavenExe, arguments);
final ExecutableOutput mvnOutput = executableRunner.execute(mvnExecutable);
if (mvnOutput.getReturnCode() == 0) {
final String mavenScope = detectConfiguration.getProperty(DetectProperty.DETECT_MAVEN_SCOPE, PropertyAuthority.None);
final String excludedModules = detectConfiguration.getProperty(DetectProperty.DETECT_MAVEN_EXCLUDED_MODULES, PropertyAuthority.None);
final String includedModules = detectConfiguration.getProperty(DetectProperty.DETECT_MAVEN_INCLUDED_MODULES, PropertyAuthority.None);
final List<MavenParseResult> mavenResults = mavenCodeLocationPackager.extractCodeLocations(directory.toString(), mvnOutput.getStandardOutput(), mavenScope, excludedModules, includedModules);
final List<DetectCodeLocation> codeLocations = mavenResults.stream().map(it -> it.codeLocation).collect(Collectors.toList());
final Optional<MavenParseResult> firstWithName = mavenResults.stream().filter(it -> StringUtils.isNoneBlank(it.projectName)).findFirst();
final Extraction.Builder builder = new Extraction.Builder().success(codeLocations);
if (firstWithName.isPresent()) {
builder.projectName(firstWithName.get().projectName);
builder.projectVersion(firstWithName.get().projectVersion);
}
return builder.build();
} else {
final Extraction.Builder builder = new Extraction.Builder().failure(String.format("Executing command '%s' returned a non-zero exit code %s", String.join(" ", arguments), mvnOutput.getReturnCode()));
return builder.build();
}
} catch (final Exception e) {
return new Extraction.Builder().exception(e).build();
}
}
Aggregations