Examples with DirectoryManager com.blackducksoftware.integration.hub.detect.workflow.file.DirectoryManager used on opensource projects

Example 1 with DirectoryManager

use of com.blackducksoftware.integration.hub.detect.workflow.file.DirectoryManager in project hub-detect by blackducksoftware.

the class Application method run.

@Override
public void run(final ApplicationArguments applicationArguments) throws Exception {
    final long startTime = System.currentTimeMillis();
    // Events, Status and Exit Codes are required even if boot fails.
    EventSystem eventSystem = new EventSystem();
    DetectStatusManager statusManager = new DetectStatusManager(eventSystem);
    ExitCodeUtility exitCodeUtility = new ExitCodeUtility();
    ExitCodeManager exitCodeManager = new ExitCodeManager(eventSystem, exitCodeUtility);
    ReportManager reportManager = ReportManager.createDefault(eventSystem);
    // Before boot even begins, we create a new Spring context for Detect to work within.
    logger.info("Preparing detect.");
    DetectRun detectRun = DetectRun.createDefault();
    DetectContext detectContext = new DetectContext(detectRun);
    BootResult bootResult = null;
    Optional<RunResult> runResult = Optional.empty();
    try {
        logger.info("Detect boot begin.");
        BootManager bootManager = new BootManager(new BootFactory());
        bootResult = bootManager.boot(detectRun, applicationArguments.getSourceArgs(), environment, eventSystem, detectContext);
        logger.info("Detect boot completed.");
    } catch (final Exception e) {
        logger.error("Detect boot failed.");
        exitCodeManager.requestExitCode(e);
    }
    if (bootResult != null && bootResult.bootType == BootResult.BootType.CONTINUE) {
        logger.info("Detect will attempt to run.");
        RunManager runManager = new RunManager(detectContext);
        try {
            logger.info("Detect run begin: " + detectRun.getRunId());
            runResult = Optional.ofNullable(runManager.run());
            logger.info("Detect run completed.");
        } catch (final Exception e) {
            if (e.getMessage() != null) {
                logger.error("Detect run failed: " + e.getMessage());
            } else {
                logger.error("Detect run failed: " + e.getClass().getSimpleName());
            }
            logger.debug("An exception was thrown during the detect run.", e);
            exitCodeManager.requestExitCode(e);
        }
        try {
            logger.info("Detect will attempt to shutdown.");
            DiagnosticManager diagnosticManager = detectContext.getBean(DiagnosticManager.class);
            DirectoryManager directoryManager = detectContext.getBean(DirectoryManager.class);
            DetectConfiguration detectConfiguration = detectContext.getBean(DetectConfiguration.class);
            ConnectivityManager connectivityManager = detectContext.getBean(ConnectivityManager.class);
            ShutdownManager shutdownManager = new ShutdownManager(connectivityManager, statusManager, exitCodeManager, directoryManager, detectConfiguration, reportManager, diagnosticManager);
            logger.info("Detect shutdown begin.");
            shutdownManager.shutdown(runResult);
            logger.info("Detect shutdown completed.");
        } catch (final Exception e) {
            logger.error("Detect shutdown failed.");
            exitCodeManager.requestExitCode(e);
        }
    } else {
        logger.debug("Detect will NOT attempt to run.");
    }
    logger.info("All detect actions completed.");
    // Determine how detect should actually exit
    boolean printOutput = true;
    boolean shouldForceSuccess = false;
    if (bootResult != null && bootResult.detectConfiguration != null) {
        printOutput = !bootResult.detectConfiguration.getBooleanProperty(DetectProperty.DETECT_SUPPRESS_RESULTS_OUTPUT, PropertyAuthority.None);
        shouldForceSuccess = bootResult.detectConfiguration.getBooleanProperty(DetectProperty.DETECT_FORCE_SUCCESS, PropertyAuthority.None);
    }
    // Generally, when requesting a failure status, an exit code is also requested, but if it is not, we default to an unknown error.
    if (statusManager.hasAnyFailure()) {
        eventSystem.publishEvent(Event.ExitCode, new ExitCodeRequest(ExitCodeType.FAILURE_UNKNOWN_ERROR, "A failure status was requested by one or more of Detect's tools."));
    }
    // Find the final (as requested) exit code
    ExitCodeType finalExitCode = exitCodeManager.getWinningExitCode();
    // Print detect's status
    if (printOutput) {
        reportManager.printDetectorIssues();
        statusManager.logDetectResults(new Slf4jIntLogger(logger), finalExitCode);
    }
    // Print duration of run
    final long endTime = System.currentTimeMillis();
    logger.info(String.format("Detect duration: %s", DurationFormatUtils.formatPeriod(startTime, endTime, "HH'h' mm'm' ss's' SSS'ms'")));
    // Exit with formal exit code
    if (finalExitCode != ExitCodeType.SUCCESS && shouldForceSuccess) {
        logger.warn(String.format("Forcing success: Exiting with exit code 0. Ignored exit code was %s.", finalExitCode.getExitCode()));
        System.exit(0);
    } else if (finalExitCode != ExitCodeType.SUCCESS) {
        logger.error(String.format("Exiting with code %s - %s", finalExitCode.getExitCode(), finalExitCode.toString()));
    }
    System.exit(finalExitCode.getExitCode());
}

Example 2 with DirectoryManager

use of com.blackducksoftware.integration.hub.detect.workflow.file.DirectoryManager in project hub-detect by blackducksoftware.

the class ClangExtractorTest method testMultipleCommandsDependenciesPackages.

@Test
public void testMultipleCommandsDependenciesPackages() throws ExecutableRunnerException {
    Assume.assumeFalse(SystemUtils.IS_OS_WINDOWS);
    final CompileCommand compileCommandWrapperHelloWorld = createCompileCommand("src/test/resources/clang/source/hello_world.cpp", "gcc hello_world.cpp", null);
    final CompileCommand compileCommandWrapperGoodbyeWorld = createCompileCommand("src/test/resources/clang/source/goodbye_world.cpp", "gcc goodbye_world.cpp", null);
    final Set<String> dependencyFilePathsHelloWorld = createDependencyFilePaths(new File("src/test/resources/clang/source/myinclude.h"), new File("/usr/include/nonexistentfile1.h"), new File("/usr/include/nonexistentfile2.h"));
    final Set<String> dependencyFilePathsGoodbyeWorld = createDependencyFilePaths(new File("/usr/include/nonexistentfile4.h"), new File("/usr/include/nonexistentfile3.h"));
    final ExecutableRunner executableRunner = Mockito.mock(ExecutableRunner.class);
    final DirectoryManager directoryManager = Mockito.mock(DirectoryManager.class);
    final DependenciesListFileManager dependenciesListFileManager = Mockito.mock(DependenciesListFileManager.class);
    Mockito.when(dependenciesListFileManager.generateDependencyFilePaths(outputDir, compileCommandWrapperHelloWorld, true)).thenReturn(dependencyFilePathsHelloWorld);
    Mockito.when(dependenciesListFileManager.generateDependencyFilePaths(outputDir, compileCommandWrapperGoodbyeWorld, true)).thenReturn(dependencyFilePathsGoodbyeWorld);
    Mockito.when(executableRunner.executeFromDirQuietly(Mockito.any(File.class), Mockito.anyString(), Mockito.anyList())).thenReturn(new ExecutableOutput(0, "", ""));
    final ExternalIdFactory externalIdFactory = new ExternalIdFactory();
    final CodeLocationAssembler codeLocationAssembler = new CodeLocationAssembler(externalIdFactory);
    final ClangExtractor extractor = new ClangExtractor(null, executableRunner, gson, new DetectFileFinder(), directoryManager, dependenciesListFileManager, codeLocationAssembler);
    final ClangLinuxPackageManager pkgMgr = Mockito.mock(ClangLinuxPackageManager.class);
    final File givenDir = new File("src/test/resources/clang/source/build");
    final int depth = 1;
    final ExtractionId extractionId = new ExtractionId(DetectorType.CLANG, EXTRACTION_ID);
    final File jsonCompilationDatabaseFile = new File("src/test/resources/clang/source/build/compile_commands.json");
    Mockito.when(directoryManager.getExtractionOutputDirectory(Mockito.any(ExtractionId.class))).thenReturn(outputDir);
    final List<PackageDetails> packages = new ArrayList<>();
    packages.add(new PackageDetails("testPackageName1", "testPackageVersion1", "testPackageArch1"));
    packages.add(new PackageDetails("testPackageName2", "testPackageVersion2", "testPackageArch2"));
    Mockito.when(pkgMgr.getDefaultForge()).thenReturn(Forge.CENTOS);
    Mockito.when(pkgMgr.getPackages(Mockito.any(File.class), Mockito.any(ExecutableRunner.class), Mockito.any(Set.class), Mockito.any(DependencyFileDetails.class))).thenReturn(packages);
    Mockito.when(pkgMgr.getForges()).thenReturn(Arrays.asList(Forge.CENTOS, Forge.FEDORA, Forge.REDHAT));
    final Extraction extraction = extractor.extract(pkgMgr, givenDir, depth, extractionId, jsonCompilationDatabaseFile);
    checkGeneratedDependenciesComplex(extraction);
}

Example 3 with DirectoryManager

use of com.blackducksoftware.integration.hub.detect.workflow.file.DirectoryManager in project hub-detect by blackducksoftware.

the class ClangExtractorTest method testJsonWithArgumentsNotCommand.

@Test
public void testJsonWithArgumentsNotCommand() throws ExecutableRunnerException {
    Assume.assumeFalse(SystemUtils.IS_OS_WINDOWS);
    final String[] argsHello = { "gcc", "hello_world.cpp" };
    final CompileCommand compileCommandWrapperHelloWorld = createCompileCommand("src/test/resources/clang/source/hello_world.cpp", null, argsHello);
    final String[] argsGoodbye = { "gcc", "goodbye_world.cpp" };
    final CompileCommand compileCommandWrapperGoodbyeWorld = createCompileCommand("src/test/resources/clang/source/goodbye_world.cpp", null, argsGoodbye);
    final Set<String> dependencyFilePathsHelloWorld = createDependencyFilePaths(new File("src/test/resources/clang/source/myinclude.h"), new File("/usr/include/nonexistentfile1.h"), new File("/usr/include/nonexistentfile2.h"));
    final Set<String> dependencyFilePathsGoodbyeWorld = createDependencyFilePaths(new File("/usr/include/nonexistentfile4.h"), new File("/usr/include/nonexistentfile3.h"));
    ;
    final ExecutableRunner executableRunner = Mockito.mock(ExecutableRunner.class);
    final DirectoryManager directoryManager = Mockito.mock(DirectoryManager.class);
    final DependenciesListFileManager dependenciesListFileManager = Mockito.mock(DependenciesListFileManager.class);
    Mockito.when(dependenciesListFileManager.generateDependencyFilePaths(outputDir, compileCommandWrapperHelloWorld, true)).thenReturn(dependencyFilePathsHelloWorld);
    Mockito.when(dependenciesListFileManager.generateDependencyFilePaths(outputDir, compileCommandWrapperGoodbyeWorld, true)).thenReturn(dependencyFilePathsGoodbyeWorld);
    Mockito.when(executableRunner.executeFromDirQuietly(Mockito.any(File.class), Mockito.anyString(), Mockito.anyList())).thenReturn(new ExecutableOutput(0, "", ""));
    final ExternalIdFactory externalIdFactory = new ExternalIdFactory();
    final CodeLocationAssembler codeLocationAssembler = new CodeLocationAssembler(externalIdFactory);
    final ClangExtractor extractor = new ClangExtractor(null, executableRunner, gson, new DetectFileFinder(), directoryManager, dependenciesListFileManager, codeLocationAssembler);
    final ClangLinuxPackageManager pkgMgr = Mockito.mock(ClangLinuxPackageManager.class);
    final File givenDir = new File("src/test/resources/clang/source/build");
    final int depth = 1;
    final ExtractionId extractionId = new ExtractionId(DetectorType.CLANG, EXTRACTION_ID);
    final File jsonCompilationDatabaseFile = new File("src/test/resources/clang/source/build/compile_commands_usesArguments.json");
    Mockito.when(directoryManager.getExtractionOutputDirectory(Mockito.any(ExtractionId.class))).thenReturn(outputDir);
    final List<PackageDetails> packages = new ArrayList<>();
    packages.add(new PackageDetails("testPackageName1", "testPackageVersion1", "testPackageArch1"));
    packages.add(new PackageDetails("testPackageName2", "testPackageVersion2", "testPackageArch2"));
    Mockito.when(pkgMgr.getDefaultForge()).thenReturn(Forge.CENTOS);
    Mockito.when(pkgMgr.getPackages(Mockito.any(File.class), Mockito.any(ExecutableRunner.class), Mockito.any(Set.class), Mockito.any(DependencyFileDetails.class))).thenReturn(packages);
    Mockito.when(pkgMgr.getForges()).thenReturn(Arrays.asList(Forge.CENTOS, Forge.FEDORA, Forge.REDHAT));
    final Extraction extraction = extractor.extract(pkgMgr, givenDir, depth, extractionId, jsonCompilationDatabaseFile);
    checkGeneratedDependenciesComplex(extraction);
}

Example 4 with DirectoryManager

use of com.blackducksoftware.integration.hub.detect.workflow.file.DirectoryManager in project hub-detect by blackducksoftware.

the class ClangExtractorTest method testSimple.

@Test
public void testSimple() throws ExecutableRunnerException {
    Assume.assumeFalse(SystemUtils.IS_OS_WINDOWS);
    final CompileCommand compileCommandWrapper = createCompileCommand("src/test/resources/clang/source/hello_world.cpp", "gcc hello_world.cpp", null);
    final Set<String> dependencyFilePaths = createDependencyFilePaths(new File("/usr/include/nonexistentfile1.h"), new File("src/test/resources/clang/source/myinclude.h"));
    final ExecutableRunner executableRunner = Mockito.mock(ExecutableRunner.class);
    final DirectoryManager directoryManager = Mockito.mock(DirectoryManager.class);
    final DependenciesListFileManager dependenciesListFileManager = Mockito.mock(DependenciesListFileManager.class);
    Mockito.when(dependenciesListFileManager.generateDependencyFilePaths(outputDir, compileCommandWrapper, true)).thenReturn(dependencyFilePaths);
    Mockito.when(executableRunner.executeFromDirQuietly(Mockito.any(File.class), Mockito.anyString(), Mockito.anyList())).thenReturn(new ExecutableOutput(0, "", ""));
    final ExternalIdFactory externalIdFactory = new ExternalIdFactory();
    final CodeLocationAssembler codeLocationAssembler = new CodeLocationAssembler(externalIdFactory);
    final ClangExtractor extractor = new ClangExtractor(null, executableRunner, gson, new DetectFileFinder(), directoryManager, dependenciesListFileManager, codeLocationAssembler);
    final ClangLinuxPackageManager pkgMgr = Mockito.mock(ClangLinuxPackageManager.class);
    final File givenDir = new File("src/test/resources/clang/source/build");
    final int depth = 1;
    final ExtractionId extractionId = new ExtractionId(DetectorType.CLANG, EXTRACTION_ID);
    final File jsonCompilationDatabaseFile = new File("src/test/resources/clang/source/build/compile_commands.json");
    Mockito.when(directoryManager.getExtractionOutputDirectory(Mockito.any(ExtractionId.class))).thenReturn(outputDir);
    final List<PackageDetails> packages = new ArrayList<>();
    packages.add(new PackageDetails("testPackageName", "testPackageVersion", "testPackageArch"));
    Mockito.when(pkgMgr.getDefaultForge()).thenReturn(Forge.UBUNTU);
    Mockito.when(pkgMgr.getPackages(Mockito.any(File.class), Mockito.any(ExecutableRunner.class), Mockito.any(Set.class), Mockito.any(DependencyFileDetails.class))).thenReturn(packages);
    Mockito.when(pkgMgr.getForges()).thenReturn(Arrays.asList(Forge.UBUNTU, Forge.DEBIAN));
    final Extraction extraction = extractor.extract(pkgMgr, givenDir, depth, extractionId, jsonCompilationDatabaseFile);
    checkGeneratedDependenciesSimple(extraction);
}

Example 5 with DirectoryManager

use of com.blackducksoftware.integration.hub.detect.workflow.file.DirectoryManager in project hub-detect by blackducksoftware.

the class BlackDuckSignatureScannerTool method runScanTool.

public SignatureScannerToolResult runScanTool(NameVersion projectNameVersion, Optional<File> dockerTar) throws DetectUserFriendlyException {
    DetectConfiguration detectConfiguration = detectContext.getBean(DetectConfiguration.class);
    DetectConfigurationFactory detectConfigurationFactory = detectContext.getBean(DetectConfigurationFactory.class);
    ConnectionManager connectionManager = detectContext.getBean(ConnectionManager.class);
    ConnectivityManager connectivityManager = detectContext.getBean(ConnectivityManager.class);
    DirectoryManager directoryManager = detectContext.getBean(DirectoryManager.class);
    Optional<BlackDuckServerConfig> hubServerConfig = Optional.empty();
    if (connectivityManager.isDetectOnline() && connectivityManager.getBlackDuckServerConfig().isPresent()) {
        hubServerConfig = connectivityManager.getBlackDuckServerConfig();
    }
    logger.info("Will run the signature scanner tool.");
    final String offlineLocalScannerInstallPath = detectConfiguration.getProperty(DetectProperty.DETECT_BLACKDUCK_SIGNATURE_SCANNER_OFFLINE_LOCAL_PATH, PropertyAuthority.None);
    final String onlineLocalScannerInstallPath = detectConfiguration.getProperty(DetectProperty.DETECT_BLACKDUCK_SIGNATURE_SCANNER_LOCAL_PATH, PropertyAuthority.None);
    String localScannerInstallPath = "";
    if (StringUtils.isNotBlank(offlineLocalScannerInstallPath)) {
        localScannerInstallPath = offlineLocalScannerInstallPath;
        logger.debug("Determined offline local scanner path: " + localScannerInstallPath);
    } else if (StringUtils.isNotBlank(onlineLocalScannerInstallPath)) {
        localScannerInstallPath = onlineLocalScannerInstallPath;
        logger.debug("Determined online local scanner path: " + localScannerInstallPath);
    }
    final String userProvidedScannerInstallUrl = detectConfiguration.getProperty(DetectProperty.DETECT_BLACKDUCK_SIGNATURE_SCANNER_HOST_URL, PropertyAuthority.None);
    BlackDuckSignatureScannerOptions blackDuckSignatureScannerOptions = detectConfigurationFactory.createBlackDuckSignatureScannerOptions();
    final ExecutorService executorService = Executors.newFixedThreadPool(blackDuckSignatureScannerOptions.getParrallelProcessors());
    IntEnvironmentVariables intEnvironmentVariables = new IntEnvironmentVariables();
    ScanBatchRunnerFactory scanBatchRunnerFactory = new ScanBatchRunnerFactory(intEnvironmentVariables, executorService);
    ScanBatchRunner scanBatchRunner;
    File installDirectory = directoryManager.getPermanentDirectory();
    if (hubServerConfig.isPresent() && StringUtils.isBlank(userProvidedScannerInstallUrl) && StringUtils.isBlank(localScannerInstallPath)) {
        logger.debug("Signature scanner will use the hub server to download/update the scanner - this is the most likely situation.");
        scanBatchRunner = scanBatchRunnerFactory.withHubInstall(hubServerConfig.get());
    } else {
        if (StringUtils.isNotBlank(userProvidedScannerInstallUrl)) {
            logger.debug("Signature scanner will use the provided url to download/update the scanner.");
            scanBatchRunner = scanBatchRunnerFactory.withUserProvidedUrl(userProvidedScannerInstallUrl, connectionManager);
        } else {
            logger.debug("Signature scanner either given an existing path for the scanner or is offline - either way, we won't attempt to manage the install.");
            if (StringUtils.isNotBlank(localScannerInstallPath)) {
                logger.debug("Using provided path: " + localScannerInstallPath);
                installDirectory = new File(localScannerInstallPath);
            } else {
                logger.debug("Using default scanner path.");
            }
            scanBatchRunner = scanBatchRunnerFactory.withoutInstall(installDirectory);
        }
    }
    logger.debug("Determined install directory: " + installDirectory.getAbsolutePath());
    try {
        if (hubServerConfig.isPresent()) {
            logger.debug("Signature scan is online.");
            CodeLocationCreationService codeLocationCreationService = connectivityManager.getBlackDuckServicesFactory().get().createCodeLocationCreationService();
            OnlineBlackDuckSignatureScanner blackDuckSignatureScanner = detectContext.getBean(OnlineBlackDuckSignatureScanner.class, signatureScannerOptions, scanBatchRunner, codeLocationCreationService, hubServerConfig.get());
            CodeLocationCreationData<ScanBatchOutput> codeLocationCreationData = blackDuckSignatureScanner.performOnlineScan(projectNameVersion, installDirectory, dockerTar.orElse(null));
            return SignatureScannerToolResult.createOnlineResult(codeLocationCreationData);
        } else {
            logger.debug("Signature scan is offline.");
            OfflineBlackDuckSignatureScanner blackDuckSignatureScanner = detectContext.getBean(OfflineBlackDuckSignatureScanner.class, signatureScannerOptions, scanBatchRunner);
            ScanBatchOutput scanBatchOutput = blackDuckSignatureScanner.performScanActions(projectNameVersion, installDirectory, dockerTar.orElse(null));
            return SignatureScannerToolResult.createOfflineResult(scanBatchOutput);
        }
    } catch (IOException | InterruptedException | IntegrationException e) {
        logger.info("Signature scan failed!");
        logger.debug("Signature scan error!", e);
        return SignatureScannerToolResult.createFailureResult();
    } finally {
        executorService.shutdownNow();
    }
}