Examples with ProcessExecutor com.facebook.buck.util.ProcessExecutor used on opensource projects

Example 31 with ProcessExecutor

use of com.facebook.buck.util.ProcessExecutor in project buck by facebook.

the class PythonBuckConfig method getPythonVersion.

private static PythonVersion getPythonVersion(ProcessExecutor processExecutor, Path pythonPath) throws InterruptedException {
    try {
        // Taken from pex's interpreter.py.
        String versionId = "import sys\n" + "\n" + "if hasattr(sys, 'pypy_version_info'):\n" + "  subversion = 'PyPy'\n" + "elif sys.platform.startswith('java'):\n" + "  subversion = 'Jython'\n" + "else:\n" + "  subversion = 'CPython'\n" + "\n" + "print('%s %s %s' % (subversion, sys.version_info[0], " + "sys.version_info[1]))\n";
        ProcessExecutor.Result versionResult = processExecutor.launchAndExecute(ProcessExecutorParams.builder().addCommand(pythonPath.toString(), "-").build(), EnumSet.of(ProcessExecutor.Option.EXPECTING_STD_OUT, ProcessExecutor.Option.EXPECTING_STD_ERR), Optional.of(versionId), /* timeOutMs */
        Optional.empty(), /* timeoutHandler */
        Optional.empty());
        return extractPythonVersion(pythonPath, versionResult);
    } catch (IOException e) {
        throw new HumanReadableException(e, "Could not run \"%s - < [code]\": %s", pythonPath, e.getMessage());
    }
}

Example 32 with ProcessExecutor

use of com.facebook.buck.util.ProcessExecutor in project buck by facebook.

the class Main method runMainWithExitCode.

/**
   * @param buildId an identifier for this command execution.
   * @param context an optional NGContext that is present if running inside a Nailgun server.
   * @param initTimestamp Value of System.nanoTime() when process got main()/nailMain() invoked.
   * @param unexpandedCommandLineArgs command line arguments
   * @return an exit code or {@code null} if this is a process that should not exit
   */
@SuppressWarnings("PMD.PrematureDeclaration")
public int runMainWithExitCode(BuildId buildId, Path projectRoot, Optional<NGContext> context, ImmutableMap<String, String> clientEnvironment, CommandMode commandMode, WatchmanWatcher.FreshInstanceAction watchmanFreshInstanceAction, final long initTimestamp, String... unexpandedCommandLineArgs) throws IOException, InterruptedException {
    String[] args = BuckArgsMethods.expandAtFiles(unexpandedCommandLineArgs);
    // Parse the command line args.
    BuckCommand command = new BuckCommand();
    AdditionalOptionsCmdLineParser cmdLineParser = new AdditionalOptionsCmdLineParser(command);
    try {
        cmdLineParser.parseArgument(args);
    } catch (CmdLineException e) {
        // Can't go through the console for prettification since that needs the BuckConfig, and that
        // needs to be created with the overrides, which are parsed from the command line here, which
        // required the console to print the message that parsing has failed. So just write to stderr
        // and be done with it.
        stdErr.println(e.getLocalizedMessage());
        stdErr.println("For help see 'buck --help'.");
        return 1;
    }
    {
        // Return help strings fast if the command is a help request.
        OptionalInt result = command.runHelp(stdErr);
        if (result.isPresent()) {
            return result.getAsInt();
        }
    }
    // Setup logging.
    if (commandMode.isLoggingEnabled()) {
        // Reset logging each time we run a command while daemonized.
        // This will cause us to write a new log per command.
        LOG.debug("Rotating log.");
        LogConfig.flushLogs();
        LogConfig.setupLogging(command.getLogConfig());
        if (LOG.isDebugEnabled()) {
            Long gitCommitTimestamp = Long.getLong("buck.git_commit_timestamp");
            String buildDateStr;
            if (gitCommitTimestamp == null) {
                buildDateStr = "(unknown)";
            } else {
                buildDateStr = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss Z", Locale.US).format(new Date(TimeUnit.SECONDS.toMillis(gitCommitTimestamp)));
            }
            String buildRev = System.getProperty("buck.git_commit", "(unknown)");
            LOG.debug("Starting up (build date %s, rev %s), args: %s", buildDateStr, buildRev, Arrays.toString(args));
            LOG.debug("System properties: %s", System.getProperties());
        }
    }
    // Setup filesystem and buck config.
    Path canonicalRootPath = projectRoot.toRealPath().normalize();
    Config config = Configs.createDefaultConfig(canonicalRootPath, command.getConfigOverrides().getForCell(RelativeCellName.ROOT_CELL_NAME));
    ProjectFilesystem filesystem = new ProjectFilesystem(canonicalRootPath, config);
    DefaultCellPathResolver cellPathResolver = new DefaultCellPathResolver(filesystem.getRootPath(), config);
    BuckConfig buckConfig = new BuckConfig(config, filesystem, architecture, platform, clientEnvironment, cellPathResolver);
    ImmutableSet<Path> projectWatchList = ImmutableSet.<Path>builder().add(canonicalRootPath).addAll(buckConfig.getView(ParserConfig.class).getWatchCells() ? cellPathResolver.getTransitivePathMapping().values() : ImmutableList.of()).build();
    Optional<ImmutableList<String>> allowedJavaSpecificiationVersions = buckConfig.getAllowedJavaSpecificationVersions();
    if (allowedJavaSpecificiationVersions.isPresent()) {
        String specificationVersion = System.getProperty("java.specification.version");
        boolean javaSpecificationVersionIsAllowed = allowedJavaSpecificiationVersions.get().contains(specificationVersion);
        if (!javaSpecificationVersionIsAllowed) {
            throw new HumanReadableException("Current Java version '%s' is not in the allowed java specification versions:\n%s", specificationVersion, Joiner.on(", ").join(allowedJavaSpecificiationVersions.get()));
        }
    }
    // Setup the console.
    Verbosity verbosity = VerbosityParser.parse(args);
    Optional<String> color;
    if (context.isPresent() && (context.get().getEnv() != null)) {
        String colorString = context.get().getEnv().getProperty(BUCKD_COLOR_DEFAULT_ENV_VAR);
        color = Optional.ofNullable(colorString);
    } else {
        color = Optional.empty();
    }
    final Console console = new Console(verbosity, stdOut, stdErr, buckConfig.createAnsi(color));
    // No more early outs: if this command is not read only, acquire the command semaphore to
    // become the only executing read/write command.
    // This must happen immediately before the try block to ensure that the semaphore is released.
    boolean commandSemaphoreAcquired = false;
    boolean shouldCleanUpTrash = false;
    if (!command.isReadOnly()) {
        commandSemaphoreAcquired = commandSemaphore.tryAcquire();
        if (!commandSemaphoreAcquired) {
            LOG.warn("Buck server was busy executing a command. Maybe retrying later will help.");
            return BUSY_EXIT_CODE;
        }
    }
    try {
        if (commandSemaphoreAcquired) {
            commandSemaphoreNgClient = context;
        }
        if (!command.isReadOnly()) {
            Optional<String> currentVersion = filesystem.readFileIfItExists(filesystem.getBuckPaths().getCurrentVersionFile());
            BuckPaths unconfiguredPaths = filesystem.getBuckPaths().withConfiguredBuckOut(filesystem.getBuckPaths().getBuckOut());
            if (!currentVersion.isPresent() || !currentVersion.get().equals(BuckVersion.getVersion()) || (filesystem.exists(unconfiguredPaths.getGenDir(), LinkOption.NOFOLLOW_LINKS) && (filesystem.isSymLink(unconfiguredPaths.getGenDir()) ^ buckConfig.getBuckOutCompatLink()))) {
                // Migrate any version-dependent directories (which might be huge) to a trash directory
                // so we can delete it asynchronously after the command is done.
                moveToTrash(filesystem, console, buildId, filesystem.getBuckPaths().getAnnotationDir(), filesystem.getBuckPaths().getGenDir(), filesystem.getBuckPaths().getScratchDir(), filesystem.getBuckPaths().getResDir());
                shouldCleanUpTrash = true;
                filesystem.mkdirs(filesystem.getBuckPaths().getCurrentVersionFile().getParent());
                filesystem.writeContentsToPath(BuckVersion.getVersion(), filesystem.getBuckPaths().getCurrentVersionFile());
            }
        }
        AndroidBuckConfig androidBuckConfig = new AndroidBuckConfig(buckConfig, platform);
        AndroidDirectoryResolver androidDirectoryResolver = new DefaultAndroidDirectoryResolver(filesystem.getRootPath().getFileSystem(), clientEnvironment, androidBuckConfig.getBuildToolsVersion(), androidBuckConfig.getNdkVersion());
        ProcessExecutor processExecutor = new DefaultProcessExecutor(console);
        Clock clock;
        boolean enableThreadCpuTime = buckConfig.getBooleanValue("build", "enable_thread_cpu_time", true);
        if (BUCKD_LAUNCH_TIME_NANOS.isPresent()) {
            long nanosEpoch = Long.parseLong(BUCKD_LAUNCH_TIME_NANOS.get(), 10);
            LOG.verbose("Using nanos epoch: %d", nanosEpoch);
            clock = new NanosAdjustedClock(nanosEpoch, enableThreadCpuTime);
        } else {
            clock = new DefaultClock(enableThreadCpuTime);
        }
        ParserConfig parserConfig = buckConfig.getView(ParserConfig.class);
        try (Watchman watchman = buildWatchman(context, parserConfig, projectWatchList, clientEnvironment, console, clock)) {
            final boolean isDaemon = context.isPresent() && (watchman != Watchman.NULL_WATCHMAN);
            if (!isDaemon && shouldCleanUpTrash) {
                // Clean up the trash on a background thread if this was a
                // non-buckd read-write command. (We don't bother waiting
                // for it to complete; the thread is a daemon thread which
                // will just be terminated at shutdown time.)
                TRASH_CLEANER.startCleaningDirectory(filesystem.getBuckPaths().getTrashDir());
            }
            KnownBuildRuleTypesFactory factory = new KnownBuildRuleTypesFactory(processExecutor, androidDirectoryResolver);
            Cell rootCell = CellProvider.createForLocalBuild(filesystem, watchman, buckConfig, command.getConfigOverrides(), factory).getCellByPath(filesystem.getRootPath());
            int exitCode;
            ImmutableList<BuckEventListener> eventListeners = ImmutableList.of();
            ExecutionEnvironment executionEnvironment = new DefaultExecutionEnvironment(clientEnvironment, System.getProperties());
            ImmutableList.Builder<ProjectFileHashCache> allCaches = ImmutableList.builder();
            // Build up the hash cache, which is a collection of the stateful cell cache and some
            // per-run caches.
            //
            // TODO(Coneko, ruibm, andrewjcg): Determine whether we can use the existing filesystem
            // object that is in scope instead of creating a new rootCellProjectFilesystem. The primary
            // difference appears to be that filesystem is created with a Config that is used to produce
            // ImmutableSet<PathOrGlobMatcher> and BuckPaths for the ProjectFilesystem, whereas this one
            // uses the defaults.
            ProjectFilesystem rootCellProjectFilesystem = ProjectFilesystem.createNewOrThrowHumanReadableException(rootCell.getFilesystem().getRootPath());
            if (isDaemon) {
                allCaches.addAll(getFileHashCachesFromDaemon(rootCell));
            } else {
                getTransitiveCells(rootCell).stream().map(cell -> DefaultFileHashCache.createDefaultFileHashCache(cell.getFilesystem())).forEach(allCaches::add);
                allCaches.add(DefaultFileHashCache.createBuckOutFileHashCache(rootCellProjectFilesystem, rootCell.getFilesystem().getBuckPaths().getBuckOut()));
            }
            // A cache which caches hashes of cell-relative paths which may have been ignore by
            // the main cell cache, and only serves to prevent rehashing the same file multiple
            // times in a single run.
            allCaches.add(DefaultFileHashCache.createDefaultFileHashCache(rootCellProjectFilesystem));
            allCaches.addAll(DefaultFileHashCache.createOsRootDirectoriesCaches());
            StackedFileHashCache fileHashCache = new StackedFileHashCache(allCaches.build());
            Optional<WebServer> webServer = getWebServerIfDaemon(context, rootCell);
            Optional<ConcurrentMap<String, WorkerProcessPool>> persistentWorkerPools = getPersistentWorkerPoolsIfDaemon(context, rootCell);
            TestConfig testConfig = new TestConfig(buckConfig);
            ArtifactCacheBuckConfig cacheBuckConfig = new ArtifactCacheBuckConfig(buckConfig);
            ExecutorService diskIoExecutorService = MostExecutors.newSingleThreadExecutor("Disk I/O");
            ListeningExecutorService httpWriteExecutorService = getHttpWriteExecutorService(cacheBuckConfig);
            ScheduledExecutorService counterAggregatorExecutor = Executors.newSingleThreadScheduledExecutor(new CommandThreadFactory("CounterAggregatorThread"));
            VersionControlStatsGenerator vcStatsGenerator;
            // Eventually, we'll want to get allow websocket and/or nailgun clients to specify locale
            // when connecting. For now, we'll use the default from the server environment.
            Locale locale = Locale.getDefault();
            // Create a cached thread pool for cpu intensive tasks
            Map<ExecutorPool, ListeningExecutorService> executors = new HashMap<>();
            executors.put(ExecutorPool.CPU, listeningDecorator(Executors.newCachedThreadPool()));
            // Create a thread pool for network I/O tasks
            executors.put(ExecutorPool.NETWORK, newDirectExecutorService());
            executors.put(ExecutorPool.PROJECT, listeningDecorator(MostExecutors.newMultiThreadExecutor("Project", buckConfig.getNumThreads())));
            // Create and register the event buses that should listen to broadcast events.
            // If the build doesn't have a daemon create a new instance.
            BroadcastEventListener broadcastEventListener = getBroadcastEventListener(isDaemon, rootCell, objectMapper);
            // The order of resources in the try-with-resources block is important: the BuckEventBus
            // must be the last resource, so that it is closed first and can deliver its queued events
            // to the other resources before they are closed.
            InvocationInfo invocationInfo = InvocationInfo.of(buildId, isSuperConsoleEnabled(console), isDaemon, command.getSubCommandNameForLogging(), filesystem.getBuckPaths().getLogDir());
            try (GlobalStateManager.LoggerIsMappedToThreadScope loggerThreadMappingScope = GlobalStateManager.singleton().setupLoggers(invocationInfo, console.getStdErr(), stdErr, verbosity);
                AbstractConsoleEventBusListener consoleListener = createConsoleEventListener(clock, new SuperConsoleConfig(buckConfig), console, testConfig.getResultSummaryVerbosity(), executionEnvironment, webServer, locale, filesystem.getBuckPaths().getLogDir().resolve("test.log"));
                AsyncCloseable asyncCloseable = new AsyncCloseable(diskIoExecutorService);
                BuckEventBus buildEventBus = new BuckEventBus(clock, buildId);
                BroadcastEventListener.BroadcastEventBusClosable broadcastEventBusClosable = broadcastEventListener.addEventBus(buildEventBus);
                // stderr.
                Closeable logErrorToEventBus = loggerThreadMappingScope.setWriter(createWriterForConsole(consoleListener));
                // NOTE: This will only run during the lifetime of the process and will flush on close.
                CounterRegistry counterRegistry = new CounterRegistryImpl(counterAggregatorExecutor, buildEventBus, buckConfig.getCountersFirstFlushIntervalMillis(), buckConfig.getCountersFlushIntervalMillis());
                PerfStatsTracking perfStatsTracking = new PerfStatsTracking(buildEventBus, invocationInfo);
                ProcessTracker processTracker = buckConfig.isProcessTrackerEnabled() && platform != Platform.WINDOWS ? new ProcessTracker(buildEventBus, invocationInfo, isDaemon, buckConfig.isProcessTrackerDeepEnabled()) : null) {
                LOG.debug(invocationInfo.toLogLine(args));
                buildEventBus.register(HANG_MONITOR.getHangMonitor());
                ArtifactCaches artifactCacheFactory = new ArtifactCaches(cacheBuckConfig, buildEventBus, filesystem, executionEnvironment.getWifiSsid(), httpWriteExecutorService, Optional.of(asyncCloseable));
                ProgressEstimator progressEstimator = new ProgressEstimator(filesystem.resolve(filesystem.getBuckPaths().getBuckOut()).resolve(ProgressEstimator.PROGRESS_ESTIMATIONS_JSON), buildEventBus, objectMapper);
                consoleListener.setProgressEstimator(progressEstimator);
                BuildEnvironmentDescription buildEnvironmentDescription = getBuildEnvironmentDescription(executionEnvironment, buckConfig);
                Iterable<BuckEventListener> commandEventListeners = command.getSubcommand().isPresent() ? command.getSubcommand().get().getEventListeners(invocationInfo.getLogDirectoryPath(), filesystem) : ImmutableList.of();
                Supplier<BuckEventListener> missingSymbolsListenerSupplier = () -> {
                    return MissingSymbolsHandler.createListener(rootCell.getFilesystem(), rootCell.getKnownBuildRuleTypes().getAllDescriptions(), rootCell.getBuckConfig(), buildEventBus, console, buckConfig.getView(JavaBuckConfig.class).getDefaultJavacOptions(), clientEnvironment);
                };
                eventListeners = addEventListeners(buildEventBus, rootCell.getFilesystem(), invocationInfo, rootCell.getBuckConfig(), webServer, clock, consoleListener, missingSymbolsListenerSupplier, counterRegistry, commandEventListeners);
                if (commandMode == CommandMode.RELEASE && buckConfig.isPublicAnnouncementsEnabled()) {
                    PublicAnnouncementManager announcementManager = new PublicAnnouncementManager(clock, buildEventBus, consoleListener, buckConfig.getRepository().orElse("unknown"), new RemoteLogBuckConfig(buckConfig), executors.get(ExecutorPool.CPU));
                    announcementManager.getAndPostAnnouncements();
                }
                // This needs to be after the registration of the event listener so they can pick it up.
                if (watchmanFreshInstanceAction == WatchmanWatcher.FreshInstanceAction.NONE) {
                    buildEventBus.post(DaemonEvent.newDaemonInstance());
                }
                if (command.subcommand instanceof AbstractCommand) {
                    AbstractCommand subcommand = (AbstractCommand) command.subcommand;
                    VersionControlBuckConfig vcBuckConfig = new VersionControlBuckConfig(buckConfig);
                    if (!commandMode.equals(CommandMode.TEST) && (subcommand.isSourceControlStatsGatheringEnabled() || vcBuckConfig.shouldGenerateStatistics())) {
                        vcStatsGenerator = new VersionControlStatsGenerator(diskIoExecutorService, new DefaultVersionControlCmdLineInterfaceFactory(rootCell.getFilesystem().getRootPath(), new PrintStreamProcessExecutorFactory(), vcBuckConfig, buckConfig.getEnvironment()), buildEventBus);
                        vcStatsGenerator.generateStatsAsync();
                    }
                }
                ImmutableList<String> remainingArgs = args.length > 1 ? ImmutableList.copyOf(Arrays.copyOfRange(args, 1, args.length)) : ImmutableList.of();
                CommandEvent.Started startedEvent = CommandEvent.started(args.length > 0 ? args[0] : "", remainingArgs, isDaemon, getBuckPID());
                buildEventBus.post(startedEvent);
                // Create or get Parser and invalidate cached command parameters.
                Parser parser = null;
                VersionedTargetGraphCache versionedTargetGraphCache = null;
                ActionGraphCache actionGraphCache = null;
                Optional<RuleKeyCacheRecycler<RuleKey>> defaultRuleKeyFactoryCacheRecycler = Optional.empty();
                if (isDaemon) {
                    try {
                        Daemon daemon = getDaemon(rootCell, objectMapper);
                        WatchmanWatcher watchmanWatcher = createWatchmanWatcher(daemon, watchman.getProjectWatches(), daemon.getFileEventBus(), ImmutableSet.<PathOrGlobMatcher>builder().addAll(filesystem.getIgnorePaths()).addAll(DEFAULT_IGNORE_GLOBS).build(), watchman);
                        parser = getParserFromDaemon(context, rootCell, startedEvent, buildEventBus, watchmanWatcher, watchmanFreshInstanceAction);
                        versionedTargetGraphCache = daemon.getVersionedTargetGraphCache();
                        actionGraphCache = daemon.getActionGraphCache();
                        if (buckConfig.getRuleKeyCaching()) {
                            LOG.debug("Using rule key calculation caching");
                            defaultRuleKeyFactoryCacheRecycler = Optional.of(daemon.getDefaultRuleKeyFactoryCacheRecycler());
                        }
                    } catch (WatchmanWatcherException | IOException e) {
                        buildEventBus.post(ConsoleEvent.warning("Watchman threw an exception while parsing file changes.\n%s", e.getMessage()));
                    }
                }
                if (versionedTargetGraphCache == null) {
                    versionedTargetGraphCache = new VersionedTargetGraphCache();
                }
                if (actionGraphCache == null) {
                    actionGraphCache = new ActionGraphCache(broadcastEventListener);
                }
                if (parser == null) {
                    TypeCoercerFactory typeCoercerFactory = new DefaultTypeCoercerFactory(objectMapper);
                    parser = new Parser(broadcastEventListener, rootCell.getBuckConfig().getView(ParserConfig.class), typeCoercerFactory, new ConstructorArgMarshaller(typeCoercerFactory));
                }
                // Because the Parser is potentially constructed before the CounterRegistry,
                // we need to manually register its counters after it's created.
                //
                // The counters will be unregistered once the counter registry is closed.
                counterRegistry.registerCounters(parser.getCounters());
                JavaUtilsLoggingBuildListener.ensureLogFileIsWritten(rootCell.getFilesystem());
                Optional<ProcessManager> processManager;
                if (platform == Platform.WINDOWS) {
                    processManager = Optional.empty();
                } else {
                    processManager = Optional.of(new PkillProcessManager(processExecutor));
                }
                Supplier<AndroidPlatformTarget> androidPlatformTargetSupplier = createAndroidPlatformTargetSupplier(androidDirectoryResolver, androidBuckConfig, buildEventBus);
                // event-listener.
                if (command.subcommand instanceof AbstractCommand) {
                    AbstractCommand subcommand = (AbstractCommand) command.subcommand;
                    Optional<Path> eventsOutputPath = subcommand.getEventsOutputPath();
                    if (eventsOutputPath.isPresent()) {
                        BuckEventListener listener = new FileSerializationEventBusListener(eventsOutputPath.get(), objectMapper);
                        buildEventBus.register(listener);
                    }
                }
                buildEventBus.post(new BuckInitializationDurationEvent(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - initTimestamp)));
                try {
                    exitCode = command.run(CommandRunnerParams.builder().setConsole(console).setStdIn(stdIn).setCell(rootCell).setAndroidPlatformTargetSupplier(androidPlatformTargetSupplier).setArtifactCacheFactory(artifactCacheFactory).setBuckEventBus(buildEventBus).setParser(parser).setPlatform(platform).setEnvironment(clientEnvironment).setJavaPackageFinder(rootCell.getBuckConfig().getView(JavaBuckConfig.class).createDefaultJavaPackageFinder()).setObjectMapper(objectMapper).setClock(clock).setProcessManager(processManager).setPersistentWorkerPools(persistentWorkerPools).setWebServer(webServer).setBuckConfig(buckConfig).setFileHashCache(fileHashCache).setExecutors(executors).setBuildEnvironmentDescription(buildEnvironmentDescription).setVersionedTargetGraphCache(versionedTargetGraphCache).setActionGraphCache(actionGraphCache).setKnownBuildRuleTypesFactory(factory).setInvocationInfo(Optional.of(invocationInfo)).setDefaultRuleKeyFactoryCacheRecycler(defaultRuleKeyFactoryCacheRecycler).build());
                } catch (InterruptedException | ClosedByInterruptException e) {
                    exitCode = INTERRUPTED_EXIT_CODE;
                    buildEventBus.post(CommandEvent.interrupted(startedEvent, INTERRUPTED_EXIT_CODE));
                    throw e;
                }
                // Let's avoid an infinite loop
                if (exitCode == BUSY_EXIT_CODE) {
                    // Some loss of info here, but better than looping
                    exitCode = FAIL_EXIT_CODE;
                    LOG.error("Buck return with exit code %d which we use to indicate busy status. " + "This is probably propagating an exit code from a sub process or tool. " + "Coercing to %d to avoid retries.", BUSY_EXIT_CODE, FAIL_EXIT_CODE);
                }
                // Wait for HTTP writes to complete.
                closeHttpExecutorService(cacheBuckConfig, Optional.of(buildEventBus), httpWriteExecutorService);
                closeExecutorService("CounterAggregatorExecutor", counterAggregatorExecutor, COUNTER_AGGREGATOR_SERVICE_TIMEOUT_SECONDS);
                buildEventBus.post(CommandEvent.finished(startedEvent, exitCode));
            } catch (Throwable t) {
                LOG.debug(t, "Failing build on exception.");
                closeHttpExecutorService(cacheBuckConfig, Optional.empty(), httpWriteExecutorService);
                closeDiskIoExecutorService(diskIoExecutorService);
                flushEventListeners(console, buildId, eventListeners);
                throw t;
            } finally {
                if (commandSemaphoreAcquired) {
                    commandSemaphoreNgClient = Optional.empty();
                    BgProcessKiller.disarm();
                    // Allow another command to execute while outputting traces.
                    commandSemaphore.release();
                    commandSemaphoreAcquired = false;
                }
                if (isDaemon && shouldCleanUpTrash) {
                    // Clean up the trash in the background if this was a buckd
                    // read-write command. (We don't bother waiting for it to
                    // complete; the cleaner will ensure subsequent cleans are
                    // serialized with this one.)
                    TRASH_CLEANER.startCleaningDirectory(filesystem.getBuckPaths().getTrashDir());
                }
                // shut down the cached thread pools
                for (ExecutorPool p : executors.keySet()) {
                    closeExecutorService(p.toString(), executors.get(p), EXECUTOR_SERVICES_TIMEOUT_SECONDS);
                }
            }
            if (context.isPresent() && !rootCell.getBuckConfig().getFlushEventsBeforeExit()) {
                // Avoid client exit triggering client disconnection handling.
                context.get().in.close();
                // Allow nailgun client to exit while outputting traces.
                context.get().exit(exitCode);
            }
            closeDiskIoExecutorService(diskIoExecutorService);
            flushEventListeners(console, buildId, eventListeners);
            return exitCode;
        }
    } finally {
        if (commandSemaphoreAcquired) {
            commandSemaphoreNgClient = Optional.empty();
            BgProcessKiller.disarm();
            commandSemaphore.release();
        }
    }
}

Example 33 with ProcessExecutor

use of com.facebook.buck.util.ProcessExecutor in project buck by facebook.

the class JavacExecutionContextSerializerTest method testSerializingAndDeserializing.

@Test
public void testSerializingAndDeserializing() throws Exception {
    Path tmp = Files.createTempDirectory("junit-temp-path").toRealPath();
    JavacEventSink eventSink = new JavacEventSinkToBuckEventBusBridge(BuckEventBusFactory.newInstance());
    PrintStream stdErr = new PrintStream(new ByteArrayOutputStream());
    ClassLoaderCache classLoaderCache = new ClassLoaderCache();
    ObjectMapper objectMapper = new ObjectMapper();
    Verbosity verbosity = Verbosity.COMMANDS_AND_OUTPUT;
    DefaultCellPathResolver cellPathResolver = new DefaultCellPathResolver(Paths.get("/some/cell/path/resolver/path"), ImmutableMap.of("key1", Paths.get("/path/1")));
    DefaultJavaPackageFinder javaPackageFinder = new DefaultJavaPackageFinder(ImmutableSortedSet.of("paths", "from", "root"), ImmutableSet.of("path", "elements"));
    ProjectFilesystem projectFilesystem = new ProjectFilesystem(tmp);
    NoOpClassUsageFileWriter classUsageFileWriter = NoOpClassUsageFileWriter.instance();
    ImmutableMap<String, String> environment = ImmutableMap.of("k1", "v1", "k2", "v2");
    ImmutableMap<String, String> processExecutorContext = ImmutableMap.of("pek1", "pev1", "pek2", "pev2");
    ProcessExecutor processExecutor = new ContextualProcessExecutor(new DefaultProcessExecutor(new TestConsole()), processExecutorContext);
    ImmutableList<Path> pathToInputs = ImmutableList.of(Paths.get("/path/one"), Paths.get("/path/two"));
    DirectToJarOutputSettings directToJarOutputSettings = DirectToJarOutputSettings.of(Paths.get("/some/path"), ImmutableSet.of(Pattern.compile("[a-z]"), Pattern.compile("[0-9]", Pattern.MULTILINE)), ImmutableSortedSet.of(Paths.get("some/path"), Paths.get("/other path/")), Optional.of("hello I am main class"), Optional.of(Paths.get("/MANIFEST/FILE.TXT")));
    JavacExecutionContext input = JavacExecutionContext.of(eventSink, stdErr, classLoaderCache, objectMapper, verbosity, cellPathResolver, javaPackageFinder, projectFilesystem, classUsageFileWriter, environment, processExecutor, pathToInputs, Optional.of(directToJarOutputSettings));
    Map<String, Object> data = JavacExecutionContextSerializer.serialize(input);
    JavacExecutionContext output = JavacExecutionContextSerializer.deserialize(data, eventSink, stdErr, classLoaderCache, objectMapper, new TestConsole());
    assertThat(output.getEventSink(), Matchers.equalTo(eventSink));
    assertThat(output.getStdErr(), Matchers.equalTo(stdErr));
    assertThat(output.getClassLoaderCache(), Matchers.equalTo(classLoaderCache));
    assertThat(output.getObjectMapper(), Matchers.equalTo(objectMapper));
    assertThat(output.getVerbosity(), Matchers.equalTo(verbosity));
    assertThat(output.getCellPathResolver(), Matchers.instanceOf(DefaultCellPathResolver.class));
    DefaultCellPathResolver outCellPathResolver = (DefaultCellPathResolver) output.getCellPathResolver();
    assertThat(outCellPathResolver.getRoot(), Matchers.equalToObject(cellPathResolver.getRoot()));
    assertThat(outCellPathResolver.getCellPaths(), Matchers.equalToObject(cellPathResolver.getCellPaths()));
    assertThat(output.getProcessExecutor(), Matchers.instanceOf(ContextualProcessExecutor.class));
    ContextualProcessExecutor contextualProcessExecutor = (ContextualProcessExecutor) output.getProcessExecutor();
    assertThat(contextualProcessExecutor.getContext(), Matchers.equalToObject(processExecutorContext));
    assertThat(output.getJavaPackageFinder(), Matchers.instanceOf(DefaultJavaPackageFinder.class));
    DefaultJavaPackageFinder outputJavaPackageFinder = (DefaultJavaPackageFinder) output.getJavaPackageFinder();
    assertThat(outputJavaPackageFinder.getPathsFromRoot(), Matchers.equalToObject(javaPackageFinder.getPathsFromRoot()));
    assertThat(outputJavaPackageFinder.getPathElements(), Matchers.equalToObject(javaPackageFinder.getPathElements()));
    assertThat(output.getProjectFilesystem().getRootPath(), Matchers.equalToObject(projectFilesystem.getRootPath()));
    assertThat(output.getUsedClassesFileWriter(), Matchers.instanceOf(NoOpClassUsageFileWriter.class));
    assertThat(output.getEnvironment(), Matchers.equalToObject(environment));
    assertThat(output.getAbsolutePathsForInputs(), Matchers.equalToObject(pathToInputs));
    assertThat(output.getDirectToJarOutputSettings().get().getDirectToJarOutputPath(), Matchers.equalToObject(directToJarOutputSettings.getDirectToJarOutputPath()));
    assertThat(output.getDirectToJarOutputSettings().get().getEntriesToJar(), Matchers.equalToObject(directToJarOutputSettings.getEntriesToJar()));
    assertThat(output.getDirectToJarOutputSettings().get().getMainClass(), Matchers.equalToObject(directToJarOutputSettings.getMainClass()));
    assertThat(output.getDirectToJarOutputSettings().get().getManifestFile(), Matchers.equalToObject(directToJarOutputSettings.getManifestFile()));
    assertThat(output.getDirectToJarOutputSettings().get().getClassesToRemoveFromJar().size(), Matchers.equalToObject(directToJarOutputSettings.getClassesToRemoveFromJar().size()));
    ImmutableList<Pattern> inputPatterns = directToJarOutputSettings.getClassesToRemoveFromJar().asList();
    ImmutableList<Pattern> outputPatterns = output.getDirectToJarOutputSettings().get().getClassesToRemoveFromJar().asList();
    for (int i = 0; i < inputPatterns.size(); i++) {
        Pattern inputPattern = inputPatterns.get(i);
        Pattern outputPattern = outputPatterns.get(i);
        assertThat(outputPattern.pattern(), Matchers.equalToObject(inputPattern.pattern()));
        assertThat(outputPattern.flags(), Matchers.equalTo(inputPattern.flags()));
    }
}

Example 34 with ProcessExecutor

use of com.facebook.buck.util.ProcessExecutor in project buck by facebook.

the class XctoolRunTestsStep method listAndFilterTestsThenFormatXctoolParams.

private static int listAndFilterTestsThenFormatXctoolParams(ProcessExecutor processExecutor, Console console, TestSelectorList testSelectorList, ProcessExecutorParams listTestsOnlyParams, ImmutableList.Builder<String> filterParamsBuilder) throws IOException, InterruptedException {
    Preconditions.checkArgument(!testSelectorList.isEmpty());
    LOG.debug("Filtering tests with selector list: %s", testSelectorList.getExplanation());
    LOG.debug("Listing tests with command: %s", listTestsOnlyParams);
    ProcessExecutor.LaunchedProcess launchedProcess = processExecutor.launchProcess(listTestsOnlyParams);
    ListTestsOnlyHandler listTestsOnlyHandler = new ListTestsOnlyHandler();
    String stderr;
    int listTestsResult;
    try (InputStreamReader isr = new InputStreamReader(launchedProcess.getInputStream(), StandardCharsets.UTF_8);
        BufferedReader br = new BufferedReader(isr);
        InputStreamReader esr = new InputStreamReader(launchedProcess.getErrorStream(), StandardCharsets.UTF_8);
        BufferedReader ebr = new BufferedReader(esr)) {
        XctoolOutputParsing.streamOutputFromReader(br, listTestsOnlyHandler);
        stderr = CharStreams.toString(ebr).trim();
        listTestsResult = processExecutor.waitForLaunchedProcess(launchedProcess).getExitCode();
    }
    if (listTestsResult != 0) {
        if (!stderr.isEmpty()) {
            console.printErrorText(String.format(Locale.US, "xctool failed with exit code %d: %s", listTestsResult, stderr));
        } else {
            console.printErrorText(String.format(Locale.US, "xctool failed with exit code %d", listTestsResult));
        }
    } else {
        formatXctoolFilterParams(testSelectorList, listTestsOnlyHandler.testTargetsToDescriptions, filterParamsBuilder);
    }
    return listTestsResult;
}

Example 35 with ProcessExecutor

use of com.facebook.buck.util.ProcessExecutor in project buck by facebook.

the class XctestRunTestsStep method execute.

@Override
public StepExecutionResult execute(ExecutionContext context) throws InterruptedException {
    ProcessExecutorParams.Builder builder = ProcessExecutorParams.builder().addAllCommand(getCommand()).setDirectory(filesystem.getRootPath().toAbsolutePath()).setRedirectErrorStream(true).setEnvironment(getEnv(context));
    ProcessExecutorParams params = builder.build();
    LOG.debug("xctest command: %s", Joiner.on(" ").join(params.getCommand()));
    try {
        ProcessExecutor executor = context.getProcessExecutor();
        ProcessExecutor.LaunchedProcess launchedProcess = executor.launchProcess(params);
        int exitCode;
        try (OutputStream outputStream = filesystem.newFileOutputStream(outputPath);
            TeeInputStream outputWrapperStream = new TeeInputStream(launchedProcess.getInputStream(), outputStream)) {
            if (outputReadingCallback.isPresent()) {
                // The caller is responsible for reading all the data, which TeeInputStream will
                // copy to outputStream.
                outputReadingCallback.get().readOutput(outputWrapperStream);
            } else {
                // Nobody's going to read from outputWrapperStream, so close it and copy
                // the process's stdout and stderr to outputPath directly.
                outputWrapperStream.close();
                ByteStreams.copy(launchedProcess.getInputStream(), outputStream);
            }
            exitCode = executor.waitForLaunchedProcess(launchedProcess).getExitCode();
            // There's no way to distinguish a test failure from an xctest issue. We don't
            // want to fail the step on a test failure, so return 0 for any xctest exit code.
            exitCode = 0;
            LOG.debug("Finished running command, exit code %d", exitCode);
        } finally {
            context.getProcessExecutor().destroyLaunchedProcess(launchedProcess);
            context.getProcessExecutor().waitForLaunchedProcess(launchedProcess);
        }
        if (exitCode != 0) {
            context.getConsole().printErrorText(String.format(Locale.US, "xctest failed with exit code %d", exitCode));
        }
        return StepExecutionResult.of(exitCode);
    } catch (IOException e) {
        LOG.error(e, "Exception while running %s", params.getCommand());
        MoreThrowables.propagateIfInterrupt(e);
        context.getConsole().printBuildFailureWithStacktrace(e);
        return StepExecutionResult.ERROR;
    }
}