Examples with LogLineBatchRequest com.facebook.buck.distributed.thrift.LogLineBatchRequest used on opensource projects

Example 1 with LogLineBatchRequest

use of com.facebook.buck.distributed.thrift.LogLineBatchRequest in project buck by facebook.

the class DistBuildClientExecutor method executeAndPrintFailuresToEventBus.

public int executeAndPrintFailuresToEventBus(final WeightedListeningExecutorService executorService, ProjectFilesystem projectFilesystem, FileHashCache fileHashCache, BuckEventBus eventBus) throws IOException, InterruptedException {
    BuildJob job = distBuildService.createBuild();
    final StampedeId id = job.getStampedeId();
    LOG.info("Created job. Build id = " + id.getId());
    logDebugInfo(job);
    List<ListenableFuture<Void>> asyncJobs = new LinkedList<>();
    LOG.info("Uploading local changes.");
    asyncJobs.add(distBuildService.uploadMissingFiles(buildJobState.fileHashes, executorService));
    LOG.info("Uploading target graph.");
    asyncJobs.add(distBuildService.uploadTargetGraph(buildJobState, id, executorService));
    LOG.info("Uploading buck dot-files.");
    asyncJobs.add(distBuildService.uploadBuckDotFiles(id, projectFilesystem, fileHashCache, executorService));
    try {
        Futures.allAsList(asyncJobs).get();
    } catch (ExecutionException e) {
        LOG.error("Upload failed.");
        throw new RuntimeException(e);
    }
    distBuildService.setBuckVersion(id, buckVersion);
    LOG.info("Set Buck Version. Build status: " + job.getStatus().toString());
    job = distBuildService.startBuild(id);
    LOG.info("Started job. Build status: " + job.getStatus().toString());
    logDebugInfo(job);
    Stopwatch stopwatch = Stopwatch.createStarted();
    // Keep polling until the build is complete or failed.
    do {
        job = distBuildService.getCurrentBuildJobState(id);
        LOG.info("Got build status: " + job.getStatus().toString());
        DistBuildStatus distBuildStatus = prepareStatusFromJob(job).setETAMillis(MAX_BUILD_DURATION_MILLIS - stopwatch.elapsed(TimeUnit.MILLISECONDS)).build();
        eventBus.post(new DistBuildStatusEvent(distBuildStatus));
        List<LogLineBatchRequest> newLogLineRequests = distBuildLogStateTracker.createRealtimeLogRequests(job.getSlaveInfoByRunId().values());
        MultiGetBuildSlaveRealTimeLogsResponse slaveLogsResponse = distBuildService.fetchSlaveLogLines(job.stampedeId, newLogLineRequests);
        distBuildLogStateTracker.processStreamLogs(slaveLogsResponse.getMultiStreamLogs());
        try {
            // TODO(shivanker): Get rid of sleeps in methods which we want to unit test
            Thread.sleep(millisBetweenStatusPoll);
        } catch (InterruptedException e) {
            LOG.error(e, "BuildStatus polling sleep call has been interrupted unexpectedly.");
        }
    } while (!(job.getStatus().equals(BuildStatus.FINISHED_SUCCESSFULLY) || job.getStatus().equals(BuildStatus.FAILED)));
    try {
        MultiGetBuildSlaveLogDirResponse logDirsResponse = distBuildService.fetchBuildSlaveLogDir(job.stampedeId, distBuildLogStateTracker.runIdsToMaterializeLogDirsFor(job.slaveInfoByRunId.values()));
        distBuildLogStateTracker.materializeLogDirs(logDirsResponse.getLogDirs());
    } catch (IOException ex) {
        LOG.error(ex);
    }
    LOG.info("Build was " + (job.getStatus().equals(BuildStatus.FINISHED_SUCCESSFULLY) ? "" : "not ") + "successful!");
    logDebugInfo(job);
    DistBuildStatus distBuildStatus = prepareStatusFromJob(job).setETAMillis(0).build();
    eventBus.post(new DistBuildStatusEvent(distBuildStatus));
    return job.getStatus().equals(BuildStatus.FINISHED_SUCCESSFULLY) ? 0 : 1;
}

Example 2 with LogLineBatchRequest

use of com.facebook.buck.distributed.thrift.LogLineBatchRequest in project buck by facebook.

the class DistBuildLogStateTrackerTest method testStreamsLogsForRuns.

@Test
public void testStreamsLogsForRuns() throws IOException {
    BuildSlaveInfo runOneSlaveInfo = new BuildSlaveInfo();
    RunId runOneId = new RunId();
    runOneId.setId(RUN_ONE_ID);
    runOneSlaveInfo.setRunId(runOneId);
    runOneSlaveInfo.setStdOutCurrentBatchNumber(0);
    runOneSlaveInfo.setStdOutCurrentBatchLineCount(0);
    runOneSlaveInfo.setStdErrCurrentBatchNumber(1);
    runOneSlaveInfo.setStdErrCurrentBatchLineCount(1);
    BuildSlaveInfo runTwoSlaveInfo = new BuildSlaveInfo();
    RunId runTwoId = new RunId();
    runTwoId.setId(RUN_TWO_ID);
    runTwoSlaveInfo.setRunId(runTwoId);
    runTwoSlaveInfo.setStdOutCurrentBatchNumber(2);
    runTwoSlaveInfo.setStdOutCurrentBatchLineCount(2);
    runTwoSlaveInfo.setStdErrCurrentBatchNumber(0);
    runTwoSlaveInfo.setStdErrCurrentBatchLineCount(0);
    // runOne has stdErr, and runTwo has stdOut to download.
    List<BuildSlaveInfo> buildSlaveInfos = ImmutableList.of(runOneSlaveInfo, runTwoSlaveInfo);
    List<LogLineBatchRequest> requestsOne = distBuildLogStateTracker.createRealtimeLogRequests(buildSlaveInfos);
    assertThat(requestsOne.size(), Matchers.equalTo(2));
    // Request runOne/stdErr from batch 1
    assertTrue(requestsOne.stream().anyMatch(r -> r.slaveStream.runId.equals(runOneId) && r.slaveStream.streamType.equals(LogStreamType.STDERR) && r.batchNumber == 1));
    // Request runTwo/stdOut from batch 1
    assertTrue(requestsOne.stream().anyMatch(r -> r.slaveStream.runId.equals(runTwoId) && r.slaveStream.streamType.equals(LogStreamType.STDOUT) && r.batchNumber == 1));
    // Process new logs
    SlaveStream runOneStdErrStream = new SlaveStream();
    runOneStdErrStream.setRunId(runOneId);
    runOneStdErrStream.setStreamType(LogStreamType.STDERR);
    SlaveStream runOneStdOutStream = new SlaveStream();
    runOneStdOutStream.setRunId(runOneId);
    runOneStdOutStream.setStreamType(LogStreamType.STDOUT);
    SlaveStream runTwoStdOutStream = new SlaveStream();
    runTwoStdOutStream.setRunId(runTwoId);
    runTwoStdOutStream.setStreamType(LogStreamType.STDOUT);
    StreamLogs runOneStdErrLogs = new StreamLogs();
    runOneStdErrLogs.setSlaveStream(runOneStdErrStream);
    LogLineBatch runOneStdErrLogsBatchOne = new LogLineBatch();
    runOneStdErrLogsBatchOne.setBatchNumber(1);
    runOneStdErrLogsBatchOne.setLines(ImmutableList.of("runOneStdErrLine1\n", "runOneStdErrLine2\n"));
    runOneStdErrLogs.setLogLineBatches(ImmutableList.of(runOneStdErrLogsBatchOne));
    StreamLogs runTwoStdOutLogs = new StreamLogs();
    runTwoStdOutLogs.setSlaveStream(runTwoStdOutStream);
    LogLineBatch runTwoStdOutLogsBatchOne = new LogLineBatch();
    runTwoStdOutLogsBatchOne.setBatchNumber(1);
    runTwoStdOutLogsBatchOne.setLines(ImmutableList.of("runTwoStdOutLine1\n"));
    LogLineBatch runTwoStdOutLogsBatchTwo = new LogLineBatch();
    runTwoStdOutLogsBatchTwo.setBatchNumber(2);
    runTwoStdOutLogsBatchTwo.setLines(ImmutableList.of("runTwoStdOutLine2\n", "runTwoStdOutLine3\n"));
    runTwoStdOutLogs.setLogLineBatches(ImmutableList.of(runTwoStdOutLogsBatchOne, runTwoStdOutLogsBatchTwo));
    List<StreamLogs> streamLogsOne = ImmutableList.of(runOneStdErrLogs, runTwoStdOutLogs);
    distBuildLogStateTracker.processStreamLogs(streamLogsOne);
    assertLogLines(RUN_ONE_STD_ERR_LOG, ImmutableList.of("runOneStdErrLine1", "runOneStdErrLine2"));
    assertLogLines(RUN_TWO_STD_OUT_LOG, ImmutableList.of("runTwoStdOutLine1", "runTwoStdOutLine2", "runTwoStdOutLine3"));
    // New build status arrives.
    // runOne/stdErr has same values as last time (so fewer lines than already processed).
    // => ignore
    // runTwo/stdOut updated within existing batch 2
    // => fetch batch 2 and process new lines
    runTwoSlaveInfo.setStdOutCurrentBatchNumber(2);
    runTwoSlaveInfo.setStdOutCurrentBatchLineCount(3);
    buildSlaveInfos = ImmutableList.of(runOneSlaveInfo, runTwoSlaveInfo);
    List<LogLineBatchRequest> requestsTwo = distBuildLogStateTracker.createRealtimeLogRequests(buildSlaveInfos);
    assertThat(requestsTwo.size(), Matchers.equalTo(1));
    // Request runTwo/stdOut from batch 2
    assertTrue(requestsTwo.stream().anyMatch(r -> r.slaveStream.runId.equals(runTwoId) && r.slaveStream.streamType.equals(LogStreamType.STDOUT) && r.batchNumber == 2));
    // Process new logs
    runTwoStdOutLogs = new StreamLogs();
    runTwoStdOutLogs.setSlaveStream(runTwoStdOutStream);
    runTwoStdOutLogsBatchTwo = new LogLineBatch();
    runTwoStdOutLogsBatchTwo.setBatchNumber(2);
    runTwoStdOutLogsBatchTwo.setLines(ImmutableList.of("runTwoStdOutLine2\n", "runTwoStdOutLine3\n", "runTwoStdOutLine4\n"));
    runTwoStdOutLogs.setLogLineBatches(ImmutableList.of(runTwoStdOutLogsBatchTwo));
    List<StreamLogs> streamLogsTwo = ImmutableList.of(runTwoStdOutLogs);
    distBuildLogStateTracker.processStreamLogs(streamLogsTwo);
    assertLogLines(RUN_TWO_STD_OUT_LOG, ImmutableList.of("runTwoStdOutLine1", "runTwoStdOutLine2", "runTwoStdOutLine3", "runTwoStdOutLine4"));
    // New build status arrives.
    // runOne/stdOut has now been populated with 2 batches
    // runOne/stdErr updated to new batch 2, with changes to batch 1 too
    // => fetch 1 and 2, processing new lines in batch 1 and all lines in batch 2.
    // runTwo/stdOut updated to new batch 3, no changes to existing batches
    // => fetch batch 2 and 3, processing only changes in batch 3
    runOneSlaveInfo.setStdOutCurrentBatchNumber(2);
    runOneSlaveInfo.setStdOutCurrentBatchLineCount(2);
    runOneSlaveInfo.setStdErrCurrentBatchNumber(2);
    runOneSlaveInfo.setStdErrCurrentBatchLineCount(1);
    runTwoSlaveInfo.setStdOutCurrentBatchNumber(3);
    runTwoSlaveInfo.setStdOutCurrentBatchLineCount(2);
    // runOne has stdErr, and runTwo has stdOut to download.
    buildSlaveInfos = ImmutableList.of(runOneSlaveInfo, runTwoSlaveInfo);
    List<LogLineBatchRequest> requestsThree = distBuildLogStateTracker.createRealtimeLogRequests(buildSlaveInfos);
    assertThat(requestsThree.size(), Matchers.equalTo(3));
    // Request runOne/stdErr from batch 1
    assertTrue(requestsThree.stream().anyMatch(r -> r.slaveStream.runId.equals(runOneId) && r.slaveStream.streamType.equals(LogStreamType.STDERR) && r.batchNumber == 1));
    // Request runOne/stdOut from batch 1
    assertTrue(requestsThree.stream().anyMatch(r -> r.slaveStream.runId.equals(runOneId) && r.slaveStream.streamType.equals(LogStreamType.STDOUT) && r.batchNumber == 1));
    // Request runTwo/stdOut from batch 2
    assertTrue(requestsThree.stream().anyMatch(r -> r.slaveStream.runId.equals(runTwoId) && r.slaveStream.streamType.equals(LogStreamType.STDOUT) && r.batchNumber == 2));
    // Process new logs
    // runOne/stdErr
    runOneStdErrLogs = new StreamLogs();
    runOneStdErrLogs.setSlaveStream(runOneStdErrStream);
    runOneStdErrLogsBatchOne = new LogLineBatch();
    runOneStdErrLogsBatchOne.setBatchNumber(1);
    runOneStdErrLogsBatchOne.setLines(ImmutableList.of("runOneStdErrLine1\n", "runOneStdErrLine2\n", "runOneStdErrLine3\n"));
    LogLineBatch runOneStdErrLogsBatchTwo = new LogLineBatch();
    runOneStdErrLogsBatchTwo.setBatchNumber(2);
    runOneStdErrLogsBatchTwo.setLines(ImmutableList.of("runOneStdErrLine4\n"));
    runOneStdErrLogs.setLogLineBatches(ImmutableList.of(runOneStdErrLogsBatchOne, runOneStdErrLogsBatchTwo));
    // runOne/stdOut
    StreamLogs runOneStdOutLogs = new StreamLogs();
    runOneStdOutLogs.setSlaveStream(runOneStdOutStream);
    LogLineBatch runOneStdOutLogsBatchOne = new LogLineBatch();
    runOneStdOutLogsBatchOne.setBatchNumber(1);
    runOneStdOutLogsBatchOne.setLines(ImmutableList.of("runOneStdOutLine1\n", "runOneStdOutLine2\n"));
    LogLineBatch runOneStdOutLogsBatchTwo = new LogLineBatch();
    runOneStdOutLogsBatchTwo.setBatchNumber(2);
    runOneStdOutLogsBatchTwo.setLines(ImmutableList.of("runOneStdOutLine3\n", "runOneStdOutLine4\n"));
    runOneStdOutLogs.setLogLineBatches(ImmutableList.of(runOneStdOutLogsBatchOne, runOneStdOutLogsBatchTwo));
    // runTwo/stdOut
    runTwoStdOutLogs = new StreamLogs();
    runTwoStdOutLogs.setSlaveStream(runTwoStdOutStream);
    runTwoStdOutLogsBatchTwo = new LogLineBatch();
    runTwoStdOutLogsBatchTwo.setBatchNumber(2);
    runTwoStdOutLogsBatchTwo.setLines(ImmutableList.of("runTwoStdOutLine2\n", "runTwoStdOutLine3\n", "runTwoStdOutLine4\n"));
    LogLineBatch runTwoStdOutLogsBatchThree = new LogLineBatch();
    runTwoStdOutLogsBatchThree.setBatchNumber(3);
    runTwoStdOutLogsBatchThree.setLines(ImmutableList.of("runTwoStdOutLine5\n", "runTwoStdOutLine6\n"));
    runTwoStdOutLogs.setLogLineBatches(ImmutableList.of(runTwoStdOutLogsBatchTwo, runTwoStdOutLogsBatchThree));
    List<StreamLogs> streamLogsThree = ImmutableList.of(runOneStdErrLogs, runOneStdOutLogs, runTwoStdOutLogs);
    distBuildLogStateTracker.processStreamLogs(streamLogsThree);
    assertLogLines(RUN_ONE_STD_OUT_LOG, ImmutableList.of("runOneStdOutLine1", "runOneStdOutLine2", "runOneStdOutLine3", "runOneStdOutLine4"));
    assertLogLines(RUN_ONE_STD_ERR_LOG, ImmutableList.of("runOneStdErrLine1", "runOneStdErrLine2", "runOneStdErrLine3", "runOneStdErrLine4"));
    assertLogLines(RUN_TWO_STD_OUT_LOG, ImmutableList.of("runTwoStdOutLine1", "runTwoStdOutLine2", "runTwoStdOutLine3", "runTwoStdOutLine4", "runTwoStdOutLine5", "runTwoStdOutLine6"));
}

Example 3 with LogLineBatchRequest

use of com.facebook.buck.distributed.thrift.LogLineBatchRequest in project buck by facebook.

the class DistBuildLogStateTracker method createRequest.

private static LogLineBatchRequest createRequest(SlaveStream slaveStream, int batchNumber) {
    LogLineBatchRequest request = new LogLineBatchRequest();
    request.setSlaveStream(slaveStream);
    request.setBatchNumber(batchNumber);
    return request;
}