Examples with LogLineBatch com.facebook.buck.distributed.thrift.LogLineBatch used on opensource projects

Example 1 with LogLineBatch

use of com.facebook.buck.distributed.thrift.LogLineBatch in project buck by facebook.

the class DistBuildLogStateTracker method processStreamLogs.

/*
   *******************************
   *  Helpers
   *******************************
   */
private void processStreamLogs(StreamLogs streamLogs) {
    if (!seenSlaveLogs.containsKey(streamLogs.slaveStream)) {
        seenSlaveLogs.put(streamLogs.slaveStream, new SlaveStreamState());
    }
    SlaveStreamState seenStreamState = Preconditions.checkNotNull(seenSlaveLogs.get(streamLogs.slaveStream));
    LogLineBatch lastReceivedBatch = streamLogs.logLineBatches.get(streamLogs.logLineBatches.size() - 1);
    if (seenStreamState.seenBatchNumber > lastReceivedBatch.batchNumber || (seenStreamState.seenBatchNumber == lastReceivedBatch.batchNumber && seenStreamState.seenBatchLineCount >= lastReceivedBatch.lines.size())) {
        LOG.warn("Received stale logs for runID [%s] and stream [%s]", streamLogs.slaveStream.runId, streamLogs.slaveStream.streamType);
        return;
    }
    // Determines which log lines need writing, and then writes them to disk.
    List<String> newLines = Lists.newArrayList();
    for (LogLineBatch batch : streamLogs.logLineBatches) {
        if (batch.batchNumber < seenStreamState.seenBatchNumber) {
            continue;
        }
        if (batch.batchNumber == seenStreamState.seenBatchNumber) {
            if (batch.lines.size() == seenStreamState.seenBatchLineCount) {
                continue;
            }
            newLines.addAll(batch.lines.subList(seenStreamState.seenBatchLineCount, batch.lines.size()));
        } else {
            newLines.addAll(batch.lines);
        }
    }
    writeLogStreamLinesToDisk(streamLogs.slaveStream, newLines);
    seenStreamState.seenBatchNumber = lastReceivedBatch.batchNumber;
    seenStreamState.seenBatchLineCount = lastReceivedBatch.lines.size();
}

Example 2 with LogLineBatch

use of com.facebook.buck.distributed.thrift.LogLineBatch in project buck by facebook.

the class DistBuildLogStateTrackerTest method testStreamsLogsForRuns.

@Test
public void testStreamsLogsForRuns() throws IOException {
    BuildSlaveInfo runOneSlaveInfo = new BuildSlaveInfo();
    RunId runOneId = new RunId();
    runOneId.setId(RUN_ONE_ID);
    runOneSlaveInfo.setRunId(runOneId);
    runOneSlaveInfo.setStdOutCurrentBatchNumber(0);
    runOneSlaveInfo.setStdOutCurrentBatchLineCount(0);
    runOneSlaveInfo.setStdErrCurrentBatchNumber(1);
    runOneSlaveInfo.setStdErrCurrentBatchLineCount(1);
    BuildSlaveInfo runTwoSlaveInfo = new BuildSlaveInfo();
    RunId runTwoId = new RunId();
    runTwoId.setId(RUN_TWO_ID);
    runTwoSlaveInfo.setRunId(runTwoId);
    runTwoSlaveInfo.setStdOutCurrentBatchNumber(2);
    runTwoSlaveInfo.setStdOutCurrentBatchLineCount(2);
    runTwoSlaveInfo.setStdErrCurrentBatchNumber(0);
    runTwoSlaveInfo.setStdErrCurrentBatchLineCount(0);
    // runOne has stdErr, and runTwo has stdOut to download.
    List<BuildSlaveInfo> buildSlaveInfos = ImmutableList.of(runOneSlaveInfo, runTwoSlaveInfo);
    List<LogLineBatchRequest> requestsOne = distBuildLogStateTracker.createRealtimeLogRequests(buildSlaveInfos);
    assertThat(requestsOne.size(), Matchers.equalTo(2));
    // Request runOne/stdErr from batch 1
    assertTrue(requestsOne.stream().anyMatch(r -> r.slaveStream.runId.equals(runOneId) && r.slaveStream.streamType.equals(LogStreamType.STDERR) && r.batchNumber == 1));
    // Request runTwo/stdOut from batch 1
    assertTrue(requestsOne.stream().anyMatch(r -> r.slaveStream.runId.equals(runTwoId) && r.slaveStream.streamType.equals(LogStreamType.STDOUT) && r.batchNumber == 1));
    // Process new logs
    SlaveStream runOneStdErrStream = new SlaveStream();
    runOneStdErrStream.setRunId(runOneId);
    runOneStdErrStream.setStreamType(LogStreamType.STDERR);
    SlaveStream runOneStdOutStream = new SlaveStream();
    runOneStdOutStream.setRunId(runOneId);
    runOneStdOutStream.setStreamType(LogStreamType.STDOUT);
    SlaveStream runTwoStdOutStream = new SlaveStream();
    runTwoStdOutStream.setRunId(runTwoId);
    runTwoStdOutStream.setStreamType(LogStreamType.STDOUT);
    StreamLogs runOneStdErrLogs = new StreamLogs();
    runOneStdErrLogs.setSlaveStream(runOneStdErrStream);
    LogLineBatch runOneStdErrLogsBatchOne = new LogLineBatch();
    runOneStdErrLogsBatchOne.setBatchNumber(1);
    runOneStdErrLogsBatchOne.setLines(ImmutableList.of("runOneStdErrLine1\n", "runOneStdErrLine2\n"));
    runOneStdErrLogs.setLogLineBatches(ImmutableList.of(runOneStdErrLogsBatchOne));
    StreamLogs runTwoStdOutLogs = new StreamLogs();
    runTwoStdOutLogs.setSlaveStream(runTwoStdOutStream);
    LogLineBatch runTwoStdOutLogsBatchOne = new LogLineBatch();
    runTwoStdOutLogsBatchOne.setBatchNumber(1);
    runTwoStdOutLogsBatchOne.setLines(ImmutableList.of("runTwoStdOutLine1\n"));
    LogLineBatch runTwoStdOutLogsBatchTwo = new LogLineBatch();
    runTwoStdOutLogsBatchTwo.setBatchNumber(2);
    runTwoStdOutLogsBatchTwo.setLines(ImmutableList.of("runTwoStdOutLine2\n", "runTwoStdOutLine3\n"));
    runTwoStdOutLogs.setLogLineBatches(ImmutableList.of(runTwoStdOutLogsBatchOne, runTwoStdOutLogsBatchTwo));
    List<StreamLogs> streamLogsOne = ImmutableList.of(runOneStdErrLogs, runTwoStdOutLogs);
    distBuildLogStateTracker.processStreamLogs(streamLogsOne);
    assertLogLines(RUN_ONE_STD_ERR_LOG, ImmutableList.of("runOneStdErrLine1", "runOneStdErrLine2"));
    assertLogLines(RUN_TWO_STD_OUT_LOG, ImmutableList.of("runTwoStdOutLine1", "runTwoStdOutLine2", "runTwoStdOutLine3"));
    // New build status arrives.
    // runOne/stdErr has same values as last time (so fewer lines than already processed).
    // => ignore
    // runTwo/stdOut updated within existing batch 2
    // => fetch batch 2 and process new lines
    runTwoSlaveInfo.setStdOutCurrentBatchNumber(2);
    runTwoSlaveInfo.setStdOutCurrentBatchLineCount(3);
    buildSlaveInfos = ImmutableList.of(runOneSlaveInfo, runTwoSlaveInfo);
    List<LogLineBatchRequest> requestsTwo = distBuildLogStateTracker.createRealtimeLogRequests(buildSlaveInfos);
    assertThat(requestsTwo.size(), Matchers.equalTo(1));
    // Request runTwo/stdOut from batch 2
    assertTrue(requestsTwo.stream().anyMatch(r -> r.slaveStream.runId.equals(runTwoId) && r.slaveStream.streamType.equals(LogStreamType.STDOUT) && r.batchNumber == 2));
    // Process new logs
    runTwoStdOutLogs = new StreamLogs();
    runTwoStdOutLogs.setSlaveStream(runTwoStdOutStream);
    runTwoStdOutLogsBatchTwo = new LogLineBatch();
    runTwoStdOutLogsBatchTwo.setBatchNumber(2);
    runTwoStdOutLogsBatchTwo.setLines(ImmutableList.of("runTwoStdOutLine2\n", "runTwoStdOutLine3\n", "runTwoStdOutLine4\n"));
    runTwoStdOutLogs.setLogLineBatches(ImmutableList.of(runTwoStdOutLogsBatchTwo));
    List<StreamLogs> streamLogsTwo = ImmutableList.of(runTwoStdOutLogs);
    distBuildLogStateTracker.processStreamLogs(streamLogsTwo);
    assertLogLines(RUN_TWO_STD_OUT_LOG, ImmutableList.of("runTwoStdOutLine1", "runTwoStdOutLine2", "runTwoStdOutLine3", "runTwoStdOutLine4"));
    // New build status arrives.
    // runOne/stdOut has now been populated with 2 batches
    // runOne/stdErr updated to new batch 2, with changes to batch 1 too
    // => fetch 1 and 2, processing new lines in batch 1 and all lines in batch 2.
    // runTwo/stdOut updated to new batch 3, no changes to existing batches
    // => fetch batch 2 and 3, processing only changes in batch 3
    runOneSlaveInfo.setStdOutCurrentBatchNumber(2);
    runOneSlaveInfo.setStdOutCurrentBatchLineCount(2);
    runOneSlaveInfo.setStdErrCurrentBatchNumber(2);
    runOneSlaveInfo.setStdErrCurrentBatchLineCount(1);
    runTwoSlaveInfo.setStdOutCurrentBatchNumber(3);
    runTwoSlaveInfo.setStdOutCurrentBatchLineCount(2);
    // runOne has stdErr, and runTwo has stdOut to download.
    buildSlaveInfos = ImmutableList.of(runOneSlaveInfo, runTwoSlaveInfo);
    List<LogLineBatchRequest> requestsThree = distBuildLogStateTracker.createRealtimeLogRequests(buildSlaveInfos);
    assertThat(requestsThree.size(), Matchers.equalTo(3));
    // Request runOne/stdErr from batch 1
    assertTrue(requestsThree.stream().anyMatch(r -> r.slaveStream.runId.equals(runOneId) && r.slaveStream.streamType.equals(LogStreamType.STDERR) && r.batchNumber == 1));
    // Request runOne/stdOut from batch 1
    assertTrue(requestsThree.stream().anyMatch(r -> r.slaveStream.runId.equals(runOneId) && r.slaveStream.streamType.equals(LogStreamType.STDOUT) && r.batchNumber == 1));
    // Request runTwo/stdOut from batch 2
    assertTrue(requestsThree.stream().anyMatch(r -> r.slaveStream.runId.equals(runTwoId) && r.slaveStream.streamType.equals(LogStreamType.STDOUT) && r.batchNumber == 2));
    // Process new logs
    // runOne/stdErr
    runOneStdErrLogs = new StreamLogs();
    runOneStdErrLogs.setSlaveStream(runOneStdErrStream);
    runOneStdErrLogsBatchOne = new LogLineBatch();
    runOneStdErrLogsBatchOne.setBatchNumber(1);
    runOneStdErrLogsBatchOne.setLines(ImmutableList.of("runOneStdErrLine1\n", "runOneStdErrLine2\n", "runOneStdErrLine3\n"));
    LogLineBatch runOneStdErrLogsBatchTwo = new LogLineBatch();
    runOneStdErrLogsBatchTwo.setBatchNumber(2);
    runOneStdErrLogsBatchTwo.setLines(ImmutableList.of("runOneStdErrLine4\n"));
    runOneStdErrLogs.setLogLineBatches(ImmutableList.of(runOneStdErrLogsBatchOne, runOneStdErrLogsBatchTwo));
    // runOne/stdOut
    StreamLogs runOneStdOutLogs = new StreamLogs();
    runOneStdOutLogs.setSlaveStream(runOneStdOutStream);
    LogLineBatch runOneStdOutLogsBatchOne = new LogLineBatch();
    runOneStdOutLogsBatchOne.setBatchNumber(1);
    runOneStdOutLogsBatchOne.setLines(ImmutableList.of("runOneStdOutLine1\n", "runOneStdOutLine2\n"));
    LogLineBatch runOneStdOutLogsBatchTwo = new LogLineBatch();
    runOneStdOutLogsBatchTwo.setBatchNumber(2);
    runOneStdOutLogsBatchTwo.setLines(ImmutableList.of("runOneStdOutLine3\n", "runOneStdOutLine4\n"));
    runOneStdOutLogs.setLogLineBatches(ImmutableList.of(runOneStdOutLogsBatchOne, runOneStdOutLogsBatchTwo));
    // runTwo/stdOut
    runTwoStdOutLogs = new StreamLogs();
    runTwoStdOutLogs.setSlaveStream(runTwoStdOutStream);
    runTwoStdOutLogsBatchTwo = new LogLineBatch();
    runTwoStdOutLogsBatchTwo.setBatchNumber(2);
    runTwoStdOutLogsBatchTwo.setLines(ImmutableList.of("runTwoStdOutLine2\n", "runTwoStdOutLine3\n", "runTwoStdOutLine4\n"));
    LogLineBatch runTwoStdOutLogsBatchThree = new LogLineBatch();
    runTwoStdOutLogsBatchThree.setBatchNumber(3);
    runTwoStdOutLogsBatchThree.setLines(ImmutableList.of("runTwoStdOutLine5\n", "runTwoStdOutLine6\n"));
    runTwoStdOutLogs.setLogLineBatches(ImmutableList.of(runTwoStdOutLogsBatchTwo, runTwoStdOutLogsBatchThree));
    List<StreamLogs> streamLogsThree = ImmutableList.of(runOneStdErrLogs, runOneStdOutLogs, runTwoStdOutLogs);
    distBuildLogStateTracker.processStreamLogs(streamLogsThree);
    assertLogLines(RUN_ONE_STD_OUT_LOG, ImmutableList.of("runOneStdOutLine1", "runOneStdOutLine2", "runOneStdOutLine3", "runOneStdOutLine4"));
    assertLogLines(RUN_ONE_STD_ERR_LOG, ImmutableList.of("runOneStdErrLine1", "runOneStdErrLine2", "runOneStdErrLine3", "runOneStdErrLine4"));
    assertLogLines(RUN_TWO_STD_OUT_LOG, ImmutableList.of("runTwoStdOutLine1", "runTwoStdOutLine2", "runTwoStdOutLine3", "runTwoStdOutLine4", "runTwoStdOutLine5", "runTwoStdOutLine6"));
}