Examples with SlaveStream com.facebook.buck.distributed.thrift.SlaveStream used on opensource projects

Example 1 with SlaveStream

use of com.facebook.buck.distributed.thrift.SlaveStream in project buck by facebook.

the class DistBuildLogStateTracker method createRealtimeLogRequests.

private void createRealtimeLogRequests(BuildSlaveInfo buildSlaveInfo, LogStreamType streamType, List<LogLineBatchRequest> requests) {
    RunId runId = buildSlaveInfo.runId;
    SlaveStream slaveStream = new SlaveStream();
    slaveStream.setRunId(runId);
    slaveStream.setStreamType(streamType);
    int latestBatchNumber = getLatestBatchNumber(buildSlaveInfo, streamType);
    // No logs have been created for this slave stream yet
    if (latestBatchNumber == 0) {
        return;
    }
    // Logs exist, but no requests have been made yet => request everything
    if (!seenSlaveLogs.containsKey(slaveStream)) {
        requests.add(createRequest(slaveStream, 1));
        return;
    }
    int latestBatchLineNumber = getLatestBatchLineNumber(buildSlaveInfo, streamType);
    SlaveStreamState seenState = seenSlaveLogs.get(slaveStream);
    // Logs exists, but we have seen them all already.
    if (seenState.seenBatchNumber > latestBatchNumber || (seenState.seenBatchNumber == latestBatchNumber && seenState.seenBatchLineCount >= latestBatchLineNumber)) {
        return;
    }
    // New logs exists, that we haven't seen yet.
    requests.add(createRequest(slaveStream, seenState.seenBatchNumber));
}

Example 2 with SlaveStream

use of com.facebook.buck.distributed.thrift.SlaveStream in project buck by facebook.

the class DistBuildLogStateTrackerTest method testStreamsLogsForRuns.

@Test
public void testStreamsLogsForRuns() throws IOException {
    BuildSlaveInfo runOneSlaveInfo = new BuildSlaveInfo();
    RunId runOneId = new RunId();
    runOneId.setId(RUN_ONE_ID);
    runOneSlaveInfo.setRunId(runOneId);
    runOneSlaveInfo.setStdOutCurrentBatchNumber(0);
    runOneSlaveInfo.setStdOutCurrentBatchLineCount(0);
    runOneSlaveInfo.setStdErrCurrentBatchNumber(1);
    runOneSlaveInfo.setStdErrCurrentBatchLineCount(1);
    BuildSlaveInfo runTwoSlaveInfo = new BuildSlaveInfo();
    RunId runTwoId = new RunId();
    runTwoId.setId(RUN_TWO_ID);
    runTwoSlaveInfo.setRunId(runTwoId);
    runTwoSlaveInfo.setStdOutCurrentBatchNumber(2);
    runTwoSlaveInfo.setStdOutCurrentBatchLineCount(2);
    runTwoSlaveInfo.setStdErrCurrentBatchNumber(0);
    runTwoSlaveInfo.setStdErrCurrentBatchLineCount(0);
    // runOne has stdErr, and runTwo has stdOut to download.
    List<BuildSlaveInfo> buildSlaveInfos = ImmutableList.of(runOneSlaveInfo, runTwoSlaveInfo);
    List<LogLineBatchRequest> requestsOne = distBuildLogStateTracker.createRealtimeLogRequests(buildSlaveInfos);
    assertThat(requestsOne.size(), Matchers.equalTo(2));
    // Request runOne/stdErr from batch 1
    assertTrue(requestsOne.stream().anyMatch(r -> r.slaveStream.runId.equals(runOneId) && r.slaveStream.streamType.equals(LogStreamType.STDERR) && r.batchNumber == 1));
    // Request runTwo/stdOut from batch 1
    assertTrue(requestsOne.stream().anyMatch(r -> r.slaveStream.runId.equals(runTwoId) && r.slaveStream.streamType.equals(LogStreamType.STDOUT) && r.batchNumber == 1));
    // Process new logs
    SlaveStream runOneStdErrStream = new SlaveStream();
    runOneStdErrStream.setRunId(runOneId);
    runOneStdErrStream.setStreamType(LogStreamType.STDERR);
    SlaveStream runOneStdOutStream = new SlaveStream();
    runOneStdOutStream.setRunId(runOneId);
    runOneStdOutStream.setStreamType(LogStreamType.STDOUT);
    SlaveStream runTwoStdOutStream = new SlaveStream();
    runTwoStdOutStream.setRunId(runTwoId);
    runTwoStdOutStream.setStreamType(LogStreamType.STDOUT);
    StreamLogs runOneStdErrLogs = new StreamLogs();
    runOneStdErrLogs.setSlaveStream(runOneStdErrStream);
    LogLineBatch runOneStdErrLogsBatchOne = new LogLineBatch();
    runOneStdErrLogsBatchOne.setBatchNumber(1);
    runOneStdErrLogsBatchOne.setLines(ImmutableList.of("runOneStdErrLine1\n", "runOneStdErrLine2\n"));
    runOneStdErrLogs.setLogLineBatches(ImmutableList.of(runOneStdErrLogsBatchOne));
    StreamLogs runTwoStdOutLogs = new StreamLogs();
    runTwoStdOutLogs.setSlaveStream(runTwoStdOutStream);
    LogLineBatch runTwoStdOutLogsBatchOne = new LogLineBatch();
    runTwoStdOutLogsBatchOne.setBatchNumber(1);
    runTwoStdOutLogsBatchOne.setLines(ImmutableList.of("runTwoStdOutLine1\n"));
    LogLineBatch runTwoStdOutLogsBatchTwo = new LogLineBatch();
    runTwoStdOutLogsBatchTwo.setBatchNumber(2);
    runTwoStdOutLogsBatchTwo.setLines(ImmutableList.of("runTwoStdOutLine2\n", "runTwoStdOutLine3\n"));
    runTwoStdOutLogs.setLogLineBatches(ImmutableList.of(runTwoStdOutLogsBatchOne, runTwoStdOutLogsBatchTwo));
    List<StreamLogs> streamLogsOne = ImmutableList.of(runOneStdErrLogs, runTwoStdOutLogs);
    distBuildLogStateTracker.processStreamLogs(streamLogsOne);
    assertLogLines(RUN_ONE_STD_ERR_LOG, ImmutableList.of("runOneStdErrLine1", "runOneStdErrLine2"));
    assertLogLines(RUN_TWO_STD_OUT_LOG, ImmutableList.of("runTwoStdOutLine1", "runTwoStdOutLine2", "runTwoStdOutLine3"));
    // New build status arrives.
    // runOne/stdErr has same values as last time (so fewer lines than already processed).
    // => ignore
    // runTwo/stdOut updated within existing batch 2
    // => fetch batch 2 and process new lines
    runTwoSlaveInfo.setStdOutCurrentBatchNumber(2);
    runTwoSlaveInfo.setStdOutCurrentBatchLineCount(3);
    buildSlaveInfos = ImmutableList.of(runOneSlaveInfo, runTwoSlaveInfo);
    List<LogLineBatchRequest> requestsTwo = distBuildLogStateTracker.createRealtimeLogRequests(buildSlaveInfos);
    assertThat(requestsTwo.size(), Matchers.equalTo(1));
    // Request runTwo/stdOut from batch 2
    assertTrue(requestsTwo.stream().anyMatch(r -> r.slaveStream.runId.equals(runTwoId) && r.slaveStream.streamType.equals(LogStreamType.STDOUT) && r.batchNumber == 2));
    // Process new logs
    runTwoStdOutLogs = new StreamLogs();
    runTwoStdOutLogs.setSlaveStream(runTwoStdOutStream);
    runTwoStdOutLogsBatchTwo = new LogLineBatch();
    runTwoStdOutLogsBatchTwo.setBatchNumber(2);
    runTwoStdOutLogsBatchTwo.setLines(ImmutableList.of("runTwoStdOutLine2\n", "runTwoStdOutLine3\n", "runTwoStdOutLine4\n"));
    runTwoStdOutLogs.setLogLineBatches(ImmutableList.of(runTwoStdOutLogsBatchTwo));
    List<StreamLogs> streamLogsTwo = ImmutableList.of(runTwoStdOutLogs);
    distBuildLogStateTracker.processStreamLogs(streamLogsTwo);
    assertLogLines(RUN_TWO_STD_OUT_LOG, ImmutableList.of("runTwoStdOutLine1", "runTwoStdOutLine2", "runTwoStdOutLine3", "runTwoStdOutLine4"));
    // New build status arrives.
    // runOne/stdOut has now been populated with 2 batches
    // runOne/stdErr updated to new batch 2, with changes to batch 1 too
    // => fetch 1 and 2, processing new lines in batch 1 and all lines in batch 2.
    // runTwo/stdOut updated to new batch 3, no changes to existing batches
    // => fetch batch 2 and 3, processing only changes in batch 3
    runOneSlaveInfo.setStdOutCurrentBatchNumber(2);
    runOneSlaveInfo.setStdOutCurrentBatchLineCount(2);
    runOneSlaveInfo.setStdErrCurrentBatchNumber(2);
    runOneSlaveInfo.setStdErrCurrentBatchLineCount(1);
    runTwoSlaveInfo.setStdOutCurrentBatchNumber(3);
    runTwoSlaveInfo.setStdOutCurrentBatchLineCount(2);
    // runOne has stdErr, and runTwo has stdOut to download.
    buildSlaveInfos = ImmutableList.of(runOneSlaveInfo, runTwoSlaveInfo);
    List<LogLineBatchRequest> requestsThree = distBuildLogStateTracker.createRealtimeLogRequests(buildSlaveInfos);
    assertThat(requestsThree.size(), Matchers.equalTo(3));
    // Request runOne/stdErr from batch 1
    assertTrue(requestsThree.stream().anyMatch(r -> r.slaveStream.runId.equals(runOneId) && r.slaveStream.streamType.equals(LogStreamType.STDERR) && r.batchNumber == 1));
    // Request runOne/stdOut from batch 1
    assertTrue(requestsThree.stream().anyMatch(r -> r.slaveStream.runId.equals(runOneId) && r.slaveStream.streamType.equals(LogStreamType.STDOUT) && r.batchNumber == 1));
    // Request runTwo/stdOut from batch 2
    assertTrue(requestsThree.stream().anyMatch(r -> r.slaveStream.runId.equals(runTwoId) && r.slaveStream.streamType.equals(LogStreamType.STDOUT) && r.batchNumber == 2));
    // Process new logs
    // runOne/stdErr
    runOneStdErrLogs = new StreamLogs();
    runOneStdErrLogs.setSlaveStream(runOneStdErrStream);
    runOneStdErrLogsBatchOne = new LogLineBatch();
    runOneStdErrLogsBatchOne.setBatchNumber(1);
    runOneStdErrLogsBatchOne.setLines(ImmutableList.of("runOneStdErrLine1\n", "runOneStdErrLine2\n", "runOneStdErrLine3\n"));
    LogLineBatch runOneStdErrLogsBatchTwo = new LogLineBatch();
    runOneStdErrLogsBatchTwo.setBatchNumber(2);
    runOneStdErrLogsBatchTwo.setLines(ImmutableList.of("runOneStdErrLine4\n"));
    runOneStdErrLogs.setLogLineBatches(ImmutableList.of(runOneStdErrLogsBatchOne, runOneStdErrLogsBatchTwo));
    // runOne/stdOut
    StreamLogs runOneStdOutLogs = new StreamLogs();
    runOneStdOutLogs.setSlaveStream(runOneStdOutStream);
    LogLineBatch runOneStdOutLogsBatchOne = new LogLineBatch();
    runOneStdOutLogsBatchOne.setBatchNumber(1);
    runOneStdOutLogsBatchOne.setLines(ImmutableList.of("runOneStdOutLine1\n", "runOneStdOutLine2\n"));
    LogLineBatch runOneStdOutLogsBatchTwo = new LogLineBatch();
    runOneStdOutLogsBatchTwo.setBatchNumber(2);
    runOneStdOutLogsBatchTwo.setLines(ImmutableList.of("runOneStdOutLine3\n", "runOneStdOutLine4\n"));
    runOneStdOutLogs.setLogLineBatches(ImmutableList.of(runOneStdOutLogsBatchOne, runOneStdOutLogsBatchTwo));
    // runTwo/stdOut
    runTwoStdOutLogs = new StreamLogs();
    runTwoStdOutLogs.setSlaveStream(runTwoStdOutStream);
    runTwoStdOutLogsBatchTwo = new LogLineBatch();
    runTwoStdOutLogsBatchTwo.setBatchNumber(2);
    runTwoStdOutLogsBatchTwo.setLines(ImmutableList.of("runTwoStdOutLine2\n", "runTwoStdOutLine3\n", "runTwoStdOutLine4\n"));
    LogLineBatch runTwoStdOutLogsBatchThree = new LogLineBatch();
    runTwoStdOutLogsBatchThree.setBatchNumber(3);
    runTwoStdOutLogsBatchThree.setLines(ImmutableList.of("runTwoStdOutLine5\n", "runTwoStdOutLine6\n"));
    runTwoStdOutLogs.setLogLineBatches(ImmutableList.of(runTwoStdOutLogsBatchTwo, runTwoStdOutLogsBatchThree));
    List<StreamLogs> streamLogsThree = ImmutableList.of(runOneStdErrLogs, runOneStdOutLogs, runTwoStdOutLogs);
    distBuildLogStateTracker.processStreamLogs(streamLogsThree);
    assertLogLines(RUN_ONE_STD_OUT_LOG, ImmutableList.of("runOneStdOutLine1", "runOneStdOutLine2", "runOneStdOutLine3", "runOneStdOutLine4"));
    assertLogLines(RUN_ONE_STD_ERR_LOG, ImmutableList.of("runOneStdErrLine1", "runOneStdErrLine2", "runOneStdErrLine3", "runOneStdErrLine4"));
    assertLogLines(RUN_TWO_STD_OUT_LOG, ImmutableList.of("runTwoStdOutLine1", "runTwoStdOutLine2", "runTwoStdOutLine3", "runTwoStdOutLine4", "runTwoStdOutLine5", "runTwoStdOutLine6"));
}