Examples with GetLogEventsRequest com.amazonaws.services.logs.model.GetLogEventsRequest used on opensource projects

Example 1 with GetLogEventsRequest

use of com.amazonaws.services.logs.model.GetLogEventsRequest in project aws-athena-query-federation by awslabs.

the class CloudwatchRecordHandlerTest method setUp.

@Before
public void setUp() throws Exception {
    schemaForRead = CloudwatchMetadataHandler.CLOUDWATCH_SCHEMA;
    mockS3Storage = new ArrayList<>();
    allocator = new BlockAllocatorImpl();
    handler = new CloudwatchRecordHandler(mockS3, mockSecretsManager, mockAthena, mockAwsLogs);
    spillReader = new S3BlockSpillReader(mockS3, allocator);
    when(mockS3.putObject(anyObject(), anyObject(), anyObject(), anyObject())).thenAnswer((InvocationOnMock invocationOnMock) -> {
        InputStream inputStream = (InputStream) invocationOnMock.getArguments()[2];
        ByteHolder byteHolder = new ByteHolder();
        byteHolder.setBytes(ByteStreams.toByteArray(inputStream));
        synchronized (mockS3Storage) {
            mockS3Storage.add(byteHolder);
            logger.info("puObject: total size " + mockS3Storage.size());
        }
        return mock(PutObjectResult.class);
    });
    when(mockS3.getObject(anyString(), anyString())).thenAnswer((InvocationOnMock invocationOnMock) -> {
        S3Object mockObject = mock(S3Object.class);
        ByteHolder byteHolder;
        synchronized (mockS3Storage) {
            byteHolder = mockS3Storage.get(0);
            mockS3Storage.remove(0);
            logger.info("getObject: total size " + mockS3Storage.size());
        }
        when(mockObject.getObjectContent()).thenReturn(new S3ObjectInputStream(new ByteArrayInputStream(byteHolder.getBytes()), null));
        return mockObject;
    });
    when(mockAwsLogs.getLogEvents(any(GetLogEventsRequest.class))).thenAnswer((InvocationOnMock invocationOnMock) -> {
        GetLogEventsRequest request = (GetLogEventsRequest) invocationOnMock.getArguments()[0];
        // Check that predicate pushdown was propagated to cloudwatch
        assertNotNull(request.getStartTime());
        assertNotNull(request.getEndTime());
        GetLogEventsResult result = new GetLogEventsResult();
        Integer nextToken;
        if (request.getNextToken() == null) {
            nextToken = 1;
        } else if (Integer.valueOf(request.getNextToken()) < 3) {
            nextToken = Integer.valueOf(request.getNextToken()) + 1;
        } else {
            nextToken = null;
        }
        List<OutputLogEvent> logEvents = new ArrayList<>();
        if (request.getNextToken() == null || Integer.valueOf(request.getNextToken()) < 3) {
            long continuation = request.getNextToken() == null ? 0 : Integer.valueOf(request.getNextToken());
            for (int i = 0; i < 100_000; i++) {
                OutputLogEvent outputLogEvent = new OutputLogEvent();
                outputLogEvent.setMessage("message-" + (continuation * i));
                outputLogEvent.setTimestamp(i * 100L);
                logEvents.add(outputLogEvent);
            }
        }
        result.withEvents(logEvents);
        if (nextToken != null) {
            result.setNextForwardToken(String.valueOf(nextToken));
        }
        return result;
    });
}

Example 2 with GetLogEventsRequest

use of com.amazonaws.services.logs.model.GetLogEventsRequest in project aws-codebuild-jenkins-plugin by awslabs.

the class CloudWatchMonitorTest method testFormatLogsTwoCalls.

@Test
public void testFormatLogsTwoCalls() throws Exception {
    CloudWatchMonitor c = getMockCloudWatchMonitor();
    List<OutputLogEvent> logsFirst = new ArrayList();
    logsFirst.add(new OutputLogEvent().withMessage("[Container] entry 1"));
    logsFirst.add(new OutputLogEvent().withMessage("[Container] entry2").withTimestamp(1L));
    List<OutputLogEvent> logsSecond = new ArrayList();
    logsSecond.add(new OutputLogEvent().withMessage("[Container] entry 3").withTimestamp(3L));
    GetLogEventsResult resultFirst = new GetLogEventsResult().withEvents(logsFirst);
    GetLogEventsResult resultSecond = new GetLogEventsResult().withEvents(logsSecond).withNextForwardToken(null);
    GetLogEventsRequest requestFirst = new GetLogEventsRequest().withStartTime(0L).withStartFromHead(true).withLogGroupName(mockGroup).withLogStreamName(mockStream);
    GetLogEventsRequest requestSecond = new GetLogEventsRequest().withStartTime(2L).withStartFromHead(true).withLogGroupName(mockGroup).withLogStreamName(mockStream);
    when(mockClient.getLogEvents(requestFirst)).thenReturn(resultFirst);
    when(mockClient.getLogEvents(requestSecond)).thenReturn(resultSecond);
    c.pollForLogs(listener);
    assert (c.getLatestLogs().size() == 2);
    assert (c.getLatestLogs().get(0).equals("entry 1"));
    assert (c.getLatestLogs().get(1).equals("entry2"));
    assert (c.getLastPollTime() == 2L);
    c.pollForLogs(listener);
    assert (c.getLatestLogs().size() == 1);
    assert (c.getLatestLogs().get(0).equals("entry 3"));
    assert (c.getLastPollTime() == 4L);
}

Example 3 with GetLogEventsRequest

use of com.amazonaws.services.logs.model.GetLogEventsRequest in project aws-athena-query-federation by awslabs.

the class CloudwatchRecordHandler method readWithConstraint.

/**
 * Scans Cloudwatch Logs using the LogStream and optional Time stamp filters.
 *
 * @see RecordHandler
 */
@Override
protected void readWithConstraint(BlockSpiller spiller, ReadRecordsRequest recordsRequest, QueryStatusChecker queryStatusChecker) throws TimeoutException {
    String continuationToken = null;
    TableName tableName = recordsRequest.getTableName();
    Split split = recordsRequest.getSplit();
    invoker.setBlockSpiller(spiller);
    do {
        final String actualContinuationToken = continuationToken;
        GetLogEventsResult logEventsResult = invoker.invoke(() -> awsLogs.getLogEvents(pushDownConstraints(recordsRequest.getConstraints(), new GetLogEventsRequest().withLogGroupName(split.getProperty(LOG_GROUP_FIELD)).withLogStreamName(split.getProperty(LOG_STREAM_FIELD)).withNextToken(actualContinuationToken).withStartFromHead(true))));
        if (continuationToken == null || !continuationToken.equals(logEventsResult.getNextForwardToken())) {
            continuationToken = logEventsResult.getNextForwardToken();
        } else {
            continuationToken = null;
        }
        for (OutputLogEvent ole : logEventsResult.getEvents()) {
            spiller.writeRows((Block block, int rowNum) -> {
                boolean matched = true;
                matched &= block.offerValue(LOG_STREAM_FIELD, rowNum, split.getProperty(LOG_STREAM_FIELD));
                matched &= block.offerValue(LOG_TIME_FIELD, rowNum, ole.getTimestamp());
                matched &= block.offerValue(LOG_MSG_FIELD, rowNum, ole.getMessage());
                return matched ? 1 : 0;
            });
        }
        logger.info("readWithConstraint: LogGroup[{}] LogStream[{}] Continuation[{}] rows[{}]", tableName.getSchemaName(), tableName.getTableName(), continuationToken, logEventsResult.getEvents().size());
    } while (continuationToken != null && queryStatusChecker.isQueryRunning());
}