Search in sources :

Example 1 with LogGroup

use of com.amazonaws.services.logs.model.LogGroup in project aws-athena-query-federation by awslabs.

the class CloudwatchTableResolver method loadLogGroups.

/**
 * Loads the requested LogGroup as identified by the schemaName.
 *
 * @param schemaName The schemaName to load.
 * @return The actual LogGroup name in cloudwatch.
 * @note This method also primes the cache with other LogGroups found along the way while scanning Cloudwatch.
 */
private String loadLogGroups(String schemaName) throws TimeoutException {
    // As an optimization, see if the table name is an exact match (meaning likely no casing issues)
    String result = loadLogGroup(schemaName);
    if (result != null) {
        return result;
    }
    logger.info("loadLogGroups: Did not find a match for the schema, falling back to LogGroup scan for  {}", schemaName);
    DescribeLogGroupsRequest validateSchemaRequest = new DescribeLogGroupsRequest();
    DescribeLogGroupsResult validateSchemaResult;
    do {
        validateSchemaResult = invoker.invoke(() -> awsLogs.describeLogGroups(validateSchemaRequest));
        for (LogGroup next : validateSchemaResult.getLogGroups()) {
            String nextLogGroupName = next.getLogGroupName();
            schemaCache.put(schemaName.toLowerCase(), nextLogGroupName);
            if (nextLogGroupName.equalsIgnoreCase(schemaName)) {
                logger.info("loadLogGroups: Matched {} for {}", nextLogGroupName, schemaName);
                return nextLogGroupName;
            }
        }
        validateSchemaRequest.setNextToken(validateSchemaResult.getNextToken());
    } while (validateSchemaResult.getNextToken() != null);
    // We could not find a match
    throw new IllegalArgumentException("No such schema " + schemaName);
}
Also used : DescribeLogGroupsResult(com.amazonaws.services.logs.model.DescribeLogGroupsResult) LogGroup(com.amazonaws.services.logs.model.LogGroup) DescribeLogGroupsRequest(com.amazonaws.services.logs.model.DescribeLogGroupsRequest)

Example 2 with LogGroup

use of com.amazonaws.services.logs.model.LogGroup in project aws-athena-query-federation by awslabs.

the class CloudwatchMetadataHandlerTest method setUp.

@Before
public void setUp() throws Exception {
    when(mockAwsLogs.describeLogStreams(any(DescribeLogStreamsRequest.class))).thenAnswer((InvocationOnMock invocationOnMock) -> {
        return new DescribeLogStreamsResult().withLogStreams(new LogStream().withLogStreamName("table-9"), new LogStream().withLogStreamName("table-10"));
    });
    when(mockAwsLogs.describeLogGroups(any(DescribeLogGroupsRequest.class))).thenAnswer((InvocationOnMock invocationOnMock) -> {
        return new DescribeLogGroupsResult().withLogGroups(new LogGroup().withLogGroupName("schema-1"), new LogGroup().withLogGroupName("schema-20"));
    });
    handler = new CloudwatchMetadataHandler(mockAwsLogs, new LocalKeyFactory(), mockSecretsManager, mockAthena, "spillBucket", "spillPrefix");
    allocator = new BlockAllocatorImpl();
}
Also used : DescribeLogGroupsResult(com.amazonaws.services.logs.model.DescribeLogGroupsResult) LogGroup(com.amazonaws.services.logs.model.LogGroup) BlockAllocatorImpl(com.amazonaws.athena.connector.lambda.data.BlockAllocatorImpl) InvocationOnMock(org.mockito.invocation.InvocationOnMock) DescribeLogStreamsRequest(com.amazonaws.services.logs.model.DescribeLogStreamsRequest) LogStream(com.amazonaws.services.logs.model.LogStream) DescribeLogStreamsResult(com.amazonaws.services.logs.model.DescribeLogStreamsResult) DescribeLogGroupsRequest(com.amazonaws.services.logs.model.DescribeLogGroupsRequest) LocalKeyFactory(com.amazonaws.athena.connector.lambda.security.LocalKeyFactory) Before(org.junit.Before)

Example 3 with LogGroup

use of com.amazonaws.services.logs.model.LogGroup in project aws-athena-query-federation by awslabs.

the class CloudwatchMetadataHandlerTest method doListSchemaNames.

@Test
public void doListSchemaNames() throws TimeoutException {
    logger.info("doListSchemas - enter");
    when(mockAwsLogs.describeLogGroups(any(DescribeLogGroupsRequest.class))).thenAnswer((InvocationOnMock invocationOnMock) -> {
        DescribeLogGroupsRequest request = (DescribeLogGroupsRequest) invocationOnMock.getArguments()[0];
        DescribeLogGroupsResult result = new DescribeLogGroupsResult();
        Integer nextToken;
        if (request.getNextToken() == null) {
            nextToken = 1;
        } else if (Integer.valueOf(request.getNextToken()) < 3) {
            nextToken = Integer.valueOf(request.getNextToken()) + 1;
        } else {
            nextToken = null;
        }
        List<LogGroup> logGroups = new ArrayList<>();
        if (request.getNextToken() == null || Integer.valueOf(request.getNextToken()) < 3) {
            for (int i = 0; i < 10; i++) {
                LogGroup nextLogGroup = new LogGroup();
                nextLogGroup.setLogGroupName("schema-" + String.valueOf(i));
                logGroups.add(nextLogGroup);
            }
        }
        result.withLogGroups(logGroups);
        if (nextToken != null) {
            result.setNextToken(String.valueOf(nextToken));
        }
        return result;
    });
    ListSchemasRequest req = new ListSchemasRequest(identity, "queryId", "default");
    ListSchemasResponse res = handler.doListSchemaNames(allocator, req);
    logger.info("doListSchemas - {}", res.getSchemas());
    assertTrue(res.getSchemas().size() == 30);
    verify(mockAwsLogs, times(4)).describeLogGroups(any(DescribeLogGroupsRequest.class));
    verifyNoMoreInteractions(mockAwsLogs);
    logger.info("doListSchemas - exit");
}
Also used : DescribeLogGroupsResult(com.amazonaws.services.logs.model.DescribeLogGroupsResult) ListSchemasRequest(com.amazonaws.athena.connector.lambda.metadata.ListSchemasRequest) LogGroup(com.amazonaws.services.logs.model.LogGroup) InvocationOnMock(org.mockito.invocation.InvocationOnMock) ArrayList(java.util.ArrayList) ListSchemasResponse(com.amazonaws.athena.connector.lambda.metadata.ListSchemasResponse) DescribeLogGroupsRequest(com.amazonaws.services.logs.model.DescribeLogGroupsRequest) Test(org.junit.Test)

Example 4 with LogGroup

use of com.amazonaws.services.logs.model.LogGroup in project cloudwatch-logback-appender by j256.

the class CloudWatchAppenderTest method testMoreAwsCalls.

@Test(timeout = 10000)
public void testMoreAwsCalls() throws InterruptedException {
    CloudWatchAppender appender = new CloudWatchAppender();
    AWSLogs logsClient = createMock(AWSLogs.class);
    AmazonEC2 ec2Client = createMock(AmazonEC2.class);
    appender.setTestAwsLogsClient(logsClient);
    appender.setTestAmazonEc2Client(ec2Client);
    appender.setMaxBatchSize(1);
    appender.setRegion("region");
    final String logGroup = "pfqoejpfqe";
    appender.setLogGroup(logGroup);
    final String logStream = "pffqjfqjpoqoejpfqe";
    appender.setLogStream(logStream);
    appender.setContext(LOGGER_CONTEXT);
    PatternLayout layout = new PatternLayout();
    layout.setContext(LOGGER_CONTEXT);
    layout.setPattern("[%thread] %level %logger{20} - %msg%n%xThrowable");
    layout.start();
    appender.setLayout(layout);
    final String loggerName = "name";
    final Level level = Level.DEBUG;
    String message = "kuykregddwqwef4wve";
    LoggingEvent event = createEvent(loggerName, level, message, System.currentTimeMillis());
    String threadName = Thread.currentThread().getName();
    final String fullMessage = "[" + threadName + "] " + level + " " + loggerName + " - " + message + "\n";
    DescribeLogGroupsResult logGroupsResult = new DescribeLogGroupsResult().withLogGroups(Arrays.asList(new LogGroup().withLogGroupName(logGroup)));
    expect(logsClient.describeLogGroups(isA(DescribeLogGroupsRequest.class))).andReturn(logGroupsResult);
    DescribeLogStreamsResult logStreamsResult = new DescribeLogStreamsResult().withLogStreams(Arrays.asList(new LogStream().withLogStreamName(logStream)));
    expect(logsClient.describeLogStreams(isA(DescribeLogStreamsRequest.class))).andReturn(logStreamsResult);
    final PutLogEventsResult putLogEventsResult = new PutLogEventsResult();
    String sequence = "ewopjfewfj";
    putLogEventsResult.setNextSequenceToken(sequence);
    expect(logsClient.putLogEvents(isA(PutLogEventsRequest.class))).andAnswer(new IAnswer<PutLogEventsResult>() {

        @Override
        public PutLogEventsResult answer() {
            PutLogEventsRequest request = (PutLogEventsRequest) getCurrentArguments()[0];
            assertEquals(logGroup, request.getLogGroupName());
            assertEquals(logStream, request.getLogStreamName());
            List<InputLogEvent> events = request.getLogEvents();
            assertEquals(1, events.size());
            assertEquals(fullMessage, events.get(0).getMessage());
            return putLogEventsResult;
        }
    }).times(2);
    logsClient.shutdown();
    // =====================================
    replay(logsClient, ec2Client);
    appender.start();
    // for coverage
    appender.start();
    appender.append(event);
    Thread.sleep(100);
    appender.append(event);
    while (appender.getEventsWrittenCount() < 2) {
        Thread.sleep(100);
    }
    appender.stop();
    verify(logsClient, ec2Client);
}
Also used : DescribeLogGroupsResult(com.amazonaws.services.logs.model.DescribeLogGroupsResult) InputLogEvent(com.amazonaws.services.logs.model.InputLogEvent) PatternLayout(ch.qos.logback.classic.PatternLayout) PutLogEventsResult(com.amazonaws.services.logs.model.PutLogEventsResult) AmazonEC2(com.amazonaws.services.ec2.AmazonEC2) LogStream(com.amazonaws.services.logs.model.LogStream) AWSLogs(com.amazonaws.services.logs.AWSLogs) ILoggingEvent(ch.qos.logback.classic.spi.ILoggingEvent) LoggingEvent(ch.qos.logback.classic.spi.LoggingEvent) IAnswer(org.easymock.IAnswer) LogGroup(com.amazonaws.services.logs.model.LogGroup) Level(ch.qos.logback.classic.Level) DescribeLogStreamsRequest(com.amazonaws.services.logs.model.DescribeLogStreamsRequest) DescribeLogGroupsRequest(com.amazonaws.services.logs.model.DescribeLogGroupsRequest) DescribeLogStreamsResult(com.amazonaws.services.logs.model.DescribeLogStreamsResult) PutLogEventsRequest(com.amazonaws.services.logs.model.PutLogEventsRequest) Test(org.junit.Test)

Example 5 with LogGroup

use of com.amazonaws.services.logs.model.LogGroup in project aws-athena-query-federation by awslabs.

the class CloudwatchTableResolver method loadLogGroup.

/**
 * Optomizaiton that attempts to load a specific  LogStream as identified by the TableName.
 *
 * @param schemaName The schemaName to load.
 * @return The CloudwatchTableName or null if not found.
 */
private String loadLogGroup(String schemaName) throws TimeoutException {
    DescribeLogGroupsRequest request = new DescribeLogGroupsRequest().withLogGroupNamePrefix(schemaName);
    DescribeLogGroupsResult result = invoker.invoke(() -> awsLogs.describeLogGroups(request));
    for (LogGroup next : result.getLogGroups()) {
        String nextLogGroupName = next.getLogGroupName();
        if (nextLogGroupName.equalsIgnoreCase(schemaName)) {
            logger.info("loadLogGroup: Matched {} for {}", nextLogGroupName, schemaName);
            return nextLogGroupName;
        }
    }
    return null;
}
Also used : DescribeLogGroupsResult(com.amazonaws.services.logs.model.DescribeLogGroupsResult) LogGroup(com.amazonaws.services.logs.model.LogGroup) DescribeLogGroupsRequest(com.amazonaws.services.logs.model.DescribeLogGroupsRequest)

Aggregations

DescribeLogGroupsRequest (com.amazonaws.services.logs.model.DescribeLogGroupsRequest)5 DescribeLogGroupsResult (com.amazonaws.services.logs.model.DescribeLogGroupsResult)5 LogGroup (com.amazonaws.services.logs.model.LogGroup)5 DescribeLogStreamsRequest (com.amazonaws.services.logs.model.DescribeLogStreamsRequest)2 DescribeLogStreamsResult (com.amazonaws.services.logs.model.DescribeLogStreamsResult)2 LogStream (com.amazonaws.services.logs.model.LogStream)2 Test (org.junit.Test)2 InvocationOnMock (org.mockito.invocation.InvocationOnMock)2 Level (ch.qos.logback.classic.Level)1 PatternLayout (ch.qos.logback.classic.PatternLayout)1 ILoggingEvent (ch.qos.logback.classic.spi.ILoggingEvent)1 LoggingEvent (ch.qos.logback.classic.spi.LoggingEvent)1 BlockAllocatorImpl (com.amazonaws.athena.connector.lambda.data.BlockAllocatorImpl)1 ListSchemasRequest (com.amazonaws.athena.connector.lambda.metadata.ListSchemasRequest)1 ListSchemasResponse (com.amazonaws.athena.connector.lambda.metadata.ListSchemasResponse)1 LocalKeyFactory (com.amazonaws.athena.connector.lambda.security.LocalKeyFactory)1 AmazonEC2 (com.amazonaws.services.ec2.AmazonEC2)1 AWSLogs (com.amazonaws.services.logs.AWSLogs)1 InputLogEvent (com.amazonaws.services.logs.model.InputLogEvent)1 PutLogEventsRequest (com.amazonaws.services.logs.model.PutLogEventsRequest)1