use of com.amazonaws.services.logs.model.DescribeLogGroupsResult in project aws-athena-query-federation by awslabs.
the class CloudwatchTableResolver method loadLogGroups.
/**
* Loads the requested LogGroup as identified by the schemaName.
*
* @param schemaName The schemaName to load.
* @return The actual LogGroup name in cloudwatch.
* @note This method also primes the cache with other LogGroups found along the way while scanning Cloudwatch.
*/
private String loadLogGroups(String schemaName) throws TimeoutException {
// As an optimization, see if the table name is an exact match (meaning likely no casing issues)
String result = loadLogGroup(schemaName);
if (result != null) {
return result;
}
logger.info("loadLogGroups: Did not find a match for the schema, falling back to LogGroup scan for {}", schemaName);
DescribeLogGroupsRequest validateSchemaRequest = new DescribeLogGroupsRequest();
DescribeLogGroupsResult validateSchemaResult;
do {
validateSchemaResult = invoker.invoke(() -> awsLogs.describeLogGroups(validateSchemaRequest));
for (LogGroup next : validateSchemaResult.getLogGroups()) {
String nextLogGroupName = next.getLogGroupName();
schemaCache.put(schemaName.toLowerCase(), nextLogGroupName);
if (nextLogGroupName.equalsIgnoreCase(schemaName)) {
logger.info("loadLogGroups: Matched {} for {}", nextLogGroupName, schemaName);
return nextLogGroupName;
}
}
validateSchemaRequest.setNextToken(validateSchemaResult.getNextToken());
} while (validateSchemaResult.getNextToken() != null);
// We could not find a match
throw new IllegalArgumentException("No such schema " + schemaName);
}
use of com.amazonaws.services.logs.model.DescribeLogGroupsResult in project aws-athena-query-federation by awslabs.
the class CloudwatchMetadataHandlerTest method setUp.
@Before
public void setUp() throws Exception {
when(mockAwsLogs.describeLogStreams(any(DescribeLogStreamsRequest.class))).thenAnswer((InvocationOnMock invocationOnMock) -> {
return new DescribeLogStreamsResult().withLogStreams(new LogStream().withLogStreamName("table-9"), new LogStream().withLogStreamName("table-10"));
});
when(mockAwsLogs.describeLogGroups(any(DescribeLogGroupsRequest.class))).thenAnswer((InvocationOnMock invocationOnMock) -> {
return new DescribeLogGroupsResult().withLogGroups(new LogGroup().withLogGroupName("schema-1"), new LogGroup().withLogGroupName("schema-20"));
});
handler = new CloudwatchMetadataHandler(mockAwsLogs, new LocalKeyFactory(), mockSecretsManager, mockAthena, "spillBucket", "spillPrefix");
allocator = new BlockAllocatorImpl();
}
use of com.amazonaws.services.logs.model.DescribeLogGroupsResult in project aws-athena-query-federation by awslabs.
the class CloudwatchMetadataHandlerTest method doListSchemaNames.
@Test
public void doListSchemaNames() throws TimeoutException {
logger.info("doListSchemas - enter");
when(mockAwsLogs.describeLogGroups(any(DescribeLogGroupsRequest.class))).thenAnswer((InvocationOnMock invocationOnMock) -> {
DescribeLogGroupsRequest request = (DescribeLogGroupsRequest) invocationOnMock.getArguments()[0];
DescribeLogGroupsResult result = new DescribeLogGroupsResult();
Integer nextToken;
if (request.getNextToken() == null) {
nextToken = 1;
} else if (Integer.valueOf(request.getNextToken()) < 3) {
nextToken = Integer.valueOf(request.getNextToken()) + 1;
} else {
nextToken = null;
}
List<LogGroup> logGroups = new ArrayList<>();
if (request.getNextToken() == null || Integer.valueOf(request.getNextToken()) < 3) {
for (int i = 0; i < 10; i++) {
LogGroup nextLogGroup = new LogGroup();
nextLogGroup.setLogGroupName("schema-" + String.valueOf(i));
logGroups.add(nextLogGroup);
}
}
result.withLogGroups(logGroups);
if (nextToken != null) {
result.setNextToken(String.valueOf(nextToken));
}
return result;
});
ListSchemasRequest req = new ListSchemasRequest(identity, "queryId", "default");
ListSchemasResponse res = handler.doListSchemaNames(allocator, req);
logger.info("doListSchemas - {}", res.getSchemas());
assertTrue(res.getSchemas().size() == 30);
verify(mockAwsLogs, times(4)).describeLogGroups(any(DescribeLogGroupsRequest.class));
verifyNoMoreInteractions(mockAwsLogs);
logger.info("doListSchemas - exit");
}
use of com.amazonaws.services.logs.model.DescribeLogGroupsResult in project cloudwatch-logback-appender by j256.
the class CloudWatchAppenderTest method testMoreAwsCalls.
@Test(timeout = 10000)
public void testMoreAwsCalls() throws InterruptedException {
CloudWatchAppender appender = new CloudWatchAppender();
AWSLogs logsClient = createMock(AWSLogs.class);
AmazonEC2 ec2Client = createMock(AmazonEC2.class);
appender.setTestAwsLogsClient(logsClient);
appender.setTestAmazonEc2Client(ec2Client);
appender.setMaxBatchSize(1);
appender.setRegion("region");
final String logGroup = "pfqoejpfqe";
appender.setLogGroup(logGroup);
final String logStream = "pffqjfqjpoqoejpfqe";
appender.setLogStream(logStream);
appender.setContext(LOGGER_CONTEXT);
PatternLayout layout = new PatternLayout();
layout.setContext(LOGGER_CONTEXT);
layout.setPattern("[%thread] %level %logger{20} - %msg%n%xThrowable");
layout.start();
appender.setLayout(layout);
final String loggerName = "name";
final Level level = Level.DEBUG;
String message = "kuykregddwqwef4wve";
LoggingEvent event = createEvent(loggerName, level, message, System.currentTimeMillis());
String threadName = Thread.currentThread().getName();
final String fullMessage = "[" + threadName + "] " + level + " " + loggerName + " - " + message + "\n";
DescribeLogGroupsResult logGroupsResult = new DescribeLogGroupsResult().withLogGroups(Arrays.asList(new LogGroup().withLogGroupName(logGroup)));
expect(logsClient.describeLogGroups(isA(DescribeLogGroupsRequest.class))).andReturn(logGroupsResult);
DescribeLogStreamsResult logStreamsResult = new DescribeLogStreamsResult().withLogStreams(Arrays.asList(new LogStream().withLogStreamName(logStream)));
expect(logsClient.describeLogStreams(isA(DescribeLogStreamsRequest.class))).andReturn(logStreamsResult);
final PutLogEventsResult putLogEventsResult = new PutLogEventsResult();
String sequence = "ewopjfewfj";
putLogEventsResult.setNextSequenceToken(sequence);
expect(logsClient.putLogEvents(isA(PutLogEventsRequest.class))).andAnswer(new IAnswer<PutLogEventsResult>() {
@Override
public PutLogEventsResult answer() {
PutLogEventsRequest request = (PutLogEventsRequest) getCurrentArguments()[0];
assertEquals(logGroup, request.getLogGroupName());
assertEquals(logStream, request.getLogStreamName());
List<InputLogEvent> events = request.getLogEvents();
assertEquals(1, events.size());
assertEquals(fullMessage, events.get(0).getMessage());
return putLogEventsResult;
}
}).times(2);
logsClient.shutdown();
// =====================================
replay(logsClient, ec2Client);
appender.start();
// for coverage
appender.start();
appender.append(event);
Thread.sleep(100);
appender.append(event);
while (appender.getEventsWrittenCount() < 2) {
Thread.sleep(100);
}
appender.stop();
verify(logsClient, ec2Client);
}
use of com.amazonaws.services.logs.model.DescribeLogGroupsResult in project aws-athena-query-federation by awslabs.
the class CloudwatchMetadataHandler method doListSchemaNames.
/**
* List LogGroups in your Cloudwatch account treating each as a 'schema' (aka database)
*
* @see MetadataHandler
*/
@Override
public ListSchemasResponse doListSchemaNames(BlockAllocator blockAllocator, ListSchemasRequest listSchemasRequest) throws TimeoutException {
DescribeLogGroupsRequest request = new DescribeLogGroupsRequest();
DescribeLogGroupsResult result;
List<String> schemas = new ArrayList<>();
do {
if (schemas.size() > MAX_RESULTS) {
throw new RuntimeException("Too many log groups, exceeded max metadata results for schema count.");
}
result = invoker.invoke(() -> awsLogs.describeLogGroups(request));
result.getLogGroups().forEach(next -> schemas.add(next.getLogGroupName().toLowerCase()));
request.setNextToken(result.getNextToken());
logger.info("doListSchemaNames: Listing log groups {} {}", result.getNextToken(), schemas.size());
} while (result.getNextToken() != null);
return new ListSchemasResponse(listSchemasRequest.getCatalogName(), schemas);
}
Aggregations