Search in sources :

Example 11 with ListTablesRequest

use of com.amazonaws.athena.connector.lambda.metadata.ListTablesRequest in project aws-athena-query-federation by awslabs.

the class HiveMuxMetadataHandlerTest method doListTables.

@Test
public void doListTables() {
    ListTablesRequest listTablesRequest = Mockito.mock(ListTablesRequest.class);
    Mockito.when(listTablesRequest.getCatalogName()).thenReturn("metaHive");
    this.jdbcMetadataHandler.doListTables(this.allocator, listTablesRequest);
    Mockito.verify(this.hiveMetadataHandler, Mockito.times(1)).doListTables(Mockito.eq(this.allocator), Mockito.eq(listTablesRequest));
}
Also used : ListTablesRequest(com.amazonaws.athena.connector.lambda.metadata.ListTablesRequest) Test(org.junit.Test)

Example 12 with ListTablesRequest

use of com.amazonaws.athena.connector.lambda.metadata.ListTablesRequest in project aws-athena-query-federation by awslabs.

the class DataLakeGen2MuxMetadataHandlerTest method doListTables.

@Test
public void doListTables() {
    ListTablesRequest listTablesRequest = Mockito.mock(ListTablesRequest.class);
    Mockito.when(listTablesRequest.getCatalogName()).thenReturn("fakedatabase");
    this.jdbcMetadataHandler.doListTables(this.allocator, listTablesRequest);
    Mockito.verify(this.dataLakeGen2MetadataHandler, Mockito.times(1)).doListTables(Mockito.eq(this.allocator), Mockito.eq(listTablesRequest));
}
Also used : ListTablesRequest(com.amazonaws.athena.connector.lambda.metadata.ListTablesRequest) Test(org.junit.Test)

Example 13 with ListTablesRequest

use of com.amazonaws.athena.connector.lambda.metadata.ListTablesRequest in project aws-athena-query-federation by awslabs.

the class DynamoDBMetadataHandlerTest method doListTablesGlueAndDynamo.

@Test
public void doListTablesGlueAndDynamo() throws Exception {
    List<String> tableNames = new ArrayList<>();
    tableNames.add("table1");
    tableNames.add("table2");
    tableNames.add("table3");
    GetTablesResult mockResult = new GetTablesResult();
    List<Table> tableList = new ArrayList<>();
    tableList.add(new Table().withName("table1").withParameters(ImmutableMap.of("classification", "dynamodb")).withStorageDescriptor(new StorageDescriptor().withLocation("some.location")));
    tableList.add(new Table().withName("table2").withParameters(ImmutableMap.of()).withStorageDescriptor(new StorageDescriptor().withLocation("some.location").withParameters(ImmutableMap.of("classification", "dynamodb"))));
    tableList.add(new Table().withName("table3").withParameters(ImmutableMap.of()).withStorageDescriptor(new StorageDescriptor().withLocation("arn:aws:dynamodb:us-east-1:012345678910:table/table3")));
    tableList.add(new Table().withName("notADynamoTable").withParameters(ImmutableMap.of()).withStorageDescriptor(new StorageDescriptor().withParameters(ImmutableMap.of()).withLocation("some_location")));
    mockResult.setTableList(tableList);
    when(glueClient.getTables(any())).thenReturn(mockResult);
    ListTablesRequest req = new ListTablesRequest(TEST_IDENTITY, TEST_QUERY_ID, TEST_CATALOG_NAME, DEFAULT_SCHEMA, null, UNLIMITED_PAGE_SIZE_VALUE);
    ListTablesResponse res = handler.doListTables(allocator, req);
    logger.info("doListTables - {}", res.getTables());
    List<TableName> expectedTables = tableNames.stream().map(table -> new TableName(DEFAULT_SCHEMA, table)).collect(Collectors.toList());
    expectedTables.add(TEST_TABLE_NAME);
    expectedTables.add(new TableName(DEFAULT_SCHEMA, "test_table2"));
    expectedTables.add(new TableName(DEFAULT_SCHEMA, "test_table3"));
    expectedTables.add(new TableName(DEFAULT_SCHEMA, "test_table4"));
    assertThat(new HashSet<>(res.getTables()), equalTo(new HashSet<>(expectedTables)));
}
Also used : Types(org.apache.arrow.vector.types.Types) LoggerFactory(org.slf4j.LoggerFactory) SEGMENT_COUNT_METADATA(com.amazonaws.athena.connectors.dynamodb.constants.DynamoDBConstants.SEGMENT_COUNT_METADATA) BlockAllocator(com.amazonaws.athena.connector.lambda.data.BlockAllocator) SOURCE_TABLE_PROPERTY(com.amazonaws.athena.connector.lambda.handlers.GlueMetadataHandler.SOURCE_TABLE_PROPERTY) GetDatabasesResult(com.amazonaws.services.glue.model.GetDatabasesResult) AttributeValue(com.amazonaws.services.dynamodbv2.model.AttributeValue) MAX_SPLITS_PER_REQUEST(com.amazonaws.athena.connectors.dynamodb.DynamoDBMetadataHandler.MAX_SPLITS_PER_REQUEST) After(org.junit.After) Map(java.util.Map) DATETIME_FORMAT_MAPPING_PROPERTY_NORMALIZED(com.amazonaws.athena.connector.lambda.handlers.GlueMetadataHandler.DATETIME_FORMAT_MAPPING_PROPERTY_NORMALIZED) DYNAMO_DB_FLAG(com.amazonaws.athena.connectors.dynamodb.DynamoDBMetadataHandler.DYNAMO_DB_FLAG) ValueSet(com.amazonaws.athena.connector.lambda.domain.predicate.ValueSet) ZoneOffset(java.time.ZoneOffset) BlockAllocatorImpl(com.amazonaws.athena.connector.lambda.data.BlockAllocatorImpl) GetTableRequest(com.amazonaws.athena.connector.lambda.metadata.GetTableRequest) GetTablesResult(com.amazonaws.services.glue.model.GetTablesResult) COLUMN_NAME_MAPPING_PROPERTY(com.amazonaws.athena.connector.lambda.handlers.GlueMetadataHandler.COLUMN_NAME_MAPPING_PROPERTY) RANGE_KEY_FILTER_METADATA(com.amazonaws.athena.connectors.dynamodb.constants.DynamoDBConstants.RANGE_KEY_FILTER_METADATA) ListSchemasResponse(com.amazonaws.athena.connector.lambda.metadata.ListSchemasResponse) AmazonServiceException(com.amazonaws.AmazonServiceException) ImmutableMap(com.google.common.collect.ImmutableMap) TimeZone(java.util.TimeZone) Split(com.amazonaws.athena.connector.lambda.domain.Split) Column(com.amazonaws.services.glue.model.Column) PARTITION_TYPE_METADATA(com.amazonaws.athena.connectors.dynamodb.constants.DynamoDBConstants.PARTITION_TYPE_METADATA) EXPRESSION_NAMES_METADATA(com.amazonaws.athena.connectors.dynamodb.constants.DynamoDBConstants.EXPRESSION_NAMES_METADATA) Instant(java.time.Instant) QUERY_PARTITION_TYPE(com.amazonaws.athena.connectors.dynamodb.constants.DynamoDBConstants.QUERY_PARTITION_TYPE) Collectors(java.util.stream.Collectors) TableName(com.amazonaws.athena.connector.lambda.domain.TableName) Matchers.any(org.mockito.Matchers.any) GetTableLayoutResponse(com.amazonaws.athena.connector.lambda.metadata.GetTableLayoutResponse) EXPRESSION_VALUES_METADATA(com.amazonaws.athena.connectors.dynamodb.constants.DynamoDBConstants.EXPRESSION_VALUES_METADATA) RANGE_KEY_NAME_METADATA(com.amazonaws.athena.connectors.dynamodb.constants.DynamoDBConstants.RANGE_KEY_NAME_METADATA) List(java.util.List) GetTableLayoutRequest(com.amazonaws.athena.connector.lambda.metadata.GetTableLayoutRequest) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Matchers.is(org.hamcrest.Matchers.is) ListSchemasRequest(com.amazonaws.athena.connector.lambda.metadata.ListSchemasRequest) GetSplitsResponse(com.amazonaws.athena.connector.lambda.metadata.GetSplitsResponse) Iterables(com.google.common.collect.Iterables) DATETIME_FORMAT_MAPPING_PROPERTY(com.amazonaws.athena.connector.lambda.handlers.GlueMetadataHandler.DATETIME_FORMAT_MAPPING_PROPERTY) SCAN_PARTITION_TYPE(com.amazonaws.athena.connectors.dynamodb.constants.DynamoDBConstants.SCAN_PARTITION_TYPE) Mock(org.mockito.Mock) ArrowType(org.apache.arrow.vector.types.pojo.ArrowType) EquatableValueSet(com.amazonaws.athena.connector.lambda.domain.predicate.EquatableValueSet) RunWith(org.junit.runner.RunWith) LocalDateTime(java.time.LocalDateTime) HashMap(java.util.HashMap) Jackson(com.amazonaws.util.json.Jackson) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Range(com.amazonaws.athena.connector.lambda.domain.predicate.Range) MetadataResponse(com.amazonaws.athena.connector.lambda.metadata.MetadataResponse) ImmutableList(com.google.common.collect.ImmutableList) DEFAULT_SCHEMA(com.amazonaws.athena.connectors.dynamodb.constants.DynamoDBConstants.DEFAULT_SCHEMA) AWSGlue(com.amazonaws.services.glue.AWSGlue) SchemaBuilder(com.amazonaws.athena.connector.lambda.data.SchemaBuilder) TestName(org.junit.rules.TestName) LocalKeyFactory(com.amazonaws.athena.connector.lambda.security.LocalKeyFactory) ItemUtils(com.amazonaws.services.dynamodbv2.document.ItemUtils) Table(com.amazonaws.services.glue.model.Table) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) INDEX_METADATA(com.amazonaws.athena.connectors.dynamodb.constants.DynamoDBConstants.INDEX_METADATA) ListTablesResponse(com.amazonaws.athena.connector.lambda.metadata.ListTablesResponse) NON_KEY_FILTER_METADATA(com.amazonaws.athena.connectors.dynamodb.constants.DynamoDBConstants.NON_KEY_FILTER_METADATA) GetSplitsRequest(com.amazonaws.athena.connector.lambda.metadata.GetSplitsRequest) Before(org.junit.Before) HASH_KEY_NAME_METADATA(com.amazonaws.athena.connectors.dynamodb.constants.DynamoDBConstants.HASH_KEY_NAME_METADATA) Logger(org.slf4j.Logger) AmazonAthena(com.amazonaws.services.athena.AmazonAthena) SortedRangeSet(com.amazonaws.athena.connector.lambda.domain.predicate.SortedRangeSet) AWSSecretsManager(com.amazonaws.services.secretsmanager.AWSSecretsManager) Test(org.junit.Test) UNLIMITED_PAGE_SIZE_VALUE(com.amazonaws.athena.connector.lambda.metadata.ListTablesRequest.UNLIMITED_PAGE_SIZE_VALUE) SEGMENT_ID_PROPERTY(com.amazonaws.athena.connectors.dynamodb.constants.DynamoDBConstants.SEGMENT_ID_PROPERTY) Mockito.when(org.mockito.Mockito.when) MetadataRequestType(com.amazonaws.athena.connector.lambda.metadata.MetadataRequestType) GetTableResult(com.amazonaws.services.glue.model.GetTableResult) TABLE_METADATA(com.amazonaws.athena.connectors.dynamodb.constants.DynamoDBConstants.TABLE_METADATA) Constraints(com.amazonaws.athena.connector.lambda.domain.predicate.Constraints) StorageDescriptor(com.amazonaws.services.glue.model.StorageDescriptor) Rule(org.junit.Rule) MockitoJUnitRunner(org.mockito.runners.MockitoJUnitRunner) ChronoUnit(java.time.temporal.ChronoUnit) ListTablesRequest(com.amazonaws.athena.connector.lambda.metadata.ListTablesRequest) Database(com.amazonaws.services.glue.model.Database) Collections(java.util.Collections) GetTableResponse(com.amazonaws.athena.connector.lambda.metadata.GetTableResponse) Table(com.amazonaws.services.glue.model.Table) GetTablesResult(com.amazonaws.services.glue.model.GetTablesResult) ArrayList(java.util.ArrayList) StorageDescriptor(com.amazonaws.services.glue.model.StorageDescriptor) TableName(com.amazonaws.athena.connector.lambda.domain.TableName) ListTablesRequest(com.amazonaws.athena.connector.lambda.metadata.ListTablesRequest) ListTablesResponse(com.amazonaws.athena.connector.lambda.metadata.ListTablesResponse) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 14 with ListTablesRequest

use of com.amazonaws.athena.connector.lambda.metadata.ListTablesRequest in project aws-athena-query-federation by awslabs.

the class ExampleMetadataHandlerTest method doListTables.

@Test
public void doListTables() {
    logger.info("doListTables - enter");
    // Test request with unlimited page size
    logger.info("doListTables - Test unlimited page size");
    ListTablesRequest req = new ListTablesRequest(IdentityUtil.fakeIdentity(), "queryId", "default", "schema", null, UNLIMITED_PAGE_SIZE_VALUE);
    ListTablesResponse expectedResponse = new ListTablesResponse("default", new ImmutableList.Builder<TableName>().add(new TableName("schema", "table1")).add(new TableName("schema", "table2")).add(new TableName("schema", "table3")).add(new TableName("schema", "table4")).add(new TableName("schema", "table5")).build(), null);
    ObjectMapperUtil.assertSerialization(req);
    ListTablesResponse res = metadataHandler.doListTables(allocator, req);
    ObjectMapperUtil.assertSerialization(res);
    logger.info("doListTables - {}", res);
    assertEquals("Expecting a different response", expectedResponse, res);
    // Test first paginated request with pageSize: 3, nextToken: null
    logger.info("doListTables - Test first pagination request");
    req = new ListTablesRequest(IdentityUtil.fakeIdentity(), "queryId", "default", "schema", null, 3);
    expectedResponse = new ListTablesResponse("default", new ImmutableList.Builder<TableName>().add(new TableName("schema", "table1")).add(new TableName("schema", "table2")).add(new TableName("schema", "table3")).build(), "table4");
    ObjectMapperUtil.assertSerialization(req);
    res = metadataHandler.doListTables(allocator, req);
    ObjectMapperUtil.assertSerialization(res);
    logger.info("doListTables - {}", res);
    assertEquals("Expecting a different response", expectedResponse, res);
    // Test second paginated request with pageSize: 3, nextToken: res.getNextToken()
    logger.info("doListTables - Test second pagination request");
    req = new ListTablesRequest(IdentityUtil.fakeIdentity(), "queryId", "default", "schema", res.getNextToken(), 3);
    expectedResponse = new ListTablesResponse("default", new ImmutableList.Builder<TableName>().add(new TableName("schema", "table4")).add(new TableName("schema", "table5")).build(), null);
    ObjectMapperUtil.assertSerialization(req);
    res = metadataHandler.doListTables(allocator, req);
    ObjectMapperUtil.assertSerialization(res);
    logger.info("doListTables - {}", res);
    assertEquals("Expecting a different response", expectedResponse, res);
    logger.info("doListTables - exit");
}
Also used : TableName(com.amazonaws.athena.connector.lambda.domain.TableName) ImmutableList(com.google.common.collect.ImmutableList) SchemaBuilder(com.amazonaws.athena.connector.lambda.data.SchemaBuilder) ListTablesRequest(com.amazonaws.athena.connector.lambda.metadata.ListTablesRequest) ListTablesResponse(com.amazonaws.athena.connector.lambda.metadata.ListTablesResponse) Test(org.junit.Test)

Example 15 with ListTablesRequest

use of com.amazonaws.athena.connector.lambda.metadata.ListTablesRequest in project aws-athena-query-federation by awslabs.

the class HbaseMetadataHandlerTest method doListTables.

@Test
public void doListTables() throws IOException {
    logger.info("doListTables - enter");
    String schema = "schema1";
    org.apache.hadoop.hbase.TableName[] tables = { org.apache.hadoop.hbase.TableName.valueOf("schema1", "table1"), org.apache.hadoop.hbase.TableName.valueOf("schema1", "table2"), org.apache.hadoop.hbase.TableName.valueOf("schema1", "table3") };
    Set<String> tableNames = new HashSet<>();
    tableNames.add("table1");
    tableNames.add("table2");
    tableNames.add("table3");
    when(mockClient.listTableNamesByNamespace(eq(schema))).thenReturn(tables);
    ListTablesRequest req = new ListTablesRequest(IDENTITY, QUERY_ID, DEFAULT_CATALOG, schema, null, UNLIMITED_PAGE_SIZE_VALUE);
    ListTablesResponse res = handler.doListTables(allocator, req);
    logger.info("doListTables - {}", res.getTables());
    for (TableName next : res.getTables()) {
        assertEquals(schema, next.getSchemaName());
        assertTrue(tableNames.contains(next.getTableName()));
    }
    assertEquals(tableNames.size(), res.getTables().size());
}
Also used : TableName(com.amazonaws.athena.connector.lambda.domain.TableName) ListTablesRequest(com.amazonaws.athena.connector.lambda.metadata.ListTablesRequest) Matchers.anyString(org.mockito.Matchers.anyString) ListTablesResponse(com.amazonaws.athena.connector.lambda.metadata.ListTablesResponse) HashSet(java.util.HashSet) Test(org.junit.Test)

Aggregations

ListTablesRequest (com.amazonaws.athena.connector.lambda.metadata.ListTablesRequest)33 Test (org.junit.Test)31 ListTablesResponse (com.amazonaws.athena.connector.lambda.metadata.ListTablesResponse)17 TableName (com.amazonaws.athena.connector.lambda.domain.TableName)13 ArrayList (java.util.ArrayList)5 ImmutableList (com.google.common.collect.ImmutableList)4 SchemaBuilder (com.amazonaws.athena.connector.lambda.data.SchemaBuilder)3 BlockAllocatorImpl (com.amazonaws.athena.connector.lambda.data.BlockAllocatorImpl)2 GetTablesResult (com.amazonaws.services.glue.model.GetTablesResult)2 Table (com.amazonaws.services.glue.model.Table)2 Matchers.anyString (org.mockito.Matchers.anyString)2 InvocationOnMock (org.mockito.invocation.InvocationOnMock)2 AmazonServiceException (com.amazonaws.AmazonServiceException)1 BlockAllocator (com.amazonaws.athena.connector.lambda.data.BlockAllocator)1 Split (com.amazonaws.athena.connector.lambda.domain.Split)1 Constraints (com.amazonaws.athena.connector.lambda.domain.predicate.Constraints)1 EquatableValueSet (com.amazonaws.athena.connector.lambda.domain.predicate.EquatableValueSet)1 Range (com.amazonaws.athena.connector.lambda.domain.predicate.Range)1 SortedRangeSet (com.amazonaws.athena.connector.lambda.domain.predicate.SortedRangeSet)1 ValueSet (com.amazonaws.athena.connector.lambda.domain.predicate.ValueSet)1