Search in sources :

Example 1 with TableName

use of com.amazonaws.athena.connector.lambda.domain.TableName in project aws-athena-query-federation by awslabs.

the class ExampleMetadataHandlerTest method doGetTableFail.

@Test(expected = LambdaFunctionException.class)
public void doGetTableFail() {
    try {
        logger.info("doGetTableFail - enter");
        GetTableRequest req = new GetTableRequest(IdentityUtil.fakeIdentity(), "queryId", "default", new TableName("lambda", "fake"));
        metadataHandler.doGetTable(allocator, req);
    } catch (Exception ex) {
        logger.info("doGetTableFail: ", ex);
        throw new LambdaFunctionException(ex.getMessage(), false, "repackaged");
    }
}
Also used : GetTableRequest(com.amazonaws.athena.connector.lambda.metadata.GetTableRequest) TableName(com.amazonaws.athena.connector.lambda.domain.TableName) LambdaFunctionException(com.amazonaws.services.lambda.invoke.LambdaFunctionException) LambdaFunctionException(com.amazonaws.services.lambda.invoke.LambdaFunctionException) Test(org.junit.Test)

Example 2 with TableName

use of com.amazonaws.athena.connector.lambda.domain.TableName in project aws-athena-query-federation by awslabs.

the class ExampleRecordHandlerTest method doReadRecordsNoSpill.

@Test
public void doReadRecordsNoSpill() {
    logger.info("doReadRecordsNoSpill: enter");
    for (int i = 0; i < 2; i++) {
        EncryptionKey encryptionKey = (i % 2 == 0) ? keyFactory.create() : null;
        logger.info("doReadRecordsNoSpill: Using encryptionKey[" + encryptionKey + "]");
        Map<String, ValueSet> constraintsMap = new HashMap<>();
        constraintsMap.put("col3", SortedRangeSet.copyOf(Types.MinorType.FLOAT8.getType(), ImmutableList.of(Range.equal(allocator, Types.MinorType.FLOAT8.getType(), 22.0D)), false));
        ReadRecordsRequest request = new ReadRecordsRequest(IdentityUtil.fakeIdentity(), "catalog", "queryId-" + System.currentTimeMillis(), new TableName("schema", "table"), schemaForRead, Split.newBuilder(makeSpillLocation(), encryptionKey).add("year", "10").add("month", "10").add("day", "10").build(), new Constraints(constraintsMap), // 100GB don't expect this to spill
        100_000_000_000L, 100_000_000_000L);
        ObjectMapperUtil.assertSerialization(request);
        RecordResponse rawResponse = recordService.readRecords(request);
        ObjectMapperUtil.assertSerialization(rawResponse);
        assertTrue(rawResponse instanceof ReadRecordsResponse);
        ReadRecordsResponse response = (ReadRecordsResponse) rawResponse;
        logger.info("doReadRecordsNoSpill: rows[{}]", response.getRecordCount());
        assertTrue(response.getRecords().getRowCount() == 1);
        logger.info("doReadRecordsNoSpill: {}", BlockUtils.rowToString(response.getRecords(), 0));
    }
    logger.info("doReadRecordsNoSpill: exit");
}
Also used : TableName(com.amazonaws.athena.connector.lambda.domain.TableName) ReadRecordsRequest(com.amazonaws.athena.connector.lambda.records.ReadRecordsRequest) Constraints(com.amazonaws.athena.connector.lambda.domain.predicate.Constraints) HashMap(java.util.HashMap) ReadRecordsResponse(com.amazonaws.athena.connector.lambda.records.ReadRecordsResponse) RemoteReadRecordsResponse(com.amazonaws.athena.connector.lambda.records.RemoteReadRecordsResponse) EncryptionKey(com.amazonaws.athena.connector.lambda.security.EncryptionKey) Matchers.anyString(org.mockito.Matchers.anyString) RecordResponse(com.amazonaws.athena.connector.lambda.records.RecordResponse) ValueSet(com.amazonaws.athena.connector.lambda.domain.predicate.ValueSet) EquatableValueSet(com.amazonaws.athena.connector.lambda.domain.predicate.EquatableValueSet) AllOrNoneValueSet(com.amazonaws.athena.connector.lambda.domain.predicate.AllOrNoneValueSet) Test(org.junit.Test)

Example 3 with TableName

use of com.amazonaws.athena.connector.lambda.domain.TableName in project aws-athena-query-federation by awslabs.

the class CompositeHandlerTest method doReadRecords.

@Test
public void doReadRecords() throws Exception {
    ReadRecordsRequest req = new ReadRecordsRequest(IdentityUtil.fakeIdentity(), "catalog", "queryId-" + System.currentTimeMillis(), new TableName("schema", "table"), schemaForRead, Split.newBuilder(S3SpillLocation.newBuilder().withBucket("athena-virtuoso-test").withPrefix("lambda-spill").withQueryId(UUID.randomUUID().toString()).withSplitId(UUID.randomUUID().toString()).withIsDirectory(true).build(), null).build(), new Constraints(new HashMap<>()), // 100GB don't expect this to spill
    100_000_000_000L, 100_000_000_000L);
    compositeHandler.handleRequest(allocator, req, new ByteArrayOutputStream(), objectMapper);
    verify(mockRecordHandler, times(1)).doReadRecords(any(BlockAllocator.class), any(ReadRecordsRequest.class));
}
Also used : TableName(com.amazonaws.athena.connector.lambda.domain.TableName) ReadRecordsRequest(com.amazonaws.athena.connector.lambda.records.ReadRecordsRequest) Constraints(com.amazonaws.athena.connector.lambda.domain.predicate.Constraints) HashMap(java.util.HashMap) BlockAllocator(com.amazonaws.athena.connector.lambda.data.BlockAllocator) ByteArrayOutputStream(java.io.ByteArrayOutputStream) Test(org.junit.Test)

Example 4 with TableName

use of com.amazonaws.athena.connector.lambda.domain.TableName in project aws-athena-query-federation by awslabs.

the class CompositeHandlerTest method setUp.

@Before
public void setUp() throws Exception {
    logger.info("{}: enter", testName.getMethodName());
    allocator = new BlockAllocatorImpl();
    objectMapper = ObjectMapperFactory.create(allocator);
    mockMetadataHandler = mock(MetadataHandler.class);
    mockRecordHandler = mock(RecordHandler.class);
    schemaForRead = SchemaBuilder.newBuilder().addField("col1", new ArrowType.Int(32, true)).build();
    when(mockMetadataHandler.doGetTableLayout(any(BlockAllocatorImpl.class), any(GetTableLayoutRequest.class))).thenReturn(new GetTableLayoutResponse("catalog", new TableName("schema", "table"), BlockUtils.newBlock(allocator, "col1", Types.MinorType.BIGINT.getType(), 1L)));
    when(mockMetadataHandler.doListTables(any(BlockAllocatorImpl.class), any(ListTablesRequest.class))).thenReturn(new ListTablesResponse("catalog", Collections.singletonList(new TableName("schema", "table")), null));
    when(mockMetadataHandler.doGetTable(any(BlockAllocatorImpl.class), any(GetTableRequest.class))).thenReturn(new GetTableResponse("catalog", new TableName("schema", "table"), SchemaBuilder.newBuilder().addStringField("col1").build()));
    when(mockMetadataHandler.doListSchemaNames(any(BlockAllocatorImpl.class), any(ListSchemasRequest.class))).thenReturn(new ListSchemasResponse("catalog", Collections.singleton("schema1")));
    when(mockMetadataHandler.doGetSplits(any(BlockAllocatorImpl.class), any(GetSplitsRequest.class))).thenReturn(new GetSplitsResponse("catalog", Split.newBuilder(null, null).build()));
    when(mockMetadataHandler.doPing(any(PingRequest.class))).thenReturn(new PingResponse("catalog", "queryId", "type", 23, 2));
    when(mockRecordHandler.doReadRecords(any(BlockAllocatorImpl.class), any(ReadRecordsRequest.class))).thenReturn(new ReadRecordsResponse("catalog", BlockUtils.newEmptyBlock(allocator, "col", new ArrowType.Int(32, true))));
    compositeHandler = new CompositeHandler(mockMetadataHandler, mockRecordHandler);
}
Also used : PingRequest(com.amazonaws.athena.connector.lambda.request.PingRequest) GetSplitsRequest(com.amazonaws.athena.connector.lambda.metadata.GetSplitsRequest) ReadRecordsResponse(com.amazonaws.athena.connector.lambda.records.ReadRecordsResponse) ArrowType(org.apache.arrow.vector.types.pojo.ArrowType) PingResponse(com.amazonaws.athena.connector.lambda.request.PingResponse) TableName(com.amazonaws.athena.connector.lambda.domain.TableName) GetTableRequest(com.amazonaws.athena.connector.lambda.metadata.GetTableRequest) ListSchemasRequest(com.amazonaws.athena.connector.lambda.metadata.ListSchemasRequest) ReadRecordsRequest(com.amazonaws.athena.connector.lambda.records.ReadRecordsRequest) GetTableLayoutResponse(com.amazonaws.athena.connector.lambda.metadata.GetTableLayoutResponse) BlockAllocatorImpl(com.amazonaws.athena.connector.lambda.data.BlockAllocatorImpl) GetTableResponse(com.amazonaws.athena.connector.lambda.metadata.GetTableResponse) GetSplitsResponse(com.amazonaws.athena.connector.lambda.metadata.GetSplitsResponse) GetTableLayoutRequest(com.amazonaws.athena.connector.lambda.metadata.GetTableLayoutRequest) ListTablesRequest(com.amazonaws.athena.connector.lambda.metadata.ListTablesRequest) ListTablesResponse(com.amazonaws.athena.connector.lambda.metadata.ListTablesResponse) ListSchemasResponse(com.amazonaws.athena.connector.lambda.metadata.ListSchemasResponse) Before(org.junit.Before)

Example 5 with TableName

use of com.amazonaws.athena.connector.lambda.domain.TableName in project aws-athena-query-federation by awslabs.

the class GlueMetadataHandlerTest method doListTablesWithPagination.

@Test
public void doListTablesWithPagination() throws Exception {
    logger.info("First paginated request");
    ListTablesRequest req = new ListTablesRequest(IdentityUtil.fakeIdentity(), queryId, catalog, schema, null, 3);
    logger.info("Request - {}", req);
    ListTablesResponse expectedResponse = new ListTablesResponse(req.getCatalogName(), new ImmutableList.Builder<TableName>().add(new TableName(req.getSchemaName(), "table1")).add(new TableName(req.getSchemaName(), "table2")).add(new TableName(req.getSchemaName(), "table3")).build(), "table4");
    ListTablesResponse actualResponse = handler.doListTables(allocator, req);
    logger.info("Response - {}", actualResponse);
    assertEquals("Lists do not match.", expectedResponse, actualResponse);
    logger.info("Second paginated request");
    req = new ListTablesRequest(IdentityUtil.fakeIdentity(), queryId, catalog, schema, actualResponse.getNextToken(), 3);
    logger.info("Request - {}", req);
    expectedResponse = new ListTablesResponse(req.getCatalogName(), new ImmutableList.Builder<TableName>().add(new TableName(req.getSchemaName(), "table4")).add(new TableName(req.getSchemaName(), "table5")).build(), null);
    actualResponse = handler.doListTables(allocator, req);
    logger.info("Response - {}", actualResponse);
    assertEquals("Lists do not match.", expectedResponse, actualResponse);
}
Also used : GlueMetadataHandler.getSourceTableName(com.amazonaws.athena.connector.lambda.handlers.GlueMetadataHandler.getSourceTableName) TableName(com.amazonaws.athena.connector.lambda.domain.TableName) ImmutableList(com.google.common.collect.ImmutableList) ListTablesRequest(com.amazonaws.athena.connector.lambda.metadata.ListTablesRequest) ListTablesResponse(com.amazonaws.athena.connector.lambda.metadata.ListTablesResponse) Test(org.junit.Test)

Aggregations

TableName (com.amazonaws.athena.connector.lambda.domain.TableName)279 Test (org.junit.Test)229 Schema (org.apache.arrow.vector.types.pojo.Schema)154 Constraints (com.amazonaws.athena.connector.lambda.domain.predicate.Constraints)150 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)70 BlockAllocator (com.amazonaws.athena.connector.lambda.data.BlockAllocator)55 GetTableLayoutRequest (com.amazonaws.athena.connector.lambda.metadata.GetTableLayoutRequest)55 Split (com.amazonaws.athena.connector.lambda.domain.Split)54 ArrayList (java.util.ArrayList)54 SchemaBuilder (com.amazonaws.athena.connector.lambda.data.SchemaBuilder)51 BlockAllocatorImpl (com.amazonaws.athena.connector.lambda.data.BlockAllocatorImpl)50 HashMap (java.util.HashMap)48 ValueSet (com.amazonaws.athena.connector.lambda.domain.predicate.ValueSet)47 PreparedStatement (java.sql.PreparedStatement)44 ResultSet (java.sql.ResultSet)44 GetTableLayoutResponse (com.amazonaws.athena.connector.lambda.metadata.GetTableLayoutResponse)41 ReadRecordsRequest (com.amazonaws.athena.connector.lambda.records.ReadRecordsRequest)41 GetTableResponse (com.amazonaws.athena.connector.lambda.metadata.GetTableResponse)40 GetTableRequest (com.amazonaws.athena.connector.lambda.metadata.GetTableRequest)32 Block (com.amazonaws.athena.connector.lambda.data.Block)30