Examples with S3BlockSpillReader com.amazonaws.athena.connector.lambda.data.S3BlockSpillReader used on opensource projects

Example 11 with S3BlockSpillReader

use of com.amazonaws.athena.connector.lambda.data.S3BlockSpillReader in project aws-athena-query-federation by awslabs.

the class MetricsRecordHandlerTest method setUp.

@Before
public void setUp() throws Exception {
    mockS3Storage = new ArrayList<>();
    allocator = new BlockAllocatorImpl();
    handler = new MetricsRecordHandler(mockS3, mockSecretsManager, mockAthena, mockMetrics);
    spillReader = new S3BlockSpillReader(mockS3, allocator);
    when(mockS3.putObject(anyObject(), anyObject(), anyObject(), anyObject())).thenAnswer((InvocationOnMock invocationOnMock) -> {
        InputStream inputStream = (InputStream) invocationOnMock.getArguments()[2];
        ByteHolder byteHolder = new ByteHolder();
        byteHolder.setBytes(ByteStreams.toByteArray(inputStream));
        synchronized (mockS3Storage) {
            mockS3Storage.add(byteHolder);
            logger.info("puObject: total size " + mockS3Storage.size());
        }
        return mock(PutObjectResult.class);
    });
    when(mockS3.getObject(anyString(), anyString())).thenAnswer((InvocationOnMock invocationOnMock) -> {
        S3Object mockObject = mock(S3Object.class);
        ByteHolder byteHolder;
        synchronized (mockS3Storage) {
            byteHolder = mockS3Storage.get(0);
            mockS3Storage.remove(0);
            logger.info("getObject: total size " + mockS3Storage.size());
        }
        when(mockObject.getObjectContent()).thenReturn(new S3ObjectInputStream(new ByteArrayInputStream(byteHolder.getBytes()), null));
        return mockObject;
    });
}

Example 12 with S3BlockSpillReader

use of com.amazonaws.athena.connector.lambda.data.S3BlockSpillReader in project aws-athena-query-federation by awslabs.

the class DocDBRecordHandlerTest method setUp.

@Before
public void setUp() {
    logger.info("{}: enter", testName.getMethodName());
    schemaForRead = SchemaBuilder.newBuilder().addField("col1", new ArrowType.Int(32, true)).addField("col2", new ArrowType.Utf8()).addField("col3", new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)).addField("int", Types.MinorType.INT.getType()).addField("tinyint", Types.MinorType.TINYINT.getType()).addField("smallint", Types.MinorType.SMALLINT.getType()).addField("bigint", Types.MinorType.BIGINT.getType()).addField("uint1", Types.MinorType.UINT1.getType()).addField("uint2", Types.MinorType.UINT2.getType()).addField("uint4", Types.MinorType.UINT4.getType()).addField("uint8", Types.MinorType.UINT8.getType()).addField("float4", Types.MinorType.FLOAT4.getType()).addField("float8", Types.MinorType.FLOAT8.getType()).addField("bit", Types.MinorType.BIT.getType()).addField("varchar", Types.MinorType.VARCHAR.getType()).addField("varbinary", Types.MinorType.VARBINARY.getType()).addField("decimal", new ArrowType.Decimal(10, 2)).addField("decimalLong", new ArrowType.Decimal(36, 2)).addField("unsupported", Types.MinorType.VARCHAR.getType()).addStructField("struct").addChildField("struct", "struct_string", Types.MinorType.VARCHAR.getType()).addChildField("struct", "struct_int", Types.MinorType.INT.getType()).addListField("list", Types.MinorType.VARCHAR.getType()).build();
    when(connectionFactory.getOrCreateConn(anyString())).thenReturn(mockClient);
    allocator = new BlockAllocatorImpl();
    amazonS3 = mock(AmazonS3.class);
    mockDatabase = mock(MongoDatabase.class);
    mockCollection = mock(MongoCollection.class);
    mockIterable = mock(FindIterable.class);
    when(mockClient.getDatabase(eq(DEFAULT_SCHEMA))).thenReturn(mockDatabase);
    when(mockDatabase.getCollection(eq(TEST_TABLE))).thenReturn(mockCollection);
    when(amazonS3.putObject(anyObject(), anyObject(), anyObject(), anyObject())).thenAnswer((InvocationOnMock invocationOnMock) -> {
        InputStream inputStream = (InputStream) invocationOnMock.getArguments()[2];
        ByteHolder byteHolder = new ByteHolder();
        byteHolder.setBytes(ByteStreams.toByteArray(inputStream));
        synchronized (mockS3Storage) {
            mockS3Storage.add(byteHolder);
            logger.info("puObject: total size " + mockS3Storage.size());
        }
        return mock(PutObjectResult.class);
    });
    when(amazonS3.getObject(anyString(), anyString())).thenAnswer((InvocationOnMock invocationOnMock) -> {
        S3Object mockObject = mock(S3Object.class);
        ByteHolder byteHolder;
        synchronized (mockS3Storage) {
            byteHolder = mockS3Storage.get(0);
            mockS3Storage.remove(0);
            logger.info("getObject: total size " + mockS3Storage.size());
        }
        when(mockObject.getObjectContent()).thenReturn(new S3ObjectInputStream(new ByteArrayInputStream(byteHolder.getBytes()), null));
        return mockObject;
    });
    handler = new DocDBRecordHandler(amazonS3, mockSecretsManager, mockAthena, connectionFactory);
    spillReader = new S3BlockSpillReader(amazonS3, allocator);
    mdHandler = new DocDBMetadataHandler(awsGlue, connectionFactory, new LocalKeyFactory(), secretsManager, mockAthena, "spillBucket", "spillPrefix");
}