use of com.amazonaws.athena.connector.lambda.metadata.MetadataResponse in project aws-athena-query-federation by awslabs.
the class ExampleMetadataHandlerTest method doGetSplits.
/**
* The goal of this test is to test happy case for getting splits and also to exercise the continuation token
* logic specifically.
*/
@Test
public void doGetSplits() {
logger.info("doGetSplits: enter");
String yearCol = "year";
String monthCol = "month";
String dayCol = "day";
// This is the schema that ExampleMetadataHandler has layed out for a 'Partition' so we need to populate this
// minimal set of info here.
Schema schema = SchemaBuilder.newBuilder().addField(yearCol, new ArrowType.Int(16, false)).addField(monthCol, new ArrowType.Int(16, false)).addField(dayCol, new ArrowType.Int(16, false)).addField(ExampleMetadataHandler.PARTITION_LOCATION, new ArrowType.Utf8()).addField(ExampleMetadataHandler.SERDE, new ArrowType.Utf8()).build();
List<String> partitionCols = new ArrayList<>();
partitionCols.add(yearCol);
partitionCols.add(monthCol);
partitionCols.add(dayCol);
Map<String, ValueSet> constraintsMap = new HashMap<>();
constraintsMap.put(dayCol, SortedRangeSet.copyOf(Types.MinorType.INT.getType(), ImmutableList.of(Range.greaterThan(allocator, Types.MinorType.INT.getType(), 20)), false));
Block partitions = allocator.createBlock(schema);
int num_partitions = 100;
for (int i = 0; i < num_partitions; i++) {
BlockUtils.setValue(partitions.getFieldVector(yearCol), i, 2016 + i);
BlockUtils.setValue(partitions.getFieldVector(monthCol), i, (i % 12) + 1);
BlockUtils.setValue(partitions.getFieldVector(dayCol), i, (i % 28) + 1);
BlockUtils.setValue(partitions.getFieldVector(ExampleMetadataHandler.PARTITION_LOCATION), i, String.valueOf(i));
BlockUtils.setValue(partitions.getFieldVector(ExampleMetadataHandler.SERDE), i, "TextInputType");
}
partitions.setRowCount(num_partitions);
String continuationToken = null;
GetSplitsRequest originalReq = new GetSplitsRequest(IdentityUtil.fakeIdentity(), "queryId", "catalog_name", new TableName("schema", "table_name"), partitions, partitionCols, new Constraints(constraintsMap), continuationToken);
int numContinuations = 0;
do {
GetSplitsRequest req = new GetSplitsRequest(originalReq, continuationToken);
ObjectMapperUtil.assertSerialization(req);
logger.info("doGetSplits: req[{}]", req);
metadataHandler.setEncryption(numContinuations % 2 == 0);
logger.info("doGetSplits: Toggle encryption " + (numContinuations % 2 == 0));
MetadataResponse rawResponse = metadataHandler.doGetSplits(allocator, req);
ObjectMapperUtil.assertSerialization(rawResponse);
assertEquals(MetadataRequestType.GET_SPLITS, rawResponse.getRequestType());
GetSplitsResponse response = (GetSplitsResponse) rawResponse;
continuationToken = response.getContinuationToken();
logger.info("doGetSplits: continuationToken[{}] - numSplits[{}] - maxSplits[{}]", new Object[] { continuationToken, response.getSplits().size(), MAX_SPLITS_PER_REQUEST });
for (Split nextSplit : response.getSplits()) {
if (numContinuations % 2 == 0) {
assertNotNull(nextSplit.getEncryptionKey());
} else {
assertNull(nextSplit.getEncryptionKey());
}
assertNotNull(nextSplit.getProperty(SplitProperties.LOCATION.getId()));
assertNotNull(nextSplit.getProperty(SplitProperties.SERDE.getId()));
assertNotNull(nextSplit.getProperty(SplitProperties.SPLIT_PART.getId()));
}
assertTrue("Continuation criteria violated", (response.getSplits().size() == MAX_SPLITS_PER_REQUEST && response.getContinuationToken() != null) || response.getSplits().size() < MAX_SPLITS_PER_REQUEST);
if (continuationToken != null) {
numContinuations++;
}
} while (continuationToken != null);
assertTrue(numContinuations > 0);
logger.info("doGetSplits: exit");
}
use of com.amazonaws.athena.connector.lambda.metadata.MetadataResponse in project aws-athena-query-federation by awslabs.
the class HbaseMetadataHandlerTest method doGetSplits.
@Test
public void doGetSplits() throws IOException {
List<HRegionInfo> regionServers = new ArrayList<>();
regionServers.add(TestUtils.makeRegion(1, "schema1", "table1"));
regionServers.add(TestUtils.makeRegion(2, "schema1", "table1"));
regionServers.add(TestUtils.makeRegion(3, "schema1", "table1"));
regionServers.add(TestUtils.makeRegion(4, "schema1", "table1"));
when(mockClient.getTableRegions(any())).thenReturn(regionServers);
List<String> partitionCols = new ArrayList<>();
Block partitions = BlockUtils.newBlock(allocator, "partitionId", Types.MinorType.INT.getType(), 0);
String continuationToken = null;
GetSplitsRequest originalReq = new GetSplitsRequest(IDENTITY, QUERY_ID, DEFAULT_CATALOG, TABLE_NAME, partitions, partitionCols, new Constraints(new HashMap<>()), null);
GetSplitsRequest req = new GetSplitsRequest(originalReq, continuationToken);
logger.info("doGetSplits: req[{}]", req);
MetadataResponse rawResponse = handler.doGetSplits(allocator, req);
assertEquals(MetadataRequestType.GET_SPLITS, rawResponse.getRequestType());
GetSplitsResponse response = (GetSplitsResponse) rawResponse;
continuationToken = response.getContinuationToken();
logger.info("doGetSplits: continuationToken[{}] - numSplits[{}]", new Object[] { continuationToken, response.getSplits().size() });
assertTrue("Continuation criteria violated", response.getSplits().size() == 4);
assertTrue("Continuation criteria violated", response.getContinuationToken() == null);
}
use of com.amazonaws.athena.connector.lambda.metadata.MetadataResponse in project aws-athena-query-federation by awslabs.
the class RedisMetadataHandlerTest method doGetSplitsZset.
@Test
public void doGetSplitsZset() {
// 3 prefixes for this table
String prefixes = "prefix1-*,prefix2-*, prefix3-*";
// 4 zsets per prefix
when(mockSyncCommands.scan(any(ScanCursor.class), any(ScanArgs.class))).then((InvocationOnMock invocationOnMock) -> {
ScanCursor cursor = (ScanCursor) invocationOnMock.getArguments()[0];
if (cursor == null || cursor.getCursor().equals("0")) {
List<String> result = new ArrayList<>();
result.add(UUID.randomUUID().toString());
result.add(UUID.randomUUID().toString());
result.add(UUID.randomUUID().toString());
MockKeyScanCursor<String> scanCursor = new MockKeyScanCursor<>();
scanCursor.setCursor("1");
scanCursor.setKeys(result);
return scanCursor;
} else {
List<String> result = new ArrayList<>();
result.add(UUID.randomUUID().toString());
MockKeyScanCursor<String> scanCursor = new MockKeyScanCursor<>();
scanCursor.setCursor("0");
scanCursor.setKeys(result);
scanCursor.setFinished(true);
return scanCursor;
}
});
// 100 keys per zset
when(mockSyncCommands.zcount(anyString(), any(Range.class))).thenReturn(200L);
List<String> partitionCols = new ArrayList<>();
Schema schema = SchemaBuilder.newBuilder().addField("partitionId", Types.MinorType.INT.getType()).addStringField(REDIS_ENDPOINT_PROP).addStringField(VALUE_TYPE_TABLE_PROP).addStringField(KEY_PREFIX_TABLE_PROP).addStringField(ZSET_KEYS_TABLE_PROP).addStringField(REDIS_SSL_FLAG).addStringField(REDIS_CLUSTER_FLAG).addStringField(REDIS_DB_NUMBER).build();
Block partitions = allocator.createBlock(schema);
partitions.setValue(REDIS_ENDPOINT_PROP, 0, endpoint);
partitions.setValue(VALUE_TYPE_TABLE_PROP, 0, "literal");
partitions.setValue(KEY_PREFIX_TABLE_PROP, 0, null);
partitions.setValue(ZSET_KEYS_TABLE_PROP, 0, prefixes);
partitions.setValue(REDIS_SSL_FLAG, 0, null);
partitions.setValue(REDIS_CLUSTER_FLAG, 0, null);
partitions.setValue(REDIS_DB_NUMBER, 0, null);
partitions.setRowCount(1);
String continuationToken = null;
GetSplitsRequest originalReq = new GetSplitsRequest(IDENTITY, QUERY_ID, DEFAULT_CATALOG, TABLE_NAME, partitions, partitionCols, new Constraints(new HashMap<>()), null);
GetSplitsRequest req = new GetSplitsRequest(originalReq, continuationToken);
logger.info("doGetSplitsPrefix: req[{}]", req);
MetadataResponse rawResponse = handler.doGetSplits(allocator, req);
assertEquals(MetadataRequestType.GET_SPLITS, rawResponse.getRequestType());
GetSplitsResponse response = (GetSplitsResponse) rawResponse;
continuationToken = response.getContinuationToken();
logger.info("doGetSplitsPrefix: continuationToken[{}] - numSplits[{}]", new Object[] { continuationToken, response.getSplits().size() });
assertEquals("Continuation criteria violated", 120, response.getSplits().size());
assertTrue("Continuation criteria violated", response.getContinuationToken() == null);
verify(mockSyncCommands, times(6)).scan(any(ScanCursor.class), any(ScanArgs.class));
}
use of com.amazonaws.athena.connector.lambda.metadata.MetadataResponse in project aws-athena-query-federation by awslabs.
the class RedisMetadataHandlerTest method doGetSplitsPrefix.
@Test
public void doGetSplitsPrefix() {
Schema schema = SchemaBuilder.newBuilder().addField("partitionId", Types.MinorType.INT.getType()).addStringField(REDIS_ENDPOINT_PROP).addStringField(VALUE_TYPE_TABLE_PROP).addStringField(KEY_PREFIX_TABLE_PROP).addStringField(ZSET_KEYS_TABLE_PROP).addStringField(REDIS_SSL_FLAG).addStringField(REDIS_CLUSTER_FLAG).addStringField(REDIS_DB_NUMBER).build();
Block partitions = allocator.createBlock(schema);
partitions.setValue(REDIS_ENDPOINT_PROP, 0, endpoint);
partitions.setValue(VALUE_TYPE_TABLE_PROP, 0, "literal");
partitions.setValue(KEY_PREFIX_TABLE_PROP, 0, "prefix1-*,prefix2-*, prefix3-*");
partitions.setValue(ZSET_KEYS_TABLE_PROP, 0, null);
partitions.setValue(REDIS_SSL_FLAG, 0, null);
partitions.setValue(REDIS_CLUSTER_FLAG, 0, null);
partitions.setValue(REDIS_DB_NUMBER, 0, null);
partitions.setRowCount(1);
String continuationToken = null;
GetSplitsRequest originalReq = new GetSplitsRequest(IDENTITY, QUERY_ID, DEFAULT_CATALOG, TABLE_NAME, partitions, new ArrayList<>(), new Constraints(new HashMap<>()), null);
GetSplitsRequest req = new GetSplitsRequest(originalReq, continuationToken);
logger.info("doGetSplitsPrefix: req[{}]", req);
MetadataResponse rawResponse = handler.doGetSplits(allocator, req);
assertEquals(MetadataRequestType.GET_SPLITS, rawResponse.getRequestType());
GetSplitsResponse response = (GetSplitsResponse) rawResponse;
continuationToken = response.getContinuationToken();
logger.info("doGetSplitsPrefix: continuationToken[{}] - numSplits[{}]", new Object[] { continuationToken, response.getSplits().size() });
assertTrue("Continuation criteria violated", response.getSplits().size() == 3);
assertTrue("Continuation criteria violated", response.getContinuationToken() == null);
}
use of com.amazonaws.athena.connector.lambda.metadata.MetadataResponse in project aws-athena-query-federation by awslabs.
the class TimestreamMetadataHandlerTest method doGetSplits.
@Test
public void doGetSplits() throws Exception {
logger.info("doGetSplits - enter");
List<String> partitionCols = new ArrayList<>();
Block partitions = BlockUtils.newBlock(allocator, "partition_id", Types.MinorType.INT.getType(), 0);
String continuationToken = null;
GetSplitsRequest originalReq = new GetSplitsRequest(identity, "query-id", defaultSchema, new TableName("database1", "table1"), partitions, partitionCols, new Constraints(new HashMap<>()), null);
GetSplitsRequest req = new GetSplitsRequest(originalReq, continuationToken);
logger.info("doGetSplits: req[{}]", req);
MetadataResponse rawResponse = handler.doGetSplits(allocator, req);
assertEquals(MetadataRequestType.GET_SPLITS, rawResponse.getRequestType());
GetSplitsResponse response = (GetSplitsResponse) rawResponse;
continuationToken = response.getContinuationToken();
logger.info("doGetSplits: continuationToken[{}] - numSplits[{}]", new Object[] { continuationToken, response.getSplits().size() });
assertTrue("Continuation criteria violated", response.getSplits().size() == 1);
assertTrue("Continuation criteria violated", response.getContinuationToken() == null);
logger.info("doGetSplits - exit");
}
Aggregations