use of com.github.ambry.network.Send in project ambry by linkedin.
the class AmbryRequests method handleGetRequest.
public void handleGetRequest(Request request) throws IOException, InterruptedException {
GetRequest getRequest = GetRequest.readFrom(new DataInputStream(request.getInputStream()), clusterMap);
Histogram responseQueueTime = null;
Histogram responseSendTime = null;
Histogram responseTotalTime = null;
long requestQueueTime = SystemTime.getInstance().milliseconds() - request.getStartTimeInMs();
long totalTimeSpent = requestQueueTime;
if (getRequest.getMessageFormatFlag() == MessageFormatFlags.Blob) {
metrics.getBlobRequestQueueTimeInMs.update(requestQueueTime);
metrics.getBlobRequestRate.mark();
responseQueueTime = metrics.getBlobResponseQueueTimeInMs;
responseSendTime = metrics.getBlobSendTimeInMs;
responseTotalTime = metrics.getBlobTotalTimeInMs;
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobProperties) {
metrics.getBlobPropertiesRequestQueueTimeInMs.update(requestQueueTime);
metrics.getBlobPropertiesRequestRate.mark();
responseQueueTime = metrics.getBlobPropertiesResponseQueueTimeInMs;
responseSendTime = metrics.getBlobPropertiesSendTimeInMs;
responseTotalTime = metrics.getBlobPropertiesTotalTimeInMs;
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobUserMetadata) {
metrics.getBlobUserMetadataRequestQueueTimeInMs.update(requestQueueTime);
metrics.getBlobUserMetadataRequestRate.mark();
responseQueueTime = metrics.getBlobUserMetadataResponseQueueTimeInMs;
responseSendTime = metrics.getBlobUserMetadataSendTimeInMs;
responseTotalTime = metrics.getBlobUserMetadataTotalTimeInMs;
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobInfo) {
metrics.getBlobInfoRequestQueueTimeInMs.update(requestQueueTime);
metrics.getBlobInfoRequestRate.mark();
responseQueueTime = metrics.getBlobInfoResponseQueueTimeInMs;
responseSendTime = metrics.getBlobInfoSendTimeInMs;
responseTotalTime = metrics.getBlobInfoTotalTimeInMs;
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.All) {
metrics.getBlobAllRequestQueueTimeInMs.update(requestQueueTime);
metrics.getBlobAllRequestRate.mark();
responseQueueTime = metrics.getBlobAllResponseQueueTimeInMs;
responseSendTime = metrics.getBlobAllSendTimeInMs;
responseTotalTime = metrics.getBlobAllTotalTimeInMs;
}
long startTime = SystemTime.getInstance().milliseconds();
GetResponse response = null;
try {
List<Send> messagesToSendList = new ArrayList<Send>(getRequest.getPartitionInfoList().size());
List<PartitionResponseInfo> partitionResponseInfoList = new ArrayList<PartitionResponseInfo>(getRequest.getPartitionInfoList().size());
for (PartitionRequestInfo partitionRequestInfo : getRequest.getPartitionInfoList()) {
ServerErrorCode error = validateRequest(partitionRequestInfo.getPartition(), RequestOrResponseType.GetRequest);
if (error != ServerErrorCode.No_Error) {
logger.error("Validating get request failed for partition {} with error {}", partitionRequestInfo.getPartition(), error);
PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), error);
partitionResponseInfoList.add(partitionResponseInfo);
} else {
try {
Store storeToGet = storageManager.getStore(partitionRequestInfo.getPartition());
EnumSet<StoreGetOptions> storeGetOptions = EnumSet.noneOf(StoreGetOptions.class);
// Currently only one option is supported.
if (getRequest.getGetOption() == GetOption.Include_Expired_Blobs) {
storeGetOptions = EnumSet.of(StoreGetOptions.Store_Include_Expired);
}
if (getRequest.getGetOption() == GetOption.Include_Deleted_Blobs) {
storeGetOptions = EnumSet.of(StoreGetOptions.Store_Include_Deleted);
}
if (getRequest.getGetOption() == GetOption.Include_All) {
storeGetOptions = EnumSet.of(StoreGetOptions.Store_Include_Deleted, StoreGetOptions.Store_Include_Expired);
}
StoreInfo info = storeToGet.get(partitionRequestInfo.getBlobIds(), storeGetOptions);
MessageFormatSend blobsToSend = new MessageFormatSend(info.getMessageReadSet(), getRequest.getMessageFormatFlag(), messageFormatMetrics, storeKeyFactory);
PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), info.getMessageReadSetInfo(), blobsToSend.getMessageMetadataList());
messagesToSendList.add(blobsToSend);
partitionResponseInfoList.add(partitionResponseInfo);
} catch (StoreException e) {
boolean logInErrorLevel = false;
if (e.getErrorCode() == StoreErrorCodes.ID_Not_Found) {
metrics.idNotFoundError.inc();
} else if (e.getErrorCode() == StoreErrorCodes.TTL_Expired) {
metrics.ttlExpiredError.inc();
} else if (e.getErrorCode() == StoreErrorCodes.ID_Deleted) {
metrics.idDeletedError.inc();
} else if (e.getErrorCode() == StoreErrorCodes.Authorization_Failure) {
metrics.getAuthorizationFailure.inc();
} else {
metrics.unExpectedStoreGetError.inc();
logInErrorLevel = true;
}
if (logInErrorLevel) {
logger.error("Store exception on a get with error code {} for partition {}", e.getErrorCode(), partitionRequestInfo.getPartition(), e);
} else {
logger.trace("Store exception on a get with error code {} for partition {}", e.getErrorCode(), partitionRequestInfo.getPartition(), e);
}
PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), ErrorMapping.getStoreErrorMapping(e.getErrorCode()));
partitionResponseInfoList.add(partitionResponseInfo);
} catch (MessageFormatException e) {
logger.error("Message format exception on a get with error code " + e.getErrorCode() + " for partitionRequestInfo " + partitionRequestInfo, e);
if (e.getErrorCode() == MessageFormatErrorCodes.Data_Corrupt) {
metrics.dataCorruptError.inc();
} else if (e.getErrorCode() == MessageFormatErrorCodes.Unknown_Format_Version) {
metrics.unknownFormatError.inc();
}
PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), ErrorMapping.getMessageFormatErrorMapping(e.getErrorCode()));
partitionResponseInfoList.add(partitionResponseInfo);
}
}
}
CompositeSend compositeSend = new CompositeSend(messagesToSendList);
response = new GetResponse(getRequest.getCorrelationId(), getRequest.getClientId(), partitionResponseInfoList, compositeSend, ServerErrorCode.No_Error);
} catch (Exception e) {
logger.error("Unknown exception for request " + getRequest, e);
response = new GetResponse(getRequest.getCorrelationId(), getRequest.getClientId(), ServerErrorCode.Unknown_Error);
} finally {
long processingTime = SystemTime.getInstance().milliseconds() - startTime;
totalTimeSpent += processingTime;
publicAccessLogger.info("{} {} processingTime {}", getRequest, response, processingTime);
if (getRequest.getMessageFormatFlag() == MessageFormatFlags.Blob) {
metrics.getBlobProcessingTimeInMs.update(processingTime);
metrics.updateGetBlobProcessingTimeBySize(response.sizeInBytes(), processingTime);
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobProperties) {
metrics.getBlobPropertiesProcessingTimeInMs.update(processingTime);
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobUserMetadata) {
metrics.getBlobUserMetadataProcessingTimeInMs.update(processingTime);
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobInfo) {
metrics.getBlobInfoProcessingTimeInMs.update(processingTime);
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.All) {
metrics.getBlobAllProcessingTimeInMs.update(processingTime);
metrics.updateGetBlobProcessingTimeBySize(response.sizeInBytes(), processingTime);
}
}
sendGetResponse(requestResponseChannel, response, request, responseQueueTime, responseSendTime, responseTotalTime, totalTimeSpent, response.sizeInBytes(), getRequest.getMessageFormatFlag(), metrics);
}
use of com.github.ambry.network.Send in project ambry by linkedin.
the class AmbryRequests method handleGetRequest.
@Override
public void handleGetRequest(NetworkRequest request) throws IOException, InterruptedException {
GetRequest getRequest;
if (request instanceof LocalChannelRequest) {
// This is a case where handleGetRequest is called when frontends are reading from Azure. In this case, this method
// is called by request handler threads running within the frontend router itself. So, the request can be directly
// referenced as java objects without any need for deserialization.
getRequest = (GetRequest) ((LocalChannelRequest) request).getRequestInfo().getRequest();
} else {
getRequest = GetRequest.readFrom(new DataInputStream(request.getInputStream()), clusterMap);
}
Histogram responseQueueTime = null;
Histogram responseSendTime = null;
Histogram responseTotalTime = null;
long requestQueueTime = SystemTime.getInstance().milliseconds() - request.getStartTimeInMs();
long totalTimeSpent = requestQueueTime;
boolean isReplicaRequest = getRequest.getClientId().startsWith(GetRequest.Replication_Client_Id_Prefix);
if (getRequest.getMessageFormatFlag() == MessageFormatFlags.Blob) {
metrics.getBlobRequestQueueTimeInMs.update(requestQueueTime);
metrics.getBlobRequestRate.mark();
responseQueueTime = metrics.getBlobResponseQueueTimeInMs;
responseSendTime = metrics.getBlobSendTimeInMs;
responseTotalTime = metrics.getBlobTotalTimeInMs;
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobProperties) {
metrics.getBlobPropertiesRequestQueueTimeInMs.update(requestQueueTime);
metrics.getBlobPropertiesRequestRate.mark();
responseQueueTime = metrics.getBlobPropertiesResponseQueueTimeInMs;
responseSendTime = metrics.getBlobPropertiesSendTimeInMs;
responseTotalTime = metrics.getBlobPropertiesTotalTimeInMs;
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobUserMetadata) {
metrics.getBlobUserMetadataRequestQueueTimeInMs.update(requestQueueTime);
metrics.getBlobUserMetadataRequestRate.mark();
responseQueueTime = metrics.getBlobUserMetadataResponseQueueTimeInMs;
responseSendTime = metrics.getBlobUserMetadataSendTimeInMs;
responseTotalTime = metrics.getBlobUserMetadataTotalTimeInMs;
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobInfo) {
metrics.getBlobInfoRequestQueueTimeInMs.update(requestQueueTime);
metrics.getBlobInfoRequestRate.mark();
responseQueueTime = metrics.getBlobInfoResponseQueueTimeInMs;
responseSendTime = metrics.getBlobInfoSendTimeInMs;
responseTotalTime = metrics.getBlobInfoTotalTimeInMs;
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.All) {
if (isReplicaRequest) {
metrics.getBlobAllByReplicaRequestQueueTimeInMs.update(requestQueueTime);
metrics.getBlobAllByReplicaRequestRate.mark();
responseQueueTime = metrics.getBlobAllByReplicaResponseQueueTimeInMs;
responseSendTime = metrics.getBlobAllByReplicaSendTimeInMs;
responseTotalTime = metrics.getBlobAllByReplicaTotalTimeInMs;
} else {
metrics.getBlobAllRequestQueueTimeInMs.update(requestQueueTime);
metrics.getBlobAllRequestRate.mark();
responseQueueTime = metrics.getBlobAllResponseQueueTimeInMs;
responseSendTime = metrics.getBlobAllSendTimeInMs;
responseTotalTime = metrics.getBlobAllTotalTimeInMs;
}
}
long startTime = SystemTime.getInstance().milliseconds();
GetResponse response = null;
try {
List<Send> messagesToSendList = new ArrayList<>(getRequest.getPartitionInfoList().size());
List<PartitionResponseInfo> partitionResponseInfoList = new ArrayList<>(getRequest.getPartitionInfoList().size());
for (PartitionRequestInfo partitionRequestInfo : getRequest.getPartitionInfoList()) {
ServerErrorCode error = validateRequest(partitionRequestInfo.getPartition(), RequestOrResponseType.GetRequest, false);
if (error != ServerErrorCode.No_Error) {
logger.error("Validating get request failed for partition {} with error {}", partitionRequestInfo.getPartition(), error);
PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), error);
partitionResponseInfoList.add(partitionResponseInfo);
} else {
try {
Store storeToGet = storeManager.getStore(partitionRequestInfo.getPartition());
EnumSet<StoreGetOptions> storeGetOptions = EnumSet.noneOf(StoreGetOptions.class);
// Currently only one option is supported.
if (getRequest.getGetOption() == GetOption.Include_Expired_Blobs) {
storeGetOptions = EnumSet.of(StoreGetOptions.Store_Include_Expired);
}
if (getRequest.getGetOption() == GetOption.Include_Deleted_Blobs) {
storeGetOptions = EnumSet.of(StoreGetOptions.Store_Include_Deleted);
}
if (getRequest.getGetOption() == GetOption.Include_All) {
storeGetOptions = EnumSet.of(StoreGetOptions.Store_Include_Deleted, StoreGetOptions.Store_Include_Expired);
}
List<StoreKey> convertedStoreKeys = getConvertedStoreKeys(partitionRequestInfo.getBlobIds());
List<StoreKey> dedupedStoreKeys = convertedStoreKeys.size() > 1 ? convertedStoreKeys.stream().distinct().collect(Collectors.toList()) : convertedStoreKeys;
StoreInfo info = storeToGet.get(dedupedStoreKeys, storeGetOptions);
MessageFormatSend blobsToSend = new MessageFormatSend(info.getMessageReadSet(), getRequest.getMessageFormatFlag(), messageFormatMetrics, storeKeyFactory);
PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), info.getMessageReadSetInfo(), blobsToSend.getMessageMetadataList());
messagesToSendList.add(blobsToSend);
partitionResponseInfoList.add(partitionResponseInfo);
} catch (StoreException e) {
boolean logInErrorLevel = false;
if (e.getErrorCode() == StoreErrorCodes.ID_Not_Found) {
metrics.idNotFoundError.inc();
} else if (e.getErrorCode() == StoreErrorCodes.TTL_Expired) {
metrics.ttlExpiredError.inc();
} else if (e.getErrorCode() == StoreErrorCodes.ID_Deleted) {
metrics.idDeletedError.inc();
} else if (e.getErrorCode() == StoreErrorCodes.Authorization_Failure) {
metrics.getAuthorizationFailure.inc();
} else {
metrics.unExpectedStoreGetError.inc();
logInErrorLevel = true;
}
if (logInErrorLevel) {
logger.error("Store exception on a get with error code {} for partition {}", e.getErrorCode(), partitionRequestInfo.getPartition(), e);
} else {
logger.trace("Store exception on a get with error code {} for partition {}", e.getErrorCode(), partitionRequestInfo.getPartition(), e);
}
PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), ErrorMapping.getStoreErrorMapping(e.getErrorCode()));
partitionResponseInfoList.add(partitionResponseInfo);
} catch (MessageFormatException e) {
logger.error("Message format exception on a get with error code {} for partitionRequestInfo {}", e.getErrorCode(), partitionRequestInfo, e);
if (e.getErrorCode() == MessageFormatErrorCodes.Data_Corrupt) {
metrics.dataCorruptError.inc();
} else if (e.getErrorCode() == MessageFormatErrorCodes.Unknown_Format_Version) {
metrics.unknownFormatError.inc();
}
PartitionResponseInfo partitionResponseInfo = new PartitionResponseInfo(partitionRequestInfo.getPartition(), ErrorMapping.getMessageFormatErrorMapping(e.getErrorCode()));
partitionResponseInfoList.add(partitionResponseInfo);
}
}
}
CompositeSend compositeSend = new CompositeSend(messagesToSendList);
response = new GetResponse(getRequest.getCorrelationId(), getRequest.getClientId(), partitionResponseInfoList, compositeSend, ServerErrorCode.No_Error);
} catch (Exception e) {
logger.error("Unknown exception for request {}", getRequest, e);
response = new GetResponse(getRequest.getCorrelationId(), getRequest.getClientId(), ServerErrorCode.Unknown_Error);
} finally {
long processingTime = SystemTime.getInstance().milliseconds() - startTime;
totalTimeSpent += processingTime;
publicAccessLogger.info("{} {} processingTime {}", getRequest, response, processingTime);
long responseSize = response != null ? response.sizeInBytes() : 0;
if (getRequest.getMessageFormatFlag() == MessageFormatFlags.Blob) {
metrics.getBlobProcessingTimeInMs.update(processingTime);
metrics.updateGetBlobProcessingTimeBySize(responseSize, processingTime);
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobProperties) {
metrics.getBlobPropertiesProcessingTimeInMs.update(processingTime);
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobUserMetadata) {
metrics.getBlobUserMetadataProcessingTimeInMs.update(processingTime);
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.BlobInfo) {
metrics.getBlobInfoProcessingTimeInMs.update(processingTime);
} else if (getRequest.getMessageFormatFlag() == MessageFormatFlags.All) {
if (isReplicaRequest) {
metrics.getBlobAllByReplicaProcessingTimeInMs.update(processingTime);
// client id now has dc name at the end, for example: ClientId=replication-fetch-abc.example.com[dc1]
String[] clientStrs = getRequest.getClientId().split("\\[");
if (clientStrs.length > 1) {
String clientDc = clientStrs[1].substring(0, clientStrs[1].length() - 1);
if (!currentNode.getDatacenterName().equals(clientDc)) {
metrics.updateCrossColoFetchBytesRate(clientDc, responseSize);
}
}
} else {
metrics.getBlobAllProcessingTimeInMs.update(processingTime);
metrics.updateGetBlobProcessingTimeBySize(responseSize, processingTime);
}
}
}
sendGetResponse(requestResponseChannel, response, request, responseQueueTime, responseSendTime, responseTotalTime, totalTimeSpent, response.sizeInBytes(), getRequest.getMessageFormatFlag(), metrics);
}
use of com.github.ambry.network.Send in project ambry by linkedin.
the class AmbrySendToHttp2AdaptorTest method testServerWrite.
/**
* Test when write {@link Send} from server to client. There are two cases tested in this method
* 1. {@link Send} returns a non-null value for {@link Send#content()} method.
* 2. {@link Send} returns a null value for {@link Send#content()} method.
* Make sure both cases don't leak memory.
* @throws Exception
*/
@Test
public void testServerWrite() throws Exception {
int maxFrameSize = 2001;
EmbeddedChannel channel = new EmbeddedChannel(new AmbrySendToHttp2Adaptor(true, maxFrameSize));
int contentSize = 7000;
byte[] byteArray = new byte[contentSize];
new Random().nextBytes(byteArray);
ByteBuf content = PooledByteBufAllocator.DEFAULT.heapBuffer(contentSize).writeBytes(byteArray);
// Retain the ByteBuf for data comparison
content.retain();
Send send = new MockSend(content);
channel.writeOutbound(send);
DefaultHttp2HeadersFrame header = channel.readOutbound();
Assert.assertNotNull(header.headers());
Assert.assertEquals(header.headers().status().toString(), "200");
byte[] resultArray = new byte[contentSize];
int i;
for (i = 0; i < contentSize / maxFrameSize; i++) {
DefaultHttp2DataFrame data = channel.readOutbound();
data.content().readBytes(resultArray, i * maxFrameSize, maxFrameSize);
data.content().release();
}
// The last frame
DefaultHttp2DataFrame data = channel.readOutbound();
data.content().readBytes(resultArray, i * maxFrameSize, data.content().readableBytes());
data.content().release();
Assert.assertArrayEquals(byteArray, resultArray);
content.release();
}
use of com.github.ambry.network.Send in project ambry by linkedin.
the class AmbrySendToHttp2Adaptor method write.
/**
* Handles conversion of {@link Send} to HTTP/2 frames.
*/
@Override
public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) {
if (!ctx.channel().isOpen()) {
logger.debug("Channel closed when write. Channel: {}", ctx.channel());
promise.setFailure(new ChannelException("Channel has been closed when write."));
}
if (!(msg instanceof Send)) {
ctx.write(msg, promise);
return;
}
Send send = (Send) msg;
Http2Headers http2Headers;
if (forServer) {
logger.trace("Write content to channel as server {}", ctx.channel());
http2Headers = new DefaultHttp2Headers().status(HttpResponseStatus.OK.codeAsText());
} else {
logger.trace("Write content to channel as client {}", ctx.channel());
http2Headers = new DefaultHttp2Headers().method(HttpMethod.POST.asciiName()).scheme("https").path("/");
}
DefaultHttp2HeadersFrame headersFrame = new DefaultHttp2HeadersFrame(http2Headers, false);
ctx.write(headersFrame);
// Referencing counting for derived {@link ByteBuf}: https://netty.io/wiki/reference-counted-objects.html#derived-buffers
try {
while (send.content().isReadable(maxFrameSize)) {
ByteBuf slice = send.content().readSlice(maxFrameSize);
slice.retain();
DefaultHttp2DataFrame dataFrame = new DefaultHttp2DataFrame(slice, false);
ctx.write(dataFrame);
}
// The last slice
ByteBuf slice = send.content().readSlice(send.content().readableBytes());
slice.retain();
DefaultHttp2DataFrame dataFrame = new DefaultHttp2DataFrame(slice, true);
ctx.write(dataFrame, promise);
} catch (Exception e) {
logger.error("Error while processing frames. Channel: {}", ctx.channel(), e);
} finally {
send.content().release();
}
}
use of com.github.ambry.network.Send in project ambry by linkedin.
the class StoredBlob method send.
/**
* Take in a request in the form of {@link Send} and return a response in the form of a
* {@link BoundedNettyByteBufReceive}.
* @param send the request.
* @return the response.
* @throws IOException if there was an error in interpreting the request.
*/
public BoundedNettyByteBufReceive send(Send send) throws IOException {
if (!shouldRespond) {
return null;
}
ServerErrorCode serverError = hardError != null ? hardError : serverErrors.size() > 0 ? serverErrors.poll() : ServerErrorCode.No_Error;
RequestOrResponseType type = ((RequestOrResponse) send).getRequestType();
RequestOrResponse response;
requestCounts.computeIfAbsent(type, k -> new LongAdder()).increment();
switch(type) {
case PutRequest:
response = makePutResponse((PutRequest) send, serverError);
break;
case GetRequest:
response = makeGetResponse((GetRequest) send, serverError);
break;
case DeleteRequest:
response = makeDeleteResponse((DeleteRequest) send, serverError);
break;
case TtlUpdateRequest:
response = makeTtlUpdateResponse((TtlUpdateRequest) send, serverError);
break;
case UndeleteRequest:
response = makeUndeleteResponse((UndeleteRequest) send, serverError);
break;
default:
throw new IOException("Unknown request type received");
}
ByteBufferChannel channel = new ByteBufferChannel(ByteBuffer.allocate((int) response.sizeInBytes()));
response.writeTo(channel);
response.release();
ByteBuffer payload = channel.getBuffer();
payload.flip();
BoundedNettyByteBufReceive receive = new BoundedNettyByteBufReceive(100 * 1024 * 1024);
receive.readFrom(Channels.newChannel(new ByteBufferInputStream(payload)));
return receive;
}
Aggregations