use of com.microsoft.azure.cosmosdb.FeedOptions in project ambry by linkedin.
the class CosmosDataAccessor method getDeadBlobs.
/**
* Get the list of blobs in the specified partition that have been deleted or expired for at least the
* configured retention period.
* @param partitionPath the partition to query.
* @param fieldName the field name to query on. Allowed values are {@link CloudBlobMetadata#FIELD_DELETION_TIME} and
* {@link CloudBlobMetadata#FIELD_EXPIRATION_TIME}.
* @param startTime the start of the query time range.
* @param endTime the end of the query time range.
* @param maxEntries the max number of metadata records to return.
* @return a List of {@link CloudBlobMetadata} referencing the dead blobs found.
* @throws DocumentClientException
*/
List<CloudBlobMetadata> getDeadBlobs(String partitionPath, String fieldName, long startTime, long endTime, int maxEntries) throws DocumentClientException {
String deadBlobsQuery;
if (fieldName.equals(CloudBlobMetadata.FIELD_DELETION_TIME)) {
deadBlobsQuery = DELETED_BLOBS_QUERY;
} else if (fieldName.equals(CloudBlobMetadata.FIELD_EXPIRATION_TIME)) {
deadBlobsQuery = EXPIRED_BLOBS_QUERY;
} else {
throw new IllegalArgumentException("Invalid field: " + fieldName);
}
SqlQuerySpec querySpec = new SqlQuerySpec(deadBlobsQuery, new SqlParameterCollection(new SqlParameter(LIMIT_PARAM, maxEntries), new SqlParameter(START_TIME_PARAM, startTime), new SqlParameter(END_TIME_PARAM, endTime)));
FeedOptions feedOptions = new FeedOptions();
feedOptions.setMaxItemCount(maxEntries);
feedOptions.setResponseContinuationTokenLimitInKb(continuationTokenLimitKb);
feedOptions.setPartitionKey(new PartitionKey(partitionPath));
try {
Iterator<FeedResponse<Document>> iterator = executeCosmosQuery(partitionPath, querySpec, feedOptions, azureMetrics.deadBlobsQueryTime).getIterator();
List<CloudBlobMetadata> deadBlobsList = new ArrayList<>();
double requestCharge = 0.0;
while (iterator.hasNext()) {
FeedResponse<Document> response = iterator.next();
requestCharge += response.getRequestCharge();
response.getResults().iterator().forEachRemaining(doc -> deadBlobsList.add(createMetadataFromDocument(doc)));
}
if (requestCharge >= requestChargeThreshold) {
logger.info("Dead blobs query partition {} endTime {} request charge {} for {} records", partitionPath, new Date(endTime), requestCharge, deadBlobsList.size());
}
return deadBlobsList;
} catch (RuntimeException rex) {
if (rex.getCause() instanceof DocumentClientException) {
logger.warn("Dead blobs query {} partition {} got {}", deadBlobsQuery, partitionPath, ((DocumentClientException) rex.getCause()).getStatusCode());
throw (DocumentClientException) rex.getCause();
}
throw rex;
}
}
use of com.microsoft.azure.cosmosdb.FeedOptions in project ambry by linkedin.
the class CosmosDataAccessor method getDeprecatedContainers.
/**
* Fetch a {@link Set} of {@link CosmosContainerDeletionEntry} objects from cosmos db that are not marked as deleted.
* @param maxEntries Max number of entries to fetch on one query.
* @return {@link Set} of {@link CosmosContainerDeletionEntry} objects.
* @throws DocumentClientException in case of any error.
*/
public Set<CosmosContainerDeletionEntry> getDeprecatedContainers(int maxEntries) throws DocumentClientException {
SqlQuerySpec querySpec = new SqlQuerySpec(DEPRECATED_CONTAINERS_QUERY, new SqlParameterCollection(new SqlParameter(LIMIT_PARAM, maxEntries)));
Timer timer = new Timer();
Set<CosmosContainerDeletionEntry> containerDeletionEntries = new HashSet<>();
try {
Iterator<FeedResponse<Document>> iterator = executeCosmosQuery(cosmosDeletedContainerCollectionLink, null, querySpec, new FeedOptions(), timer).getIterator();
while (iterator.hasNext()) {
FeedResponse<Document> response = iterator.next();
response.getResults().iterator().forEachRemaining(doc -> containerDeletionEntries.add(CosmosContainerDeletionEntry.fromJson(new JSONObject(doc.toJson()))));
}
} catch (RuntimeException rex) {
if (rex.getCause() instanceof DocumentClientException) {
logger.warn("Get deprecated containers query {} got {}", querySpec.getQueryText(), ((DocumentClientException) rex.getCause()).getStatusCode());
throw (DocumentClientException) rex.getCause();
}
throw rex;
}
return containerDeletionEntries;
}
use of com.microsoft.azure.cosmosdb.FeedOptions in project ambry by linkedin.
the class CosmosDataAccessor method getContainerBlobs.
/**
* Get the list of blobs in the specified partition that belong to the specified container.
* @param partitionPath the partition to query.
* @param accountId account id of the container.
* @param containerId container id of the container.
* @param queryLimit max number of blobs to return.
* @return a List of {@link CloudBlobMetadata} referencing the blobs belonging to the deprecated containers.
* @throws DocumentClientException in case of any error.
*/
List<CloudBlobMetadata> getContainerBlobs(String partitionPath, short accountId, short containerId, int queryLimit) throws DocumentClientException {
SqlQuerySpec querySpec = new SqlQuerySpec(CONTAINER_BLOBS_QUERY, new SqlParameterCollection(new SqlParameter(LIMIT_PARAM, queryLimit), new SqlParameter(CONTAINER_ID_PARAM, containerId), new SqlParameter(ACCOUNT_ID_PARAM, accountId)));
FeedOptions feedOptions = new FeedOptions();
feedOptions.setMaxItemCount(queryLimit);
feedOptions.setResponseContinuationTokenLimitInKb(continuationTokenLimitKb);
feedOptions.setPartitionKey(new PartitionKey(partitionPath));
try {
Iterator<FeedResponse<Document>> iterator = executeCosmosQuery(partitionPath, querySpec, feedOptions, azureMetrics.deletedContainerBlobsQueryTime).getIterator();
List<CloudBlobMetadata> containerBlobsList = new ArrayList<>();
double requestCharge = 0.0;
while (iterator.hasNext()) {
FeedResponse<Document> response = iterator.next();
requestCharge += response.getRequestCharge();
response.getResults().iterator().forEachRemaining(doc -> containerBlobsList.add(createMetadataFromDocument(doc)));
}
if (requestCharge >= requestChargeThreshold) {
logger.info("Deleted container blobs query partition {} containerId {} accountId {} request charge {} for {} records", partitionPath, containerId, accountId, requestCharge, containerBlobsList.size());
}
return containerBlobsList;
} catch (RuntimeException rex) {
if (rex.getCause() instanceof DocumentClientException) {
logger.warn("Dead blobs query {} partition {} got {}", querySpec.getQueryText(), partitionPath, ((DocumentClientException) rex.getCause()).getStatusCode());
throw (DocumentClientException) rex.getCause();
}
throw rex;
}
}
use of com.microsoft.azure.cosmosdb.FeedOptions in project ambry by linkedin.
the class CosmosDataAccessor method queryMetadata.
/**
* Get the list of blobs in the specified partition matching the specified DocumentDB query spec.
* @param partitionPath the partition to query.
* @param querySpec the DocumentDB {@link SqlQuerySpec} to execute.
* @param timer the {@link Timer} to use to record query time (excluding waiting).
* @return a List of {@link CloudBlobMetadata} referencing the matching blobs.
*/
List<CloudBlobMetadata> queryMetadata(String partitionPath, SqlQuerySpec querySpec, Timer timer) throws DocumentClientException {
FeedOptions feedOptions = new FeedOptions();
// TODO: set maxItemCount
feedOptions.setResponseContinuationTokenLimitInKb(continuationTokenLimitKb);
feedOptions.setPartitionKey(new PartitionKey(partitionPath));
// TODO: consolidate error count here
try {
Iterator<FeedResponse<Document>> iterator = executeCosmosQuery(partitionPath, querySpec, feedOptions, timer).getIterator();
List<CloudBlobMetadata> metadataList = new ArrayList<>();
double requestCharge = 0.0;
while (iterator.hasNext()) {
FeedResponse<Document> response = iterator.next();
requestCharge += response.getRequestCharge();
response.getResults().iterator().forEachRemaining(doc -> metadataList.add(createMetadataFromDocument(doc)));
}
if (requestCharge >= requestChargeThreshold) {
logger.info("Query partition {} request charge {} for {} records", partitionPath, requestCharge, metadataList.size());
}
return metadataList;
} catch (RuntimeException rex) {
if (rex.getCause() instanceof DocumentClientException) {
logger.warn("Query {} on partition {} got {}", querySpec.getQueryText(), partitionPath, ((DocumentClientException) rex.getCause()).getStatusCode());
throw (DocumentClientException) rex.getCause();
}
throw rex;
}
}
use of com.microsoft.azure.cosmosdb.FeedOptions in project ambry by linkedin.
the class CloudTestUtil method cleanupPartition.
/**
* Cleanup the specified partition in azure by deleting all the blobs of the partition.
* @param azureCloudConfig Properties containing the credentials needed for connection to azure.
* @param partitionId partition to be deleted.
*/
static void cleanupPartition(AzureCloudConfig azureCloudConfig, PartitionId partitionId) {
ConnectionPolicy connectionPolicy = new ConnectionPolicy();
AsyncDocumentClient asyncDocumentClient = new AsyncDocumentClient.Builder().withServiceEndpoint(azureCloudConfig.cosmosEndpoint).withMasterKeyOrResourceToken(azureCloudConfig.cosmosKey).withConnectionPolicy(connectionPolicy).withConsistencyLevel(ConsistencyLevel.Session).build();
SqlQuerySpec sqlQuerySpec = new SqlQuerySpec("select * from c where c.partitionId=\"" + partitionId.toPathString() + "\"");
FeedOptions feedOptions = new FeedOptions();
feedOptions.setPartitionKey(new PartitionKey(partitionId.toPathString()));
Iterator<FeedResponse<Document>> iterator = asyncDocumentClient.queryDocuments(azureCloudConfig.cosmosCollectionLink, sqlQuerySpec, feedOptions).toBlocking().getIterator();
RequestOptions requestOptions = new RequestOptions();
requestOptions.setPartitionKey(new PartitionKey(partitionId.toPathString()));
while (iterator.hasNext()) {
FeedResponse<Document> response = iterator.next();
response.getResults().forEach(document -> asyncDocumentClient.deleteDocument(azureCloudConfig.cosmosCollectionLink + "/docs/" + document.getId(), requestOptions).toBlocking().single());
}
}
Aggregations