Examples with MetadataConflictException io.cdap.cdap.common.metadata.MetadataConflictException used on opensource projects

Example 1 with MetadataConflictException

use of io.cdap.cdap.common.metadata.MetadataConflictException in project cdap by caskdata.

the class ElasticsearchMetadataStorage method batch.

@Override
public List<MetadataChange> batch(List<? extends MetadataMutation> mutations, MutationOptions options) throws IOException {
    if (mutations.isEmpty()) {
        return Collections.emptyList();
    }
    if (mutations.size() == 1) {
        return Collections.singletonList(apply(mutations.get(0), options));
    }
    // first detect whether there are duplicate entity ids. If so, execute in sequence
    Set<MetadataEntity> entities = new HashSet<>();
    LinkedHashMap<MetadataEntity, MetadataMutation> mutationMap = new LinkedHashMap<>(mutations.size());
    boolean duplicate = false;
    for (MetadataMutation mutation : mutations) {
        if (!entities.add(mutation.getEntity())) {
            duplicate = true;
            break;
        }
        mutationMap.put(mutation.getEntity(), mutation);
    }
    if (duplicate) {
        // if there are multiple mutations for the same entity, execute all in sequence
        List<MetadataChange> changes = new ArrayList<>(mutations.size());
        for (MetadataMutation mutation : mutations) {
            changes.add(apply(mutation, options));
        }
        return changes;
    }
    // collect all changes in an order-preserving map. The first time doBatch() is called, it will
    // enter all entities in the map. Every time it is retried, the change may get updated, but that
    // will not change the order of the map.
    LinkedHashMap<MetadataEntity, MetadataChange> changes = new LinkedHashMap<>(mutations.size());
    try {
        // repeatedly try to read current metadata, apply the mutations and reindex, until there is no conflict
        return Retries.callWithRetries(() -> doBatch(mutationMap, changes, options), RetryStrategies.limit(50, RetryStrategies.fixDelay(100, TimeUnit.MILLISECONDS)), e -> e instanceof MetadataConflictException);
    } catch (MetadataConflictException e) {
        throw new MetadataConflictException("After retries: " + e.getRawMessage(), e.getConflictingEntities());
    }
}

Example 2 with MetadataConflictException

use of io.cdap.cdap.common.metadata.MetadataConflictException in project cdap by caskdata.

the class ElasticsearchMetadataStorage method executeMutation.

/**
 * Executes an ElasticSearch request to modify a document (index or delete), and handles possible failure.
 *
 * @throws MetadataConflictException if a conflict occurs
 * @throws IOException for any other problem encountered
 */
private void executeMutation(MetadataEntity entity, WriteRequest<?> writeRequest, MutationOptions options) throws IOException {
    String requestType = null;
    DocWriteResponse response;
    setRefreshPolicy(writeRequest, options);
    RestHighLevelClient client = getClient();
    try {
        if (writeRequest instanceof DeleteRequest) {
            requestType = "Delete";
            response = client.delete((DeleteRequest) writeRequest, RequestOptions.DEFAULT);
        } else if (writeRequest instanceof IndexRequest) {
            requestType = "Index";
            response = client.index((IndexRequest) writeRequest, RequestOptions.DEFAULT);
        } else {
            throw new IllegalStateException("Unexpected WriteRequest of type " + writeRequest.getClass().getName());
        }
    } catch (ElasticsearchStatusException e) {
        if (isConflict(e.status())) {
            LOG.debug("Encountered conflict in {} request for entity {}", requestType, entity);
            throw new MetadataConflictException(requestType + " conflict for ${conflicting}", entity);
        }
        throw e;
    }
    if (isNotFound(response.status())) {
        // ignore entities that do not exist - only happens for deletes
        return;
    }
    if (isConflict(response.status())) {
        LOG.debug("Encountered conflict in {} request for entity {}", requestType, entity);
        throw new MetadataConflictException(requestType + " conflict for ${conflicting}", entity);
    }
    if (isFailure(response)) {
        throw new IOException(String.format(requestType + " request unsuccessful for entity %s: %s", entity, response));
    }
}

Example 3 with MetadataConflictException

use of io.cdap.cdap.common.metadata.MetadataConflictException in project cdap by caskdata.

the class ElasticsearchMetadataStorage method executeBulk.

/**
 * Executes a bulk request and handles the responses for possible failures, and removes all successful
 * mutations from the mutations map.
 *
 * @param mutations the mutations represented by this bulk. Every mutation that is successfully executed
 *                  is removed from this map, even if an exception is thrown.
 *
 * @throws MetadataConflictException if a conflict occurs for any of the operations in the bulk
 * @throws IOException for any other problem encountered
 */
private void executeBulk(BulkRequest bulkRequest, LinkedHashMap<MetadataEntity, MetadataMutation> mutations) throws IOException {
    RestHighLevelClient client = getClient();
    BulkResponse response = client.bulk(bulkRequest, RequestOptions.DEFAULT);
    if (response.hasFailures()) {
        IOException ioe = null;
        List<MetadataEntity> conflictEntities = new ArrayList<>();
        for (BulkItemResponse itemResponse : response) {
            MetadataEntity entityId;
            try {
                entityId = toMetadataEntity(itemResponse.getId());
            } catch (Exception e) {
                LOG.warn("Cannot parse entity id from document id {} in bulk response", itemResponse.getId());
                continue;
            }
            if (!itemResponse.isFailed()) {
                // remove this mutation - it was successful
                mutations.remove(entityId);
                continue;
            }
            if (isNotFound(itemResponse.status())) {
                // only happens for deletes - we consider this successful
                mutations.remove(entityId);
                continue;
            }
            // this mutation failed
            BulkItemResponse.Failure failure = itemResponse.getFailure();
            if (isConflict(failure.getStatus())) {
                conflictEntities.add(entityId);
                continue;
            }
            // not a conflict -> true failure
            if (ioe == null) {
                ioe = new IOException("Bulk request unsuccessful");
            }
            ioe.addSuppressed(new IOException(String.format("%s request unsuccessful for entity %s: %s", itemResponse.getOpType(), entityId, failure.getMessage())));
        }
        if (ioe != null) {
            throw ioe;
        }
        if (!conflictEntities.isEmpty()) {
            LOG.debug("Encountered conflicts in batch mutation for entities {}", conflictEntities);
            throw new MetadataConflictException("Bulk request conflicts for entities ${conflicting}", conflictEntities);
        }
    }
}

Example 4 with MetadataConflictException

use of io.cdap.cdap.common.metadata.MetadataConflictException in project cdap by caskdata.

the class ElasticsearchMetadataStorage method apply.

@Override
public MetadataChange apply(MetadataMutation mutation, MutationOptions options) throws IOException {
    MetadataEntity entity = mutation.getEntity();
    try {
        // repeatedly try to read current metadata, apply the mutation and reindex, until there is no conflict
        return Retries.callWithRetries(() -> {
            VersionedMetadata before = readFromIndex(entity);
            RequestAndChange intermediary = applyMutation(before, mutation);
            executeMutation(mutation.getEntity(), intermediary.getRequest(), options);
            return intermediary.getChange();
        }, retryStrategyOnConflict, e -> e instanceof MetadataConflictException);
    } catch (MetadataConflictException e) {
        throw new MetadataConflictException("After retries: " + e.getRawMessage(), e.getConflictingEntities());
    }
}