Examples with VersionedObjectList org.apache.geode.internal.cache.tier.sockets.VersionedObjectList used on opensource projects

Example 16 with VersionedObjectList

use of org.apache.geode.internal.cache.tier.sockets.VersionedObjectList in project geode by apache.

the class PartitionedRegion method tryToSendOnePutAllMessage.

public VersionedObjectList tryToSendOnePutAllMessage(PutAllPRMessage prMsg, InternalDistributedMember currentTarget) throws DataLocationException {
    boolean putResult = false;
    VersionedObjectList versions = null;
    final boolean isLocal = (this.localMaxMemory > 0) && currentTarget.equals(getMyId());
    if (isLocal) {
        // local
        // It might throw retry exception when one key failed
        // InternalDS has to be set for each msg
        prMsg.initMessage(this, null, false, null);
        putResult = prMsg.doLocalPutAll(this, this.getDistributionManager().getDistributionManagerId(), 0L);
        versions = prMsg.getVersions();
    } else {
        PutAllPRMessage.PutAllResponse response = (PutAllPRMessage.PutAllResponse) prMsg.send(currentTarget, this);
        PutAllPRMessage.PutAllResult pr = null;
        if (response != null) {
            this.prStats.incPartitionMessagesSent();
            try {
                pr = response.waitForResult();
                putResult = pr.returnValue;
                versions = pr.versions;
            } catch (RegionDestroyedException rde) {
                if (logger.isDebugEnabled()) {
                    logger.debug("prMsg.send: caught RegionDestroyedException", rde);
                }
                throw new RegionDestroyedException(toString(), getFullPath());
            } catch (CacheException ce) {
                // Fix for bug 36014
                throw new PartitionedRegionDistributionException("prMsg.send on " + currentTarget + " failed", ce);
            }
        } else {
            // follow the same behavior of putRemotely()
            putResult = true;
        }
    }
    if (!putResult) {
        // retry exception when msg failed in waitForResult()
        ForceReattemptException fre = new ForceReattemptException("false result in PutAllMessage.send - retrying");
        fre.setHash(0);
        throw fre;
    }
    return versions;
}

Example 17 with VersionedObjectList

use of org.apache.geode.internal.cache.tier.sockets.VersionedObjectList in project geode by apache.

the class GetAll70 method fillAndSendGetAllResponseChunks.

private void fillAndSendGetAllResponseChunks(Region region, String regionName, Object[] keys, ServerConnection servConn, boolean requestSerializedValues) throws IOException {
    // Interpret null keys object as a request to get all key,value entry pairs
    // of the region; otherwise iterate each key and perform the get behavior.
    Iterator allKeysIter;
    int numKeys;
    if (keys != null) {
        allKeysIter = null;
        numKeys = keys.length;
    } else {
        Set allKeys = region.keySet();
        allKeysIter = allKeys.iterator();
        numKeys = allKeys.size();
    }
    // Shouldn't it be 'keys != null' below?
    // The answer is no.
    // Note that the current implementation of client/server getAll the "keys" will always be
    // non-null.
    // The server callects and returns the values in the same order as the keys it received.
    // So the server does not need to send the keys back to the client.
    // When the client receives the server's "values" it calls setKeys using the key list the client
    // already has.
    // So the only reason we would tell the VersionedObjectList that it needs to track keys is if we
    // are running
    // in the old mode (which may be impossible since we only used that mode pre 7.0) in which the
    // client told us
    // to get and return all the keys and values. I think this was used for register interest.
    VersionedObjectList values = new VersionedObjectList(MAXIMUM_CHUNK_SIZE, keys == null, region.getAttributes().getConcurrencyChecksEnabled(), requestSerializedValues);
    try {
        AuthorizeRequest authzRequest = servConn.getAuthzRequest();
        AuthorizeRequestPP postAuthzRequest = servConn.getPostAuthzRequest();
        Get70 request = (Get70) Get70.getCommand();
        final boolean isDebugEnabled = logger.isDebugEnabled();
        for (int i = 0; i < numKeys; i++) {
            // Send the intermediate chunk if necessary
            if (values.size() == MAXIMUM_CHUNK_SIZE) {
                // Send the chunk and clear the list
                values.setKeys(null);
                sendGetAllResponseChunk(region, values, false, servConn);
                values.clear();
            }
            Object key;
            boolean keyNotPresent = false;
            if (keys != null) {
                key = keys[i];
            } else {
                key = allKeysIter.next();
            }
            if (isDebugEnabled) {
                logger.debug("{}: Getting value for key={}", servConn.getName(), key);
            }
            // Determine if the user authorized to get this key
            GetOperationContext getContext = null;
            if (authzRequest != null) {
                try {
                    getContext = authzRequest.getAuthorize(regionName, key, null);
                    if (isDebugEnabled) {
                        logger.debug("{}: Passed GET pre-authorization for key={}", servConn.getName(), key);
                    }
                } catch (NotAuthorizedException ex) {
                    logger.warn(LocalizedMessage.create(LocalizedStrings.GetAll_0_CAUGHT_THE_FOLLOWING_EXCEPTION_ATTEMPTING_TO_GET_VALUE_FOR_KEY_1, new Object[] { servConn.getName(), key }), ex);
                    values.addExceptionPart(key, ex);
                    continue;
                }
            }
            try {
                this.securityService.authorizeRegionRead(regionName, key.toString());
            } catch (NotAuthorizedException ex) {
                logger.warn(LocalizedMessage.create(LocalizedStrings.GetAll_0_CAUGHT_THE_FOLLOWING_EXCEPTION_ATTEMPTING_TO_GET_VALUE_FOR_KEY_1, new Object[] { servConn.getName(), key }), ex);
                values.addExceptionPart(key, ex);
                continue;
            }
            // Get the value and update the statistics. Do not deserialize
            // the value if it is a byte[].
            // Getting a value in serialized form is pretty nasty. I split this out
            // so the logic can be re-used by the CacheClientProxy.
            Get70.Entry entry = request.getEntry(region, key, null, servConn);
            @Retained final Object originalData = entry.value;
            Object data = originalData;
            if (logger.isDebugEnabled()) {
                logger.debug("retrieved key={} {}", key, entry);
            }
            boolean addedToValues = false;
            try {
                boolean isObject = entry.isObject;
                VersionTag versionTag = entry.versionTag;
                keyNotPresent = entry.keyNotPresent;
                if (postAuthzRequest != null) {
                    try {
                        getContext = postAuthzRequest.getAuthorize(regionName, key, data, isObject, getContext);
                        GetOperationContextImpl gci = (GetOperationContextImpl) getContext;
                        Object newData = gci.getRawValue();
                        if (newData != data) {
                            // user changed the value
                            isObject = getContext.isObject();
                            data = newData;
                        }
                    } catch (NotAuthorizedException ex) {
                        logger.warn(LocalizedMessage.create(LocalizedStrings.GetAll_0_CAUGHT_THE_FOLLOWING_EXCEPTION_ATTEMPTING_TO_GET_VALUE_FOR_KEY_1, new Object[] { servConn.getName(), key }), ex);
                        values.addExceptionPart(key, ex);
                        continue;
                    } finally {
                        if (getContext != null) {
                            ((GetOperationContextImpl) getContext).release();
                        }
                    }
                }
                data = this.securityService.postProcess(regionName, key, data, entry.isObject);
                // Add the entry to the list that will be returned to the client
                if (keyNotPresent) {
                    values.addObjectPartForAbsentKey(key, data, versionTag);
                    addedToValues = true;
                } else {
                    values.addObjectPart(key, data, isObject, versionTag);
                    addedToValues = true;
                }
            } finally {
                if (!addedToValues || data != originalData) {
                    OffHeapHelper.release(originalData);
                }
            }
        }
        // Send the last chunk even if the list is of zero size.
        if (Version.GFE_701.compareTo(servConn.getClientVersion()) <= 0) {
            // 7.0.1 and later clients do not expect the keys in the response
            values.setKeys(null);
        }
        sendGetAllResponseChunk(region, values, true, servConn);
        servConn.setAsTrue(RESPONDED);
    } finally {
        values.release();
    }
}

Example 18 with VersionedObjectList

use of org.apache.geode.internal.cache.tier.sockets.VersionedObjectList in project geode by apache.

the class GetAllWithCallback method fillAndSendGetAllResponseChunks.

private void fillAndSendGetAllResponseChunks(Region region, String regionName, Object[] keys, ServerConnection servConn, Object callback) throws IOException {
    assert keys != null;
    int numKeys = keys.length;
    VersionedObjectList values = new VersionedObjectList(MAXIMUM_CHUNK_SIZE, false, region.getAttributes().getConcurrencyChecksEnabled(), false);
    try {
        AuthorizeRequest authzRequest = servConn.getAuthzRequest();
        AuthorizeRequestPP postAuthzRequest = servConn.getPostAuthzRequest();
        Get70 request = (Get70) Get70.getCommand();
        for (int i = 0; i < numKeys; i++) {
            // Send the intermediate chunk if necessary
            if (values.size() == MAXIMUM_CHUNK_SIZE) {
                // Send the chunk and clear the list
                sendGetAllResponseChunk(region, values, false, servConn);
                values.clear();
            }
            Object key;
            boolean keyNotPresent = false;
            key = keys[i];
            if (logger.isDebugEnabled()) {
                logger.debug("{}: Getting value for key={}", servConn.getName(), key);
            }
            // Determine if the user authorized to get this key
            GetOperationContext getContext = null;
            if (authzRequest != null) {
                try {
                    getContext = authzRequest.getAuthorize(regionName, key, callback);
                    if (logger.isDebugEnabled()) {
                        logger.debug("{}: Passed GET pre-authorization for key={}", servConn.getName(), key);
                    }
                } catch (NotAuthorizedException ex) {
                    logger.warn(LocalizedMessage.create(LocalizedStrings.GetAll_0_CAUGHT_THE_FOLLOWING_EXCEPTION_ATTEMPTING_TO_GET_VALUE_FOR_KEY_1, new Object[] { servConn.getName(), key }), ex);
                    values.addExceptionPart(key, ex);
                    continue;
                }
            }
            try {
                this.securityService.authorizeRegionRead(regionName, key.toString());
            } catch (NotAuthorizedException ex) {
                logger.warn(LocalizedMessage.create(LocalizedStrings.GetAll_0_CAUGHT_THE_FOLLOWING_EXCEPTION_ATTEMPTING_TO_GET_VALUE_FOR_KEY_1, new Object[] { servConn.getName(), key }), ex);
                values.addExceptionPart(key, ex);
                continue;
            }
            // Get the value and update the statistics. Do not deserialize
            // the value if it is a byte[].
            // Getting a value in serialized form is pretty nasty. I split this out
            // so the logic can be re-used by the CacheClientProxy.
            Get70.Entry entry = request.getEntry(region, key, callback, servConn);
            @Retained final Object originalData = entry.value;
            Object data = originalData;
            if (logger.isDebugEnabled()) {
                logger.debug("retrieved key={} {}", key, entry);
            }
            boolean addedToValues = false;
            try {
                boolean isObject = entry.isObject;
                VersionTag versionTag = entry.versionTag;
                keyNotPresent = entry.keyNotPresent;
                if (postAuthzRequest != null) {
                    try {
                        getContext = postAuthzRequest.getAuthorize(regionName, key, data, isObject, getContext);
                        GetOperationContextImpl gci = (GetOperationContextImpl) getContext;
                        Object newData = gci.getRawValue();
                        if (newData != data) {
                            // user changed the value
                            isObject = getContext.isObject();
                            data = newData;
                        }
                    } catch (NotAuthorizedException ex) {
                        logger.warn(LocalizedMessage.create(LocalizedStrings.GetAll_0_CAUGHT_THE_FOLLOWING_EXCEPTION_ATTEMPTING_TO_GET_VALUE_FOR_KEY_1, new Object[] { servConn.getName(), key }), ex);
                        values.addExceptionPart(key, ex);
                        continue;
                    } finally {
                        if (getContext != null) {
                            ((GetOperationContextImpl) getContext).release();
                        }
                    }
                }
                // Add the entry to the list that will be returned to the client
                if (keyNotPresent) {
                    values.addObjectPartForAbsentKey(key, data, versionTag);
                    addedToValues = true;
                } else {
                    values.addObjectPart(key, data, isObject, versionTag);
                    addedToValues = true;
                }
            } finally {
                if (!addedToValues || data != originalData) {
                    OffHeapHelper.release(originalData);
                }
            }
        }
        // Send the last chunk even if the list is of zero size.
        sendGetAllResponseChunk(region, values, true, servConn);
        servConn.setAsTrue(RESPONDED);
    } finally {
        values.release();
    }
}

Example 19 with VersionedObjectList

use of org.apache.geode.internal.cache.tier.sockets.VersionedObjectList in project geode by apache.

the class RemoveAllOp method execute.

/**
   * Does a region put on a server using connections from the given pool to communicate with the
   * server.
   * 
   * @param pool the pool to use to communicate with the server.
   * @param region the name of the region to do the removeAll on
   * @param keys the Collection of keys to remove
   * @param eventId the event id for this removeAll
   */
public static VersionedObjectList execute(ExecutablePool pool, Region region, Collection<Object> keys, EventID eventId, int retryAttempts, Object callbackArg) {
    final boolean isDebugEnabled = logger.isDebugEnabled();
    ClientMetadataService cms = ((LocalRegion) region).getCache().getClientMetadataService();
    Map<ServerLocation, HashSet> serverToFilterMap = cms.getServerToFilterMap(keys, region, true);
    if (serverToFilterMap == null || serverToFilterMap.isEmpty()) {
        AbstractOp op = new RemoveAllOpImpl(region, keys, eventId, ((PoolImpl) pool).getPRSingleHopEnabled(), callbackArg);
        op.initMessagePart();
        return (VersionedObjectList) pool.execute(op);
    }
    List callableTasks = constructAndGetRemoveAllTasks(region, eventId, serverToFilterMap, (PoolImpl) pool, callbackArg);
    if (isDebugEnabled) {
        logger.debug("RemoveAllOp#execute : Number of removeAll tasks is :{}", callableTasks.size());
    }
    HashMap<ServerLocation, RuntimeException> failedServers = new HashMap<ServerLocation, RuntimeException>();
    PutAllPartialResult result = new PutAllPartialResult(keys.size());
    try {
        Map<ServerLocation, Object> results = SingleHopClientExecutor.submitBulkOp(callableTasks, cms, (LocalRegion) region, failedServers);
        for (Map.Entry<ServerLocation, Object> entry : results.entrySet()) {
            Object value = entry.getValue();
            if (value instanceof PutAllPartialResultException) {
                PutAllPartialResultException pap = (PutAllPartialResultException) value;
                if (isDebugEnabled) {
                    logger.debug("RemoveAll SingleHop encountered BulkOpPartialResultException exception: {}, failedServers are {}", pap, failedServers.keySet());
                }
                result.consolidate(pap.getResult());
            } else {
                if (value != null) {
                    VersionedObjectList list = (VersionedObjectList) value;
                    result.addKeysAndVersions(list);
                }
            }
        }
    } catch (RuntimeException ex) {
        logger.debug("single-hop removeAll encountered unexpected exception: {}", ex);
        throw ex;
    }
    if (!failedServers.isEmpty()) {
        if (retryAttempts == 0) {
            throw failedServers.values().iterator().next();
        }
        // add them to the partial result set
        if (result.getSucceededKeysAndVersions().size() == 0) {
            // if there're failed servers, we need to save the succeed keys in submitRemoveAll
            // if retry succeeded, everything is ok, otherwise, the saved "succeeded
            // keys" should be consolidated into PutAllPartialResultException
            // succeedKeySet is used to send back to client in PartialResult case
            // so it's not a must to use LinkedHashSet
            Set succeedKeySet = new LinkedHashSet();
            Set<ServerLocation> serverSet = serverToFilterMap.keySet();
            for (ServerLocation server : serverSet) {
                if (!failedServers.containsKey(server)) {
                    succeedKeySet.addAll(serverToFilterMap.get(server));
                }
            }
            // save succeedKeys, but if retries all succeeded, discard the PutAllPartialResult
            result.addKeys(succeedKeySet);
        }
        // send maps for the failed servers one by one instead of merging
        // them into one big map. The reason is, we have to keep the same event
        // ids for each sub map. There is a unit test in PutAllCSDUnitTest for
        // the otherwise case.
        boolean oneSubMapRetryFailed = false;
        Set<ServerLocation> failedServerSet = failedServers.keySet();
        for (ServerLocation failedServer : failedServerSet) {
            // Throwable failedServers.values().iterator().next();
            RuntimeException savedRTE = failedServers.get(failedServer);
            if (savedRTE instanceof PutAllPartialResultException) {
                // will not retry for BulkOpPartialResultException
                // but it means at least one sub map ever failed
                oneSubMapRetryFailed = true;
                continue;
            }
            Collection<Object> newKeys = serverToFilterMap.get(failedServer);
            try {
                VersionedObjectList v = RemoveAllOp.execute(pool, region, newKeys, eventId, true, callbackArg);
                if (v == null) {
                    result.addKeys(newKeys);
                } else {
                    result.addKeysAndVersions(v);
                }
            } catch (PutAllPartialResultException pre) {
                oneSubMapRetryFailed = true;
                logger.debug("Retry failed with BulkOpPartialResultException: {} Before retry: {}", pre, result.getKeyListString());
                result.consolidate(pre.getResult());
            } catch (Exception rte) {
                oneSubMapRetryFailed = true;
                Object firstKey = newKeys.iterator().next();
                result.saveFailedKey(firstKey, rte);
            }
        }
        // If all retries succeeded, the PRE in first tries can be ignored
        if (oneSubMapRetryFailed && result.hasFailure()) {
            PutAllPartialResultException pre = new PutAllPartialResultException(result);
            throw pre;
        }
    }
    return result.getSucceededKeysAndVersions();
}

Example 20 with VersionedObjectList

use of org.apache.geode.internal.cache.tier.sockets.VersionedObjectList in project geode by apache.

the class PartitionedRegion method postPutAllSend.

/**
   * Create PutAllPRMsgs for each bucket, and send them.
   * 
   * @param putAllOp DistributedPutAllOperation object.
   * @param successfulPuts not used in PartitionedRegion.
   */
@Override
public long postPutAllSend(DistributedPutAllOperation putAllOp, VersionedObjectList successfulPuts) {
    final boolean isDebugEnabled = logger.isDebugEnabled();
    if (cache.isCacheAtShutdownAll()) {
        throw new CacheClosedException("Cache is shutting down");
    }
    final long startTime = PartitionedRegionStats.startTime();
    // build all the msgs by bucketid
    HashMap prMsgMap = putAllOp.createPRMessages();
    PutAllPartialResult partialKeys = new PutAllPartialResult(putAllOp.putAllDataSize);
    // clear the successfulPuts list since we're actually doing the puts here
    // and the basicPutAll work was just a way to build the DPAO object
    Map<Object, VersionTag> keyToVersionMap = new HashMap<Object, VersionTag>(successfulPuts.size());
    successfulPuts.clearVersions();
    Iterator itor = prMsgMap.entrySet().iterator();
    while (itor.hasNext()) {
        Map.Entry mapEntry = (Map.Entry) itor.next();
        Integer bucketId = (Integer) mapEntry.getKey();
        PutAllPRMessage prMsg = (PutAllPRMessage) mapEntry.getValue();
        checkReadiness();
        long then = 0;
        if (isDebugEnabled) {
            then = System.currentTimeMillis();
        }
        try {
            VersionedObjectList versions = sendMsgByBucket(bucketId, prMsg);
            if (versions.size() > 0) {
                partialKeys.addKeysAndVersions(versions);
                versions.saveVersions(keyToVersionMap);
            } else if (!this.concurrencyChecksEnabled) {
                // no keys returned if not versioned
                Set keys = prMsg.getKeys();
                partialKeys.addKeys(keys);
            }
        } catch (PutAllPartialResultException pre) {
            // sendMsgByBucket applied partial keys
            if (isDebugEnabled) {
                logger.debug("PR.postPutAll encountered PutAllPartialResultException, ", pre);
            }
            partialKeys.consolidate(pre.getResult());
        } catch (Exception ex) {
            // If failed at other exception
            if (isDebugEnabled) {
                logger.debug("PR.postPutAll encountered exception at sendMsgByBucket, ", ex);
            }
            @Released EntryEventImpl firstEvent = prMsg.getFirstEvent(this);
            try {
                partialKeys.saveFailedKey(firstEvent.getKey(), ex);
            } finally {
                firstEvent.release();
            }
        }
        if (isDebugEnabled) {
            long now = System.currentTimeMillis();
            if ((now - then) >= 10000) {
                logger.debug("PR.sendMsgByBucket took " + (now - then) + " ms");
            }
        }
    }
    this.prStats.endPutAll(startTime);
    if (!keyToVersionMap.isEmpty()) {
        for (Iterator it = successfulPuts.getKeys().iterator(); it.hasNext(); ) {
            successfulPuts.addVersion(keyToVersionMap.get(it.next()));
        }
        keyToVersionMap.clear();
    }
    if (partialKeys.hasFailure()) {
        logger.info(LocalizedMessage.create(LocalizedStrings.Region_PutAll_Applied_PartialKeys_0_1, new Object[] { getFullPath(), partialKeys }));
        if (putAllOp.isBridgeOperation()) {
            if (partialKeys.getFailure() instanceof CancelException) {
                throw (CancelException) partialKeys.getFailure();
            } else {
                throw new PutAllPartialResultException(partialKeys);
            }
        } else {
            if (partialKeys.getFailure() instanceof RuntimeException) {
                throw (RuntimeException) partialKeys.getFailure();
            } else {
                throw new RuntimeException(partialKeys.getFailure());
            }
        }
    }
    return -1;
}