Examples with VersionTag org.apache.geode.internal.cache.versions.VersionTag used on opensource projects

Example 81 with VersionTag

use of org.apache.geode.internal.cache.versions.VersionTag in project geode by apache.

the class BaseCommand method handleKVList.

private static void handleKVList(final LocalRegion region, final List keyList, boolean serializeValues, final ServerConnection servConn) throws IOException {
    if (region instanceof PartitionedRegion) {
        handleKVKeysPR((PartitionedRegion) region, keyList, serializeValues, servConn);
        return;
    }
    VersionedObjectList values = new VersionedObjectList(MAXIMUM_CHUNK_SIZE, true, region == null || region.getAttributes().getConcurrencyChecksEnabled(), serializeValues);
    // Handle list of keys
    if (region != null) {
        for (Object key : keyList) {
            if (region.containsKey(key) || region.containsTombstone(key)) {
                VersionTagHolder versionHolder = createVersionTagHolder();
                ClientProxyMembershipID id = servConn == null ? null : servConn.getProxyID();
                Object data = region.get(key, null, true, true, true, id, versionHolder, true);
                VersionTag versionTag = versionHolder.getVersionTag();
                updateValues(values, key, data, versionTag);
                if (values.size() == MAXIMUM_CHUNK_SIZE) {
                    // Send the chunk and clear the list
                    // values.setKeys(null); // Now we need to send keys too.
                    sendNewRegisterInterestResponseChunk(region, keyList, values, false, servConn);
                    values.clear();
                }
            }
        }
    }
    // Send the last chunk (the only chunk for individual and list keys)
    // always send it back, even if the list is of zero size.
    sendNewRegisterInterestResponseChunk(region, keyList, values, true, servConn);
}

Example 82 with VersionTag

use of org.apache.geode.internal.cache.versions.VersionTag in project geode by apache.

the class BaseCommand method handleKVSingleton.

/**
   * Handles both RR and PR cases
   */
@SuppressWarnings(value = "NP_NULL_PARAM_DEREF", justification = "Null value handled in sendNewRegisterInterestResponseChunk()")
private static void handleKVSingleton(LocalRegion region, Object entryKey, boolean serializeValues, ServerConnection servConn) throws IOException {
    VersionedObjectList values = new VersionedObjectList(MAXIMUM_CHUNK_SIZE, true, region == null || region.getAttributes().getConcurrencyChecksEnabled(), serializeValues);
    if (region != null) {
        if (region.containsKey(entryKey) || region.containsTombstone(entryKey)) {
            VersionTagHolder versionHolder = createVersionTagHolder();
            ClientProxyMembershipID id = servConn == null ? null : servConn.getProxyID();
            // From Get70.getValueAndIsObject()
            Object data = region.get(entryKey, null, true, true, true, id, versionHolder, true);
            VersionTag vt = versionHolder.getVersionTag();
            updateValues(values, entryKey, data, vt);
        }
    }
    // Send the last chunk (the only chunk for individual and list keys)
    // always send it back, even if the list is of zero size.
    sendNewRegisterInterestResponseChunk(region, entryKey, values, true, servConn);
}

Example 83 with VersionTag

use of org.apache.geode.internal.cache.versions.VersionTag in project geode by apache.

the class LocalRegion method basicPutAll.

// TODO: refactor basicPutAll
public VersionedObjectList basicPutAll(final Map<?, ?> map, final DistributedPutAllOperation putAllOp, final Map<Object, VersionTag> retryVersions) {
    final boolean isDebugEnabled = logger.isDebugEnabled();
    final EntryEventImpl event = putAllOp.getBaseEvent();
    EventID eventId = event.getEventId();
    if (eventId == null && generateEventID()) {
        // We need to "reserve" the eventIds for the entries in map here
        event.reserveNewEventId(this.cache.getDistributedSystem(), map.size());
        eventId = event.getEventId();
    }
    verifyPutAllMap(map);
    VersionedObjectList proxyResult = null;
    boolean partialResult = false;
    RuntimeException runtimeException = null;
    if (hasServerProxy()) {
        // send message to bridge server
        if (isTX()) {
            TXStateProxyImpl txState = (TXStateProxyImpl) this.cache.getTxManager().getTXState();
            txState.getRealDeal(null, this);
        }
        try {
            proxyResult = getServerProxy().putAll(map, eventId, !event.isGenerateCallbacks(), event.getCallbackArgument());
            if (isDebugEnabled) {
                logger.debug("PutAll received response from server: {}", proxyResult);
            }
        } catch (PutAllPartialResultException e) {
            // adjust the map to only add succeeded entries, then apply the adjustedMap
            proxyResult = e.getSucceededKeysAndVersions();
            partialResult = true;
            if (isDebugEnabled) {
                logger.debug("putAll in client encountered a PutAllPartialResultException:{}{}. Adjusted keys are: {}", e.getMessage(), getLineSeparator(), proxyResult.getKeys());
            }
            Throwable txException = e.getFailure();
            while (txException != null) {
                if (txException instanceof TransactionException) {
                    runtimeException = (RuntimeException) txException;
                    break;
                }
                txException = txException.getCause();
            }
            if (runtimeException == null) {
                // for cache close
                runtimeException = getCancelCriterion().generateCancelledException(e.getFailure());
                if (runtimeException == null) {
                    runtimeException = new ServerOperationException(LocalizedStrings.Region_PutAll_Applied_PartialKeys_At_Server_0.toLocalizedString(getFullPath()), e.getFailure());
                }
            }
        }
    }
    final VersionedObjectList succeeded = new VersionedObjectList(map.size(), true, this.concurrencyChecksEnabled);
    // if this is a transactional putAll, we will not have version information as it is only
    // generated at commit
    // so treat transactional putAll as if the server is not versioned
    final boolean serverIsVersioned = proxyResult != null && proxyResult.regionIsVersioned() && !isTX() && this.dataPolicy != DataPolicy.EMPTY;
    if (!serverIsVersioned && !partialResult) {
        // we don't need server information if it isn't versioned or if the region is empty
        proxyResult = null;
    }
    lockRVVForBulkOp();
    try {
        try {
            int size = proxyResult == null ? map.size() : proxyResult.size();
            if (isDebugEnabled) {
                logger.debug("size of put result is {} maps is {} proxyResult is {}", size, map, proxyResult);
            }
            final PutAllPartialResult partialKeys = new PutAllPartialResult(size);
            final Iterator iterator;
            final boolean isVersionedResults;
            if (proxyResult != null) {
                iterator = proxyResult.iterator();
                isVersionedResults = true;
            } else {
                iterator = map.entrySet().iterator();
                isVersionedResults = false;
            }
            // TODO: refactor this mess
            Runnable task = new Runnable() {

                @Override
                public void run() {
                    int offset = 0;
                    VersionTagHolder tagHolder = new VersionTagHolder();
                    while (iterator.hasNext()) {
                        stopper.checkCancelInProgress(null);
                        Map.Entry mapEntry = (Map.Entry) iterator.next();
                        Object key = mapEntry.getKey();
                        VersionTag versionTag = null;
                        tagHolder.setVersionTag(null);
                        final Object value;
                        boolean overwritten = false;
                        if (isVersionedResults) {
                            versionTag = ((VersionedObjectList.Entry) mapEntry).getVersionTag();
                            value = map.get(key);
                            if (isDebugEnabled) {
                                logger.debug("putAll key {} -> {} version={}", key, value, versionTag);
                            }
                            if (versionTag == null && serverIsVersioned && concurrencyChecksEnabled && dataPolicy.withStorage()) {
                                // entry since we don't know what its state should be (but the server should)
                                if (isDebugEnabled) {
                                    logger.debug("server returned no version information for {}", key);
                                }
                                localDestroyNoCallbacks(key);
                                // to be consistent we need to fetch the current entry
                                get(key, event.getCallbackArgument(), false, null);
                                overwritten = true;
                            }
                        } else {
                            value = mapEntry.getValue();
                            if (isDebugEnabled) {
                                logger.debug("putAll {} -> {}", key, value);
                            }
                        }
                        try {
                            if (serverIsVersioned) {
                                if (isDebugEnabled) {
                                    logger.debug("associating version tag with {} version={}", key, versionTag);
                                }
                                // If we have received a version tag from a server, add it to the event
                                tagHolder.setVersionTag(versionTag);
                                tagHolder.setFromServer(true);
                            } else if (retryVersions != null && retryVersions.containsKey(key)) {
                                // If this is a retried event, and we have a version tag for the retry,
                                // add it to the event.
                                tagHolder.setVersionTag(retryVersions.get(key));
                            }
                            if (!overwritten) {
                                basicEntryPutAll(key, value, putAllOp, offset, tagHolder);
                            }
                            // now we must check again since the cache may have closed during
                            // distribution (causing this process to not receive and queue the
                            // event for clients
                            stopper.checkCancelInProgress(null);
                            succeeded.addKeyAndVersion(key, tagHolder.getVersionTag());
                        } catch (Exception ex) {
                            if (isDebugEnabled) {
                                logger.debug("PutAll operation encountered exception for key {}", key, ex);
                            }
                            partialKeys.saveFailedKey(key, ex);
                        }
                        offset++;
                    }
                }
            };
            syncBulkOp(task, eventId);
            if (partialKeys.hasFailure()) {
                // Bug 51725: Now succeeded contains an order key list, may be missing the version tags.
                // Save reference of succeeded into partialKeys. The succeeded may be modified by
                // postPutAll() to fill in the version tags.
                partialKeys.setSucceededKeysAndVersions(succeeded);
                logger.info(LocalizedMessage.create(LocalizedStrings.Region_PutAll_Applied_PartialKeys_0_1, new Object[] { getFullPath(), partialKeys }));
                if (isDebugEnabled) {
                    logger.debug(partialKeys.detailString());
                }
                if (runtimeException == null) {
                    // if received exception from server first, ignore local exception
                    if (putAllOp.isBridgeOperation()) {
                        if (partialKeys.getFailure() instanceof CancelException) {
                            runtimeException = (RuntimeException) partialKeys.getFailure();
                        } else if (partialKeys.getFailure() instanceof LowMemoryException) {
                            // fix for #43589
                            throw partialKeys.getFailure();
                        } else {
                            runtimeException = new PutAllPartialResultException(partialKeys);
                            if (isDebugEnabled) {
                                logger.debug("basicPutAll:" + partialKeys.detailString());
                            }
                        }
                    } else {
                        throw partialKeys.getFailure();
                    }
                }
            }
        } catch (LowMemoryException lme) {
            throw lme;
        } catch (RuntimeException ex) {
            runtimeException = ex;
        } catch (Exception ex) {
            runtimeException = new RuntimeException(ex);
        } finally {
            putAllOp.getBaseEvent().release();
            putAllOp.freeOffHeapResources();
        }
        getDataView().postPutAll(putAllOp, succeeded, this);
    } finally {
        unlockRVVForBulkOp();
    }
    if (runtimeException != null) {
        throw runtimeException;
    }
    return succeeded;
}

Example 84 with VersionTag

use of org.apache.geode.internal.cache.versions.VersionTag in project geode by apache.

the class Oplog method readKrf.

private boolean readKrf(OplogEntryIdSet deletedIds, boolean recoverValues, boolean recoverValuesSync, Set<Oplog> oplogsNeedingValueRecovery, boolean latestOplog) {
    File f = new File(this.diskFile.getPath() + KRF_FILE_EXT);
    if (!f.exists()) {
        return false;
    }
    if (!getParent().getDiskInitFile().hasKrf(this.oplogId)) {
        logger.info(LocalizedMessage.create(LocalizedStrings.Oplog_REMOVING_INCOMPLETE_KRF, new Object[] { f.getName(), this.oplogId, getParent().getName() }));
        f.delete();
    }
    // Set krfCreated to true since we have a krf.
    this.krfCreated.set(true);
    // so that we don't try to recreate the krf.
    if (recoverValuesSync) {
        return false;
    }
    FileInputStream fis;
    try {
        fis = new FileInputStream(f);
    } catch (FileNotFoundException ignore) {
        return false;
    }
    try {
        if (getParent().isOffline() && !getParent().FORCE_KRF_RECOVERY) {
            return false;
        }
        logger.info(LocalizedMessage.create(LocalizedStrings.DiskRegion_RECOVERING_OPLOG_0_1_2, new Object[] { toString(), f.getAbsolutePath(), getParent().getName() }));
        this.recoverNewEntryId = DiskStoreImpl.INVALID_ID;
        this.recoverModEntryId = DiskStoreImpl.INVALID_ID;
        this.recoverModEntryIdHWM = DiskStoreImpl.INVALID_ID;
        long oplogKeyIdHWM = DiskStoreImpl.INVALID_ID;
        int krfEntryCount = 0;
        DataInputStream dis = new DataInputStream(new BufferedInputStream(fis, 1024 * 1024));
        final Version version = getProductVersionIfOld();
        final ByteArrayDataInput in = new ByteArrayDataInput();
        try {
            try {
                validateOpcode(dis, OPLOG_MAGIC_SEQ_ID);
                readOplogMagicSeqRecord(dis, f, OPLOG_TYPE.KRF);
                validateOpcode(dis, OPLOG_DISK_STORE_ID);
                readDiskStoreRecord(dis, f);
            } catch (DiskAccessException ignore) {
                // Failed to read the file. There are two possibilities. Either this
                // file is in old format which does not have a magic seq in the
                // beginning or this is not a valid file at all. Try reading it as a
                // file in old format
                fis.close();
                fis = new FileInputStream(f);
                dis = new DataInputStream(new BufferedInputStream(fis, 1024 * 1024));
                readDiskStoreRecord(dis, f);
            } catch (IllegalStateException ignore) {
                // Failed to read the file. There are two possibilities. Either this
                // is in new format which has a magic seq in the beginning or this is
                // not a valid file at all
                fis.close();
                fis = new FileInputStream(f);
                dis = new DataInputStream(new BufferedInputStream(fis, 1024 * 1024));
                readDiskStoreRecord(dis, f);
            }
            readGemfireVersionRecord(dis, f);
            readTotalCountRecord(dis, f);
            readRVVRecord(dis, f, false, latestOplog);
            long lastOffset = 0;
            byte[] keyBytes = DataSerializer.readByteArray(dis);
            while (keyBytes != null) {
                byte userBits = dis.readByte();
                int valueLength = InternalDataSerializer.readArrayLength(dis);
                byte[] valueBytes = null;
                long drId = DiskInitFile.readDiskRegionID(dis);
                DiskRecoveryStore drs = getOplogSet().getCurrentlyRecovering(drId);
                // read version
                VersionTag tag = null;
                if (EntryBits.isWithVersions(userBits)) {
                    tag = readVersionsFromOplog(dis);
                    if (drs != null && !drs.getDiskRegionView().getFlags().contains(DiskRegionFlag.IS_WITH_VERSIONING)) {
                        // 50044 Remove version tag from entry if we don't want versioning
                        // for this region
                        tag = null;
                        userBits = EntryBits.setWithVersions(userBits, false);
                    } else {
                        // Update the RVV with the new entry
                        if (drs != null) {
                            drs.recordRecoveredVersionTag(tag);
                        }
                    }
                }
                long oplogKeyId = InternalDataSerializer.readVLOld(dis);
                long oplogOffset;
                if (EntryBits.isAnyInvalid(userBits) || EntryBits.isTombstone(userBits)) {
                    oplogOffset = -1;
                } else {
                    oplogOffset = lastOffset + InternalDataSerializer.readVLOld(dis);
                    lastOffset = oplogOffset;
                }
                if (oplogKeyId > oplogKeyIdHWM) {
                    oplogKeyIdHWM = oplogKeyId;
                }
                if (okToSkipModifyRecord(deletedIds, drId, drs, oplogKeyId, true, tag).skip()) {
                    if (logger.isTraceEnabled(LogMarker.PERSIST_RECOVERY)) {
                        logger.trace(LogMarker.PERSIST_RECOVERY, "readNewEntry skipping oplogKeyId=<{}> drId={} userBits={} oplogOffset={} valueLen={}", oplogKeyId, drId, userBits, oplogOffset, valueLength);
                    }
                    this.stats.incRecoveryRecordsSkipped();
                    incSkipped();
                } else {
                    if (EntryBits.isAnyInvalid(userBits)) {
                        if (EntryBits.isInvalid(userBits)) {
                            valueBytes = DiskEntry.INVALID_BYTES;
                        } else {
                            valueBytes = DiskEntry.LOCAL_INVALID_BYTES;
                        }
                    } else if (EntryBits.isTombstone(userBits)) {
                        valueBytes = DiskEntry.TOMBSTONE_BYTES;
                    }
                    Object key = deserializeKey(keyBytes, version, in);
                    {
                        Object oldValue = getRecoveryMap().put(oplogKeyId, key);
                        if (oldValue != null) {
                            throw new AssertionError(LocalizedStrings.Oplog_DUPLICATE_CREATE.toLocalizedString(oplogKeyId));
                        }
                    }
                    DiskEntry de = drs.getDiskEntry(key);
                    if (de == null) {
                        if (logger.isTraceEnabled(LogMarker.PERSIST_RECOVERY)) {
                            logger.trace(LogMarker.PERSIST_RECOVERY, "readNewEntry oplogKeyId=<{}> drId={} userBits={} oplogOffset={} valueLen={}", oplogKeyId, drId, userBits, oplogOffset, valueLength);
                        }
                        DiskEntry.RecoveredEntry re = createRecoveredEntry(valueBytes, valueLength, userBits, getOplogId(), oplogOffset, oplogKeyId, false, version, in);
                        if (tag != null) {
                            re.setVersionTag(tag);
                        }
                        initRecoveredEntry(drs.getDiskRegionView(), drs.initializeRecoveredEntry(key, re));
                        drs.getDiskRegionView().incRecoveredEntryCount();
                        this.stats.incRecoveredEntryCreates();
                        krfEntryCount++;
                    } else {
                        DiskId curdid = de.getDiskId();
                        // assert curdid.getOplogId() != getOplogId();
                        if (logger.isTraceEnabled(LogMarker.PERSIST_RECOVERY)) {
                            logger.trace(LogMarker.PERSIST_RECOVERY, "ignore readNewEntry because getOplogId()={} != curdid.getOplogId()={} for drId={} key={}", getOplogId(), curdid.getOplogId(), drId, key);
                        }
                    }
                }
                keyBytes = DataSerializer.readByteArray(dis);
            }
            // while
            setRecoverNewEntryId(oplogKeyIdHWM);
        } catch (IOException ex) {
            try {
                fis.close();
                fis = null;
            } catch (IOException ignore) {
            }
            throw new DiskAccessException("Unable to recover from krf file for oplogId=" + oplogId + ", file=" + f.getName() + ". This file is corrupt, but may be safely deleted.", ex, getParent());
        }
        if (recoverValues && krfEntryCount > 0) {
            oplogsNeedingValueRecovery.add(this);
        // TODO optimize this code and make it async
        // It should also honor the lru limit
        // The fault in logic might not work until
        // the region is actually created.
        // Instead of reading the crf it might be better to iterate the live
        // entry
        // list that was built during KRF recovery. Just fault values in until
        // we
        // hit the LRU limit (if we have one). Only fault in values for entries
        // recovered from disk that are still in this oplog.
        // Defer faulting in values until all oplogs for the ds have been
        // recovered.
        }
    } finally {
        // fix for bug 42776
        if (fis != null) {
            try {
                fis.close();
                fis = null;
            } catch (IOException ignore) {
            }
        }
    }
    return true;
}

Example 85 with VersionTag

use of org.apache.geode.internal.cache.versions.VersionTag in project geode by apache.

the class Oplog method readVersionTagOnlyEntry.

private void readVersionTagOnlyEntry(CountingDataInputStream dis, byte opcode) throws IOException {
    long oplogOffset = -1;
    long drId = DiskInitFile.readDiskRegionID(dis);
    DiskRecoveryStore drs = getOplogSet().getCurrentlyRecovering(drId);
    // read versions
    VersionTag tag = readVersionsFromOplog(dis);
    if (logger.isTraceEnabled(LogMarker.PERSIST_RECOVERY)) {
        logger.trace(LogMarker.PERSIST_RECOVERY, "readVersionTagOnlyEntry drId={} tag={}", drId, tag);
    }
    readEndOfRecord(dis);
    // Update the RVV with the new entry
    if (drs != null) {
        drs.recordRecoveredVersionTag(tag);
    }
}