Examples with TableFullException com.ms.silverking.cloud.dht.collection.TableFullException used on opensource projects

Example 1 with TableFullException

use of com.ms.silverking.cloud.dht.collection.TableFullException in project SilverKing by Morgan-Stanley.

the class WritableSegmentBase method _put.

public SegmentStorageResult _put(DHTKey key, int offset, long version, byte[] valueCreator, NamespaceOptions nsOptions) {
    OffsetList offsetList;
    int existingOffset;
    existingOffset = keyToOffset.get(key);
    if (debugPut) {
        Log.warning("segmentNumber: ", segmentNumber);
        Log.warning("existingOffset: ", existingOffset);
    }
    if (existingOffset == CuckooBase.keyNotFound) {
        // no offset for the key; add the mapping
        if (nsOptions.getVersionMode() == NamespaceVersionMode.SINGLE_VERSION || nsOptions.getStorageType() == StorageType.RAM) {
            if (debugPut) {
                Log.warning("initial mapping: ", KeyUtil.keyToString(key));
            }
            try {
                keyToOffset.put(key, offset);
                if (debugPut) {
                    if (keyToOffset.get(key) != offset) {
                        Log.warning("sanity check failed" + keyToOffset.get(key) + " " + offset);
                    }
                }
            } catch (TableFullException tfe) {
                Log.warning("Segment pkc full. Creating new table");
                keyToOffset = IntArrayCuckoo.rehashAndAdd((IntArrayCuckoo) keyToOffset, key, offset);
            }
        } else {
            long creationTime;
            // Recovery takes too long if we need to look all over for the version
            // For disk-based ns, we currently always create an offset list
            // Similar logic in NamespaceStore.putSegmentNumberAndVersion()
            offsetList = offsetListStore.newOffsetList();
            if (nsOptions.getRevisionMode() == RevisionMode.UNRESTRICTED_REVISIONS) {
                creationTime = getCreationTime(offset);
            } else {
                creationTime = 0;
            }
            offsetList.putOffset(version, offset, creationTime);
            try {
                keyToOffset.put(key, -((RAMOffsetList) offsetList).getIndex());
            } catch (TableFullException tfe) {
                Log.warning("Segment pkc full. Creating new table");
                keyToOffset = IntArrayCuckoo.rehashAndAdd((IntArrayCuckoo) keyToOffset, key, -((RAMOffsetList) offsetList).getIndex());
            }
        }
    } else {
        // a single value associated with it or an offset list
        if (nsOptions.getVersionMode() == NamespaceVersionMode.SINGLE_VERSION) {
            // long    existingVersion;
            // existingVersion = getVersion(existingOffset);
            // if (version != existingVersion) {
            // return SegmentStorageResult.invalidVersion;
            // } else {
            byte[] existingChecksum;
            byte[] newChecksum;
            existingChecksum = getChecksum(existingOffset);
            newChecksum = getChecksum(offset);
            if (ArrayUtil.compare(existingChecksum, newChecksum, ArrayUtil.MismatchedLengthMode.Ignore) == 0) {
                return SegmentStorageResult.stored;
            } else {
                if (debugPut) {
                    Log.warning(String.format("Checksums failed to compare: %s %s", StringUtil.byteArrayToHexString(existingChecksum), StringUtil.byteArrayToHexString(newChecksum)));
                }
                return SegmentStorageResult.mutation;
            }
        // }
        } else {
            if (existingOffset >= 0) {
                long existingVersion;
                long existingCreationTime;
                long creationTime;
                if (debugPut) {
                    Log.warning("single key associated: ", KeyUtil.keyToString(key));
                }
                // single key is associated, create an offset list
                offsetList = offsetListStore.newOffsetList();
                existingVersion = getVersion(existingOffset);
                if (nsOptions.getRevisionMode() == RevisionMode.UNRESTRICTED_REVISIONS) {
                    existingCreationTime = getCreationTime(existingOffset);
                    creationTime = getCreationTime(offset);
                } else {
                    existingCreationTime = 0;
                    creationTime = 0;
                }
                if (nsOptions.getRevisionMode() == RevisionMode.UNRESTRICTED_REVISIONS || version > existingVersion) {
                    offsetList.putOffset(existingVersion, existingOffset, existingCreationTime);
                    offsetList.putOffset(version, offset, creationTime);
                    if (debugPut) {
                        Log.warning("removing existing mapping: ", KeyUtil.keyToString(key));
                    }
                    boolean removed;
                    removed = keyToOffset.remove(key);
                    if (debugPut || Log.levelMet(Level.FINE)) {
                        Log.warning("removed: ", removed);
                        Log.warning("pkc.get: ", keyToOffset.get(key));
                        Log.warning("putting new mapping: ", KeyUtil.keyToString(key) + " " + -((RAMOffsetList) offsetList).getIndex());
                    }
                    try {
                        keyToOffset.put(key, -((RAMOffsetList) offsetList).getIndex());
                    } catch (TableFullException tfe) {
                        Log.warning("Segment pkc full. Creating new table");
                        keyToOffset = IntArrayCuckoo.rehashAndAdd((IntArrayCuckoo) keyToOffset, key, -((RAMOffsetList) offsetList).getIndex());
                    }
                } else {
                    ValueCreator creator;
                    // FUTURE - Think about this. Important currently to allow for retries to succeed cleanly.
                    creator = getCreator(offset);
                    if (SimpleValueCreator.areEqual(creator.getBytes(), valueCreator)) {
                        byte[] existingChecksum;
                        byte[] newChecksum;
                        existingChecksum = getChecksum(existingOffset);
                        newChecksum = getChecksum(offset);
                        if (ArrayUtil.compare(existingChecksum, newChecksum) == 0) {
                            // Log.warningf("%s %d %d", key, existingVersion, version);
                            return SegmentStorageResult.duplicateStore;
                        } else {
                            if (debugPut) {
                                Log.warning(String.format("Duplicate existingVersion %d version %d, eo %d o %d, but checksums failed to compare: %s %s", existingVersion, version, existingOffset, offset, StringUtil.byteArrayToHexString(existingChecksum), StringUtil.byteArrayToHexString(newChecksum)));
                            }
                            return SegmentStorageResult.invalidVersion;
                        }
                    } else {
                        // FUTURE: Consider: allow puts of incomplete stores to continue?
                        if (debugPut) {
                            Log.warning("WritableSegmentBase._put detected invalid version b");
                            Log.warning(nsOptions);
                        }
                        return SegmentStorageResult.invalidVersion;
                    }
                }
            } else {
                if (debugPut) {
                    Log.warning("list associated: ", KeyUtil.keyToString(key));
                }
                // offset list is associated, use the existing offset list
                offsetList = offsetListStore.getOffsetList(-existingOffset);
                if (nsOptions.getRevisionMode() == RevisionMode.UNRESTRICTED_REVISIONS || version >= offsetList.getLatestVersion()) {
                    // note: > since we want new versions to be added
                    // == since we might need to store over an incomplete store
                    // so >= to cover both
                    long creationTime;
                    if (debugPut || Log.levelMet(Level.FINE)) {
                        Log.warning("adding offset: ", KeyUtil.keyToString(key) + " " + offset);
                    }
                    if (nsOptions.getRevisionMode() == RevisionMode.UNRESTRICTED_REVISIONS) {
                        creationTime = getCreationTime(offset);
                    } else {
                        creationTime = 0;
                    }
                    offsetList.putOffset(version, offset, creationTime);
                } else {
                    return SegmentStorageResult.invalidVersion;
                }
            }
        }
    }
    // }
    return SegmentStorageResult.stored;
}

Example 2 with TableFullException

use of com.ms.silverking.cloud.dht.collection.TableFullException in project SilverKing by Morgan-Stanley.

the class IntRehashTest method rehashTest.

public static void rehashTest(int size) {
    IntArrayCuckoo map;
    map = new IntArrayCuckoo(new WritableCuckooConfig(totalEntries, numSubTables, entriesPerBucket, cuckooLimit));
    for (int i = 0; i < size; i++) {
        try {
            map.put(new SimpleKey(0, i), i);
        } catch (TableFullException tfe) {
            System.out.println("rehashing");
            map = IntArrayCuckoo.rehashAndAdd(map, new SimpleKey(0, i), i);
        }
    }
}