Examples with ArrayByteIterable jetbrains.exodus.ArrayByteIterable used on opensource projects

Example 16 with ArrayByteIterable

use of jetbrains.exodus.ArrayByteIterable in project xodus by JetBrains.

the class GarbageCollectorTestInMemory method testTextIndexLikeWithDeletions.

private void testTextIndexLikeWithDeletions(boolean useExpirationChecker) {
    final long started = System.currentTimeMillis();
    prepare(useExpirationChecker);
    final Transaction txn = env.beginTransaction();
    final Store store = env.openStore("store", getStoreConfig(false), txn);
    final Store storeDups = env.openStore("storeDups", getStoreConfig(true), txn);
    txn.commit();
    try {
        while (System.currentTimeMillis() - started < TEST_DURATION) {
            env.executeInTransaction(new TransactionalExecutable() {

                @Override
                public void execute(@NotNull final Transaction txn) {
                    int randomInt = rnd.nextInt() & 0x3fffffff;
                    final int count = 4 + (randomInt) & 0x1f;
                    for (int j = 0; j < count; randomInt += ++j) {
                        final int intKey = randomInt & 0x3fff;
                        final ArrayByteIterable key = IntegerBinding.intToCompressedEntry(intKey);
                        final int valueLength = 50 + (randomInt % 100);
                        store.put(txn, key, new ArrayByteIterable(new byte[valueLength]));
                        storeDups.put(txn, key, IntegerBinding.intToEntry(randomInt % 32));
                    }
                    randomInt = (randomInt * randomInt) & 0x3fffffff;
                    for (int j = 0; j < count / 2; randomInt += ++j) {
                        final int intKey = randomInt & 0x3fff;
                        final ArrayByteIterable key = IntegerBinding.intToCompressedEntry(intKey);
                        store.delete(txn, key);
                        try (Cursor cursor = storeDups.openCursor(txn)) {
                            if (cursor.getSearchBoth(key, IntegerBinding.intToEntry(randomInt % 32))) {
                                cursor.deleteCurrent();
                            }
                        }
                    }
                }
            });
            Thread.sleep(0);
        }
    } catch (Throwable t) {
        memory.dump(new File(System.getProperty("user.home"), "dump"));
        logger.error("User code exception: ", t);
        Assert.assertTrue(false);
    }
}

Example 17 with ArrayByteIterable

use of jetbrains.exodus.ArrayByteIterable in project xodus by JetBrains.

the class GarbageCollectorTestInMemory method testTextIndexLike.

private void testTextIndexLike(boolean useExpirationChecker) {
    final long started = System.currentTimeMillis();
    prepare(useExpirationChecker);
    final Transaction txn = env.beginTransaction();
    final Store store = env.openStore("store", getStoreConfig(false), txn);
    final Store storeDups = env.openStore("storeDups", getStoreConfig(true), txn);
    txn.commit();
    try {
        while (System.currentTimeMillis() - started < TEST_DURATION) {
            env.executeInTransaction(new TransactionalExecutable() {

                @Override
                public void execute(@NotNull final Transaction txn) {
                    int randomInt = rnd.nextInt() & 0x3fffffff;
                    final int count = 4 + (randomInt) & 0x1f;
                    for (int j = 0; j < count; randomInt += ++j) {
                        final int intKey = randomInt & 0x3fff;
                        final ArrayByteIterable key = IntegerBinding.intToCompressedEntry(intKey);
                        final int valueLength = 50 + (randomInt % 100);
                        store.put(txn, key, new ArrayByteIterable(new byte[valueLength]));
                        storeDups.put(txn, key, IntegerBinding.intToEntry(randomInt % 32));
                    }
                }
            });
            Thread.sleep(0);
        }
    } catch (Throwable t) {
        memory.dump(new File(System.getProperty("user.home"), "dump"));
        logger.error("User code exception: ", t);
        Assert.assertTrue(false);
    }
}

Example 18 with ArrayByteIterable

use of jetbrains.exodus.ArrayByteIterable in project xodus by JetBrains.

the class PatriciaTreeMutable method putRight.

@Override
public void putRight(@NotNull final ByteIterable key, @NotNull final ByteIterable value) {
    final ByteIterator it = key.iterator();
    MutableNode node = root;
    MutableNode prev = null;
    byte prevFirstByte = (byte) 0;
    while (true) {
        final long matchResult = node.matchesKeySequence(it);
        final int matchingLength = NodeBase.MatchResult.getMatchingLength(matchResult);
        if (matchingLength < 0) {
            if (!NodeBase.MatchResult.hasNext(matchResult)) {
                throw new IllegalArgumentException();
            }
            final MutableNode prefix = node.splitKey(-matchingLength - 1, NodeBase.MatchResult.getKeyByte(matchResult));
            prefix.hangRight(NodeBase.MatchResult.getNextByte(matchResult), it).setValue(value);
            if (prev == null) {
                root = new MutableRoot(prefix, root.sourceAddress);
            } else {
                prev.setChild(prevFirstByte, prefix);
            }
            ++size;
            break;
        }
        if (!it.hasNext()) {
            if (node.hasChildren() || node.hasValue()) {
                throw new IllegalArgumentException();
            }
            node.setValue(value);
            ++size;
            break;
        }
        final byte nextByte = it.next();
        final NodeBase child = node.getRightChild(this, nextByte);
        if (child == null) {
            if (node.hasChildren() || node.hasKey() || node.hasValue()) {
                node.hangRight(nextByte, it).setValue(value);
            } else {
                node.setKeySequence(new ArrayByteIterable(nextByte, it));
                node.setValue(value);
            }
            ++size;
            break;
        }
        prev = node;
        prevFirstByte = nextByte;
        final MutableNode mutableChild = child.getMutableCopy(this);
        if (!child.isMutable()) {
            node.setRightChild(nextByte, mutableChild);
        }
        node = mutableChild;
    }
}

Example 19 with ArrayByteIterable

use of jetbrains.exodus.ArrayByteIterable in project xodus by JetBrains.

the class PersistentEntityStoreRefactorings method refactorMakePropTablesConsistent.

void refactorMakePropTablesConsistent() {
    store.executeInReadonlyTransaction(new StoreTransactionalExecutable() {

        @Override
        public void execute(@NotNull final StoreTransaction tx) {
            final PersistentStoreTransaction txn = (PersistentStoreTransaction) tx;
            for (final String entityType : store.getEntityTypes(txn)) {
                if (logger.isInfoEnabled()) {
                    logger.info("Refactoring making props' tables consistent for [" + entityType + ']');
                }
                runReadonlyTransactionSafeForEntityType(entityType, new Runnable() {

                    @Override
                    public void run() {
                        final int entityTypeId = store.getEntityTypeId(txn, entityType, false);
                        final PropertiesTable propTable = store.getPropertiesTable(txn, entityTypeId);
                        final Transaction envTxn = txn.getEnvironmentTransaction();
                        final IntHashMap<LongHashMap<PropertyValue>> props = new IntHashMap<>();
                        final Cursor cursor = store.getPrimaryPropertyIndexCursor(txn, propTable);
                        final PropertyTypes propertyTypes = store.getPropertyTypes();
                        while (cursor.getNext()) {
                            final PropertyKey propKey = PropertyKey.entryToPropertyKey(cursor.getKey());
                            final PropertyValue propValue = propertyTypes.entryToPropertyValue(cursor.getValue());
                            final int propId = propKey.getPropertyId();
                            LongHashMap<PropertyValue> entitiesToValues = props.get(propId);
                            if (entitiesToValues == null) {
                                entitiesToValues = new LongHashMap<>();
                                props.put(propId, entitiesToValues);
                            }
                            entitiesToValues.put(propKey.getEntityLocalId(), propValue);
                        }
                        cursor.close();
                        final List<Pair<Integer, Pair<ByteIterable, ByteIterable>>> missingPairs = new ArrayList<>();
                        final IntHashMap<Set<Long>> allPropsMap = new IntHashMap<>();
                        for (final int propId : props.keySet()) {
                            final Store valueIndex = propTable.getValueIndex(txn, propId, false);
                            final Cursor valueCursor = valueIndex == null ? null : valueIndex.openCursor(envTxn);
                            final LongHashMap<PropertyValue> entitiesToValues = props.get(propId);
                            final Set<Long> localIdSet = entitiesToValues.keySet();
                            final TreeSet<Long> sortedLocalIdSet = new TreeSet<>(localIdSet);
                            allPropsMap.put(propId, sortedLocalIdSet);
                            final Long[] localIds = sortedLocalIdSet.toArray(new Long[entitiesToValues.size()]);
                            for (final long localId : localIds) {
                                final PropertyValue propValue = entitiesToValues.get(localId);
                                for (final ByteIterable secondaryKey : PropertiesTable.createSecondaryKeys(propertyTypes, PropertyTypes.propertyValueToEntry(propValue), propValue.getType())) {
                                    final ByteIterable secondaryValue = LongBinding.longToCompressedEntry(localId);
                                    if (valueCursor == null || !valueCursor.getSearchBoth(secondaryKey, secondaryValue)) {
                                        missingPairs.add(new Pair<>(propId, new Pair<>(secondaryKey, secondaryValue)));
                                    }
                                }
                            }
                            if (valueCursor != null) {
                                valueCursor.close();
                            }
                        }
                        if (!missingPairs.isEmpty()) {
                            store.executeInTransaction(new StoreTransactionalExecutable() {

                                @Override
                                public void execute(@NotNull final StoreTransaction tx) {
                                    final PersistentStoreTransaction txn = (PersistentStoreTransaction) tx;
                                    for (final Pair<Integer, Pair<ByteIterable, ByteIterable>> pair : missingPairs) {
                                        final Store valueIndex = propTable.getValueIndex(txn, pair.getFirst(), true);
                                        final Pair<ByteIterable, ByteIterable> missing = pair.getSecond();
                                        if (valueIndex == null) {
                                            throw new NullPointerException("Can't be");
                                        }
                                        valueIndex.put(txn.getEnvironmentTransaction(), missing.getFirst(), missing.getSecond());
                                    }
                                }
                            });
                            if (logger.isInfoEnabled()) {
                                logger.info(missingPairs.size() + " missing secondary keys found and fixed for [" + entityType + ']');
                            }
                        }
                        final List<Pair<Integer, Pair<ByteIterable, ByteIterable>>> phantomPairs = new ArrayList<>();
                        for (final Map.Entry<Integer, Store> entry : propTable.getValueIndices()) {
                            final int propId = entry.getKey();
                            final LongHashMap<PropertyValue> entitiesToValues = props.get(propId);
                            final Cursor c = entry.getValue().openCursor(envTxn);
                            while (c.getNext()) {
                                final ByteIterable keyEntry = c.getKey();
                                final ByteIterable valueEntry = c.getValue();
                                final PropertyValue propValue = entitiesToValues.get(LongBinding.compressedEntryToLong(valueEntry));
                                if (propValue != null) {
                                    final Comparable data = propValue.getData();
                                    final int typeId = propValue.getType().getTypeId();
                                    final Class<? extends Comparable> dataClass;
                                    final ComparableBinding objectBinding;
                                    if (typeId == ComparableValueType.COMPARABLE_SET_VALUE_TYPE) {
                                        // noinspection unchecked
                                        dataClass = ((ComparableSet) data).getItemClass();
                                        // noinspection ConstantConditions
                                        objectBinding = propertyTypes.getPropertyType(dataClass).getBinding();
                                    } else {
                                        dataClass = data.getClass();
                                        objectBinding = propValue.getBinding();
                                    }
                                    final Comparable value;
                                    try {
                                        value = objectBinding.entryToObject(keyEntry);
                                        if (dataClass.equals(value.getClass())) {
                                            if (typeId == ComparableValueType.COMPARABLE_SET_VALUE_TYPE) {
                                                // noinspection unchecked
                                                if (((ComparableSet) data).containsItem(value)) {
                                                    continue;
                                                }
                                            } else // noinspection unchecked
                                            if (PropertyTypes.toLowerCase(data).compareTo(value) == 0) {
                                                continue;
                                            }
                                        }
                                    } catch (Throwable t) {
                                        logger.error("Error reading property value index ", t);
                                        throwJVMError(t);
                                    }
                                }
                                phantomPairs.add(new Pair<>(propId, new Pair<>(keyEntry, valueEntry)));
                            }
                            c.close();
                        }
                        if (!phantomPairs.isEmpty()) {
                            store.executeInTransaction(new StoreTransactionalExecutable() {

                                @Override
                                public void execute(@NotNull final StoreTransaction tx) {
                                    final PersistentStoreTransaction txn = (PersistentStoreTransaction) tx;
                                    final Transaction envTxn = txn.getEnvironmentTransaction();
                                    for (final Pair<Integer, Pair<ByteIterable, ByteIterable>> pair : phantomPairs) {
                                        final Store valueIndex = propTable.getValueIndex(txn, pair.getFirst(), true);
                                        final Pair<ByteIterable, ByteIterable> phantom = pair.getSecond();
                                        if (valueIndex == null) {
                                            throw new NullPointerException("Can't be");
                                        }
                                        deletePair(valueIndex.openCursor(envTxn), phantom.getFirst(), phantom.getSecond());
                                    }
                                }
                            });
                            if (logger.isInfoEnabled()) {
                                logger.info(phantomPairs.size() + " phantom secondary keys found and fixed for [" + entityType + ']');
                            }
                        }
                        final List<Pair<Integer, Long>> phantomIds = new ArrayList<>();
                        final Cursor c = propTable.getAllPropsIndex().openCursor(envTxn);
                        while (c.getNext()) {
                            final int propId = IntegerBinding.compressedEntryToInt(c.getKey());
                            final long localId = LongBinding.compressedEntryToLong(c.getValue());
                            final Set<Long> localIds = allPropsMap.get(propId);
                            if (localIds == null || !localIds.remove(localId)) {
                                phantomIds.add(new Pair<>(propId, localId));
                            } else {
                                if (localIds.isEmpty()) {
                                    allPropsMap.remove(propId);
                                }
                            }
                        }
                        c.close();
                        if (!allPropsMap.isEmpty()) {
                            final int[] added = { 0 };
                            store.executeInTransaction(new StoreTransactionalExecutable() {

                                @Override
                                public void execute(@NotNull final StoreTransaction txn) {
                                    int count = 0;
                                    final Store allPropsIndex = propTable.getAllPropsIndex();
                                    final Transaction envTxn = ((PersistentStoreTransaction) txn).getEnvironmentTransaction();
                                    for (Map.Entry<Integer, Set<Long>> entry : allPropsMap.entrySet()) {
                                        final ArrayByteIterable keyEntry = IntegerBinding.intToCompressedEntry(entry.getKey());
                                        for (long localId : entry.getValue()) {
                                            allPropsIndex.put(envTxn, keyEntry, LongBinding.longToCompressedEntry(localId));
                                            ++count;
                                        }
                                    }
                                    added[0] = count;
                                }
                            });
                            if (logger.isInfoEnabled()) {
                                logger.info(added[0] + " missing id pairs found and fixed for [" + entityType + ']');
                            }
                        }
                        if (!phantomIds.isEmpty()) {
                            store.executeInTransaction(new StoreTransactionalExecutable() {

                                @Override
                                public void execute(@NotNull final StoreTransaction txn) {
                                    final Store allPropsIndex = propTable.getAllPropsIndex();
                                    final Transaction envTxn = ((PersistentStoreTransaction) txn).getEnvironmentTransaction();
                                    final Cursor c = allPropsIndex.openCursor(envTxn);
                                    for (final Pair<Integer, Long> phantom : phantomIds) {
                                        if (!c.getSearchBoth(IntegerBinding.intToCompressedEntry(phantom.getFirst()), LongBinding.longToCompressedEntry(phantom.getSecond()))) {
                                            throw new EntityStoreException("Can't be");
                                        }
                                        c.deleteCurrent();
                                    }
                                    c.close();
                                }
                            });
                            if (logger.isInfoEnabled()) {
                                logger.info(phantomIds.size() + " phantom id pairs found and fixed for [" + entityType + ']');
                            }
                        }
                    }
                });
            }
        }
    });
}

Example 20 with ArrayByteIterable

use of jetbrains.exodus.ArrayByteIterable in project xodus by JetBrains.

the class PersistentEntityStoreRefactorings method transactionalCopyAndRemoveEntitiesStore.

private void transactionalCopyAndRemoveEntitiesStore(@NotNull final String sourceName, @NotNull final String targetName) {
    final Environment env = store.getEnvironment();
    env.executeInTransaction(new TransactionalExecutable() {

        @Override
        public void execute(@NotNull final Transaction txn) {
            final Store store = env.openStore(sourceName, StoreConfig.USE_EXISTING, txn);
            final Store storeCopy = env.openStore(targetName, StoreConfig.WITHOUT_DUPLICATES_WITH_PREFIXING, txn);
            final Cursor cursor = store.openCursor(txn);
            ArrayByteIterable lastKey = null;
            while (cursor.getNext()) {
                final ArrayByteIterable key = new ArrayByteIterable(cursor.getKey());
                if (lastKey != null && lastKey.compareTo(key) >= 0) {
                    throw new IllegalStateException("Invalid key order");
                }
                storeCopy.putRight(txn, key, new ArrayByteIterable(cursor.getValue()));
                lastKey = key;
            }
            cursor.close();
            env.removeStore(sourceName, txn);
        }
    });
}