Examples with ByteIterable jetbrains.exodus.ByteIterable used on opensource projects

Example 56 with ByteIterable

use of jetbrains.exodus.ByteIterable in project xodus by JetBrains.

the class GarbageCollectorTest method updateSameKeyDeleteWithoutDuplicates.

@Test
public void updateSameKeyDeleteWithoutDuplicates() {
    set1KbFileWithoutGC();
    ByteIterable key = StringBinding.stringToEntry("key");
    final Store store = openStoreAutoCommit("updateSameKey");
    for (int i = 0; i < 1000; ++i) {
        putAutoCommit(store, key, key);
    }
    deleteAutoCommit(store, key);
    Assert.assertTrue(env.getLog().getNumberOfFiles() > 1);
    env.getGC().cleanWholeLog();
    Assert.assertEquals(1L, env.getLog().getNumberOfFiles());
}

Example 57 with ByteIterable

use of jetbrains.exodus.ByteIterable in project pwm by pwm-project.

the class XodusLocalDB method putAll.

@Override
public void putAll(final LocalDB.DB db, final Map<String, String> keyValueMap) throws LocalDBException {
    checkStatus(true);
    environment.executeInTransaction(transaction -> {
        final Store store = getStore(db);
        for (final Map.Entry<String, String> entry : keyValueMap.entrySet()) {
            final ByteIterable k = bindMachine.keyToEntry(entry.getKey());
            final ByteIterable v = bindMachine.valueToEntry(entry.getValue());
            store.put(transaction, k, v);
        }
    });
    outputLogExecutor.conditionallyExecuteTask();
}

Example 58 with ByteIterable

use of jetbrains.exodus.ByteIterable in project pwm by pwm-project.

the class XodusLocalDB method get.

@Override
public String get(final LocalDB.DB db, final String key) throws LocalDBException {
    checkStatus(false);
    return environment.computeInReadonlyTransaction(transaction -> {
        final Store store = getStore(db);
        final ByteIterable returnValue = store.get(transaction, bindMachine.keyToEntry(key));
        if (returnValue != null) {
            return bindMachine.entryToValue(returnValue);
        }
        return null;
    });
}

Example 59 with ByteIterable

use of jetbrains.exodus.ByteIterable in project xodus by JetBrains.

the class PersistentEntityStoreRefactorings method refactorMakePropTablesConsistent.

void refactorMakePropTablesConsistent() {
    store.executeInReadonlyTransaction(new StoreTransactionalExecutable() {

        @Override
        public void execute(@NotNull final StoreTransaction tx) {
            final PersistentStoreTransaction txn = (PersistentStoreTransaction) tx;
            for (final String entityType : store.getEntityTypes(txn)) {
                if (logger.isInfoEnabled()) {
                    logger.info("Refactoring making props' tables consistent for [" + entityType + ']');
                }
                runReadonlyTransactionSafeForEntityType(entityType, new Runnable() {

                    @Override
                    public void run() {
                        final int entityTypeId = store.getEntityTypeId(txn, entityType, false);
                        final PropertiesTable propTable = store.getPropertiesTable(txn, entityTypeId);
                        final Transaction envTxn = txn.getEnvironmentTransaction();
                        final IntHashMap<LongHashMap<PropertyValue>> props = new IntHashMap<>();
                        final Cursor cursor = store.getPrimaryPropertyIndexCursor(txn, propTable);
                        final PropertyTypes propertyTypes = store.getPropertyTypes();
                        while (cursor.getNext()) {
                            final PropertyKey propKey = PropertyKey.entryToPropertyKey(cursor.getKey());
                            final PropertyValue propValue = propertyTypes.entryToPropertyValue(cursor.getValue());
                            final int propId = propKey.getPropertyId();
                            LongHashMap<PropertyValue> entitiesToValues = props.get(propId);
                            if (entitiesToValues == null) {
                                entitiesToValues = new LongHashMap<>();
                                props.put(propId, entitiesToValues);
                            }
                            entitiesToValues.put(propKey.getEntityLocalId(), propValue);
                        }
                        cursor.close();
                        final List<Pair<Integer, Pair<ByteIterable, ByteIterable>>> missingPairs = new ArrayList<>();
                        final IntHashMap<Set<Long>> allPropsMap = new IntHashMap<>();
                        for (final int propId : props.keySet()) {
                            final Store valueIndex = propTable.getValueIndex(txn, propId, false);
                            final Cursor valueCursor = valueIndex == null ? null : valueIndex.openCursor(envTxn);
                            final LongHashMap<PropertyValue> entitiesToValues = props.get(propId);
                            final Set<Long> localIdSet = entitiesToValues.keySet();
                            final TreeSet<Long> sortedLocalIdSet = new TreeSet<>(localIdSet);
                            allPropsMap.put(propId, sortedLocalIdSet);
                            final Long[] localIds = sortedLocalIdSet.toArray(new Long[entitiesToValues.size()]);
                            for (final long localId : localIds) {
                                final PropertyValue propValue = entitiesToValues.get(localId);
                                for (final ByteIterable secondaryKey : PropertiesTable.createSecondaryKeys(propertyTypes, PropertyTypes.propertyValueToEntry(propValue), propValue.getType())) {
                                    final ByteIterable secondaryValue = LongBinding.longToCompressedEntry(localId);
                                    if (valueCursor == null || !valueCursor.getSearchBoth(secondaryKey, secondaryValue)) {
                                        missingPairs.add(new Pair<>(propId, new Pair<>(secondaryKey, secondaryValue)));
                                    }
                                }
                            }
                            if (valueCursor != null) {
                                valueCursor.close();
                            }
                        }
                        if (!missingPairs.isEmpty()) {
                            store.executeInTransaction(new StoreTransactionalExecutable() {

                                @Override
                                public void execute(@NotNull final StoreTransaction tx) {
                                    final PersistentStoreTransaction txn = (PersistentStoreTransaction) tx;
                                    for (final Pair<Integer, Pair<ByteIterable, ByteIterable>> pair : missingPairs) {
                                        final Store valueIndex = propTable.getValueIndex(txn, pair.getFirst(), true);
                                        final Pair<ByteIterable, ByteIterable> missing = pair.getSecond();
                                        if (valueIndex == null) {
                                            throw new NullPointerException("Can't be");
                                        }
                                        valueIndex.put(txn.getEnvironmentTransaction(), missing.getFirst(), missing.getSecond());
                                    }
                                }
                            });
                            if (logger.isInfoEnabled()) {
                                logger.info(missingPairs.size() + " missing secondary keys found and fixed for [" + entityType + ']');
                            }
                        }
                        final List<Pair<Integer, Pair<ByteIterable, ByteIterable>>> phantomPairs = new ArrayList<>();
                        for (final Map.Entry<Integer, Store> entry : propTable.getValueIndices()) {
                            final int propId = entry.getKey();
                            final LongHashMap<PropertyValue> entitiesToValues = props.get(propId);
                            final Cursor c = entry.getValue().openCursor(envTxn);
                            while (c.getNext()) {
                                final ByteIterable keyEntry = c.getKey();
                                final ByteIterable valueEntry = c.getValue();
                                final PropertyValue propValue = entitiesToValues.get(LongBinding.compressedEntryToLong(valueEntry));
                                if (propValue != null) {
                                    final Comparable data = propValue.getData();
                                    final int typeId = propValue.getType().getTypeId();
                                    final Class<? extends Comparable> dataClass;
                                    final ComparableBinding objectBinding;
                                    if (typeId == ComparableValueType.COMPARABLE_SET_VALUE_TYPE) {
                                        // noinspection unchecked
                                        dataClass = ((ComparableSet) data).getItemClass();
                                        // noinspection ConstantConditions
                                        objectBinding = propertyTypes.getPropertyType(dataClass).getBinding();
                                    } else {
                                        dataClass = data.getClass();
                                        objectBinding = propValue.getBinding();
                                    }
                                    final Comparable value;
                                    try {
                                        value = objectBinding.entryToObject(keyEntry);
                                        if (dataClass.equals(value.getClass())) {
                                            if (typeId == ComparableValueType.COMPARABLE_SET_VALUE_TYPE) {
                                                // noinspection unchecked
                                                if (((ComparableSet) data).containsItem(value)) {
                                                    continue;
                                                }
                                            } else // noinspection unchecked
                                            if (PropertyTypes.toLowerCase(data).compareTo(value) == 0) {
                                                continue;
                                            }
                                        }
                                    } catch (Throwable t) {
                                        logger.error("Error reading property value index ", t);
                                        throwJVMError(t);
                                    }
                                }
                                phantomPairs.add(new Pair<>(propId, new Pair<>(keyEntry, valueEntry)));
                            }
                            c.close();
                        }
                        if (!phantomPairs.isEmpty()) {
                            store.executeInTransaction(new StoreTransactionalExecutable() {

                                @Override
                                public void execute(@NotNull final StoreTransaction tx) {
                                    final PersistentStoreTransaction txn = (PersistentStoreTransaction) tx;
                                    final Transaction envTxn = txn.getEnvironmentTransaction();
                                    for (final Pair<Integer, Pair<ByteIterable, ByteIterable>> pair : phantomPairs) {
                                        final Store valueIndex = propTable.getValueIndex(txn, pair.getFirst(), true);
                                        final Pair<ByteIterable, ByteIterable> phantom = pair.getSecond();
                                        if (valueIndex == null) {
                                            throw new NullPointerException("Can't be");
                                        }
                                        deletePair(valueIndex.openCursor(envTxn), phantom.getFirst(), phantom.getSecond());
                                    }
                                }
                            });
                            if (logger.isInfoEnabled()) {
                                logger.info(phantomPairs.size() + " phantom secondary keys found and fixed for [" + entityType + ']');
                            }
                        }
                        final List<Pair<Integer, Long>> phantomIds = new ArrayList<>();
                        final Cursor c = propTable.getAllPropsIndex().openCursor(envTxn);
                        while (c.getNext()) {
                            final int propId = IntegerBinding.compressedEntryToInt(c.getKey());
                            final long localId = LongBinding.compressedEntryToLong(c.getValue());
                            final Set<Long> localIds = allPropsMap.get(propId);
                            if (localIds == null || !localIds.remove(localId)) {
                                phantomIds.add(new Pair<>(propId, localId));
                            } else {
                                if (localIds.isEmpty()) {
                                    allPropsMap.remove(propId);
                                }
                            }
                        }
                        c.close();
                        if (!allPropsMap.isEmpty()) {
                            final int[] added = { 0 };
                            store.executeInTransaction(new StoreTransactionalExecutable() {

                                @Override
                                public void execute(@NotNull final StoreTransaction txn) {
                                    int count = 0;
                                    final Store allPropsIndex = propTable.getAllPropsIndex();
                                    final Transaction envTxn = ((PersistentStoreTransaction) txn).getEnvironmentTransaction();
                                    for (Map.Entry<Integer, Set<Long>> entry : allPropsMap.entrySet()) {
                                        final ArrayByteIterable keyEntry = IntegerBinding.intToCompressedEntry(entry.getKey());
                                        for (long localId : entry.getValue()) {
                                            allPropsIndex.put(envTxn, keyEntry, LongBinding.longToCompressedEntry(localId));
                                            ++count;
                                        }
                                    }
                                    added[0] = count;
                                }
                            });
                            if (logger.isInfoEnabled()) {
                                logger.info(added[0] + " missing id pairs found and fixed for [" + entityType + ']');
                            }
                        }
                        if (!phantomIds.isEmpty()) {
                            store.executeInTransaction(new StoreTransactionalExecutable() {

                                @Override
                                public void execute(@NotNull final StoreTransaction txn) {
                                    final Store allPropsIndex = propTable.getAllPropsIndex();
                                    final Transaction envTxn = ((PersistentStoreTransaction) txn).getEnvironmentTransaction();
                                    final Cursor c = allPropsIndex.openCursor(envTxn);
                                    for (final Pair<Integer, Long> phantom : phantomIds) {
                                        if (!c.getSearchBoth(IntegerBinding.intToCompressedEntry(phantom.getFirst()), LongBinding.longToCompressedEntry(phantom.getSecond()))) {
                                            throw new EntityStoreException("Can't be");
                                        }
                                        c.deleteCurrent();
                                    }
                                    c.close();
                                }
                            });
                            if (logger.isInfoEnabled()) {
                                logger.info(phantomIds.size() + " phantom id pairs found and fixed for [" + entityType + ']');
                            }
                        }
                    }
                });
            }
        }
    });
}

Example 60 with ByteIterable

use of jetbrains.exodus.ByteIterable in project xodus by JetBrains.

the class EntitiesOfTypeRangeIterable method countImpl.

@Override
protected long countImpl(@NotNull final PersistentStoreTransaction txn) {
    final Cursor cursor = openCursor(txn);
    if (cursor == null) {
        return 0;
    }
    try {
        final ByteIterable key = LongBinding.longToCompressedEntry(min);
        long result = 0;
        boolean success = cursor.getSearchKeyRange(key) != null;
        while (success) {
            if (max > LongBinding.compressedEntryToLong(cursor.getKey())) {
                break;
            }
            result++;
            success = cursor.getNextNoDup();
        }
        return result;
    } finally {
        cursor.close();
    }
}