Examples with MemoryTracker org.neo4j.memory.MemoryTracker used on opensource projects

Example 21 with MemoryTracker

use of org.neo4j.memory.MemoryTracker in project neo4j by neo4j.

the class RelationshipModifier method acquireMostOfTheNodeAndGroupsLocks.

private void acquireMostOfTheNodeAndGroupsLocks(RelationshipModifications modifications, RecordAccessSet recordChanges, ResourceLocker locks, LockTracer lockTracer, MutableLongObjectMap<NodeContext> contexts, MappedNodeDataLookup nodeDataLookup) {
    /* Here we're going to figure out if we need to make changes to any node and/or relationship group records and lock them if we do. */
    // We check modifications for each node, it might need locking. The iteration here is always sorted by node id
    modifications.forEachSplit(byNode -> {
        long nodeId = byNode.nodeId();
        RecordProxy<NodeRecord, Void> nodeProxy = recordChanges.getNodeRecords().getOrLoad(nodeId, null, cursorContext);
        // optimistic (unlocked) read
        NodeRecord node = nodeProxy.forReadingLinkage();
        boolean nodeIsAddedInTx = node.isCreated();
        if (// we can not trust this as the node is not locked
        !node.isDense()) {
            if (// to avoid locking unnecessarily
            !nodeIsAddedInTx) {
                // lock and re-read, now we can trust it
                locks.acquireExclusive(lockTracer, NODE, nodeId);
                nodeProxy = recordChanges.getNodeRecords().getOrLoad(nodeId, null, cursorContext);
                node = nodeProxy.forReadingLinkage();
                if (node.isDense()) {
                    // another transaction just turned this node dense, unlock and let it be handled below
                    locks.releaseExclusive(NODE, nodeId);
                } else if (byNode.hasCreations()) {
                    // Sparse node with added relationships. We might turn this node dense, at which point the group lock will be needed, so lock it
                    locks.acquireExclusive(lockTracer, RELATIONSHIP_GROUP, nodeId);
                }
            }
        }
        if (// the node is not locked but the dense node is a one-way transform so we can trust it
        node.isDense()) {
            // Stabilize first in chains, in case they are deleted or needed for chain degrees.
            // We are preventing any changes to the group which in turn blocks any other relationship becomming the first in chain
            locks.acquireShared(lockTracer, RELATIONSHIP_GROUP, nodeId);
            // Creations
            NodeContext nodeContext = NodeContext.createNodeContext(nodeProxy, memoryTracker);
            contexts.put(nodeId, nodeContext);
            if (byNode.hasCreations()) {
                // We have some creations on a dense node. If the group exists we can use that, otherwise we create it
                byNode.forEachCreationSplit(byType -> {
                    RelationshipGroupGetter.RelationshipGroupPosition groupPosition = findRelationshipGroup(recordChanges, nodeContext, byType);
                    nodeContext.setCurrentGroup(groupPosition.group() != null ? groupPosition.group() : groupPosition.closestPrevious());
                    RecordProxy<RelationshipGroupRecord, Integer> groupProxy = groupPosition.group();
                    if (groupProxy == null) {
                        // The group did not exist
                        if (!nodeContext.hasExclusiveGroupLock()) {
                            // And we did not already have the lock, so we need to upgrade to exclusive create it
                            locks.releaseShared(RELATIONSHIP_GROUP, nodeId);
                            // Note the small window here where we dont hold any group lock, things might change so we can not trust previous group reads
                            locks.acquireExclusive(lockTracer, NODE, nodeId);
                            locks.acquireExclusive(lockTracer, RELATIONSHIP_GROUP, nodeId);
                        }
                        nodeContext.setNode(recordChanges.getNodeRecords().getOrLoad(nodeId, null, cursorContext));
                        long groupStartingId = nodeContext.node().forReadingLinkage().getNextRel();
                        long groupStartingPrevId = NULL_REFERENCE.longValue();
                        if (groupPosition.closestPrevious() != null) {
                            groupStartingId = groupPosition.closestPrevious().getKey();
                            groupStartingPrevId = groupPosition.closestPrevious().forReadingLinkage().getPrev();
                        }
                        // At this point the group is locked so we can create it
                        groupProxy = relGroupGetter.getOrCreateRelationshipGroup(nodeContext.node(), byType.type(), recordChanges.getRelGroupRecords(), groupStartingPrevId, groupStartingId);
                        // another transaction might beat us at this point, so we are not guaranteed to be the creator but we can trust it to exist
                        if (!nodeContext.hasExclusiveGroupLock()) {
                            nodeContext.markExclusiveGroupLock();
                        } else if (groupProxy.isCreated()) {
                            // When a new group is created we can no longer trust the pointers of the cache
                            nodeContext.clearDenseContext();
                        }
                    }
                    nodeContext.denseContext(byType.type()).setGroup(groupProxy);
                });
                if (!nodeContext.hasExclusiveGroupLock()) {
                    // No other path has given us the exclusive lock yet
                    byNode.forEachCreationSplitInterruptible(byType -> {
                        // But if we are creating relationships to a chain that does not exist on the group
                        // or we might need to flip the external degrees flag
                        RelationshipGroupRecord group = nodeContext.denseContext(byType.type()).group().forReadingLinkage();
                        if (byType.hasOut() && (!group.hasExternalDegreesOut() || isNull(group.getFirstOut())) || byType.hasIn() && (!group.hasExternalDegreesIn() || isNull(group.getFirstIn())) || byType.hasLoop() && (!group.hasExternalDegreesLoop() || isNull(group.getFirstLoop()))) {
                            // Then we need the exclusive lock to change it
                            locks.releaseShared(RELATIONSHIP_GROUP, nodeId);
                            // Note the small window here where we dont hold any group lock, things might change so we can not trust previous group reads
                            locks.acquireExclusive(lockTracer, RELATIONSHIP_GROUP, nodeId);
                            nodeContext.markExclusiveGroupLock();
                            // And we can abort the iteration as the group lock is protecting all relationship group records of the node
                            return true;
                        }
                        return false;
                    });
                }
            }
            // Deletions
            if (byNode.hasDeletions()) {
                if (// no need to do anything if it is already locked by additions
                !nodeContext.hasExclusiveGroupLock()) {
                    byNode.forEachDeletionSplitInterruptible(byType -> {
                        NodeContext.DenseContext denseContext = nodeContext.denseContext(byType.type());
                        RelationshipGroupRecord group = denseContext.getOrLoadGroup(relGroupGetter, nodeContext.node().forReadingLinkage(), byType.type(), recordChanges.getRelGroupRecords(), cursorContext);
                        // here we have the shared lock, so we can trust the read
                        if (byType.hasOut() && !group.hasExternalDegreesOut() || byType.hasIn() && !group.hasExternalDegreesIn() || byType.hasLoop() && !group.hasExternalDegreesLoop()) {
                            // We have deletions but without external degrees, we might need to flip that so we lock it
                            locks.releaseShared(RELATIONSHIP_GROUP, nodeId);
                            // Note the small window here where we dont hold any group lock, things might change so we can not trust previous group reads
                            locks.acquireExclusive(lockTracer, RELATIONSHIP_GROUP, nodeId);
                            nodeContext.markExclusiveGroupLock();
                            return true;
                        } else {
                            // We have deletions and only external degrees
                            boolean hasAnyFirst = batchContains(byType.out(), group.getFirstOut()) || batchContains(byType.in(), group.getFirstIn()) || batchContains(byType.loop(), group.getFirstLoop());
                            if (hasAnyFirst) {
                                // But we're deleting the first in the chain so the group needs to be updated
                                locks.releaseShared(RELATIONSHIP_GROUP, nodeId);
                                // Note the small window here where we dont hold any group lock, things might change so we can not trust previous group reads
                                locks.acquireExclusive(lockTracer, RELATIONSHIP_GROUP, nodeId);
                                nodeContext.markExclusiveGroupLock();
                                return true;
                            }
                        }
                        return false;
                    });
                }
            }
            // Look for an opportunity to delete empty groups that we noticed while looking for groups above
            if (nodeContext.hasExclusiveGroupLock() && nodeContext.hasAnyEmptyGroup()) {
                // There may be one or more empty groups that we can delete
                if (locks.tryExclusiveLock(NODE_RELATIONSHIP_GROUP_DELETE, nodeId)) {
                    // We got the EXCLUSIVE group lock so we can go ahead and try to remove any potentially empty groups
                    if (!nodeContext.hasEmptyFirstGroup() || locks.tryExclusiveLock(NODE, nodeId)) {
                        if (nodeContext.hasEmptyFirstGroup()) {
                            // It's possible that we need to delete the first group, i.e. we just now locked the node and therefore need to re-read it
                            nodeContext.setNode(recordChanges.getNodeRecords().getOrLoad(nodeId, null, cursorContext));
                        }
                        Predicate<RelationshipGroupRecord> canDeleteGroup = group -> !byNode.hasCreations(group.getType());
                        if (RelationshipGroupGetter.deleteEmptyGroups(nodeContext.node(), canDeleteGroup, nodeDataLookup)) {
                            nodeContext.clearDenseContext();
                        }
                    }
                }
            }
        }
    });
}

Example 22 with MemoryTracker

use of org.neo4j.memory.MemoryTracker in project neo4j by neo4j.

the class AbstractCypherResource method commitNewTransaction.

@POST
@Path("/commit")
public Response commitNewTransaction(InputEventStream inputEventStream, @Context HttpServletRequest request, @Context HttpHeaders headers) {
    try (var memoryTracker = createMemoryTracker()) {
        InputEventStream inputStream = ensureNotNull(inputEventStream);
        Optional<GraphDatabaseAPI> graphDatabaseAPI = httpTransactionManager.getGraphDatabaseAPI(databaseName);
        return graphDatabaseAPI.map(databaseAPI -> {
            if (isDatabaseNotAvailable(databaseAPI)) {
                return createNonAvailableDatabaseResponse(inputStream.getParameters());
            }
            memoryTracker.allocateHeap(Invocation.SHALLOW_SIZE);
            final TransactionFacade transactionFacade = httpTransactionManager.createTransactionFacade(databaseAPI, memoryTracker);
            TransactionHandle transactionHandle = createNewTransactionHandle(transactionFacade, request, headers, memoryTracker, true);
            Invocation invocation = new Invocation(log, transactionHandle, null, memoryPool, inputStream, true);
            OutputEventStreamImpl outputStream = new OutputEventStreamImpl(inputStream.getParameters(), transactionHandle, uriScheme, invocation::execute);
            return Response.ok(outputStream).build();
        }).orElse(createNonExistentDatabaseResponse(inputStream.getParameters()));
    }
}

Example 23 with MemoryTracker

use of org.neo4j.memory.MemoryTracker in project neo4j by neo4j.

the class AbstractCypherResource method executeInExistingTransaction.

private Response executeInExistingTransaction(long transactionId, InputEventStream inputEventStream, MemoryTracker memoryTracker, boolean finishWithCommit) {
    InputEventStream inputStream = ensureNotNull(inputEventStream);
    Optional<GraphDatabaseAPI> graphDatabaseAPI = httpTransactionManager.getGraphDatabaseAPI(databaseName);
    return graphDatabaseAPI.map(databaseAPI -> {
        if (isDatabaseNotAvailable(databaseAPI)) {
            return createNonAvailableDatabaseResponse(inputStream.getParameters());
        }
        memoryTracker.allocateHeap(Invocation.SHALLOW_SIZE);
        final TransactionFacade transactionFacade = httpTransactionManager.createTransactionFacade(databaseAPI, memoryTracker);
        TransactionHandle transactionHandle;
        try {
            transactionHandle = transactionFacade.findTransactionHandle(transactionId);
        } catch (TransactionLifecycleException e) {
            return invalidTransaction(e, inputStream.getParameters());
        }
        Invocation invocation = new Invocation(log, transactionHandle, uriScheme.txCommitUri(transactionHandle.getId()), memoryPool, inputStream, finishWithCommit);
        OutputEventStreamImpl outputEventStream = new OutputEventStreamImpl(inputStream.getParameters(), transactionHandle, uriScheme, invocation::execute);
        return Response.ok(outputEventStream).build();
    }).orElse(createNonExistentDatabaseResponse(inputStream.getParameters()));
}

Example 24 with MemoryTracker

use of org.neo4j.memory.MemoryTracker in project neo4j by neo4j.

the class IndexTxStateUpdater method onLabelChange.

/**
 * A label has been changed, figure out what updates are needed to tx state.
 *
 * @param labelId The id of the changed label
 * @param existingPropertyKeyIds all property key ids the node has, sorted by id
 * @param node cursor to the node where the change was applied
 * @param propertyCursor cursor to the properties of node
 * @param changeType The type of change event
 */
void onLabelChange(int labelId, int[] existingPropertyKeyIds, NodeCursor node, PropertyCursor propertyCursor, LabelChangeType changeType) {
    assert noSchemaChangedInTx();
    // Check all indexes of the changed label
    Collection<IndexDescriptor> indexes = storageReader.valueIndexesGetRelated(new long[] { labelId }, existingPropertyKeyIds, NODE);
    if (!indexes.isEmpty()) {
        MutableIntObjectMap<Value> materializedProperties = IntObjectMaps.mutable.empty();
        for (IndexDescriptor index : indexes) {
            MemoryTracker memoryTracker = read.txState().memoryTracker();
            int[] indexPropertyIds = index.schema().getPropertyIds();
            Value[] values = getValueTuple(new NodeCursorWrapper(node), propertyCursor, NO_SUCH_PROPERTY_KEY, NO_VALUE, indexPropertyIds, materializedProperties, memoryTracker);
            ValueTuple valueTuple = ValueTuple.of(values);
            memoryTracker.allocateHeap(valueTuple.getShallowSize());
            switch(changeType) {
                case ADDED_LABEL:
                    indexingService.validateBeforeCommit(index, values, node.nodeReference());
                    read.txState().indexDoUpdateEntry(index.schema(), node.nodeReference(), null, valueTuple);
                    break;
                case REMOVED_LABEL:
                    read.txState().indexDoUpdateEntry(index.schema(), node.nodeReference(), valueTuple, null);
                    break;
                default:
                    throw new IllegalStateException(changeType + " is not a supported event");
            }
        }
    }
}

Example 25 with MemoryTracker

use of org.neo4j.memory.MemoryTracker in project neo4j by neo4j.

the class RecordStorageEngineFactory method createMigrationTargetSchemaRuleAccess.

public static SchemaRuleMigrationAccess createMigrationTargetSchemaRuleAccess(NeoStores stores, CursorContext cursorContext, MemoryTracker memoryTracker) {
    SchemaStore dstSchema = stores.getSchemaStore();
    TokenCreator propertyKeyTokenCreator = (name, internal) -> {
        PropertyKeyTokenStore keyTokenStore = stores.getPropertyKeyTokenStore();
        DynamicStringStore nameStore = keyTokenStore.getNameStore();
        byte[] bytes = PropertyStore.encodeString(name);
        List<DynamicRecord> nameRecords = new ArrayList<>();
        AbstractDynamicStore.allocateRecordsFromBytes(nameRecords, bytes, nameStore, cursorContext, memoryTracker);
        nameRecords.forEach(record -> nameStore.prepareForCommit(record, cursorContext));
        nameRecords.forEach(record -> nameStore.updateRecord(record, cursorContext));
        nameRecords.forEach(record -> nameStore.setHighestPossibleIdInUse(record.getId()));
        int nameId = Iterables.first(nameRecords).getIntId();
        PropertyKeyTokenRecord keyTokenRecord = keyTokenStore.newRecord();
        long tokenId = keyTokenStore.nextId(cursorContext);
        keyTokenRecord.setId(tokenId);
        keyTokenRecord.initialize(true, nameId);
        keyTokenRecord.setInternal(internal);
        keyTokenRecord.setCreated();
        keyTokenStore.prepareForCommit(keyTokenRecord, cursorContext);
        keyTokenStore.updateRecord(keyTokenRecord, cursorContext);
        keyTokenStore.setHighestPossibleIdInUse(keyTokenRecord.getId());
        return Math.toIntExact(tokenId);
    };
    TokenHolders dstTokenHolders = tokenHoldersForSchemaStore(stores, propertyKeyTokenCreator, cursorContext);
    return new SchemaRuleMigrationAccessImpl(stores, new SchemaStorage(dstSchema, dstTokenHolders, () -> KernelVersion.LATEST), cursorContext, memoryTracker);
}