Examples with MemoryTracker org.neo4j.memory.MemoryTracker used on opensource projects

Example 11 with MemoryTracker

use of org.neo4j.memory.MemoryTracker in project neo4j by neo4j.

the class TxStateTransactionDataSnapshotIT method countChangedNodeInTransactionStateSnapshot.

@Test
void countChangedNodeInTransactionStateSnapshot() {
    long nodeIdToChange;
    int attachedPropertySize = (int) ByteUnit.mebiBytes(1);
    int doublePropertySize = attachedPropertySize * 2;
    Label label1 = label("label1");
    Label label2 = label("label2");
    final String property = "a";
    final String doubleProperty = "b";
    try (Transaction transaction = database.beginTx()) {
        var node = transaction.createNode(label1, label2);
        node.setProperty(property, randomAscii(attachedPropertySize));
        node.setProperty(doubleProperty, randomAscii(doublePropertySize));
        nodeIdToChange = node.getId();
        transaction.commit();
    }
    try (Transaction transaction = database.beginTx()) {
        var node = transaction.getNodeById(nodeIdToChange);
        node.removeLabel(label1);
        node.setProperty(doubleProperty, randomAscii(attachedPropertySize));
        node.removeProperty(property);
        node.addLabel(Label.label("newLabel"));
        var kernelTransaction = getKernelTransaction(transaction);
        var transactionState = kernelTransaction.txState();
        final MemoryTracker memoryTracker = kernelTransaction.memoryTracker();
        // reset to count only snapshot memory
        var trackingData = resetMemoryTracker(memoryTracker);
        try (var snapshot = new TxStateTransactionDataSnapshot(transactionState, kernelTransaction.newStorageReader(), kernelTransaction)) {
            assertThat(memoryTracker.usedNativeMemory()).isZero();
            assertThat(memoryTracker.estimatedHeapMemory()).isGreaterThanOrEqualTo(emptySnapshotSize + (attachedPropertySize + doublePropertySize) + (2 * NodePropertyEntryView.SHALLOW_SIZE) + (2 * LabelEntryView.SHALLOW_SIZE));
        } finally {
            restoreMemoryTracker(memoryTracker, trackingData);
        }
    }
}

Example 12 with MemoryTracker

use of org.neo4j.memory.MemoryTracker in project neo4j by neo4j.

the class ProbeTable method createProbeTable.

public static <K extends Measurable, V extends Measurable> ProbeTable<K, V> createProbeTable(MemoryTracker memoryTracker) {
    MemoryTracker scopedMemoryTracker = memoryTracker.getScopedMemoryTracker();
    scopedMemoryTracker.allocateHeap(SHALLOW_SIZE + SCOPED_MEMORY_TRACKER_SHALLOW_SIZE);
    return new ProbeTable<>(scopedMemoryTracker);
}

Example 13 with MemoryTracker

use of org.neo4j.memory.MemoryTracker in project neo4j by neo4j.

the class HeapTrackingLongEnumerationListTest method addPutRemoveScenario.

@Test
void addPutRemoveScenario() {
    MemoryTracker memoryTracker = new LocalMemoryTracker();
    HeapTrackingLongEnumerationList<Long> table = HeapTrackingLongEnumerationList.create(memoryTracker, 4);
    assertEmpty(table);
    table.add(0L);
    table.put(2L, 2L);
    table.put(1L, 1L);
    assertContainsOnly(table, 0, 2);
    table.remove(0L);
    table.remove(1L);
    assertContainsOnly(table, 2);
    table.remove(2);
    assertEmpty(table);
    table.add(9999L);
    table.put(4L, 4L);
    // Replace
    assertEquals(table.put(3L, 3L), 9999L);
    table.add(5L);
    assertContainsOnly(table, 3, 5);
    table.remove(5);
    assertContainsOnly(table, 3, 4);
    table.remove(4);
    assertContainsOnly(table, 3);
    long measured1 = meter.measureDeep(table) - measuredMemoryTracker;
    assertEquals(measured1, memoryTracker.estimatedHeapMemory() + HeapEstimator.LONG_SIZE);
    table.put(6L, 6L);
    assertContainsOnly(table, new long[] { 3L, 6L });
    long measured2 = meter.measureDeep(table) - measuredMemoryTracker;
    assertEquals(measured2, memoryTracker.estimatedHeapMemory() + 2 * HeapEstimator.LONG_SIZE);
    assertEquals(HeapEstimator.LONG_SIZE, measured2 - measured1);
    // New chunk needed and extra chunk allocated because of poor alignment in tail chunks
    table.put(8L, 8L);
    assertContainsOnly(table, new long[] { 3L, 6L, 8L });
    table.put(7L, 7L);
    assertContainsOnly(table, new long[] { 3L, 6L, 7L, 8L });
    table.remove(6L);
    assertContainsOnly(table, new long[] { 3L, 7L, 8L });
    table.remove(3L);
    assertContainsOnly(table, new long[] { 7L, 8L });
    table.remove(7L);
    assertContainsOnly(table, 8L);
    // Memory should be back to one chunk + one value
    long measured3 = meter.measureDeep(table) - measuredMemoryTracker;
    assertEquals(measured3, memoryTracker.estimatedHeapMemory() + HeapEstimator.LONG_SIZE);
    assertEquals(measured1, measured3);
    table.put(10L, 10L);
    table.put(9L, 9L);
    table.add(11L);
    assertContainsOnly(table, 8, 11);
    table.remove(8L);
    table.remove(10L);
    table.remove(11L);
    assertContainsOnly(table, 9);
    table.put(12L, 12L);
    assertContainsOnly(table, new long[] { 9L, 12L });
    table.remove(9L);
    assertContainsOnly(table, 12);
    // Memory should be one chunk + one value
    long measured4 = meter.measureDeep(table) - measuredMemoryTracker;
    assertEquals(measured4, memoryTracker.estimatedHeapMemory() + HeapEstimator.LONG_SIZE);
    assertEquals(measured3, measured4);
    table.close();
}

Example 14 with MemoryTracker

use of org.neo4j.memory.MemoryTracker in project neo4j by neo4j.

the class IndexPopulationJobTest method shouldCloseMultiPopulatorOnFailedPopulation.

@Test
void shouldCloseMultiPopulatorOnFailedPopulation() {
    // given
    NullLogProvider logProvider = NullLogProvider.getInstance();
    IndexStoreView failingStoreView = new IndexStoreView.Adaptor() {

        @Override
        public StoreScan visitNodes(int[] labelIds, IntPredicate propertyKeyIdFilter, PropertyScanConsumer propertyScanConsumer, TokenScanConsumer labelScanConsumer, boolean forceStoreScan, boolean parallelWrite, PageCacheTracer cacheTracer, MemoryTracker memoryTracker) {
            return new StoreScan() {

                @Override
                public void run(ExternalUpdatesCheck externalUpdatesCheck) {
                    throw new RuntimeException("Just failing");
                }

                @Override
                public void stop() {
                }

                @Override
                public PopulationProgress getProgress() {
                    return null;
                }
            };
        }
    };
    TrackingMultipleIndexPopulator populator = new TrackingMultipleIndexPopulator(failingStoreView, logProvider, EntityType.NODE, new DatabaseSchemaState(logProvider), jobScheduler, tokens);
    IndexPopulationJob populationJob = new IndexPopulationJob(populator, NO_MONITOR, false, NULL, INSTANCE, "", AUTH_DISABLED, EntityType.NODE, Config.defaults());
    // when
    populationJob.run();
    // then
    assertTrue(populator.closed);
}

Example 15 with MemoryTracker

use of org.neo4j.memory.MemoryTracker in project neo4j by neo4j.

the class IndexTxStateUpdater method onDeleteUncreated.

// PROPERTY CHANGES
/**
 * Creating an entity with its data in a transaction adds also adds that state to index transaction state (for matching indexes).
 * When deleting an entity this method will delete this state from the index transaction state.
 *
 * @param entity entity that was deleted.
 * @param propertyCursor property cursor for accessing the properties of the entity.
 * @param tokens the entity tokens this entity has.
 */
private void onDeleteUncreated(EntityCursor entity, PropertyCursor propertyCursor, long[] tokens) {
    assert noSchemaChangedInTx();
    entity.properties(propertyCursor);
    MutableIntList propertyKeyList = IntLists.mutable.empty();
    while (propertyCursor.next()) {
        propertyKeyList.add(propertyCursor.propertyKey());
    }
    // Make sure to sort the propertyKeyIds since SchemaMatcher.onMatchingSchema requires it.
    int[] propertyKeyIds = propertyKeyList.toSortedArray();
    Collection<IndexDescriptor> indexes = storageReader.valueIndexesGetRelated(tokens, propertyKeyIds, entity.entityType());
    if (!indexes.isEmpty()) {
        MutableIntObjectMap<Value> materializedProperties = IntObjectMaps.mutable.empty();
        SchemaMatcher.onMatchingSchema(indexes.iterator(), ANY_PROPERTY_KEY, propertyKeyIds, index -> {
            MemoryTracker memoryTracker = read.txState().memoryTracker();
            SchemaDescriptor schema = index.schema();
            Value[] values = getValueTuple(entity, propertyCursor, ANY_PROPERTY_KEY, NO_VALUE, schema.getPropertyIds(), materializedProperties, memoryTracker);
            ValueTuple valueTuple = ValueTuple.of(values);
            memoryTracker.allocateHeap(valueTuple.getShallowSize());
            read.txState().indexDoUpdateEntry(schema, entity.reference(), valueTuple, null);
        });
    }
}