Examples with SynchronizedArrayIdOrderingQueue org.neo4j.kernel.impl.util.SynchronizedArrayIdOrderingQueue used on opensource projects

Example 1 with SynchronizedArrayIdOrderingQueue

use of org.neo4j.kernel.impl.util.SynchronizedArrayIdOrderingQueue in project neo4j by neo4j.

the class SynchronizedArrayIdOrderingQueueTest method shouldHaveOneThreadWaitForARemoval.

@Test
public void shouldHaveOneThreadWaitForARemoval() throws Exception {
    // GIVEN
    IdOrderingQueue queue = new SynchronizedArrayIdOrderingQueue(5);
    queue.offer(3);
    queue.offer(5);
    // WHEN another thread comes in and awaits 5
    OtherThreadExecutor<Void> t2 = cleanup.add(new OtherThreadExecutor<Void>("T2", null));
    Future<Object> await5 = t2.executeDontWait(awaitHead(queue, 5));
    t2.waitUntilWaiting();
    // ... and head (3) gets removed
    queue.removeChecked(3);
    // THEN the other thread should be OK to continue
    await5.get();
}

Example 2 with SynchronizedArrayIdOrderingQueue

use of org.neo4j.kernel.impl.util.SynchronizedArrayIdOrderingQueue in project neo4j by neo4j.

the class SynchronizedArrayIdOrderingQueueTest method shouldOfferQueueABunchOfIds.

@Test
public void shouldOfferQueueABunchOfIds() throws Exception {
    // GIVEN
    IdOrderingQueue queue = new SynchronizedArrayIdOrderingQueue(5);
    // WHEN
    for (int i = 0; i < 7; i++) {
        queue.offer(i);
    }
    // THEN
    for (int i = 0; i < 7; i++) {
        assertFalse(queue.isEmpty());
        queue.waitFor(i);
        queue.removeChecked(i);
    }
    assertTrue(queue.isEmpty());
}

Example 3 with SynchronizedArrayIdOrderingQueue

use of org.neo4j.kernel.impl.util.SynchronizedArrayIdOrderingQueue in project neo4j by neo4j.

the class LegacyBatchIndexApplierTest method shouldOrderTransactionsMakingLegacyIndexChanges.

@Test
public void shouldOrderTransactionsMakingLegacyIndexChanges() throws Throwable {
    // GIVEN
    Map<String, Integer> names = MapUtil.genericMap("first", 0, "second", 1);
    Map<String, Integer> keys = MapUtil.genericMap("key", 0);
    String applierName = "test-applier";
    LegacyIndexApplierLookup applierLookup = mock(LegacyIndexApplierLookup.class);
    when(applierLookup.newApplier(anyString(), anyBoolean())).thenReturn(mock(TransactionApplier.class));
    IndexConfigStore config = newIndexConfigStore(names, applierName);
    // WHEN multiple legacy index transactions are running, they should be done in order
    SynchronizedArrayIdOrderingQueue queue = new SynchronizedArrayIdOrderingQueue(10);
    final AtomicLong lastAppliedTxId = new AtomicLong(-1);
    Race race = new Race();
    for (long i = 0; i < 100; i++) {
        final long txId = i;
        race.addContestant(() -> {
            try (LegacyBatchIndexApplier applier = new LegacyBatchIndexApplier(config, applierLookup, queue, INTERNAL)) {
                TransactionToApply txToApply = new TransactionToApply(new PhysicalTransactionRepresentation(new ArrayList<>()));
                FakeCommitment commitment = new FakeCommitment(txId, mock(TransactionIdStore.class));
                commitment.setHasLegacyIndexChanges(true);
                txToApply.commitment(commitment, txId);
                TransactionApplier txApplier = applier.startTx(txToApply);
                // Make sure threads are unordered
                Thread.sleep(ThreadLocalRandom.current().nextInt(5));
                // THEN
                assertTrue(lastAppliedTxId.compareAndSet(txId - 1, txId));
                // Closing manually instead of using try-with-resources since we have no additional work to do in
                // txApplier
                txApplier.close();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        queue.offer(txId);
    }
    race.go();
}

Example 4 with SynchronizedArrayIdOrderingQueue

use of org.neo4j.kernel.impl.util.SynchronizedArrayIdOrderingQueue in project neo4j by neo4j.

the class NeoStoreDataSource method buildTransactionLogs.

private NeoStoreTransactionLogModule buildTransactionLogs(File storeDir, Config config, LogProvider logProvider, JobScheduler scheduler, FileSystemAbstraction fileSystemAbstraction, StorageEngine storageEngine, LogEntryReader<ReadableClosablePositionAwareChannel> logEntryReader, SynchronizedArrayIdOrderingQueue legacyIndexTransactionOrdering, TransactionIdStore transactionIdStore, LogVersionRepository logVersionRepository) {
    TransactionMetadataCache transactionMetadataCache = new TransactionMetadataCache(100_000);
    LogHeaderCache logHeaderCache = new LogHeaderCache(1000);
    final PhysicalLogFiles logFiles = new PhysicalLogFiles(storeDir, PhysicalLogFile.DEFAULT_NAME, fileSystemAbstraction);
    final PhysicalLogFile logFile = life.add(new PhysicalLogFile(fileSystemAbstraction, logFiles, config.get(GraphDatabaseSettings.logical_log_rotation_threshold), transactionIdStore::getLastCommittedTransactionId, logVersionRepository, physicalLogMonitor, logHeaderCache));
    final PhysicalLogFileInformation.LogVersionToTimestamp logInformation = version -> {
        LogPosition position = LogPosition.start(version);
        try (ReadableLogChannel channel = logFile.getReader(position)) {
            LogEntry entry;
            while ((entry = logEntryReader.readLogEntry(channel)) != null) {
                if (entry instanceof LogEntryStart) {
                    return entry.<LogEntryStart>as().getTimeWritten();
                }
            }
        }
        return -1;
    };
    final LogFileInformation logFileInformation = new PhysicalLogFileInformation(logFiles, logHeaderCache, transactionIdStore::getLastCommittedTransactionId, logInformation);
    if (config.get(GraphDatabaseFacadeFactory.Configuration.ephemeral)) {
        config = config.withDefaults(stringMap(GraphDatabaseSettings.keep_logical_logs.name(), "1 files"));
    }
    String pruningConf = config.get(GraphDatabaseSettings.keep_logical_logs);
    LogPruneStrategy logPruneStrategy = fromConfigValue(fs, logFileInformation, logFiles, pruningConf);
    final LogPruning logPruning = new LogPruningImpl(logPruneStrategy, logProvider);
    final LogRotation logRotation = new LogRotationImpl(monitors.newMonitor(LogRotation.Monitor.class), logFile, databaseHealth);
    final TransactionAppender appender = life.add(new BatchingTransactionAppender(logFile, logRotation, transactionMetadataCache, transactionIdStore, legacyIndexTransactionOrdering, databaseHealth));
    final LogicalTransactionStore logicalTransactionStore = new PhysicalLogicalTransactionStore(logFile, transactionMetadataCache, logEntryReader);
    int txThreshold = config.get(GraphDatabaseSettings.check_point_interval_tx);
    final CountCommittedTransactionThreshold countCommittedTransactionThreshold = new CountCommittedTransactionThreshold(txThreshold);
    long timeMillisThreshold = config.get(GraphDatabaseSettings.check_point_interval_time);
    TimeCheckPointThreshold timeCheckPointThreshold = new TimeCheckPointThreshold(timeMillisThreshold, clock);
    CheckPointThreshold threshold = CheckPointThresholds.or(countCommittedTransactionThreshold, timeCheckPointThreshold);
    final CheckPointerImpl checkPointer = new CheckPointerImpl(transactionIdStore, threshold, storageEngine, logPruning, appender, databaseHealth, logProvider, tracers.checkPointTracer, ioLimiter, storeCopyCheckPointMutex);
    long recurringPeriod = Math.min(timeMillisThreshold, TimeUnit.SECONDS.toMillis(10));
    CheckPointScheduler checkPointScheduler = new CheckPointScheduler(checkPointer, scheduler, recurringPeriod, databaseHealth);
    life.add(checkPointer);
    life.add(checkPointScheduler);
    return new NeoStoreTransactionLogModule(logicalTransactionStore, logFileInformation, logFiles, logFile, logRotation, checkPointer, appender, legacyIndexTransactionOrdering);
}

Example 5 with SynchronizedArrayIdOrderingQueue

use of org.neo4j.kernel.impl.util.SynchronizedArrayIdOrderingQueue in project neo4j by neo4j.

the class NeoStoreDataSource method start.

@Override
public void start() throws IOException {
    dependencies = new Dependencies();
    life = new LifeSupport();
    schemaIndexProvider = dependencyResolver.resolveDependency(SchemaIndexProvider.class, HighestSelectionStrategy.getInstance());
    labelScanStoreProvider = dependencyResolver.resolveDependency(LabelScanStoreProvider.class, new NamedLabelScanStoreSelectionStrategy(config));
    dependencyResolver.resolveDependency(LabelScanStoreProvider.class, new DeleteStoresFromOtherLabelScanStoreProviders(labelScanStoreProvider));
    IndexConfigStore indexConfigStore = new IndexConfigStore(storeDir, fs);
    dependencies.satisfyDependency(lockService);
    dependencies.satisfyDependency(indexConfigStore);
    life.add(indexConfigStore);
    // Monitor listeners
    LoggingLogFileMonitor loggingLogMonitor = new LoggingLogFileMonitor(msgLog);
    monitors.addMonitorListener(loggingLogMonitor);
    life.add(new Delegate(Lifecycles.multiple(indexProviders.values())));
    // Upgrade the store before we begin
    RecordFormats formats = selectStoreFormats(config, storeDir, fs, pageCache, logService);
    upgradeStore(formats);
    // Build all modules and their services
    StorageEngine storageEngine = null;
    try {
        UpdateableSchemaState updateableSchemaState = new KernelSchemaStateStore(logProvider);
        SynchronizedArrayIdOrderingQueue legacyIndexTransactionOrdering = new SynchronizedArrayIdOrderingQueue(20);
        storageEngine = buildStorageEngine(propertyKeyTokenHolder, labelTokens, relationshipTypeTokens, legacyIndexProviderLookup, indexConfigStore, updateableSchemaState::clear, legacyIndexTransactionOrdering);
        LogEntryReader<ReadableClosablePositionAwareChannel> logEntryReader = new VersionAwareLogEntryReader<>(storageEngine.commandReaderFactory());
        TransactionIdStore transactionIdStore = dependencies.resolveDependency(TransactionIdStore.class);
        LogVersionRepository logVersionRepository = dependencies.resolveDependency(LogVersionRepository.class);
        NeoStoreTransactionLogModule transactionLogModule = buildTransactionLogs(storeDir, config, logProvider, scheduler, fs, storageEngine, logEntryReader, legacyIndexTransactionOrdering, transactionIdStore, logVersionRepository);
        transactionLogModule.satisfyDependencies(dependencies);
        buildRecovery(fs, transactionIdStore, logVersionRepository, monitors.newMonitor(Recovery.Monitor.class), monitors.newMonitor(PositionToRecoverFrom.Monitor.class), transactionLogModule.logFiles(), startupStatistics, storageEngine, logEntryReader, transactionLogModule.logicalTransactionStore());
        // At the time of writing this comes from the storage engine (IndexStoreView)
        PropertyAccessor propertyAccessor = dependencies.resolveDependency(PropertyAccessor.class);
        final NeoStoreKernelModule kernelModule = buildKernel(transactionLogModule.transactionAppender(), dependencies.resolveDependency(IndexingService.class), storageEngine.storeReadLayer(), updateableSchemaState, dependencies.resolveDependency(LabelScanStore.class), storageEngine, indexConfigStore, transactionIdStore, availabilityGuard, clock, propertyAccessor);
        kernelModule.satisfyDependencies(dependencies);
        // Do these assignments last so that we can ensure no cyclical dependencies exist
        this.storageEngine = storageEngine;
        this.transactionLogModule = transactionLogModule;
        this.kernelModule = kernelModule;
        dependencies.satisfyDependency(this);
        dependencies.satisfyDependency(updateableSchemaState);
        dependencies.satisfyDependency(storageEngine.storeReadLayer());
        dependencies.satisfyDependency(logEntryReader);
        dependencies.satisfyDependency(storageEngine);
    } catch (Throwable e) {
        // Something unexpected happened during startup
        msgLog.warn("Exception occurred while setting up store modules. Attempting to close things down.", e);
        try {
            // Close the neostore, so that locks are released properly
            if (storageEngine != null) {
                storageEngine.forceClose();
            }
        } catch (Exception closeException) {
            msgLog.error("Couldn't close neostore after startup failure", closeException);
        }
        throw Exceptions.launderedException(e);
    }
    // NOTE: please make sure this is performed after having added everything to the life, in fact we would like
    // to perform the checkpointing as first step when the life is shutdown.
    life.add(lifecycleToTriggerCheckPointOnShutdown());
    try {
        life.start();
    } catch (Throwable e) {
        // Something unexpected happened during startup
        msgLog.warn("Exception occurred while starting the datasource. Attempting to close things down.", e);
        try {
            life.shutdown();
            // Close the neostore, so that locks are released properly
            storageEngine.forceClose();
        } catch (Exception closeException) {
            msgLog.error("Couldn't close neostore after startup failure", closeException);
        }
        throw Exceptions.launderedException(e);
    }
    /*
         * At this point recovery has completed and the datasource is ready for use. Whatever panic might have
         * happened before has been healed. So we can safely set the kernel health to ok.
         * This right now has any real effect only in the case of internal restarts (for example, after a store copy
         * in the case of HA). Standalone instances will have to be restarted by the user, as is proper for all
         * kernel panics.
         */
    databaseHealth.healed();
}