Search in sources :

Example 6 with KernelVersion

use of org.neo4j.kernel.KernelVersion in project neo4j by neo4j.

the class RecordStorageEngine method createCommands.

/**
 * @throws TransactionFailureException if command generation fails or some prerequisite of some command didn't validate,
 * for example if trying to delete a node that still has relationships.
 * @throws CreateConstraintFailureException if this transaction was set to create a constraint and that failed.
 * @throws ConstraintValidationException if this transaction was set to create a constraint and some data violates that constraint.
 */
@SuppressWarnings("resource")
@Override
public void createCommands(Collection<StorageCommand> commands, ReadableTransactionState txState, StorageReader storageReader, CommandCreationContext commandCreationContext, ResourceLocker locks, LockTracer lockTracer, long lastTransactionIdWhenStarted, TxStateVisitor.Decorator additionalTxStateVisitor, CursorContext cursorContext, MemoryTracker transactionMemoryTracker) throws KernelException {
    if (txState != null) {
        KernelVersion version = neoStores.getMetaDataStore().kernelVersion();
        Preconditions.checkState(version.isAtLeast(KernelVersion.V4_2), "Can not write older version than %s. Requested %s", KernelVersion.V4_2, version);
        // We can make this cast here because we expected that the storageReader passed in here comes from
        // this storage engine itself, anything else is considered a bug. And we do know the inner workings
        // of the storage statements that we create.
        RecordStorageCommandCreationContext creationContext = (RecordStorageCommandCreationContext) commandCreationContext;
        LogCommandSerialization serialization = RecordStorageCommandReaderFactory.INSTANCE.get(version);
        TransactionRecordState recordState = creationContext.createTransactionRecordState(integrityValidator, lastTransactionIdWhenStarted, locks, lockTracer, serialization, lockVerificationFactory.create(locks, txState, neoStores, schemaRuleAccess));
        // Visit transaction state and populate these record state objects
        TxStateVisitor txStateVisitor = new TransactionToRecordStateVisitor(recordState, schemaState, schemaRuleAccess, constraintSemantics, cursorContext);
        CountsRecordState countsRecordState = new CountsRecordState(serialization);
        txStateVisitor = additionalTxStateVisitor.apply(txStateVisitor);
        txStateVisitor = new TransactionCountingStateVisitor(txStateVisitor, storageReader, txState, countsRecordState, cursorContext);
        try (TxStateVisitor visitor = txStateVisitor) {
            txState.accept(visitor);
        }
        // Convert record state into commands
        recordState.extractCommands(commands, transactionMemoryTracker);
        countsRecordState.extractCommands(commands, transactionMemoryTracker);
        // Verify sufficient locks
        CommandLockVerification commandLockVerification = commandLockVerificationFactory.create(locks, txState, neoStores, schemaRuleAccess);
        commandLockVerification.verifySufficientlyLocked(commands);
    }
}
Also used : KernelVersion(org.neo4j.kernel.KernelVersion) TxStateVisitor(org.neo4j.storageengine.api.txstate.TxStateVisitor) TransactionCountingStateVisitor(org.neo4j.storageengine.api.txstate.TransactionCountingStateVisitor)

Example 7 with KernelVersion

use of org.neo4j.kernel.KernelVersion in project neo4j by neo4j.

the class RecordStorageEngine method createUpgradeCommands.

@Override
public List<StorageCommand> createUpgradeCommands(KernelVersion versionToUpgradeTo, InjectedNLIUpgradeCallback injectedNLIUpgradeCallback) {
    MetaDataStore metaDataStore = neoStores.getMetaDataStore();
    KernelVersion currentVersion = metaDataStore.kernelVersion();
    Preconditions.checkState(currentVersion.isAtLeast(KernelVersion.V4_2), "Upgrade transaction was introduced in %s and must be done from at least %s. Tried upgrading from %s to %s", KernelVersion.V4_3_D4, KernelVersion.V4_2, currentVersion, versionToUpgradeTo);
    Preconditions.checkState(versionToUpgradeTo.isGreaterThan(currentVersion), "Can not downgrade from %s to %s", currentVersion, versionToUpgradeTo);
    int id = MetaDataStore.Position.KERNEL_VERSION.id();
    MetaDataRecord before = metaDataStore.newRecord();
    before.setId(id);
    before.initialize(true, currentVersion.version());
    MetaDataRecord after = metaDataStore.newRecord();
    after.setId(id);
    after.initialize(true, versionToUpgradeTo.version());
    // This command will be the first one in the "new" version, indicating the switch and writing it to the MetaDataStore
    LogCommandSerialization serialization = RecordStorageCommandReaderFactory.INSTANCE.get(versionToUpgradeTo);
    var commands = new ArrayList<StorageCommand>();
    commands.add(new Command.MetaDataCommand(serialization, before, after));
    // it in the schemaStore.
    if (currentVersion.isLessThan(KernelVersion.VERSION_IN_WHICH_TOKEN_INDEXES_ARE_INTRODUCED)) {
        commands.add(createSchemaUpgradeCommand(serialization, injectedNLIUpgradeCallback));
    }
    return commands;
}
Also used : KernelVersion(org.neo4j.kernel.KernelVersion) StorageCommand(org.neo4j.storageengine.api.StorageCommand) MetaDataStore(org.neo4j.kernel.impl.store.MetaDataStore) ArrayList(java.util.ArrayList) MetaDataRecord(org.neo4j.kernel.impl.store.record.MetaDataRecord)

Example 8 with KernelVersion

use of org.neo4j.kernel.KernelVersion in project neo4j by neo4j.

the class NeoStoreTransactionApplier method visitMetaDataCommand.

@Override
public boolean visitMetaDataCommand(Command.MetaDataCommand command) throws IOException {
    if (command.getAfter().getId() == MetaDataStore.Position.KERNEL_VERSION.id()) {
        KernelVersion kernelVersion = KernelVersion.getForVersion(Numbers.safeCastLongToByte(command.getAfter().getValue()));
        neoStores.getMetaDataStore().setKernelVersion(kernelVersion, cursorContext);
    } else {
        throw new UnsupportedOperationException("Unexpected meta data update " + command);
    }
    return false;
}
Also used : KernelVersion(org.neo4j.kernel.KernelVersion)

Example 9 with KernelVersion

use of org.neo4j.kernel.KernelVersion in project neo4j by neo4j.

the class DetachedCheckpointLogEntryParserTest method parseDetachedCheckpointRecord.

@Test
void parseDetachedCheckpointRecord() throws IOException {
    KernelVersion version = KernelVersion.V4_3_D4;
    var storeId = new StoreId(4, 5, 6, 7, 8);
    var channel = new InMemoryClosableChannel();
    int checkpointMillis = 3;
    String checkpointDescription = "checkpoint";
    byte[] bytes = Arrays.copyOf(checkpointDescription.getBytes(), 120);
    // For version confusion, please read LogEntryParserSetV4_3 comments
    var checkpoint = new LogEntryDetachedCheckpoint(version, new LogPosition(1, 2), checkpointMillis, storeId, checkpointDescription);
    channel.putLong(checkpoint.getLogPosition().getLogVersion()).putLong(checkpoint.getLogPosition().getByteOffset()).putLong(checkpointMillis).putLong(storeId.getCreationTime()).putLong(storeId.getRandomId()).putLong(storeId.getStoreVersion()).putLong(storeId.getUpgradeTime()).putLong(storeId.getUpgradeTxId()).putShort((short) checkpointDescription.getBytes().length).put(bytes, bytes.length);
    channel.putChecksum();
    var checkpointParser = LogEntryParserSets.parserSet(version).select(DETACHED_CHECK_POINT);
    LogEntry logEntry = checkpointParser.parse(version, channel, positionMarker, commandReader);
    assertEquals(checkpoint, logEntry);
}
Also used : KernelVersion(org.neo4j.kernel.KernelVersion) StoreId(org.neo4j.storageengine.api.StoreId) InMemoryClosableChannel(org.neo4j.kernel.impl.transaction.log.InMemoryClosableChannel) LogPosition(org.neo4j.kernel.impl.transaction.log.LogPosition) Test(org.junit.jupiter.api.Test)

Example 10 with KernelVersion

use of org.neo4j.kernel.KernelVersion in project neo4j by neo4j.

the class TransactionLogsRecoveryTest method writeSomeDataWithVersion.

private void writeSomeDataWithVersion(Path file, Visitor<Pair<LogEntryWriter, Consumer<LogPositionMarker>>, IOException> visitor, KernelVersion version) throws IOException {
    try (LogVersionedStoreChannel versionedStoreChannel = new PhysicalLogVersionedStoreChannel(fileSystem.write(file), logVersion, CURRENT_LOG_FORMAT_VERSION, file, EMPTY_ACCESSOR);
        PositionAwarePhysicalFlushableChecksumChannel writableLogChannel = new PositionAwarePhysicalFlushableChecksumChannel(versionedStoreChannel, new HeapScopedBuffer(1, KibiByte, INSTANCE))) {
        writeLogHeader(writableLogChannel, new LogHeader(logVersion, 2L, StoreId.UNKNOWN));
        writableLogChannel.beginChecksum();
        Consumer<LogPositionMarker> consumer = marker -> {
            try {
                writableLogChannel.getCurrentPosition(marker);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        };
        LogEntryWriter first = new LogEntryWriter(writableLogChannel, version);
        visitor.visit(Pair.of(first, consumer));
    }
}
Also used : HeapScopedBuffer(org.neo4j.io.memory.HeapScopedBuffer) Visitor(org.neo4j.internal.helpers.collection.Visitor) BeforeEach(org.junit.jupiter.api.BeforeEach) LifecycleAdapter(org.neo4j.kernel.lifecycle.LifecycleAdapter) MutableInt(org.apache.commons.lang3.mutable.MutableInt) Log(org.neo4j.logging.Log) DatabaseIdFactory.from(org.neo4j.kernel.database.DatabaseIdFactory.from) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) LogFilesBuilder(org.neo4j.kernel.impl.transaction.log.files.LogFilesBuilder) Config(org.neo4j.configuration.Config) LifeSupport(org.neo4j.kernel.lifecycle.LifeSupport) DatabaseLayout(org.neo4j.io.layout.DatabaseLayout) Mockito.verifyNoInteractions(org.mockito.Mockito.verifyNoInteractions) LogHeaderWriter.writeLogHeader(org.neo4j.kernel.impl.transaction.log.entry.LogHeaderWriter.writeLogHeader) LogEntryWriter(org.neo4j.kernel.impl.transaction.log.entry.LogEntryWriter) LogCheckPointEvent(org.neo4j.kernel.impl.transaction.tracing.LogCheckPointEvent) LogHeader(org.neo4j.kernel.impl.transaction.log.entry.LogHeader) Monitors(org.neo4j.monitoring.Monitors) BASE_TX_CHECKSUM(org.neo4j.storageengine.api.TransactionIdStore.BASE_TX_CHECKSUM) PageCacheTracer(org.neo4j.io.pagecache.tracing.PageCacheTracer) LogVersionRepository(org.neo4j.storageengine.api.LogVersionRepository) PositionAwarePhysicalFlushableChecksumChannel(org.neo4j.kernel.impl.transaction.log.PositionAwarePhysicalFlushableChecksumChannel) Path(java.nio.file.Path) EMPTY_CHECKER(org.neo4j.kernel.recovery.RecoveryStartupChecker.EMPTY_CHECKER) StorageEngine(org.neo4j.storageengine.api.StorageEngine) NO_RECOVERY_REQUIRED(org.neo4j.kernel.recovery.RecoveryStartInformation.NO_RECOVERY_REQUIRED) TestDirectory(org.neo4j.test.rule.TestDirectory) BASE_TX_COMMIT_TIMESTAMP(org.neo4j.storageengine.api.TransactionIdStore.BASE_TX_COMMIT_TIMESTAMP) LogPositionMarker(org.neo4j.kernel.impl.transaction.log.LogPositionMarker) LogEntryStart(org.neo4j.kernel.impl.transaction.log.entry.LogEntryStart) Instant(java.time.Instant) Neo4jLayoutExtension(org.neo4j.test.extension.Neo4jLayoutExtension) Test(org.junit.jupiter.api.Test) CommittedTransactionRepresentation(org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation) INSTANCE(org.neo4j.memory.EmptyMemoryTracker.INSTANCE) TestLogEntryReader.logEntryReader(org.neo4j.kernel.impl.transaction.log.TestLogEntryReader.logEntryReader) Assertions.assertTrue(org.junit.jupiter.api.Assertions.assertTrue) LogEntry(org.neo4j.kernel.impl.transaction.log.entry.LogEntry) DatabaseStartAbortedException(org.neo4j.dbms.database.DatabaseStartAbortedException) Mockito.mock(org.mockito.Mockito.mock) PhysicalLogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.PhysicalLogicalTransactionStore) Assertions.assertThrows(org.junit.jupiter.api.Assertions.assertThrows) Assertions.fail(org.junit.jupiter.api.Assertions.fail) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) CsvSource(org.junit.jupiter.params.provider.CsvSource) LogFiles(org.neo4j.kernel.impl.transaction.log.files.LogFiles) ArgumentMatchers.anyLong(org.mockito.ArgumentMatchers.anyLong) LogicalTransactionStore(org.neo4j.kernel.impl.transaction.log.LogicalTransactionStore) ProgressReporter(org.neo4j.common.ProgressReporter) LogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.LogVersionedStoreChannel) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) KernelVersion(org.neo4j.kernel.KernelVersion) ArrayUtils(org.apache.commons.lang3.ArrayUtils) NULL(org.neo4j.io.pagecache.tracing.PageCacheTracer.NULL) DatabaseStartupController(org.neo4j.kernel.database.DatabaseStartupController) LogFile(org.neo4j.kernel.impl.transaction.log.files.LogFile) EMPTY_ACCESSOR(org.neo4j.kernel.impl.transaction.log.files.ChannelNativeAccessor.EMPTY_ACCESSOR) LogPosition(org.neo4j.kernel.impl.transaction.log.LogPosition) Inject(org.neo4j.test.extension.Inject) PhysicalLogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel) StoreId(org.neo4j.storageengine.api.StoreId) TransactionApplicationMode(org.neo4j.storageengine.api.TransactionApplicationMode) GraphDatabaseInternalSettings(org.neo4j.configuration.GraphDatabaseInternalSettings) Assertions.assertEquals(org.junit.jupiter.api.Assertions.assertEquals) ArgumentMatchers.anyInt(org.mockito.ArgumentMatchers.anyInt) Lifecycle(org.neo4j.kernel.lifecycle.Lifecycle) LogEntryCommit(org.neo4j.kernel.impl.transaction.log.entry.LogEntryCommit) Files(java.nio.file.Files) LogEntryReader(org.neo4j.kernel.impl.transaction.log.entry.LogEntryReader) TransactionMetadataCache(org.neo4j.kernel.impl.transaction.log.TransactionMetadataCache) ExceptionUtils.getRootCause(org.apache.commons.lang3.exception.ExceptionUtils.getRootCause) IOException(java.io.IOException) Mockito.when(org.mockito.Mockito.when) CURRENT_LOG_FORMAT_VERSION(org.neo4j.kernel.impl.transaction.log.entry.LogVersions.CURRENT_LOG_FORMAT_VERSION) SimpleLogVersionRepository(org.neo4j.kernel.impl.transaction.SimpleLogVersionRepository) Mockito.verify(org.mockito.Mockito.verify) Consumer(java.util.function.Consumer) AfterEach(org.junit.jupiter.api.AfterEach) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) DefaultFileSystemAbstraction(org.neo4j.io.fs.DefaultFileSystemAbstraction) UUID.randomUUID(java.util.UUID.randomUUID) Mockito.never(org.mockito.Mockito.never) CURRENT_FORMAT_LOG_HEADER_SIZE(org.neo4j.kernel.impl.transaction.log.entry.LogVersions.CURRENT_FORMAT_LOG_HEADER_SIZE) NO_MONITOR(org.neo4j.kernel.recovery.RecoveryStartInformationProvider.NO_MONITOR) HeapScopedBuffer(org.neo4j.io.memory.HeapScopedBuffer) Pair(org.neo4j.internal.helpers.collection.Pair) KibiByte(org.neo4j.io.ByteUnit.KibiByte) TransactionIdStore(org.neo4j.storageengine.api.TransactionIdStore) SimpleTransactionIdStore(org.neo4j.kernel.impl.transaction.SimpleTransactionIdStore) LogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.LogVersionedStoreChannel) PhysicalLogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel) PositionAwarePhysicalFlushableChecksumChannel(org.neo4j.kernel.impl.transaction.log.PositionAwarePhysicalFlushableChecksumChannel) PhysicalLogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.PhysicalLogVersionedStoreChannel) LogEntryWriter(org.neo4j.kernel.impl.transaction.log.entry.LogEntryWriter) IOException(java.io.IOException) LogHeaderWriter.writeLogHeader(org.neo4j.kernel.impl.transaction.log.entry.LogHeaderWriter.writeLogHeader) LogHeader(org.neo4j.kernel.impl.transaction.log.entry.LogHeader) LogPositionMarker(org.neo4j.kernel.impl.transaction.log.LogPositionMarker)

Aggregations

KernelVersion (org.neo4j.kernel.KernelVersion)12 ArrayList (java.util.ArrayList)3 Test (org.junit.jupiter.api.Test)3 IOException (java.io.IOException)2 InMemoryClosableChannel (org.neo4j.kernel.impl.transaction.log.InMemoryClosableChannel)2 LogPosition (org.neo4j.kernel.impl.transaction.log.LogPosition)2 StoreId (org.neo4j.storageengine.api.StoreId)2 Files (java.nio.file.Files)1 Path (java.nio.file.Path)1 Instant (java.time.Instant)1 Collection (java.util.Collection)1 Iterator (java.util.Iterator)1 UUID.randomUUID (java.util.UUID.randomUUID)1 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)1 Consumer (java.util.function.Consumer)1 LongStream (java.util.stream.LongStream)1 Stream (java.util.stream.Stream)1 ArrayUtils (org.apache.commons.lang3.ArrayUtils)1 ExceptionUtils.getRootCause (org.apache.commons.lang3.exception.ExceptionUtils.getRootCause)1 MutableInt (org.apache.commons.lang3.mutable.MutableInt)1