Search in sources :

Example 21 with PhysicalLogFiles

use of org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles in project neo4j by neo4j.

the class ReadReplicaReplicationIT method shouldBeAbleToPullTxAfterHavingDownloadedANewStoreAfterPruning.

@Test
public void shouldBeAbleToPullTxAfterHavingDownloadedANewStoreAfterPruning() throws Exception {
    // given
    Map<String, String> params = stringMap(GraphDatabaseSettings.keep_logical_logs.name(), "keep_none", GraphDatabaseSettings.logical_log_rotation_threshold.name(), "1M", GraphDatabaseSettings.check_point_interval_time.name(), "100ms");
    Cluster cluster = clusterRule.withSharedCoreParams(params).startCluster();
    cluster.coreTx((db, tx) -> {
        createData(db, 10);
        tx.success();
    });
    awaitEx(() -> readReplicasUpToDateAsTheLeader(cluster.awaitLeader(), cluster.readReplicas()), 1, TimeUnit.MINUTES);
    ReadReplica readReplica = cluster.getReadReplicaById(0);
    long highestReadReplicaLogVersion = physicalLogFiles(readReplica).getHighestLogVersion();
    readReplica.shutdown();
    CoreClusterMember core;
    do {
        core = cluster.coreTx((db, tx) -> {
            createData(db, 1_000);
            tx.success();
        });
    } while (physicalLogFiles(core).getLowestLogVersion() <= highestReadReplicaLogVersion);
    readReplica.start();
    awaitEx(() -> readReplicasUpToDateAsTheLeader(cluster.awaitLeader(), cluster.readReplicas()), 1, TimeUnit.MINUTES);
    // when
    cluster.coreTx((db, tx) -> {
        createData(db, 10);
        tx.success();
    });
    // then
    assertEventually("The read replica has the same data as the core members", () -> DbRepresentation.of(readReplica.database()), equalTo(DbRepresentation.of(cluster.awaitLeader().database())), 10, TimeUnit.SECONDS);
}
Also used : ResourceIterator(org.neo4j.graphdb.ResourceIterator) AvailabilityGuard(org.neo4j.kernel.AvailabilityGuard) Log(org.neo4j.logging.Log) CausalClusteringSettings(org.neo4j.causalclustering.core.CausalClusteringSettings) ReadReplicaGraphDatabase(org.neo4j.causalclustering.readreplica.ReadReplicaGraphDatabase) CoreMatchers.startsWith(org.hamcrest.CoreMatchers.startsWith) ThrowingSupplier(org.neo4j.function.ThrowingSupplier) HighLimit(org.neo4j.kernel.impl.store.format.highlimit.HighLimit) Map(java.util.Map) Is.is(org.hamcrest.core.Is.is) CoreClusterMember(org.neo4j.causalclustering.discovery.CoreClusterMember) Assert.fail(org.junit.Assert.fail) Transaction(org.neo4j.graphdb.Transaction) Path(java.nio.file.Path) Collectors.toSet(java.util.stream.Collectors.toSet) Standard(org.neo4j.kernel.impl.store.format.standard.Standard) PageCache(org.neo4j.io.pagecache.PageCache) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) Collection(java.util.Collection) Set(java.util.Set) BinaryOperator(java.util.function.BinaryOperator) UnsatisfiedDependencyException(org.neo4j.kernel.impl.util.UnsatisfiedDependencyException) Cluster(org.neo4j.causalclustering.discovery.Cluster) GraphDatabaseAPI(org.neo4j.kernel.internal.GraphDatabaseAPI) CoreGraphDatabase(org.neo4j.causalclustering.core.CoreGraphDatabase) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) TransactionIdTracker(org.neo4j.kernel.api.txtracking.TransactionIdTracker) MetaDataStore(org.neo4j.kernel.impl.store.MetaDataStore) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Predicates.awaitEx(org.neo4j.function.Predicates.awaitEx) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) GraphDatabaseSettings(org.neo4j.graphdb.factory.GraphDatabaseSettings) SortedMap(java.util.SortedMap) Mockito.mock(org.mockito.Mockito.mock) Role(org.neo4j.causalclustering.core.consensus.roles.Role) ClusterMember(org.neo4j.causalclustering.discovery.ClusterMember) Label(org.neo4j.graphdb.Label) TransactionIdStore(org.neo4j.kernel.impl.transaction.log.TransactionIdStore) Iterables.count(org.neo4j.helpers.collection.Iterables.count) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Monitors(org.neo4j.kernel.monitoring.Monitors) MINUTES(java.util.concurrent.TimeUnit.MINUTES) ClusterRule(org.neo4j.test.causalclustering.ClusterRule) LabelScanStore(org.neo4j.kernel.api.labelscan.LabelScanStore) Duration.ofSeconds(java.time.Duration.ofSeconds) Function(java.util.function.Function) Node(org.neo4j.graphdb.Node) HashSet(java.util.HashSet) CatchupPollingProcess(org.neo4j.causalclustering.catchup.tx.CatchupPollingProcess) LifecycleException(org.neo4j.kernel.lifecycle.LifecycleException) HazelcastDiscoveryServiceFactory(org.neo4j.causalclustering.discovery.HazelcastDiscoveryServiceFactory) GraphDatabaseService(org.neo4j.graphdb.GraphDatabaseService) TIME(org.neo4j.kernel.impl.store.MetaDataStore.Position.TIME) BiConsumer(java.util.function.BiConsumer) TransactionFailureException(org.neo4j.kernel.api.exceptions.TransactionFailureException) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) MapUtil.stringMap(org.neo4j.helpers.collection.MapUtil.stringMap) FileNames(org.neo4j.causalclustering.core.consensus.log.segmented.FileNames) PhysicalLogFiles(org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles) FileCopyMonitor(org.neo4j.causalclustering.catchup.tx.FileCopyMonitor) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IOException(java.io.IOException) Assert.assertEventually(org.neo4j.test.assertion.Assert.assertEventually) File(java.io.File) TimeUnit(java.util.concurrent.TimeUnit) DefaultFileSystemAbstraction(org.neo4j.io.fs.DefaultFileSystemAbstraction) SampleData.createData(org.neo4j.causalclustering.scenarios.SampleData.createData) Rule(org.junit.Rule) Paths(java.nio.file.Paths) WriteOperationsNotAllowedException(org.neo4j.graphdb.security.WriteOperationsNotAllowedException) GraphDatabaseFacade(org.neo4j.kernel.impl.factory.GraphDatabaseFacade) DbRepresentation(org.neo4j.test.DbRepresentation) Clock(java.time.Clock) ReadReplica(org.neo4j.causalclustering.discovery.ReadReplica) SECONDS(java.util.concurrent.TimeUnit.SECONDS) Assert.assertEquals(org.junit.Assert.assertEquals) FileSystemAbstraction(org.neo4j.io.fs.FileSystemAbstraction) ReadReplica(org.neo4j.causalclustering.discovery.ReadReplica) CoreClusterMember(org.neo4j.causalclustering.discovery.CoreClusterMember) Cluster(org.neo4j.causalclustering.discovery.Cluster) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) Test(org.junit.Test)

Example 22 with PhysicalLogFiles

use of org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles in project neo4j by neo4j.

the class TransactionLogRecoveryIT method writePartialTx.

private void writePartialTx(File storeDir) throws IOException {
    PhysicalLogFiles logFiles = new PhysicalLogFiles(storeDir, fs);
    ReadOnlyLogVersionRepository logVersionRepository = new ReadOnlyLogVersionRepository(pageCache.getPageCache(fs), storeDir);
    ReadOnlyTransactionIdStore txIdStore = new ReadOnlyTransactionIdStore(pageCache.getPageCache(fs), storeDir);
    PhysicalLogFile logFile = new PhysicalLogFile(fs, logFiles, Long.MAX_VALUE, /*don't rotate*/
    txIdStore::getLastCommittedTransactionId, logVersionRepository, new Monitors().newMonitor(PhysicalLogFile.Monitor.class), new LogHeaderCache(10));
    try (Lifespan ignored = new Lifespan(logFile)) {
        LogEntryWriter writer = new LogEntryWriter(logFile.getWriter());
        writer.writeStartEntry(0, 0, 0x123456789ABCDEFL, txIdStore.getLastCommittedTransactionId() + 1, new byte[] { 0 });
    }
}
Also used : ReadOnlyTransactionIdStore(org.neo4j.kernel.impl.transaction.log.ReadOnlyTransactionIdStore) Monitors(org.neo4j.kernel.monitoring.Monitors) LogEntryWriter(org.neo4j.kernel.impl.transaction.log.entry.LogEntryWriter) ReadOnlyLogVersionRepository(org.neo4j.kernel.impl.transaction.log.ReadOnlyLogVersionRepository) LogHeaderCache(org.neo4j.kernel.impl.transaction.log.LogHeaderCache) Lifespan(org.neo4j.kernel.lifecycle.Lifespan) PhysicalLogFile(org.neo4j.kernel.impl.transaction.log.PhysicalLogFile) PhysicalLogFiles(org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles)

Example 23 with PhysicalLogFiles

use of org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles in project neo4j by neo4j.

the class TransactionLogCatchUpWriterTest method verifyTransactionsInLog.

private void verifyTransactionsInLog(long fromTxId, long endTxId) throws IOException {
    long expectedTxId = fromTxId;
    PhysicalLogFiles logFiles = new PhysicalLogFiles(storeDir, fs);
    LogVersionedStoreChannel versionedStoreChannel = PhysicalLogFile.openForVersion(logFiles, fs, 0, false);
    try (ReadableLogChannel channel = new ReadAheadLogChannel(versionedStoreChannel, LogVersionBridge.NO_MORE_CHANNELS, 1024)) {
        try (PhysicalTransactionCursor<ReadableLogChannel> txCursor = new PhysicalTransactionCursor<>(channel, new VersionAwareLogEntryReader<>())) {
            while (txCursor.next()) {
                CommittedTransactionRepresentation tx = txCursor.get();
                long txId = tx.getCommitEntry().getTxId();
                assertThat(expectedTxId, lessThanOrEqualTo(endTxId));
                assertEquals(expectedTxId, txId);
                expectedTxId++;
            }
        }
    }
}
Also used : PhysicalTransactionCursor(org.neo4j.kernel.impl.transaction.log.PhysicalTransactionCursor) ReadableLogChannel(org.neo4j.kernel.impl.transaction.log.ReadableLogChannel) CommittedTransactionRepresentation(org.neo4j.kernel.impl.transaction.CommittedTransactionRepresentation) LogVersionedStoreChannel(org.neo4j.kernel.impl.transaction.log.LogVersionedStoreChannel) ReadAheadLogChannel(org.neo4j.kernel.impl.transaction.log.ReadAheadLogChannel) PhysicalLogFiles(org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles)

Example 24 with PhysicalLogFiles

use of org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles in project neo4j by neo4j.

the class StoreCopyClient method writeTransactionsToActiveLogFile.

private void writeTransactionsToActiveLogFile(File tempStoreDir, Response<?> response) throws Exception {
    LifeSupport life = new LifeSupport();
    try {
        // Start the log and appender
        PhysicalLogFiles logFiles = new PhysicalLogFiles(tempStoreDir, fs);
        LogHeaderCache logHeaderCache = new LogHeaderCache(10);
        ReadOnlyLogVersionRepository logVersionRepository = new ReadOnlyLogVersionRepository(pageCache, tempStoreDir);
        ReadOnlyTransactionIdStore readOnlyTransactionIdStore = new ReadOnlyTransactionIdStore(pageCache, tempStoreDir);
        LogFile logFile = life.add(new PhysicalLogFile(fs, logFiles, Long.MAX_VALUE, /*don't rotate*/
        readOnlyTransactionIdStore::getLastCommittedTransactionId, logVersionRepository, new Monitors().newMonitor(PhysicalLogFile.Monitor.class), logHeaderCache));
        life.start();
        // Just write all transactions to the active log version. Remember that this is after a store copy
        // where there are no logs, and the transaction stream we're about to write will probably contain
        // transactions that goes some time back, before the last committed transaction id. So we cannot
        // use a TransactionAppender, since it has checks for which transactions one can append.
        FlushableChannel channel = logFile.getWriter();
        final TransactionLogWriter writer = new TransactionLogWriter(new LogEntryWriter(channel));
        final AtomicLong firstTxId = new AtomicLong(BASE_TX_ID);
        response.accept(new Response.Handler() {

            @Override
            public void obligation(long txId) throws IOException {
                throw new UnsupportedOperationException("Shouldn't be called");
            }

            @Override
            public Visitor<CommittedTransactionRepresentation, Exception> transactions() {
                return transaction -> {
                    long txId = transaction.getCommitEntry().getTxId();
                    if (firstTxId.compareAndSet(BASE_TX_ID, txId)) {
                        monitor.startReceivingTransactions(txId);
                    }
                    writer.append(transaction.getTransactionRepresentation(), txId);
                    return false;
                };
            }
        });
        long endTxId = firstTxId.get();
        if (endTxId != BASE_TX_ID) {
            monitor.finishReceivingTransactions(endTxId);
        }
        long currentLogVersion = logVersionRepository.getCurrentLogVersion();
        writer.checkPoint(new LogPosition(currentLogVersion, LOG_HEADER_SIZE));
        // And since we write this manually we need to set the correct transaction id in the
        // header of the log that we just wrote.
        File currentLogFile = logFiles.getLogFileForVersion(currentLogVersion);
        writeLogHeader(fs, currentLogFile, currentLogVersion, max(BASE_TX_ID, endTxId - 1));
        if (!forensics) {
            // since we just create new log and put checkpoint into it with offset equals to
            // LOG_HEADER_SIZE we need to update last transaction offset to be equal to this newly defined max
            // offset otherwise next checkpoint that use last transaction offset will be created for non
            // existing offset that is in most of the cases bigger than new log size.
            // Recovery will treat that as last checkpoint and will not try to recover store till new
            // last closed transaction offset will not overcome old one. Till that happens it will be
            // impossible for recovery process to restore the store
            File neoStore = new File(tempStoreDir, MetaDataStore.DEFAULT_NAME);
            MetaDataStore.setRecord(pageCache, neoStore, MetaDataStore.Position.LAST_CLOSED_TRANSACTION_LOG_BYTE_OFFSET, LOG_HEADER_SIZE);
        }
    } finally {
        life.shutdown();
    }
}
Also used : Visitor(org.neo4j.helpers.collection.Visitor) IOException(java.io.IOException) PhysicalLogFiles(org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles) LogFile(org.neo4j.kernel.impl.transaction.log.LogFile) PhysicalLogFile(org.neo4j.kernel.impl.transaction.log.PhysicalLogFile) Response(org.neo4j.com.Response) AtomicLong(java.util.concurrent.atomic.AtomicLong) ReadOnlyTransactionIdStore(org.neo4j.kernel.impl.transaction.log.ReadOnlyTransactionIdStore) FlushableChannel(org.neo4j.kernel.impl.transaction.log.FlushableChannel) Monitors(org.neo4j.kernel.monitoring.Monitors) LifeSupport(org.neo4j.kernel.lifecycle.LifeSupport) TransactionLogWriter(org.neo4j.kernel.impl.transaction.log.TransactionLogWriter) LogEntryWriter(org.neo4j.kernel.impl.transaction.log.entry.LogEntryWriter) ReadOnlyLogVersionRepository(org.neo4j.kernel.impl.transaction.log.ReadOnlyLogVersionRepository) LogHeaderCache(org.neo4j.kernel.impl.transaction.log.LogHeaderCache) LogFile(org.neo4j.kernel.impl.transaction.log.LogFile) File(java.io.File) PhysicalLogFile(org.neo4j.kernel.impl.transaction.log.PhysicalLogFile) PhysicalLogFile(org.neo4j.kernel.impl.transaction.log.PhysicalLogFile) LogPosition(org.neo4j.kernel.impl.transaction.log.LogPosition)

Example 25 with PhysicalLogFiles

use of org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles in project neo4j by neo4j.

the class TransactionLogCatchUpWriterTest method simulateStoreCopy.

private org.neo4j.kernel.impl.store.StoreId simulateStoreCopy() throws IOException {
    // create an empty store
    org.neo4j.kernel.impl.store.StoreId storeId;
    NeoStoreDataSource ds = dsRule.getDataSource(storeDir, fs, pageCache, emptyMap());
    try (Lifespan ignored = new Lifespan(ds)) {
        storeId = ds.getStoreId();
    }
    // we don't have log files after a store copy
    PhysicalLogFiles logFiles = new PhysicalLogFiles(storeDir, fsRule.get());
    logFiles.accept((file, version) -> file.delete());
    return storeId;
}
Also used : NeoStoreDataSource(org.neo4j.kernel.NeoStoreDataSource) Lifespan(org.neo4j.kernel.lifecycle.Lifespan) PhysicalLogFiles(org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles)

Aggregations

PhysicalLogFiles (org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles)53 File (java.io.File)38 Test (org.junit.Test)29 PhysicalLogFile (org.neo4j.kernel.impl.transaction.log.PhysicalLogFile)29 Command (org.neo4j.kernel.impl.transaction.command.Command)15 IOException (java.io.IOException)14 LogEntryWriter (org.neo4j.kernel.impl.transaction.log.entry.LogEntryWriter)12 LogPosition (org.neo4j.kernel.impl.transaction.log.LogPosition)11 NodeRecord (org.neo4j.kernel.impl.store.record.NodeRecord)9 ReadableClosablePositionAwareChannel (org.neo4j.kernel.impl.transaction.log.ReadableClosablePositionAwareChannel)8 VersionAwareLogEntryReader (org.neo4j.kernel.impl.transaction.log.entry.VersionAwareLogEntryReader)8 FileSystemAbstraction (org.neo4j.io.fs.FileSystemAbstraction)7 LatestCheckPointFinder (org.neo4j.kernel.recovery.LatestCheckPointFinder)7 LogFile (org.neo4j.kernel.impl.transaction.log.LogFile)6 TransactionLogWriter (org.neo4j.kernel.impl.transaction.log.TransactionLogWriter)6 Function (java.util.function.Function)5 Assert.assertEquals (org.junit.Assert.assertEquals)5 Assert.assertTrue (org.junit.Assert.assertTrue)5 Rule (org.junit.Rule)5 StoreChannel (org.neo4j.io.fs.StoreChannel)5