Search in sources :

Example 1 with FileCopyMonitor

use of org.neo4j.causalclustering.catchup.tx.FileCopyMonitor in project neo4j by neo4j.

the class ReadReplicaReplicationIT method shouldEventuallyPullTransactionDownToAllReadReplicas.

@Test
public void shouldEventuallyPullTransactionDownToAllReadReplicas() throws Exception {
    // given
    Cluster cluster = clusterRule.withNumberOfReadReplicas(0).startCluster();
    int nodesBeforeReadReplicaStarts = 1;
    cluster.coreTx((db, tx) -> {
        db.schema().constraintFor(Label.label("Foo")).assertPropertyIsUnique("foobar").create();
        tx.success();
    });
    // when
    for (int i = 0; i < 100; i++) {
        cluster.coreTx((db, tx) -> {
            createData(db, nodesBeforeReadReplicaStarts);
            tx.success();
        });
    }
    Set<Path> labelScanStoreFiles = new HashSet<>();
    cluster.coreTx((db, tx) -> gatherLabelScanStoreFiles(db, labelScanStoreFiles));
    AtomicBoolean labelScanStoreCorrectlyPlaced = new AtomicBoolean(false);
    Monitors monitors = new Monitors();
    ReadReplica rr = cluster.addReadReplicaWithIdAndMonitors(0, monitors);
    Path readReplicateStoreDir = rr.storeDir().toPath().toAbsolutePath();
    monitors.addMonitorListener((FileCopyMonitor) file -> {
        Path relativPath = readReplicateStoreDir.relativize(file.toPath().toAbsolutePath());
        relativPath = relativPath.subpath(1, relativPath.getNameCount());
        if (labelScanStoreFiles.contains(relativPath)) {
            labelScanStoreCorrectlyPlaced.set(true);
        }
    });
    rr.start();
    for (int i = 0; i < 100; i++) {
        cluster.coreTx((db, tx) -> {
            createData(db, nodesBeforeReadReplicaStarts);
            tx.success();
        });
    }
    // then
    for (final ReadReplica server : cluster.readReplicas()) {
        GraphDatabaseService readReplica = server.database();
        try (Transaction tx = readReplica.beginTx()) {
            ThrowingSupplier<Long, Exception> nodeCount = () -> count(readReplica.getAllNodes());
            assertEventually("node to appear on read replica", nodeCount, is(400L), 1, MINUTES);
            for (Node node : readReplica.getAllNodes()) {
                assertThat(node.getProperty("foobar").toString(), startsWith("baz_bat"));
            }
            tx.success();
        }
    }
    assertTrue(labelScanStoreCorrectlyPlaced.get());
}
Also used : Path(java.nio.file.Path) ResourceIterator(org.neo4j.graphdb.ResourceIterator) AvailabilityGuard(org.neo4j.kernel.AvailabilityGuard) Log(org.neo4j.logging.Log) CausalClusteringSettings(org.neo4j.causalclustering.core.CausalClusteringSettings) ReadReplicaGraphDatabase(org.neo4j.causalclustering.readreplica.ReadReplicaGraphDatabase) CoreMatchers.startsWith(org.hamcrest.CoreMatchers.startsWith) ThrowingSupplier(org.neo4j.function.ThrowingSupplier) HighLimit(org.neo4j.kernel.impl.store.format.highlimit.HighLimit) Map(java.util.Map) Is.is(org.hamcrest.core.Is.is) CoreClusterMember(org.neo4j.causalclustering.discovery.CoreClusterMember) Assert.fail(org.junit.Assert.fail) Transaction(org.neo4j.graphdb.Transaction) Path(java.nio.file.Path) Collectors.toSet(java.util.stream.Collectors.toSet) Standard(org.neo4j.kernel.impl.store.format.standard.Standard) PageCache(org.neo4j.io.pagecache.PageCache) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) Collection(java.util.Collection) Set(java.util.Set) BinaryOperator(java.util.function.BinaryOperator) UnsatisfiedDependencyException(org.neo4j.kernel.impl.util.UnsatisfiedDependencyException) Cluster(org.neo4j.causalclustering.discovery.Cluster) GraphDatabaseAPI(org.neo4j.kernel.internal.GraphDatabaseAPI) CoreGraphDatabase(org.neo4j.causalclustering.core.CoreGraphDatabase) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) TransactionIdTracker(org.neo4j.kernel.api.txtracking.TransactionIdTracker) MetaDataStore(org.neo4j.kernel.impl.store.MetaDataStore) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Predicates.awaitEx(org.neo4j.function.Predicates.awaitEx) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) GraphDatabaseSettings(org.neo4j.graphdb.factory.GraphDatabaseSettings) SortedMap(java.util.SortedMap) Mockito.mock(org.mockito.Mockito.mock) Role(org.neo4j.causalclustering.core.consensus.roles.Role) ClusterMember(org.neo4j.causalclustering.discovery.ClusterMember) Label(org.neo4j.graphdb.Label) TransactionIdStore(org.neo4j.kernel.impl.transaction.log.TransactionIdStore) Iterables.count(org.neo4j.helpers.collection.Iterables.count) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Monitors(org.neo4j.kernel.monitoring.Monitors) MINUTES(java.util.concurrent.TimeUnit.MINUTES) ClusterRule(org.neo4j.test.causalclustering.ClusterRule) LabelScanStore(org.neo4j.kernel.api.labelscan.LabelScanStore) Duration.ofSeconds(java.time.Duration.ofSeconds) Function(java.util.function.Function) Node(org.neo4j.graphdb.Node) HashSet(java.util.HashSet) CatchupPollingProcess(org.neo4j.causalclustering.catchup.tx.CatchupPollingProcess) LifecycleException(org.neo4j.kernel.lifecycle.LifecycleException) HazelcastDiscoveryServiceFactory(org.neo4j.causalclustering.discovery.HazelcastDiscoveryServiceFactory) GraphDatabaseService(org.neo4j.graphdb.GraphDatabaseService) TIME(org.neo4j.kernel.impl.store.MetaDataStore.Position.TIME) BiConsumer(java.util.function.BiConsumer) TransactionFailureException(org.neo4j.kernel.api.exceptions.TransactionFailureException) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) MapUtil.stringMap(org.neo4j.helpers.collection.MapUtil.stringMap) FileNames(org.neo4j.causalclustering.core.consensus.log.segmented.FileNames) PhysicalLogFiles(org.neo4j.kernel.impl.transaction.log.PhysicalLogFiles) FileCopyMonitor(org.neo4j.causalclustering.catchup.tx.FileCopyMonitor) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IOException(java.io.IOException) Assert.assertEventually(org.neo4j.test.assertion.Assert.assertEventually) File(java.io.File) TimeUnit(java.util.concurrent.TimeUnit) DefaultFileSystemAbstraction(org.neo4j.io.fs.DefaultFileSystemAbstraction) SampleData.createData(org.neo4j.causalclustering.scenarios.SampleData.createData) Rule(org.junit.Rule) Paths(java.nio.file.Paths) WriteOperationsNotAllowedException(org.neo4j.graphdb.security.WriteOperationsNotAllowedException) GraphDatabaseFacade(org.neo4j.kernel.impl.factory.GraphDatabaseFacade) DbRepresentation(org.neo4j.test.DbRepresentation) Clock(java.time.Clock) ReadReplica(org.neo4j.causalclustering.discovery.ReadReplica) SECONDS(java.util.concurrent.TimeUnit.SECONDS) Assert.assertEquals(org.junit.Assert.assertEquals) FileSystemAbstraction(org.neo4j.io.fs.FileSystemAbstraction) GraphDatabaseService(org.neo4j.graphdb.GraphDatabaseService) Node(org.neo4j.graphdb.Node) Cluster(org.neo4j.causalclustering.discovery.Cluster) UnsatisfiedDependencyException(org.neo4j.kernel.impl.util.UnsatisfiedDependencyException) LifecycleException(org.neo4j.kernel.lifecycle.LifecycleException) TransactionFailureException(org.neo4j.kernel.api.exceptions.TransactionFailureException) IOException(java.io.IOException) WriteOperationsNotAllowedException(org.neo4j.graphdb.security.WriteOperationsNotAllowedException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ReadReplica(org.neo4j.causalclustering.discovery.ReadReplica) Transaction(org.neo4j.graphdb.Transaction) Monitors(org.neo4j.kernel.monitoring.Monitors) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 2 with FileCopyMonitor

use of org.neo4j.causalclustering.catchup.tx.FileCopyMonitor in project neo4j by neo4j.

the class ReadReplicaStoreCopyIT method addStoreCopyBlockingMonitor.

private static Semaphore addStoreCopyBlockingMonitor(ReadReplica readReplica) {
    DependencyResolver dependencyResolver = readReplica.database().getDependencyResolver();
    Monitors monitors = dependencyResolver.resolveDependency(Monitors.class);
    Semaphore semaphore = new Semaphore(0);
    monitors.addMonitorListener((FileCopyMonitor) file -> {
        try {
            semaphore.acquire();
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new RuntimeException(e);
        }
    });
    return semaphore;
}
Also used : FALSE(org.neo4j.kernel.configuration.Settings.FALSE) CheckPointer(org.neo4j.kernel.impl.transaction.log.checkpoint.CheckPointer) ReadReplicaGraphDatabase(org.neo4j.causalclustering.readreplica.ReadReplicaGraphDatabase) Monitors(org.neo4j.kernel.monitoring.Monitors) TransactionFailureException(org.neo4j.graphdb.TransactionFailureException) ClusterRule(org.neo4j.test.causalclustering.ClusterRule) Assert.assertThat(org.junit.Assert.assertThat) LogRotation(org.neo4j.kernel.impl.transaction.log.rotation.LogRotation) CoreClusterMember(org.neo4j.causalclustering.discovery.CoreClusterMember) Assert.fail(org.junit.Assert.fail) FileCopyMonitor(org.neo4j.causalclustering.catchup.tx.FileCopyMonitor) Semaphore(java.util.concurrent.Semaphore) Test(org.junit.Test) IOException(java.io.IOException) Assert.assertEventually(org.neo4j.test.assertion.Assert.assertEventually) Cluster(org.neo4j.causalclustering.discovery.Cluster) SimpleTriggerInfo(org.neo4j.kernel.impl.transaction.log.checkpoint.SimpleTriggerInfo) UncheckedIOException(java.io.UncheckedIOException) TimeUnit(java.util.concurrent.TimeUnit) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) Rule(org.junit.Rule) DependencyResolver(org.neo4j.graphdb.DependencyResolver) Matchers.is(org.hamcrest.Matchers.is) GraphDatabaseSettings(org.neo4j.graphdb.factory.GraphDatabaseSettings) Matchers.containsString(org.hamcrest.Matchers.containsString) ReadReplica(org.neo4j.causalclustering.discovery.ReadReplica) Monitors(org.neo4j.kernel.monitoring.Monitors) Semaphore(java.util.concurrent.Semaphore) DependencyResolver(org.neo4j.graphdb.DependencyResolver)

Aggregations

IOException (java.io.IOException)2 TimeUnit (java.util.concurrent.TimeUnit)2 Matchers.instanceOf (org.hamcrest.Matchers.instanceOf)2 Assert.fail (org.junit.Assert.fail)2 Rule (org.junit.Rule)2 Test (org.junit.Test)2 FileCopyMonitor (org.neo4j.causalclustering.catchup.tx.FileCopyMonitor)2 Cluster (org.neo4j.causalclustering.discovery.Cluster)2 CoreClusterMember (org.neo4j.causalclustering.discovery.CoreClusterMember)2 ReadReplica (org.neo4j.causalclustering.discovery.ReadReplica)2 ReadReplicaGraphDatabase (org.neo4j.causalclustering.readreplica.ReadReplicaGraphDatabase)2 GraphDatabaseSettings (org.neo4j.graphdb.factory.GraphDatabaseSettings)2 Monitors (org.neo4j.kernel.monitoring.Monitors)2 Assert.assertEventually (org.neo4j.test.assertion.Assert.assertEventually)2 ClusterRule (org.neo4j.test.causalclustering.ClusterRule)2 File (java.io.File)1 UncheckedIOException (java.io.UncheckedIOException)1 Path (java.nio.file.Path)1 Paths (java.nio.file.Paths)1 Clock (java.time.Clock)1