Examples with ReadReplica org.neo4j.causalclustering.discovery.ReadReplica used on opensource projects

Example 11 with ReadReplica

use of org.neo4j.causalclustering.discovery.ReadReplica in project neo4j by neo4j.

the class CatchUpLoad method doWork.

@Override
protected void doWork() {
    int newMemberId = cluster.readReplicas().size();
    final ReadReplica readReplica = cluster.addReadReplicaWithId(newMemberId);
    Throwable ex = null;
    Supplier<Throwable> monitoredException = null;
    try {
        monitoredException = startAndRegisterExceptionMonitor(readReplica);
        await(// if the txId from the leader is -1, give up and retry later (leader switch?)
        this::leaderTxId, // caught up?
        (leaderTxId) -> leaderTxId < BASE_TX_ID || leaderTxId <= txId(readReplica, true), 10, TimeUnit.MINUTES);
    } catch (Throwable e) {
        ex = e;
    } finally {
        try {
            cluster.removeReadReplicaWithMemberId(newMemberId);
            if (ex == null && deleteStore) {
                fs.deleteRecursively(readReplica.storeDir());
            }
            deleteStore = !deleteStore;
        } catch (Throwable e) {
            ex = exception(ex, e);
        }
    }
    if (monitoredException != null && monitoredException.get() != null) {
        throw new RuntimeException(exception(monitoredException.get(), ex));
    }
    if (ex != null) {
        throw new RuntimeException(ex);
    }
}

Example 12 with ReadReplica

use of org.neo4j.causalclustering.discovery.ReadReplica in project neo4j by neo4j.

the class ReadReplicaReplicationIT method shouldBeAbleToDownloadANewStoreAfterPruning.

@Test
public void shouldBeAbleToDownloadANewStoreAfterPruning() throws Exception {
    // given
    Map<String, String> params = stringMap(GraphDatabaseSettings.keep_logical_logs.name(), "keep_none", GraphDatabaseSettings.logical_log_rotation_threshold.name(), "1M", GraphDatabaseSettings.check_point_interval_time.name(), "100ms");
    Cluster cluster = clusterRule.withSharedCoreParams(params).startCluster();
    cluster.coreTx((db, tx) -> {
        createData(db, 10);
        tx.success();
    });
    awaitEx(() -> readReplicasUpToDateAsTheLeader(cluster.awaitLeader(), cluster.readReplicas()), 1, TimeUnit.MINUTES);
    ReadReplica readReplica = cluster.getReadReplicaById(0);
    long highestReadReplicaLogVersion = physicalLogFiles(readReplica).getHighestLogVersion();
    // when
    readReplica.shutdown();
    CoreClusterMember core;
    do {
        core = cluster.coreTx((db, tx) -> {
            createData(db, 1_000);
            tx.success();
        });
    } while (physicalLogFiles(core).getLowestLogVersion() <= highestReadReplicaLogVersion);
    readReplica.start();
    // then
    awaitEx(() -> readReplicasUpToDateAsTheLeader(cluster.awaitLeader(), cluster.readReplicas()), 1, TimeUnit.MINUTES);
    assertEventually("The read replica has the same data as the core members", () -> DbRepresentation.of(readReplica.database()), equalTo(DbRepresentation.of(cluster.awaitLeader().database())), 10, TimeUnit.SECONDS);
}

Example 13 with ReadReplica

use of org.neo4j.causalclustering.discovery.ReadReplica in project neo4j by neo4j.

the class ReadReplicaReplicationIT method shouldEventuallyPullTransactionDownToAllReadReplicas.

@Test
public void shouldEventuallyPullTransactionDownToAllReadReplicas() throws Exception {
    // given
    Cluster cluster = clusterRule.withNumberOfReadReplicas(0).startCluster();
    int nodesBeforeReadReplicaStarts = 1;
    cluster.coreTx((db, tx) -> {
        db.schema().constraintFor(Label.label("Foo")).assertPropertyIsUnique("foobar").create();
        tx.success();
    });
    // when
    for (int i = 0; i < 100; i++) {
        cluster.coreTx((db, tx) -> {
            createData(db, nodesBeforeReadReplicaStarts);
            tx.success();
        });
    }
    Set<Path> labelScanStoreFiles = new HashSet<>();
    cluster.coreTx((db, tx) -> gatherLabelScanStoreFiles(db, labelScanStoreFiles));
    AtomicBoolean labelScanStoreCorrectlyPlaced = new AtomicBoolean(false);
    Monitors monitors = new Monitors();
    ReadReplica rr = cluster.addReadReplicaWithIdAndMonitors(0, monitors);
    Path readReplicateStoreDir = rr.storeDir().toPath().toAbsolutePath();
    monitors.addMonitorListener((FileCopyMonitor) file -> {
        Path relativPath = readReplicateStoreDir.relativize(file.toPath().toAbsolutePath());
        relativPath = relativPath.subpath(1, relativPath.getNameCount());
        if (labelScanStoreFiles.contains(relativPath)) {
            labelScanStoreCorrectlyPlaced.set(true);
        }
    });
    rr.start();
    for (int i = 0; i < 100; i++) {
        cluster.coreTx((db, tx) -> {
            createData(db, nodesBeforeReadReplicaStarts);
            tx.success();
        });
    }
    // then
    for (final ReadReplica server : cluster.readReplicas()) {
        GraphDatabaseService readReplica = server.database();
        try (Transaction tx = readReplica.beginTx()) {
            ThrowingSupplier<Long, Exception> nodeCount = () -> count(readReplica.getAllNodes());
            assertEventually("node to appear on read replica", nodeCount, is(400L), 1, MINUTES);
            for (Node node : readReplica.getAllNodes()) {
                assertThat(node.getProperty("foobar").toString(), startsWith("baz_bat"));
            }
            tx.success();
        }
    }
    assertTrue(labelScanStoreCorrectlyPlaced.get());
}

Example 14 with ReadReplica

use of org.neo4j.causalclustering.discovery.ReadReplica in project neo4j by neo4j.

the class ReadReplicaReplicationIT method shouldBeAbleToCopyStoresFromCoreToReadReplica.

@Test
public void shouldBeAbleToCopyStoresFromCoreToReadReplica() throws Exception {
    // given
    Map<String, String> params = stringMap(CausalClusteringSettings.raft_log_rotation_size.name(), "1k", CausalClusteringSettings.raft_log_pruning_frequency.name(), "500ms", CausalClusteringSettings.state_machine_flush_window_size.name(), "1", CausalClusteringSettings.raft_log_pruning_strategy.name(), "1 entries");
    Cluster cluster = clusterRule.withNumberOfReadReplicas(0).withSharedCoreParams(params).withRecordFormat(HighLimit.NAME).startCluster();
    cluster.coreTx((db, tx) -> {
        Node node = db.createNode(Label.label("L"));
        for (int i = 0; i < 10; i++) {
            node.setProperty("prop-" + i, "this is a quite long string to get to the log limit soonish");
        }
        tx.success();
    });
    long baseVersion = versionBy(cluster.awaitLeader().raftLogDirectory(), Math::max);
    CoreClusterMember coreGraphDatabase = null;
    for (int j = 0; j < 2; j++) {
        coreGraphDatabase = cluster.coreTx((db, tx) -> {
            Node node = db.createNode(Label.label("L"));
            for (int i = 0; i < 10; i++) {
                node.setProperty("prop-" + i, "this is a quite long string to get to the log limit soonish");
            }
            tx.success();
        });
    }
    File raftLogDir = coreGraphDatabase.raftLogDirectory();
    assertEventually("pruning happened", () -> versionBy(raftLogDir, Math::min), greaterThan(baseVersion), 5, SECONDS);
    // when
    cluster.addReadReplicaWithIdAndRecordFormat(4, HighLimit.NAME).start();
    // then
    for (final ReadReplica readReplica : cluster.readReplicas()) {
        assertEventually("read replica available", () -> readReplica.database().isAvailable(0), is(true), 10, SECONDS);
    }
}

Example 15 with ReadReplica

use of org.neo4j.causalclustering.discovery.ReadReplica in project neo4j by neo4j.

the class ReadReplicaReplicationIT method readReplicasShouldRestartIfTheWholeClusterIsRestarted.

@Test
public void readReplicasShouldRestartIfTheWholeClusterIsRestarted() throws Exception {
    // given
    Cluster cluster = clusterRule.startCluster();
    // when
    cluster.shutdown();
    cluster.start();
    // then
    for (final ReadReplica readReplica : cluster.readReplicas()) {
        ThrowingSupplier<Boolean, Exception> availability = () -> readReplica.database().isAvailable(0);
        assertEventually("read replica becomes available", availability, is(true), 10, SECONDS);
    }
}