Examples with Cluster org.neo4j.causalclustering.discovery.Cluster used on opensource projects

Example 31 with Cluster

use of org.neo4j.causalclustering.discovery.Cluster in project neo4j by neo4j.

the class CatchupStoreCopyInteractionStressTesting method shouldBehaveCorrectlyUnderStress.

@Test
public void shouldBehaveCorrectlyUnderStress() throws Exception {
    int numberOfCores = parseInt(fromEnv("CATCHUP_STORE_COPY_INTERACTION_STRESS_NUMBER_OF_CORES", DEFAULT_NUMBER_OF_CORES));
    int numberOfEdges = parseInt(fromEnv("CATCHUP_STORE_COPY_INTERACTION_STRESS_NUMBER_OF_EDGES", DEFAULT_NUMBER_OF_EDGES));
    long durationInMinutes = parseLong(fromEnv("CATCHUP_STORE_COPY_INTERACTION_STRESS_DURATION", DEFAULT_DURATION_IN_MINUTES));
    String workingDirectory = fromEnv("CATCHUP_STORE_COPY_INTERACTION_STRESS_WORKING_DIRECTORY", DEFAULT_WORKING_DIR);
    boolean enableIndexes = parseBoolean(fromEnv("CATCHUP_STORE_COPY_INTERACTION_STRESS_ENABLE_INDEXES", DEFAULT_ENABLE_INDEXES));
    String txPrune = fromEnv("CATCHUP_STORE_COPY_INTERACTION_STRESS_TX_PRUNE", DEFAULT_TX_PRUNE);
    File clusterDirectory = ensureExistsAndEmpty(new File(workingDirectory, "cluster"));
    Map<String, String> coreParams = enableRaftMessageLogging(configureRaftLogRotationAndPruning(configureTxLogRotationAndPruning(new HashMap<>(), txPrune)));
    Map<String, String> edgeParams = configureTxLogRotationAndPruning(new HashMap<>(), txPrune);
    HazelcastDiscoveryServiceFactory discoveryServiceFactory = new HazelcastDiscoveryServiceFactory();
    Cluster cluster = new Cluster(clusterDirectory, numberOfCores, numberOfEdges, discoveryServiceFactory, coreParams, emptyMap(), edgeParams, emptyMap(), Standard.LATEST_NAME);
    AtomicBoolean stopTheWorld = new AtomicBoolean();
    BooleanSupplier notExpired = untilTimeExpired(durationInMinutes, MINUTES);
    BooleanSupplier keepGoing = () -> !stopTheWorld.get() && notExpired.getAsBoolean();
    Runnable onFailure = () -> stopTheWorld.set(true);
    ExecutorService service = Executors.newCachedThreadPool();
    try {
        cluster.start();
        if (enableIndexes) {
            Workload.setupIndexes(cluster);
        }
        Future<Throwable> workload = service.submit(new Workload(keepGoing, onFailure, cluster));
        Future<Throwable> startStopWorker = service.submit(new StartStopLoad(fs, pageCache, keepGoing, onFailure, cluster, numberOfCores, numberOfEdges));
        Future<Throwable> catchUpWorker = service.submit(new CatchUpLoad(keepGoing, onFailure, cluster));
        long timeout = durationInMinutes + 5;
        assertNull(Exceptions.stringify(workload.get()), workload.get(timeout, MINUTES));
        assertNull(Exceptions.stringify(startStopWorker.get()), startStopWorker.get(timeout, MINUTES));
        assertNull(Exceptions.stringify(catchUpWorker.get()), catchUpWorker.get(timeout, MINUTES));
    } finally {
        cluster.shutdown();
        service.shutdown();
    }
    // let's cleanup disk space when everything went well
    FileUtils.deleteRecursively(clusterDirectory);
}

Example 32 with Cluster

use of org.neo4j.causalclustering.discovery.Cluster in project neo4j by neo4j.

the class CoreToCoreCopySnapshotIT method shouldBeAbleToDownloadToRejoinedInstanceAfterPruning.

@Test
public void shouldBeAbleToDownloadToRejoinedInstanceAfterPruning() throws Exception {
    // given
    Map<String, String> coreParams = stringMap();
    coreParams.put(raft_log_rotation_size.name(), "1K");
    coreParams.put(raft_log_pruning_strategy.name(), "keep_none");
    coreParams.put(raft_log_pruning_frequency.name(), "100ms");
    coreParams.put(state_machine_flush_window_size.name(), "64");
    int numberOfTransactions = 100;
    Timeout timeout = new Timeout(Clocks.systemClock(), 60, SECONDS);
    // start the cluster
    Cluster cluster = clusterRule.withSharedCoreParams(coreParams).startCluster();
    // accumulate some log files
    int firstServerLogFileCount;
    CoreClusterMember firstServer;
    do {
        timeout.assertNotTimedOut();
        firstServer = doSomeTransactions(cluster, numberOfTransactions);
        firstServerLogFileCount = getMostRecentLogIdOn(firstServer);
    } while (firstServerLogFileCount < 5);
    firstServer.shutdown();
    /* After shutdown we wait until we accumulate enough logs, and so that enough of the old ones
         * have been pruned, so that the rejoined instance won't be able to catch up to without a snapshot. */
    int oldestLogOnSecondServer;
    CoreClusterMember secondServer;
    do {
        timeout.assertNotTimedOut();
        secondServer = doSomeTransactions(cluster, numberOfTransactions);
        oldestLogOnSecondServer = getOldestLogIdOn(secondServer);
    } while (oldestLogOnSecondServer < firstServerLogFileCount + 5);
    // when
    firstServer.start();
    // then
    dataOnMemberEventuallyLooksLike(firstServer, secondServer);
}

Example 33 with Cluster

use of org.neo4j.causalclustering.discovery.Cluster in project neo4j by neo4j.

the class ReadReplicaReplicationIT method shouldThrowExceptionIfReadReplicaRecordFormatDiffersToCoreRecordFormat.

@Test
public void shouldThrowExceptionIfReadReplicaRecordFormatDiffersToCoreRecordFormat() throws Exception {
    // given
    Cluster cluster = clusterRule.withNumberOfReadReplicas(0).withRecordFormat(HighLimit.NAME).startCluster();
    // when
    cluster.coreTx(createSomeData);
    try {
        String format = Standard.LATEST_NAME;
        cluster.addReadReplicaWithIdAndRecordFormat(0, format).start();
        fail("starting read replica with '" + format + "' format should have failed");
    } catch (Exception e) {
        assertThat(e.getCause().getCause().getMessage(), containsString("Failed to start database with copied store"));
    }
}

Example 34 with Cluster

use of org.neo4j.causalclustering.discovery.Cluster in project neo4j by neo4j.

the class ReadReplicaReplicationIT method shouldBeAbleToDownloadANewStoreAfterPruning.

@Test
public void shouldBeAbleToDownloadANewStoreAfterPruning() throws Exception {
    // given
    Map<String, String> params = stringMap(GraphDatabaseSettings.keep_logical_logs.name(), "keep_none", GraphDatabaseSettings.logical_log_rotation_threshold.name(), "1M", GraphDatabaseSettings.check_point_interval_time.name(), "100ms");
    Cluster cluster = clusterRule.withSharedCoreParams(params).startCluster();
    cluster.coreTx((db, tx) -> {
        createData(db, 10);
        tx.success();
    });
    awaitEx(() -> readReplicasUpToDateAsTheLeader(cluster.awaitLeader(), cluster.readReplicas()), 1, TimeUnit.MINUTES);
    ReadReplica readReplica = cluster.getReadReplicaById(0);
    long highestReadReplicaLogVersion = physicalLogFiles(readReplica).getHighestLogVersion();
    // when
    readReplica.shutdown();
    CoreClusterMember core;
    do {
        core = cluster.coreTx((db, tx) -> {
            createData(db, 1_000);
            tx.success();
        });
    } while (physicalLogFiles(core).getLowestLogVersion() <= highestReadReplicaLogVersion);
    readReplica.start();
    // then
    awaitEx(() -> readReplicasUpToDateAsTheLeader(cluster.awaitLeader(), cluster.readReplicas()), 1, TimeUnit.MINUTES);
    assertEventually("The read replica has the same data as the core members", () -> DbRepresentation.of(readReplica.database()), equalTo(DbRepresentation.of(cluster.awaitLeader().database())), 10, TimeUnit.SECONDS);
}

Example 35 with Cluster

use of org.neo4j.causalclustering.discovery.Cluster in project neo4j by neo4j.

the class ReadReplicaReplicationIT method shouldEventuallyPullTransactionDownToAllReadReplicas.

@Test
public void shouldEventuallyPullTransactionDownToAllReadReplicas() throws Exception {
    // given
    Cluster cluster = clusterRule.withNumberOfReadReplicas(0).startCluster();
    int nodesBeforeReadReplicaStarts = 1;
    cluster.coreTx((db, tx) -> {
        db.schema().constraintFor(Label.label("Foo")).assertPropertyIsUnique("foobar").create();
        tx.success();
    });
    // when
    for (int i = 0; i < 100; i++) {
        cluster.coreTx((db, tx) -> {
            createData(db, nodesBeforeReadReplicaStarts);
            tx.success();
        });
    }
    Set<Path> labelScanStoreFiles = new HashSet<>();
    cluster.coreTx((db, tx) -> gatherLabelScanStoreFiles(db, labelScanStoreFiles));
    AtomicBoolean labelScanStoreCorrectlyPlaced = new AtomicBoolean(false);
    Monitors monitors = new Monitors();
    ReadReplica rr = cluster.addReadReplicaWithIdAndMonitors(0, monitors);
    Path readReplicateStoreDir = rr.storeDir().toPath().toAbsolutePath();
    monitors.addMonitorListener((FileCopyMonitor) file -> {
        Path relativPath = readReplicateStoreDir.relativize(file.toPath().toAbsolutePath());
        relativPath = relativPath.subpath(1, relativPath.getNameCount());
        if (labelScanStoreFiles.contains(relativPath)) {
            labelScanStoreCorrectlyPlaced.set(true);
        }
    });
    rr.start();
    for (int i = 0; i < 100; i++) {
        cluster.coreTx((db, tx) -> {
            createData(db, nodesBeforeReadReplicaStarts);
            tx.success();
        });
    }
    // then
    for (final ReadReplica server : cluster.readReplicas()) {
        GraphDatabaseService readReplica = server.database();
        try (Transaction tx = readReplica.beginTx()) {
            ThrowingSupplier<Long, Exception> nodeCount = () -> count(readReplica.getAllNodes());
            assertEventually("node to appear on read replica", nodeCount, is(400L), 1, MINUTES);
            for (Node node : readReplica.getAllNodes()) {
                assertThat(node.getProperty("foobar").toString(), startsWith("baz_bat"));
            }
            tx.success();
        }
    }
    assertTrue(labelScanStoreCorrectlyPlaced.get());
}