Search in sources :

Example 16 with AdvertisedSocketAddress

use of org.neo4j.helpers.AdvertisedSocketAddress in project neo4j by neo4j.

the class HazelcastCoreTopologyService method createHazelcastInstance.

private HazelcastInstance createHazelcastInstance() {
    System.setProperty(WAIT_SECONDS_BEFORE_JOIN.getName(), "1");
    JoinConfig joinConfig = new JoinConfig();
    joinConfig.getMulticastConfig().setEnabled(false);
    joinConfig.getAwsConfig().setEnabled(false);
    TcpIpConfig tcpIpConfig = joinConfig.getTcpIpConfig();
    tcpIpConfig.setEnabled(true);
    List<AdvertisedSocketAddress> initialMembers = config.get(CausalClusteringSettings.initial_discovery_members);
    for (AdvertisedSocketAddress address : initialMembers) {
        tcpIpConfig.addMember(address.toString());
    }
    Setting<ListenSocketAddress> discovery_listen_address = CausalClusteringSettings.discovery_listen_address;
    ListenSocketAddress hazelcastAddress = config.get(discovery_listen_address);
    InterfacesConfig interfaces = new InterfacesConfig();
    interfaces.addInterface(hazelcastAddress.getHostname());
    NetworkConfig networkConfig = new NetworkConfig();
    networkConfig.setInterfaces(interfaces);
    networkConfig.setPort(hazelcastAddress.getPort());
    networkConfig.setJoin(joinConfig);
    networkConfig.setPortAutoIncrement(false);
    com.hazelcast.config.Config c = new com.hazelcast.config.Config();
    c.setProperty(OPERATION_CALL_TIMEOUT_MILLIS.getName(), String.valueOf(10_000));
    c.setProperty(MERGE_NEXT_RUN_DELAY_SECONDS.getName(), "10");
    c.setProperty(MERGE_FIRST_RUN_DELAY_SECONDS.getName(), "10");
    c.setProperty(INITIAL_MIN_CLUSTER_SIZE.getName(), String.valueOf(minimumClusterSizeThatCanTolerateOneFaultForExpectedClusterSize()));
    c.setProperty(LOGGING_TYPE.getName(), "none");
    c.setNetworkConfig(networkConfig);
    MemberAttributeConfig memberAttributeConfig = HazelcastClusterTopology.buildMemberAttributesForCore(myself, config);
    c.setMemberAttributeConfig(memberAttributeConfig);
    logConnectionInfo(initialMembers);
    JobScheduler.JobHandle logJob = scheduler.schedule("HazelcastHealth", HAZELCAST_IS_HEALTHY_TIMEOUT_MS, () -> log.warn("The server has not been able to connect in a timely fashion to the " + "cluster. Please consult the logs for more details. Rebooting the server may " + "solve the problem."));
    try {
        hazelcastInstance = Hazelcast.newHazelcastInstance(c);
        logJob.cancel(true);
    } catch (HazelcastException e) {
        String errorMessage = String.format("Hazelcast was unable to start with setting: %s = %s", discovery_listen_address.name(), config.get(discovery_listen_address));
        userLog.error(errorMessage);
        log.error(errorMessage, e);
        throw new RuntimeException(e);
    }
    List<String> groups = config.get(CausalClusteringSettings.server_groups);
    refreshGroups(hazelcastInstance, myself.getUuid().toString(), groups);
    return hazelcastInstance;
}
Also used : JobScheduler(org.neo4j.kernel.impl.util.JobScheduler) InterfacesConfig(com.hazelcast.config.InterfacesConfig) HazelcastException(com.hazelcast.core.HazelcastException) JoinConfig(com.hazelcast.config.JoinConfig) MemberAttributeConfig(com.hazelcast.config.MemberAttributeConfig) Config(org.neo4j.kernel.configuration.Config) NetworkConfig(com.hazelcast.config.NetworkConfig) TcpIpConfig(com.hazelcast.config.TcpIpConfig) InterfacesConfig(com.hazelcast.config.InterfacesConfig) NetworkConfig(com.hazelcast.config.NetworkConfig) AdvertisedSocketAddress(org.neo4j.helpers.AdvertisedSocketAddress) ListenSocketAddress(org.neo4j.helpers.ListenSocketAddress) JoinConfig(com.hazelcast.config.JoinConfig) MemberAttributeConfig(com.hazelcast.config.MemberAttributeConfig) TcpIpConfig(com.hazelcast.config.TcpIpConfig)

Example 17 with AdvertisedSocketAddress

use of org.neo4j.helpers.AdvertisedSocketAddress in project neo4j by neo4j.

the class CausalClusterConfigurationValidatorTest method validateSuccess.

@Test
public void validateSuccess() throws Exception {
    // when
    Config config = Config.embeddedDefaults(stringMap(ClusterSettings.mode.name(), mode.name(), initial_discovery_members.name(), "localhost:99,remotehost:2", new BoltConnector("bolt").enabled.name(), "true"), Collections.singleton(new CausalClusterConfigurationValidator()));
    // then
    assertEquals(asList(new AdvertisedSocketAddress("localhost", 99), new AdvertisedSocketAddress("remotehost", 2)), config.get(initial_discovery_members));
}
Also used : BoltConnector(org.neo4j.kernel.configuration.BoltConnector) Config(org.neo4j.kernel.configuration.Config) AdvertisedSocketAddress(org.neo4j.helpers.AdvertisedSocketAddress) Test(org.junit.Test)

Example 18 with AdvertisedSocketAddress

use of org.neo4j.helpers.AdvertisedSocketAddress in project neo4j by neo4j.

the class BackupStoreCopyInteractionStressTesting method shouldBehaveCorrectlyUnderStress.

@Test
public void shouldBehaveCorrectlyUnderStress() throws Exception {
    int numberOfCores = parseInt(fromEnv("BACKUP_STORE_COPY_INTERACTION_STRESS_NUMBER_OF_CORES", DEFAULT_NUMBER_OF_CORES));
    int numberOfEdges = parseInt(fromEnv("BACKUP_STORE_COPY_INTERACTION_STRESS_NUMBER_OF_EDGES", DEFAULT_NUMBER_OF_EDGES));
    long durationInMinutes = parseLong(fromEnv("BACKUP_STORE_COPY_INTERACTION_STRESS_DURATION", DEFAULT_DURATION_IN_MINUTES));
    String workingDirectory = fromEnv("BACKUP_STORE_COPY_INTERACTION_STRESS_WORKING_DIRECTORY", DEFAULT_WORKING_DIR);
    int baseCoreBackupPort = parseInt(fromEnv("BACKUP_STORE_COPY_INTERACTION_STRESS_BASE_CORE_BACKUP_PORT", DEFAULT_BASE_CORE_BACKUP_PORT));
    int baseEdgeBackupPort = parseInt(fromEnv("BACKUP_STORE_COPY_INTERACTION_STRESS_BASE_EDGE_BACKUP_PORT", DEFAULT_BASE_EDGE_BACKUP_PORT));
    boolean enableIndexes = parseBoolean(fromEnv("BACKUP_STORE_COPY_INTERACTION_STRESS_ENABLE_INDEXES", DEFAULT_ENABLE_INDEXES));
    String txPrune = fromEnv("BACKUP_STORE_COPY_INTERACTION_STRESS_TX_PRUNE", DEFAULT_TX_PRUNE);
    File clusterDirectory = ensureExistsAndEmpty(new File(workingDirectory, "cluster"));
    File backupDirectory = ensureExistsAndEmpty(new File(workingDirectory, "backups"));
    BiFunction<Boolean, Integer, SocketAddress> backupAddress = (isCore, id) -> new AdvertisedSocketAddress("localhost", (isCore ? baseCoreBackupPort : baseEdgeBackupPort) + id);
    Map<String, String> coreParams = enableRaftMessageLogging(configureRaftLogRotationAndPruning(configureTxLogRotationAndPruning(new HashMap<>(), txPrune)));
    Map<String, String> readReplicaParams = configureTxLogRotationAndPruning(new HashMap<>(), txPrune);
    Map<String, IntFunction<String>> instanceCoreParams = configureBackup(new HashMap<>(), id -> backupAddress.apply(true, id));
    Map<String, IntFunction<String>> instanceReadReplicaParams = configureBackup(new HashMap<>(), id -> backupAddress.apply(false, id));
    HazelcastDiscoveryServiceFactory discoveryServiceFactory = new HazelcastDiscoveryServiceFactory();
    Cluster cluster = new Cluster(clusterDirectory, numberOfCores, numberOfEdges, discoveryServiceFactory, coreParams, instanceCoreParams, readReplicaParams, instanceReadReplicaParams, Standard.LATEST_NAME);
    AtomicBoolean stopTheWorld = new AtomicBoolean();
    BooleanSupplier notExpired = untilTimeExpired(durationInMinutes, MINUTES);
    BooleanSupplier keepGoing = () -> !stopTheWorld.get() && notExpired.getAsBoolean();
    Runnable onFailure = () -> stopTheWorld.set(true);
    ExecutorService service = Executors.newFixedThreadPool(3);
    try {
        cluster.start();
        if (enableIndexes) {
            Workload.setupIndexes(cluster);
        }
        Future<Throwable> workload = service.submit(new Workload(keepGoing, onFailure, cluster));
        Future<Throwable> startStopWorker = service.submit(new StartStopLoad(fs, pageCache, keepGoing, onFailure, cluster, numberOfCores, numberOfEdges));
        Future<Throwable> backupWorker = service.submit(new BackupLoad(keepGoing, onFailure, cluster, numberOfCores, numberOfEdges, backupDirectory, backupAddress));
        long timeout = durationInMinutes + 5;
        assertNull(Exceptions.stringify(workload.get()), workload.get(timeout, MINUTES));
        assertNull(Exceptions.stringify(startStopWorker.get()), startStopWorker.get(timeout, MINUTES));
        assertNull(Exceptions.stringify(backupWorker.get()), backupWorker.get(timeout, MINUTES));
    } finally {
        cluster.shutdown();
        service.shutdown();
    }
    // let's cleanup disk space when everything went well
    FileUtils.deleteRecursively(clusterDirectory);
    FileUtils.deleteRecursively(backupDirectory);
}
Also used : StressTestingHelper.ensureExistsAndEmpty(org.neo4j.helper.StressTestingHelper.ensureExistsAndEmpty) Suppliers.untilTimeExpired(org.neo4j.function.Suppliers.untilTimeExpired) BiFunction(java.util.function.BiFunction) Exceptions(org.neo4j.helpers.Exceptions) StressTestingHelper.fromEnv(org.neo4j.helper.StressTestingHelper.fromEnv) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) MINUTES(java.util.concurrent.TimeUnit.MINUTES) BooleanSupplier(java.util.function.BooleanSupplier) SocketAddress(org.neo4j.helpers.SocketAddress) Future(java.util.concurrent.Future) AdvertisedSocketAddress(org.neo4j.helpers.AdvertisedSocketAddress) ClusterConfiguration.configureRaftLogRotationAndPruning(org.neo4j.causalclustering.stresstests.ClusterConfiguration.configureRaftLogRotationAndPruning) HazelcastDiscoveryServiceFactory(org.neo4j.causalclustering.discovery.HazelcastDiscoveryServiceFactory) PageCacheRule(org.neo4j.test.rule.PageCacheRule) Map(java.util.Map) ClusterConfiguration.configureBackup(org.neo4j.causalclustering.stresstests.ClusterConfiguration.configureBackup) System.getProperty(java.lang.System.getProperty) ExecutorService(java.util.concurrent.ExecutorService) Before(org.junit.Before) IntFunction(java.util.function.IntFunction) Standard(org.neo4j.kernel.impl.store.format.standard.Standard) PageCache(org.neo4j.io.pagecache.PageCache) ClusterConfiguration.enableRaftMessageLogging(org.neo4j.causalclustering.stresstests.ClusterConfiguration.enableRaftMessageLogging) FileUtils(org.neo4j.io.fs.FileUtils) Test(org.junit.Test) Integer.parseInt(java.lang.Integer.parseInt) File(java.io.File) Executors(java.util.concurrent.Executors) Cluster(org.neo4j.causalclustering.discovery.Cluster) RuleChain(org.junit.rules.RuleChain) Rule(org.junit.Rule) DefaultFileSystemRule(org.neo4j.test.rule.fs.DefaultFileSystemRule) Boolean.parseBoolean(java.lang.Boolean.parseBoolean) Assert.assertNull(org.junit.Assert.assertNull) DatabaseConfiguration.configureTxLogRotationAndPruning(org.neo4j.helper.DatabaseConfiguration.configureTxLogRotationAndPruning) Long.parseLong(java.lang.Long.parseLong) FileSystemAbstraction(org.neo4j.io.fs.FileSystemAbstraction) AdvertisedSocketAddress(org.neo4j.helpers.AdvertisedSocketAddress) HazelcastDiscoveryServiceFactory(org.neo4j.causalclustering.discovery.HazelcastDiscoveryServiceFactory) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Boolean.parseBoolean(java.lang.Boolean.parseBoolean) SocketAddress(org.neo4j.helpers.SocketAddress) AdvertisedSocketAddress(org.neo4j.helpers.AdvertisedSocketAddress) BooleanSupplier(java.util.function.BooleanSupplier) Cluster(org.neo4j.causalclustering.discovery.Cluster) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) IntFunction(java.util.function.IntFunction) ExecutorService(java.util.concurrent.ExecutorService) File(java.io.File) Test(org.junit.Test)

Example 19 with AdvertisedSocketAddress

use of org.neo4j.helpers.AdvertisedSocketAddress in project neo4j by neo4j.

the class GetServersProcedureForSingleDC method coreReadEndPoints.

private Stream<AdvertisedSocketAddress> coreReadEndPoints() {
    Optional<AdvertisedSocketAddress> leader = leaderBoltAddress();
    Collection<CoreServerInfo> coreServerInfo = topologyService.coreServers().members().values();
    Stream<AdvertisedSocketAddress> boltAddresses = topologyService.coreServers().members().values().stream().map(extractBoltAddress());
    // if the leader is present and it is not alone filter it out from the read end points
    if (leader.isPresent() && coreServerInfo.size() > 1) {
        AdvertisedSocketAddress advertisedSocketAddress = leader.get();
        return boltAddresses.filter(address -> !advertisedSocketAddress.equals(address));
    }
    // or if we cannot locate the leader return all cores as read end points
    return boltAddresses;
}
Also used : AdvertisedSocketAddress(org.neo4j.helpers.AdvertisedSocketAddress) CoreServerInfo(org.neo4j.causalclustering.discovery.CoreServerInfo)

Example 20 with AdvertisedSocketAddress

use of org.neo4j.helpers.AdvertisedSocketAddress in project neo4j by neo4j.

the class LegacyGetServersProcedure method coreReadEndPoints.

private Stream<AdvertisedSocketAddress> coreReadEndPoints() {
    Optional<AdvertisedSocketAddress> leader = leaderBoltAddress();
    Collection<CoreServerInfo> coreServerInfo = topologyService.coreServers().members().values();
    Stream<AdvertisedSocketAddress> boltAddresses = topologyService.coreServers().members().values().stream().map(extractBoltAddress());
    // if the leader is present and it is not alone filter it out from the read end points
    if (leader.isPresent() && coreServerInfo.size() > 1) {
        AdvertisedSocketAddress advertisedSocketAddress = leader.get();
        return boltAddresses.filter(address -> !advertisedSocketAddress.equals(address));
    }
    // or if we cannot locate the leader return all cores as read end points
    return boltAddresses;
}
Also used : AdvertisedSocketAddress(org.neo4j.helpers.AdvertisedSocketAddress) CoreServerInfo(org.neo4j.causalclustering.discovery.CoreServerInfo)

Aggregations

AdvertisedSocketAddress (org.neo4j.helpers.AdvertisedSocketAddress)31 Test (org.junit.Test)15 HashMap (java.util.HashMap)7 MemberId (org.neo4j.causalclustering.identity.MemberId)6 BoltConnector (org.neo4j.kernel.configuration.BoltConnector)4 Config (org.neo4j.kernel.configuration.Config)4 BiFunction (java.util.function.BiFunction)3 ClientConnectorAddresses (org.neo4j.causalclustering.discovery.ClientConnectorAddresses)3 ListenSocketAddress (org.neo4j.helpers.ListenSocketAddress)3 MemberAttributeConfig (com.hazelcast.config.MemberAttributeConfig)2 Channel (io.netty.channel.Channel)2 File (java.io.File)2 InetSocketAddress (java.net.InetSocketAddress)2 URI (java.net.URI)2 HashSet (java.util.HashSet)2 Map (java.util.Map)2 Before (org.junit.Before)2 ConnectorUri (org.neo4j.causalclustering.discovery.ClientConnectorAddresses.ConnectorUri)2 Cluster (org.neo4j.causalclustering.discovery.Cluster)2 CoreServerInfo (org.neo4j.causalclustering.discovery.CoreServerInfo)2