Examples with DLSocketAddress com.twitter.distributedlog.service.DLSocketAddress used on opensource projects

Example 1 with DLSocketAddress

use of com.twitter.distributedlog.service.DLSocketAddress in project distributedlog by twitter.

the class TestConsistentHashRoutingService method testPerformServerSetChangeOnServerSet.

@Test(timeout = 60000)
public void testPerformServerSetChangeOnServerSet() throws Exception {
    TestServerSetWatcher serverSetWatcher = new TestServerSetWatcher();
    ConsistentHashRoutingService routingService = new ConsistentHashRoutingService(serverSetWatcher, 997, Integer.MAX_VALUE, NullStatsReceiver.get());
    int basePort = 3180;
    int numHosts = 4;
    Set<DLSocketAddress> addresses1 = Sets.newConcurrentHashSet();
    Set<DLSocketAddress> addresses2 = Sets.newConcurrentHashSet();
    Set<DLSocketAddress> addresses3 = Sets.newConcurrentHashSet();
    // fill up the addresses1
    for (int i = 0; i < numHosts; i++) {
        InetSocketAddress inetAddress = new InetSocketAddress("127.0.0.1", basePort + i);
        DLSocketAddress dsa = new DLSocketAddress(i, inetAddress);
        addresses1.add(dsa);
    }
    // fill up the addresses2 - overlap with addresses1
    for (int i = 0; i < numHosts; i++) {
        InetSocketAddress inetAddress = new InetSocketAddress("127.0.0.1", basePort + numHosts + i);
        DLSocketAddress dsa = new DLSocketAddress(i + 2, inetAddress);
        addresses2.add(dsa);
    }
    // fill up the addresses3 - not overlap with addresses2
    for (int i = 0; i < numHosts; i++) {
        InetSocketAddress inetAddress = new InetSocketAddress("127.0.0.1", basePort + 10 + i);
        DLSocketAddress dsa = new DLSocketAddress(i, inetAddress);
        addresses3.add(dsa);
    }
    final List<SocketAddress> leftAddresses = Lists.newArrayList();
    final List<SocketAddress> joinAddresses = Lists.newArrayList();
    RoutingService.RoutingListener routingListener = new RoutingService.RoutingListener() {

        @Override
        public void onServerLeft(SocketAddress address) {
            synchronized (leftAddresses) {
                leftAddresses.add(address);
                leftAddresses.notifyAll();
            }
        }

        @Override
        public void onServerJoin(SocketAddress address) {
            synchronized (joinAddresses) {
                joinAddresses.add(address);
                joinAddresses.notifyAll();
            }
        }
    };
    routingService.registerListener(routingListener);
    serverSetWatcher.notifyChanges(ImmutableSet.copyOf(addresses1));
    routingService.startService();
    synchronized (joinAddresses) {
        while (joinAddresses.size() < numHosts) {
            joinAddresses.wait();
        }
    }
    // validate 4 nodes joined
    synchronized (joinAddresses) {
        assertEquals(numHosts, joinAddresses.size());
    }
    synchronized (leftAddresses) {
        assertEquals(0, leftAddresses.size());
    }
    assertEquals(numHosts, routingService.shardId2Address.size());
    assertEquals(numHosts, routingService.address2ShardId.size());
    for (int i = 0; i < numHosts; i++) {
        InetSocketAddress inetAddress = new InetSocketAddress("127.0.0.1", basePort + i);
        assertTrue(routingService.address2ShardId.containsKey(inetAddress));
        int shardId = routingService.address2ShardId.get(inetAddress);
        assertEquals(i, shardId);
        SocketAddress sa = routingService.shardId2Address.get(shardId);
        assertNotNull(sa);
        assertEquals(inetAddress, sa);
    }
    // update addresses2 - 2 new hosts joined, 2 old hosts left
    serverSetWatcher.notifyChanges(ImmutableSet.copyOf(addresses2));
    synchronized (joinAddresses) {
        while (joinAddresses.size() < numHosts + 2) {
            joinAddresses.wait();
        }
    }
    synchronized (leftAddresses) {
        while (leftAddresses.size() < 2) {
            leftAddresses.wait();
        }
    }
    assertEquals(numHosts + 2, routingService.shardId2Address.size());
    assertEquals(numHosts + 2, routingService.address2ShardId.size());
    // first 2 shards should not leave
    for (int i = 0; i < 2; i++) {
        InetSocketAddress inetAddress = new InetSocketAddress("127.0.0.1", basePort + i);
        assertTrue(routingService.address2ShardId.containsKey(inetAddress));
        int shardId = routingService.address2ShardId.get(inetAddress);
        assertEquals(i, shardId);
        SocketAddress sa = routingService.shardId2Address.get(shardId);
        assertNotNull(sa);
        assertEquals(inetAddress, sa);
    }
    for (int i = 0; i < numHosts; i++) {
        InetSocketAddress inetAddress = new InetSocketAddress("127.0.0.1", basePort + numHosts + i);
        assertTrue(routingService.address2ShardId.containsKey(inetAddress));
        int shardId = routingService.address2ShardId.get(inetAddress);
        assertEquals(i + 2, shardId);
        SocketAddress sa = routingService.shardId2Address.get(shardId);
        assertNotNull(sa);
        assertEquals(inetAddress, sa);
    }
    // update addresses3
    serverSetWatcher.notifyChanges(ImmutableSet.copyOf(addresses3));
    synchronized (joinAddresses) {
        while (joinAddresses.size() < numHosts + 2 + numHosts) {
            joinAddresses.wait();
        }
    }
    synchronized (leftAddresses) {
        while (leftAddresses.size() < 2 + numHosts) {
            leftAddresses.wait();
        }
    }
    assertEquals(numHosts + 2, routingService.shardId2Address.size());
    assertEquals(numHosts + 2, routingService.address2ShardId.size());
    // first 4 shards should leave
    for (int i = 0; i < numHosts; i++) {
        InetSocketAddress inetAddress = new InetSocketAddress("127.0.0.1", basePort + 10 + i);
        assertTrue(routingService.address2ShardId.containsKey(inetAddress));
        int shardId = routingService.address2ShardId.get(inetAddress);
        assertEquals(i, shardId);
        SocketAddress sa = routingService.shardId2Address.get(shardId);
        assertNotNull(sa);
        assertEquals(inetAddress, sa);
    }
    // the other 2 shards should be still there
    for (int i = 0; i < 2; i++) {
        InetSocketAddress inetAddress = new InetSocketAddress("127.0.0.1", basePort + numHosts + 2 + i);
        assertTrue(routingService.address2ShardId.containsKey(inetAddress));
        int shardId = routingService.address2ShardId.get(inetAddress);
        assertEquals(numHosts + i, shardId);
        SocketAddress sa = routingService.shardId2Address.get(shardId);
        assertNotNull(sa);
        assertEquals(inetAddress, sa);
    }
}

Example 2 with DLSocketAddress

use of com.twitter.distributedlog.service.DLSocketAddress in project distributedlog by twitter.

the class ConsistentHashRoutingService method performServerSetChange.

@Override
protected synchronized void performServerSetChange(ImmutableSet<DLSocketAddress> serviceInstances) {
    Set<SocketAddress> joinedList = new HashSet<SocketAddress>();
    Set<SocketAddress> removedList = new HashSet<SocketAddress>();
    Map<Integer, SocketAddress> newMap = new HashMap<Integer, SocketAddress>();
    synchronized (shardId2Address) {
        for (DLSocketAddress serviceInstance : serviceInstances) {
            if (serviceInstance.getShard() >= 0) {
                newMap.put(serviceInstance.getShard(), serviceInstance.getSocketAddress());
            } else {
                Integer shard = address2ShardId.get(serviceInstance.getSocketAddress());
                if (null == shard) {
                    // Assign a random negative shardId
                    int shardId;
                    do {
                        shardId = Math.min(-1, (int) (Math.random() * Integer.MIN_VALUE));
                    } while (null != shardId2Address.get(shardId));
                    shard = shardId;
                }
                newMap.put(shard, serviceInstance.getSocketAddress());
            }
        }
    }
    Map<Integer, SocketAddress> left;
    synchronized (shardId2Address) {
        MapDifference<Integer, SocketAddress> difference = Maps.difference(shardId2Address, newMap);
        left = difference.entriesOnlyOnLeft();
        for (Map.Entry<Integer, SocketAddress> shardEntry : left.entrySet()) {
            int shard = shardEntry.getKey();
            if (shard >= 0) {
                SocketAddress host = shardId2Address.get(shard);
                if (null != host) {
                    // we don't remove those hosts that just disappered on serverset proactively,
                    // since it might be just because serverset become flaky
                    // address2ShardId.remove(host);
                    // circle.remove(shard, host);
                    logger.info("Shard {} ({}) left temporarily.", shard, host);
                }
            } else {
                // shard id is negative - they are resolved from finagle name, which instances don't have shard id
                // in this case, if they are removed from serverset, we removed them directly
                SocketAddress host = shardEntry.getValue();
                if (null != host) {
                    removeHostInternal(host, Optional.<Throwable>absent());
                    removedList.add(host);
                }
            }
        }
        // we need to find if any shards are replacing old shards
        for (Map.Entry<Integer, SocketAddress> shard : newMap.entrySet()) {
            SocketAddress oldHost = shardId2Address.get(shard.getKey());
            SocketAddress newHost = shard.getValue();
            if (!newHost.equals(oldHost)) {
                join(shard.getKey(), newHost, removedList);
                joinedList.add(newHost);
            }
        }
    }
    for (SocketAddress addr : removedList) {
        for (RoutingListener listener : listeners) {
            listener.onServerLeft(addr);
        }
    }
    for (SocketAddress addr : joinedList) {
        for (RoutingListener listener : listeners) {
            listener.onServerJoin(addr);
        }
    }
}

Example 3 with DLSocketAddress

use of com.twitter.distributedlog.service.DLSocketAddress in project distributedlog by twitter.

the class ServerSetRoutingService method performServerSetChange.

protected synchronized void performServerSetChange(ImmutableSet<DLSocketAddress> serverSet) {
    Set<SocketAddress> newSet = new HashSet<SocketAddress>();
    for (DLSocketAddress serviceInstance : serverSet) {
        newSet.add(serviceInstance.getSocketAddress());
    }
    Set<SocketAddress> removed;
    Set<SocketAddress> added;
    synchronized (hostSet) {
        removed = Sets.difference(hostSet, newSet).immutableCopy();
        added = Sets.difference(newSet, hostSet).immutableCopy();
        for (SocketAddress node : removed) {
            if (hostSet.remove(node)) {
                logger.info("Node {} left.", node);
            }
        }
        for (SocketAddress node : added) {
            if (hostSet.add(node)) {
                logger.info("Node {} joined.", node);
            }
        }
    }
    for (SocketAddress addr : removed) {
        for (RoutingListener listener : listeners) {
            listener.onServerLeft(addr);
        }
    }
    for (SocketAddress addr : added) {
        for (RoutingListener listener : listeners) {
            listener.onServerJoin(addr);
        }
    }
    synchronized (hostSet) {
        hostList = new ArrayList<SocketAddress>(hostSet);
        Collections.sort(hostList, HostComparator.instance);
        logger.info("Host list becomes : {}.", hostList);
    }
}