Examples with WritableEndpoint io.mantisrx.common.network.WritableEndpoint used on opensource projects

Example 1 with WritableEndpoint

use of io.mantisrx.common.network.WritableEndpoint in project mantis by Netflix.

the class ServerSlotManagerTest method threeNodeSameClientIdTest.

@Test
public void threeNodeSameClientIdTest() {
    WritableEndpoint<Void> n1 = new WritableEndpoint<Void>("host1", 7001);
    WritableEndpoint<Void> n2 = new WritableEndpoint<Void>("host2", 7001);
    Map<String, List<String>> params = new HashMap<String, List<String>>();
    List<String> vals = new ArrayList<String>();
    vals.add("client1");
    params.put("clientId", vals);
    ServerSlotManager<Void> ssm = new ServerSlotManager<Void>(HashFunctions.ketama());
    SlotAssignmentManager<Void> sm = ssm.registerServer(n1, params);
    SlotAssignmentManager<Void> sm2 = ssm.registerServer(n2, params);
    assertEquals(sm, sm2);
    int hostHitCountNode1 = 0;
    int hostHitCountNode2 = 0;
    int nonHitCount = 0;
    int MSG_COUNT = 1000000;
    for (int i = 0; i < MSG_COUNT; i++) {
        String msg = "msg:" + i;
        if (sm.filter(n1, msg.getBytes())) {
            hostHitCountNode1++;
        } else if (sm2.filter(n2, msg.getBytes())) {
            hostHitCountNode2++;
        } else {
            nonHitCount++;
        }
    }
    double host1HitPercentage = (double) hostHitCountNode1 / (double) MSG_COUNT;
    System.out.println("host1 hit % " + host1HitPercentage);
    assertTrue(host1HitPercentage > 0.4 && host1HitPercentage < 0.6);
    assertTrue(nonHitCount == 0);
    assertEquals(MSG_COUNT, hostHitCountNode1 + hostHitCountNode2);
    WritableEndpoint<Void> n3 = new WritableEndpoint<Void>("host3", 7001);
    // add another node
    SlotAssignmentManager<Void> sm3 = ssm.registerServer(n3, params);
    hostHitCountNode1 = 0;
    hostHitCountNode2 = 0;
    int hostHitCountNode3 = 0;
    nonHitCount = 0;
    MSG_COUNT = 1000000;
    for (int i = 0; i < MSG_COUNT; i++) {
        String msg = "msg:" + i;
        if (sm.filter(n1, msg.getBytes())) {
            hostHitCountNode1++;
        }
        if (sm2.filter(n2, msg.getBytes())) {
            hostHitCountNode2++;
        }
        if (sm3.filter(n3, msg.getBytes())) {
            hostHitCountNode3++;
        } else {
            nonHitCount++;
        }
    }
    assertEquals(MSG_COUNT, hostHitCountNode1 + hostHitCountNode2 + hostHitCountNode3);
    ssm.deregisterServer(n1, params);
    ssm.deregisterServer(n2, params);
    ssm.deregisterServer(n3, params);
}

Example 2 with WritableEndpoint

use of io.mantisrx.common.network.WritableEndpoint in project mantis by Netflix.

the class ServerSlotManagerTest method oneNodeTest.

@Test
public void oneNodeTest() {
    WritableEndpoint<Void> n1 = new WritableEndpoint<Void>("host1", 7001);
    Map<String, List<String>> params = new HashMap<String, List<String>>();
    ServerSlotManager<Void> ssm = new ServerSlotManager<Void>(HashFunctions.ketama());
    SlotAssignmentManager<Void> sam = ssm.registerServer(n1, params);
    int hostHitCountNode1 = 0;
    int nonHitCount = 0;
    int MSG_COUNT = 100000;
    for (int i = 0; i < MSG_COUNT; i++) {
        if (sam.filter(n1, ("msg:" + i).getBytes())) {
            hostHitCountNode1++;
        } else {
            nonHitCount++;
        }
    }
    assertTrue(nonHitCount == 0);
    assertEquals(MSG_COUNT, hostHitCountNode1);
    // cleanup
    ssm.deregisterServer(n1, params);
}

Example 3 with WritableEndpoint

use of io.mantisrx.common.network.WritableEndpoint in project mantis by Netflix.

the class RemoteObservableConnectionHandler method handleSubscribeRequest.

@SuppressWarnings({ "rawtypes", "unchecked" })
private Observable<Void> handleSubscribeRequest(RemoteRxEvent event, final ObservableConnection<RemoteRxEvent, List<RemoteRxEvent>> connection, MutableReference<SlottingStrategy> slottingStrategyReference, MutableReference<Subscription> unsubscribeCallbackReference, MutableReference<WritableEndpoint> slottingIdReference) {
    // check if observable exists in configs
    String observableName = event.getName();
    ServeConfig config = observables.get(observableName);
    if (config == null) {
        return Observable.error(new RemoteObservableException("No remote observable configuration found for name: " + observableName));
    }
    if (event.getType() == RemoteRxEvent.Type.subscribed) {
        String slotId = null;
        Map<String, String> subscriptionParameters = event.getSubscribeParameters();
        if (subscriptionParameters != null) {
            slotId = subscriptionParameters.get("slotId");
        }
        InetSocketAddress address = (InetSocketAddress) connection.getChannel().remoteAddress();
        WritableEndpoint endpoint = new WritableEndpoint<>(address.getHostName(), address.getPort(), slotId, connection);
        SlottingStrategy slottingStrategy = config.getSlottingStrategy();
        slottingIdReference.setValue(endpoint);
        slottingStrategyReference.setValue(slottingStrategy);
        logger.info("Connection received on server from client endpoint: " + endpoint + ", subscribed to observable: " + observableName);
        serverMetrics.incrementSubscribedCount();
        subscribe(unsubscribeCallbackReference, event, connection, config, endpoint);
        if (!slottingStrategy.addConnection(endpoint)) {
            // unable to slot connection
            logger.warn("Failed to slot connection for endpoint: " + endpoint);
            connection.close(true);
        }
    }
    return Observable.empty();
}

Example 4 with WritableEndpoint

use of io.mantisrx.common.network.WritableEndpoint in project mantis by Netflix.

the class RemoteObservableConnectionHandler method setupConnection.

@SuppressWarnings("rawtypes")
private Observable<Void> setupConnection(final ObservableConnection<RemoteRxEvent, List<RemoteRxEvent>> connection) {
    // state associated with connection
    // used to initiate 'unsubscribe' callback to subscriber
    final MutableReference<Subscription> unsubscribeCallbackReference = new MutableReference<Subscription>();
    // used to release slot when connection completes
    final MutableReference<SlottingStrategy> slottingStrategyReference = new MutableReference<SlottingStrategy>();
    // used to get slotId for connection
    final MutableReference<WritableEndpoint> slottingIdReference = new MutableReference<WritableEndpoint>();
    return connection.getInput().filter(new Func1<RemoteRxEvent, Boolean>() {

        @Override
        public Boolean call(RemoteRxEvent event) {
            boolean supportedOperation = false;
            if (event.getType() == RemoteRxEvent.Type.subscribed || event.getType() == RemoteRxEvent.Type.unsubscribed) {
                supportedOperation = true;
            }
            return supportedOperation;
        }
    }).flatMap(new Func1<RemoteRxEvent, Observable<Void>>() {

        @SuppressWarnings("unchecked")
        @Override
        public Observable<Void> call(RemoteRxEvent event) {
            if (event.getType() == RemoteRxEvent.Type.subscribed) {
                return handleSubscribeRequest(event, connection, slottingStrategyReference, unsubscribeCallbackReference, slottingIdReference);
            } else if (event.getType() == RemoteRxEvent.Type.unsubscribed) {
                Subscription subscription = unsubscribeCallbackReference.getValue();
                if (subscription != null) {
                    subscription.unsubscribe();
                }
                serverMetrics.incrementUnsubscribedCount();
                // release slot
                if (slottingStrategyReference.getValue() != null) {
                    if (!slottingStrategyReference.getValue().removeConnection(slottingIdReference.getValue())) {
                        logger.error("Failed to remove endpoint from slot,  endpoint: " + slottingIdReference.getValue());
                    }
                }
                logger.info("Connection: " + connection.getChannel().remoteAddress() + " unsubscribed, closing connection");
                connection.close(true);
            }
            return Observable.empty();
        }
    });
}

Example 5 with WritableEndpoint

use of io.mantisrx.common.network.WritableEndpoint in project mantis by Netflix.

the class RemoteObservableConnectionHandler method serveGroupedObservable.

private <K, V> Subscription serveGroupedObservable(final Observable<Group<K, V>> groups, final ObservableConnection<RemoteRxEvent, List<RemoteRxEvent>> connection, final RemoteRxEvent event, final Func1<Map<String, String>, Func1<K, Boolean>> filterFunction, final Encoder<K> keyEncoder, final Encoder<V> valueEncoder, final ServeGroupedObservable<K, V> serveConfig, final WritableEndpoint<GroupedObservable<K, V>> endpoint) {
    final MutableReference<Subscription> subReference = new MutableReference<>();
    subReference.setValue(groups.filter(new Func1<Group<K, V>, Boolean>() {

        @Override
        public Boolean call(Group<K, V> group) {
            return filterFunction.call(event.getSubscribeParameters()).call(group.getKeyValue());
        }
    }).doOnCompleted(new Action0() {

        @Override
        public void call() {
            logger.info("OnCompleted recieved in serveGroupedObservable, sending to client.");
        }
    }).doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable t1) {
            logger.info("OnError received in serveGroupedObservable, sending to client: ", t1);
        }
    }).materialize().lift(new DisableBackPressureOperator<Notification<Group<K, V>>>()).buffer(writeBufferTimeMSec, TimeUnit.MILLISECONDS).filter(new Func1<List<Notification<Group<K, V>>>, Boolean>() {

        @Override
        public Boolean call(List<Notification<Group<K, V>>> t1) {
            return t1 != null && !t1.isEmpty();
        }
    }).map(new Func1<List<Notification<Group<K, V>>>, List<RemoteRxEvent>>() {

        @Override
        public List<RemoteRxEvent> call(final List<Notification<Group<K, V>>> groupNotifications) {
            List<RemoteRxEvent> rxEvents = new ArrayList<RemoteRxEvent>(groupNotifications.size());
            for (Notification<Group<K, V>> groupNotification : groupNotifications) {
                if (Kind.OnNext == groupNotification.getKind()) {
                    // encode inner group notification
                    Group<K, V> group = groupNotification.getValue();
                    final int keyLength = group.getKeyBytes().length;
                    Notification<V> notification = groupNotification.getValue().getNotification();
                    byte[] data = null;
                    if (Kind.OnNext == notification.getKind()) {
                        V value = notification.getValue();
                        byte[] valueBytes = valueEncoder.encode(value);
                        // 1 byte for notification type,
                        // 4 bytes is to encode key length as int
                        data = ByteBuffer.allocate(1 + 4 + keyLength + valueBytes.length).put((byte) 1).putInt(keyLength).put(group.getKeyBytes()).put(valueBytes).array();
                    } else if (Kind.OnCompleted == notification.getKind()) {
                        data = ByteBuffer.allocate(1 + 4 + keyLength).put((byte) 2).putInt(keyLength).put(group.getKeyBytes()).array();
                    } else if (Kind.OnError == notification.getKind()) {
                        Throwable error = notification.getThrowable();
                        byte[] errorBytes = RemoteObservable.fromThrowableToBytes(error);
                        data = ByteBuffer.allocate(1 + 4 + keyLength + errorBytes.length).put((byte) 3).putInt(keyLength).put(group.getKeyBytes()).put(errorBytes).array();
                    }
                    rxEvents.add(RemoteRxEvent.next(event.getName(), data));
                } else if (Kind.OnCompleted == groupNotification.getKind()) {
                    rxEvents.add(RemoteRxEvent.completed(event.getName()));
                } else if (Kind.OnError == groupNotification.getKind()) {
                    rxEvents.add(RemoteRxEvent.error(event.getName(), RemoteObservable.fromThrowableToBytes(groupNotification.getThrowable())));
                } else {
                    throw new RuntimeException("Unsupported notification type: " + groupNotification.getKind());
                }
            }
            return rxEvents;
        }
    }).filter(new Func1<List<RemoteRxEvent>, Boolean>() {

        @Override
        public Boolean call(List<RemoteRxEvent> t1) {
            return t1 != null && !t1.isEmpty();
        }
    }).subscribe(new WriteBytesObserver(connection, subReference, serverMetrics, serveConfig.getSlottingStrategy(), endpoint)));
    return subReference.getValue();
}