use of io.mantisrx.common.network.WritableEndpoint in project mantis by Netflix.
the class ServerSlotManagerTest method threeNodeSameClientIdTest.
@Test
public void threeNodeSameClientIdTest() {
WritableEndpoint<Void> n1 = new WritableEndpoint<Void>("host1", 7001);
WritableEndpoint<Void> n2 = new WritableEndpoint<Void>("host2", 7001);
Map<String, List<String>> params = new HashMap<String, List<String>>();
List<String> vals = new ArrayList<String>();
vals.add("client1");
params.put("clientId", vals);
ServerSlotManager<Void> ssm = new ServerSlotManager<Void>(HashFunctions.ketama());
SlotAssignmentManager<Void> sm = ssm.registerServer(n1, params);
SlotAssignmentManager<Void> sm2 = ssm.registerServer(n2, params);
assertEquals(sm, sm2);
int hostHitCountNode1 = 0;
int hostHitCountNode2 = 0;
int nonHitCount = 0;
int MSG_COUNT = 1000000;
for (int i = 0; i < MSG_COUNT; i++) {
String msg = "msg:" + i;
if (sm.filter(n1, msg.getBytes())) {
hostHitCountNode1++;
} else if (sm2.filter(n2, msg.getBytes())) {
hostHitCountNode2++;
} else {
nonHitCount++;
}
}
double host1HitPercentage = (double) hostHitCountNode1 / (double) MSG_COUNT;
System.out.println("host1 hit % " + host1HitPercentage);
assertTrue(host1HitPercentage > 0.4 && host1HitPercentage < 0.6);
assertTrue(nonHitCount == 0);
assertEquals(MSG_COUNT, hostHitCountNode1 + hostHitCountNode2);
WritableEndpoint<Void> n3 = new WritableEndpoint<Void>("host3", 7001);
// add another node
SlotAssignmentManager<Void> sm3 = ssm.registerServer(n3, params);
hostHitCountNode1 = 0;
hostHitCountNode2 = 0;
int hostHitCountNode3 = 0;
nonHitCount = 0;
MSG_COUNT = 1000000;
for (int i = 0; i < MSG_COUNT; i++) {
String msg = "msg:" + i;
if (sm.filter(n1, msg.getBytes())) {
hostHitCountNode1++;
}
if (sm2.filter(n2, msg.getBytes())) {
hostHitCountNode2++;
}
if (sm3.filter(n3, msg.getBytes())) {
hostHitCountNode3++;
} else {
nonHitCount++;
}
}
assertEquals(MSG_COUNT, hostHitCountNode1 + hostHitCountNode2 + hostHitCountNode3);
ssm.deregisterServer(n1, params);
ssm.deregisterServer(n2, params);
ssm.deregisterServer(n3, params);
}
use of io.mantisrx.common.network.WritableEndpoint in project mantis by Netflix.
the class ServerSlotManagerTest method oneNodeTest.
@Test
public void oneNodeTest() {
WritableEndpoint<Void> n1 = new WritableEndpoint<Void>("host1", 7001);
Map<String, List<String>> params = new HashMap<String, List<String>>();
ServerSlotManager<Void> ssm = new ServerSlotManager<Void>(HashFunctions.ketama());
SlotAssignmentManager<Void> sam = ssm.registerServer(n1, params);
int hostHitCountNode1 = 0;
int nonHitCount = 0;
int MSG_COUNT = 100000;
for (int i = 0; i < MSG_COUNT; i++) {
if (sam.filter(n1, ("msg:" + i).getBytes())) {
hostHitCountNode1++;
} else {
nonHitCount++;
}
}
assertTrue(nonHitCount == 0);
assertEquals(MSG_COUNT, hostHitCountNode1);
// cleanup
ssm.deregisterServer(n1, params);
}
use of io.mantisrx.common.network.WritableEndpoint in project mantis by Netflix.
the class RemoteObservableConnectionHandler method handleSubscribeRequest.
@SuppressWarnings({ "rawtypes", "unchecked" })
private Observable<Void> handleSubscribeRequest(RemoteRxEvent event, final ObservableConnection<RemoteRxEvent, List<RemoteRxEvent>> connection, MutableReference<SlottingStrategy> slottingStrategyReference, MutableReference<Subscription> unsubscribeCallbackReference, MutableReference<WritableEndpoint> slottingIdReference) {
// check if observable exists in configs
String observableName = event.getName();
ServeConfig config = observables.get(observableName);
if (config == null) {
return Observable.error(new RemoteObservableException("No remote observable configuration found for name: " + observableName));
}
if (event.getType() == RemoteRxEvent.Type.subscribed) {
String slotId = null;
Map<String, String> subscriptionParameters = event.getSubscribeParameters();
if (subscriptionParameters != null) {
slotId = subscriptionParameters.get("slotId");
}
InetSocketAddress address = (InetSocketAddress) connection.getChannel().remoteAddress();
WritableEndpoint endpoint = new WritableEndpoint<>(address.getHostName(), address.getPort(), slotId, connection);
SlottingStrategy slottingStrategy = config.getSlottingStrategy();
slottingIdReference.setValue(endpoint);
slottingStrategyReference.setValue(slottingStrategy);
logger.info("Connection received on server from client endpoint: " + endpoint + ", subscribed to observable: " + observableName);
serverMetrics.incrementSubscribedCount();
subscribe(unsubscribeCallbackReference, event, connection, config, endpoint);
if (!slottingStrategy.addConnection(endpoint)) {
// unable to slot connection
logger.warn("Failed to slot connection for endpoint: " + endpoint);
connection.close(true);
}
}
return Observable.empty();
}
use of io.mantisrx.common.network.WritableEndpoint in project mantis by Netflix.
the class RemoteObservableConnectionHandler method setupConnection.
@SuppressWarnings("rawtypes")
private Observable<Void> setupConnection(final ObservableConnection<RemoteRxEvent, List<RemoteRxEvent>> connection) {
// state associated with connection
// used to initiate 'unsubscribe' callback to subscriber
final MutableReference<Subscription> unsubscribeCallbackReference = new MutableReference<Subscription>();
// used to release slot when connection completes
final MutableReference<SlottingStrategy> slottingStrategyReference = new MutableReference<SlottingStrategy>();
// used to get slotId for connection
final MutableReference<WritableEndpoint> slottingIdReference = new MutableReference<WritableEndpoint>();
return connection.getInput().filter(new Func1<RemoteRxEvent, Boolean>() {
@Override
public Boolean call(RemoteRxEvent event) {
boolean supportedOperation = false;
if (event.getType() == RemoteRxEvent.Type.subscribed || event.getType() == RemoteRxEvent.Type.unsubscribed) {
supportedOperation = true;
}
return supportedOperation;
}
}).flatMap(new Func1<RemoteRxEvent, Observable<Void>>() {
@SuppressWarnings("unchecked")
@Override
public Observable<Void> call(RemoteRxEvent event) {
if (event.getType() == RemoteRxEvent.Type.subscribed) {
return handleSubscribeRequest(event, connection, slottingStrategyReference, unsubscribeCallbackReference, slottingIdReference);
} else if (event.getType() == RemoteRxEvent.Type.unsubscribed) {
Subscription subscription = unsubscribeCallbackReference.getValue();
if (subscription != null) {
subscription.unsubscribe();
}
serverMetrics.incrementUnsubscribedCount();
// release slot
if (slottingStrategyReference.getValue() != null) {
if (!slottingStrategyReference.getValue().removeConnection(slottingIdReference.getValue())) {
logger.error("Failed to remove endpoint from slot, endpoint: " + slottingIdReference.getValue());
}
}
logger.info("Connection: " + connection.getChannel().remoteAddress() + " unsubscribed, closing connection");
connection.close(true);
}
return Observable.empty();
}
});
}
use of io.mantisrx.common.network.WritableEndpoint in project mantis by Netflix.
the class RemoteObservableConnectionHandler method serveGroupedObservable.
private <K, V> Subscription serveGroupedObservable(final Observable<Group<K, V>> groups, final ObservableConnection<RemoteRxEvent, List<RemoteRxEvent>> connection, final RemoteRxEvent event, final Func1<Map<String, String>, Func1<K, Boolean>> filterFunction, final Encoder<K> keyEncoder, final Encoder<V> valueEncoder, final ServeGroupedObservable<K, V> serveConfig, final WritableEndpoint<GroupedObservable<K, V>> endpoint) {
final MutableReference<Subscription> subReference = new MutableReference<>();
subReference.setValue(groups.filter(new Func1<Group<K, V>, Boolean>() {
@Override
public Boolean call(Group<K, V> group) {
return filterFunction.call(event.getSubscribeParameters()).call(group.getKeyValue());
}
}).doOnCompleted(new Action0() {
@Override
public void call() {
logger.info("OnCompleted recieved in serveGroupedObservable, sending to client.");
}
}).doOnError(new Action1<Throwable>() {
@Override
public void call(Throwable t1) {
logger.info("OnError received in serveGroupedObservable, sending to client: ", t1);
}
}).materialize().lift(new DisableBackPressureOperator<Notification<Group<K, V>>>()).buffer(writeBufferTimeMSec, TimeUnit.MILLISECONDS).filter(new Func1<List<Notification<Group<K, V>>>, Boolean>() {
@Override
public Boolean call(List<Notification<Group<K, V>>> t1) {
return t1 != null && !t1.isEmpty();
}
}).map(new Func1<List<Notification<Group<K, V>>>, List<RemoteRxEvent>>() {
@Override
public List<RemoteRxEvent> call(final List<Notification<Group<K, V>>> groupNotifications) {
List<RemoteRxEvent> rxEvents = new ArrayList<RemoteRxEvent>(groupNotifications.size());
for (Notification<Group<K, V>> groupNotification : groupNotifications) {
if (Kind.OnNext == groupNotification.getKind()) {
// encode inner group notification
Group<K, V> group = groupNotification.getValue();
final int keyLength = group.getKeyBytes().length;
Notification<V> notification = groupNotification.getValue().getNotification();
byte[] data = null;
if (Kind.OnNext == notification.getKind()) {
V value = notification.getValue();
byte[] valueBytes = valueEncoder.encode(value);
// 1 byte for notification type,
// 4 bytes is to encode key length as int
data = ByteBuffer.allocate(1 + 4 + keyLength + valueBytes.length).put((byte) 1).putInt(keyLength).put(group.getKeyBytes()).put(valueBytes).array();
} else if (Kind.OnCompleted == notification.getKind()) {
data = ByteBuffer.allocate(1 + 4 + keyLength).put((byte) 2).putInt(keyLength).put(group.getKeyBytes()).array();
} else if (Kind.OnError == notification.getKind()) {
Throwable error = notification.getThrowable();
byte[] errorBytes = RemoteObservable.fromThrowableToBytes(error);
data = ByteBuffer.allocate(1 + 4 + keyLength + errorBytes.length).put((byte) 3).putInt(keyLength).put(group.getKeyBytes()).put(errorBytes).array();
}
rxEvents.add(RemoteRxEvent.next(event.getName(), data));
} else if (Kind.OnCompleted == groupNotification.getKind()) {
rxEvents.add(RemoteRxEvent.completed(event.getName()));
} else if (Kind.OnError == groupNotification.getKind()) {
rxEvents.add(RemoteRxEvent.error(event.getName(), RemoteObservable.fromThrowableToBytes(groupNotification.getThrowable())));
} else {
throw new RuntimeException("Unsupported notification type: " + groupNotification.getKind());
}
}
return rxEvents;
}
}).filter(new Func1<List<RemoteRxEvent>, Boolean>() {
@Override
public Boolean call(List<RemoteRxEvent> t1) {
return t1 != null && !t1.isEmpty();
}
}).subscribe(new WriteBytesObserver(connection, subReference, serverMetrics, serveConfig.getSlottingStrategy(), endpoint)));
return subReference.getValue();
}
Aggregations