use of io.reactivex.mantis.remote.observable.EndpointChange in project mantis by Netflix.
the class MasterClientWrapper method main.
public static void main(String[] args) throws InterruptedException {
Properties zkProps = new Properties();
zkProps.put("mantis.zookeeper.connectString", "ec2-50-19-255-1.compute-1.amazonaws.com:2181,ec2-54-235-159-245.compute-1.amazonaws.com:2181,ec2-50-19-255-97.compute-1.amazonaws.com:2181,ec2-184-73-152-248.compute-1.amazonaws.com:2181,ec2-50-17-247-179.compute-1.amazonaws.com:2181");
zkProps.put("mantis.zookeeper.leader.announcement.path", "/leader");
zkProps.put("mantis.zookeeper.root", "/mantis/master");
String jobId = "GroupByIPNJ-12";
MasterClientWrapper clientWrapper = new MasterClientWrapper(zkProps);
clientWrapper.getMasterClientApi().flatMap(new Func1<MantisMasterClientApi, Observable<EndpointChange>>() {
@Override
public Observable<EndpointChange> call(MantisMasterClientApi mantisMasterClientApi) {
Integer sinkStage = null;
return mantisMasterClientApi.getSinkStageNum(jobId).take(// only need to figure out sink stage number once
1).flatMap(new Func1<Integer, Observable<EndpointChange>>() {
@Override
public Observable<EndpointChange> call(Integer integer) {
logger.info("Getting sink locations for " + jobId);
return clientWrapper.getSinkLocations(jobId, integer, 0, 0);
}
});
}
}).toBlocking().subscribe((ep) -> {
System.out.println("Endpoint Change -> " + ep);
});
Thread.sleep(50000);
}
use of io.reactivex.mantis.remote.observable.EndpointChange in project mantis by Netflix.
the class StageExecutorsGroupByTest method testExecuteIntermediatStage.
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testExecuteIntermediatStage() throws InterruptedException {
// Note, this test has a timing issue, client starts
// sending data before server is ready, resulting
// in a RST (connection reset by peer)
TestGroupByJob provider = new TestGroupByJob();
Job<Pair> job = provider.getJobInstance();
List<StageConfig<?, ?>> stages = job.getStages();
PortSelectorWithinRange portSelector = new PortSelectorWithinRange(8000, 9000);
final int publishPort = portSelector.acquirePort();
final int consumerPort = portSelector.acquirePort();
Observable<Observable<GroupedObservable<String, Integer>>> go = Observable.just(Observable.range(0, 10).groupBy(new Func1<Integer, String>() {
@Override
public String call(Integer t1) {
if ((t1 % 2) == 0) {
return "even";
} else {
return "odd";
}
}
}));
// mimic previous stage with a server
ServeGroupedObservable<String, Integer> config = new ServeGroupedObservable.Builder<String, Integer>().keyEncoder(Codecs.string()).valueEncoder(Codecs.integer()).observable(go).build();
RemoteRxServer server = new RemoteRxServer.Builder().addObservable(config).port(consumerPort).build();
server.start();
EndpointInjector staticEndpoints = new EndpointInjector() {
@Override
public Observable<EndpointChange> deltas() {
return Observable.create(new OnSubscribe<EndpointChange>() {
@Override
public void call(Subscriber<? super EndpointChange> subscriber) {
subscriber.onNext(new EndpointChange(EndpointChange.Type.add, new Endpoint("localhost", consumerPort, "0")));
subscriber.onNext(new EndpointChange(EndpointChange.Type.add, new Endpoint("localhost", consumerPort, "1")));
subscriber.onCompleted();
}
});
}
};
WorkerConsumer consumer = new WorkerConsumerRemoteObservable(null, staticEndpoints);
WorkerPublisher producer = new WorkerPublisherRemoteObservable(publishPort, null, Observable.just(1), null);
// execute source
StageExecutors.executeIntermediate(consumer, stages.get(1), producer, new Context());
ConnectToGroupedObservable<String, Integer> connectConfig = new ConnectToGroupedObservable.Builder<String, Integer>().host("localhost").port(publishPort).keyDecoder(Codecs.string()).valueDecoder(Codecs.integer()).build();
Iterator<GroupedObservable<String, Integer>> iter = RemoteObservable.connect(connectConfig).getObservable().toBlocking().getIterator();
// verify numbers are grouped by even/odd
// even is first due to zero
GroupedObservable<String, Integer> even = iter.next();
Assert.assertEquals("even", even.getKey());
Iterator<Integer> evenIter = even.toBlocking().getIterator();
Assert.assertEquals(0, evenIter.next().intValue());
Assert.assertEquals(4, evenIter.next().intValue());
Assert.assertEquals(16, evenIter.next().intValue());
Assert.assertEquals(36, evenIter.next().intValue());
GroupedObservable<String, Integer> odd = iter.next();
Assert.assertEquals("odd", odd.getKey());
Iterator<Integer> oddIter = odd.toBlocking().getIterator();
Assert.assertEquals(1, oddIter.next().intValue());
Assert.assertEquals(9, oddIter.next().intValue());
Assert.assertEquals(25, oddIter.next().intValue());
Assert.assertEquals(49, oddIter.next().intValue());
// should only have two groups
Assert.assertEquals(false, iter.hasNext());
}
use of io.reactivex.mantis.remote.observable.EndpointChange in project mantis by Netflix.
the class StageExecutorsTest method testExecuteIntermediatStage.
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testExecuteIntermediatStage() throws InterruptedException {
TestJob provider = new TestJob();
Job<Integer> job = provider.getJobInstance();
List<StageConfig<?, ?>> stages = job.getStages();
PortSelectorWithinRange portSelector = new PortSelectorWithinRange(8000, 9000);
final int publishPort = portSelector.acquirePort();
final int consumerPort = portSelector.acquirePort();
// mimic previous stage with a server
RemoteRxServer server1 = RemoteObservable.serve(consumerPort, Observable.range(0, 10), Codecs.integer());
server1.start();
EndpointInjector staticEndpoints = new EndpointInjector() {
@Override
public Observable<EndpointChange> deltas() {
return Observable.create(new OnSubscribe<EndpointChange>() {
@Override
public void call(Subscriber<? super EndpointChange> subscriber) {
subscriber.onNext(new EndpointChange(EndpointChange.Type.add, new Endpoint("localhost", consumerPort, "1")));
subscriber.onCompleted();
}
});
}
};
WorkerConsumer consumer = new WorkerConsumerRemoteObservable(null, staticEndpoints);
WorkerPublisher producer = new WorkerPublisherRemoteObservable(publishPort, null, Observable.just(1), null);
// execute intermediate, flatten results
StageExecutors.executeIntermediate(consumer, stages.get(1), producer, new Context());
Iterator<Integer> iter = RemoteObservable.connect(new ConnectToObservable.Builder<Integer>().host("localhost").slotId("0").port(publishPort).decoder(Codecs.integer()).build()).getObservable().toBlocking().getIterator();
// verify numbers are even
Assert.assertEquals(0, iter.next().intValue());
Assert.assertEquals(2, iter.next().intValue());
Assert.assertEquals(4, iter.next().intValue());
}
use of io.reactivex.mantis.remote.observable.EndpointChange in project mantis by Netflix.
the class StageExecutorsTest method testExecuteSink.
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testExecuteSink() throws InterruptedException {
TestJob provider = new TestJob();
Job<Integer> job = provider.getJobInstance();
List<StageConfig<?, ?>> stages = job.getStages();
PortSelectorWithinRange portSelector = new PortSelectorWithinRange(8000, 9000);
final int consumerPort = portSelector.acquirePort();
// mimic previous stage with a server
RemoteRxServer server1 = RemoteObservable.serve(consumerPort, Observable.range(0, 10), Codecs.integer());
server1.start();
EndpointInjector staticEndpoints = new EndpointInjector() {
@Override
public Observable<EndpointChange> deltas() {
return Observable.create(new OnSubscribe<EndpointChange>() {
@Override
public void call(Subscriber<? super EndpointChange> subscriber) {
subscriber.onNext(new EndpointChange(EndpointChange.Type.add, new Endpoint("localhost", consumerPort, "1")));
subscriber.onCompleted();
}
});
}
};
Action0 noOpAction = new Action0() {
@Override
public void call() {
}
};
Action1<Throwable> noOpError = new Action1<Throwable>() {
@Override
public void call(Throwable t) {
}
};
WorkerConsumer consumer = new WorkerConsumerRemoteObservable(null, staticEndpoints);
// execute source
StageExecutors.executeSink(consumer, stages.get(1), job.getSink(), new TestPortSelector(), new RxMetrics(), new Context(), noOpAction, null, null, noOpAction, noOpError);
Iterator<Integer> iter = provider.getItemsWritten().iterator();
// verify numbers are even
Assert.assertEquals(0, iter.next().intValue());
Assert.assertEquals(2, iter.next().intValue());
Assert.assertEquals(4, iter.next().intValue());
}
use of io.reactivex.mantis.remote.observable.EndpointChange in project mantis by Netflix.
the class Reconciliator method startReconciliation.
private void startReconciliation() {
if (startedReconciliation.compareAndSet(false, true)) {
logger.info("Starting reconciliation for name: " + name);
running.increment();
subscription = Observable.combineLatest(currentExpectedSet, connectionSet.activeConnections(), new Func2<Set<Endpoint>, Set<Endpoint>, Void>() {
@Override
public Void call(Set<Endpoint> expected, Set<Endpoint> actual) {
reconciliationCheck.increment();
boolean check = expected.equals(actual);
logger.debug("Check result: " + check + ", size expected: " + expected.size() + " actual: " + actual.size() + ", for values expected: " + expected + " versus actual: " + actual);
if (!check) {
// reconcile adds
Set<Endpoint> expectedDiff = new HashSet<Endpoint>(expected);
expectedDiff.removeAll(actual);
if (expectedDiff.size() > 0) {
for (Endpoint endpoint : expectedDiff) {
logger.info("Connection missing from expected set, adding missing connection: " + endpoint);
reconciledChanges.onNext(new EndpointChange(Type.add, endpoint));
}
}
// reconile removes
Set<Endpoint> actualDiff = new HashSet<Endpoint>(actual);
actualDiff.removeAll(expected);
if (actualDiff.size() > 0) {
for (Endpoint endpoint : actualDiff) {
logger.info("Unexpected connection in active set, removing connection: " + endpoint);
reconciledChanges.onNext(new EndpointChange(Type.complete, endpoint));
}
}
}
return null;
}
}).onErrorResumeNext(new Func1<Throwable, Observable<? extends Void>>() {
@Override
public Observable<? extends Void> call(Throwable throwable) {
logger.error("caught error in Reconciliation for {}", name, throwable);
return Observable.empty();
}
}).doOnCompleted(new Action0() {
@Override
public void call() {
logger.error("onComplete in Reconciliation observable chain for {}", name);
stopReconciliation();
}
}).subscribe();
} else {
logger.info("reconciliation already started for {}", name);
}
}
Aggregations