Examples with Endpoint io.mantisrx.common.network.Endpoint used on opensource projects

Example 21 with Endpoint

use of io.mantisrx.common.network.Endpoint in project mantis by Netflix.

the class ToDeltaEndpointInjectorTest method deltaTest.

@Test
public void deltaTest() {
    ReplaySubject<List<Endpoint>> subject = ReplaySubject.create();
    ToDeltaEndpointInjector service = new ToDeltaEndpointInjector(subject);
    // 1 add endpoints
    List<Endpoint> endpoints = new LinkedList<Endpoint>();
    endpoints.add(new Endpoint("localhost", 1234));
    endpoints.add(new Endpoint("localhost", 2468));
    subject.onNext(endpoints);
    // 2 remove endpoint by leaving out second endpoint
    endpoints = new LinkedList<Endpoint>();
    endpoints.add(new Endpoint("localhost", 1234));
    subject.onNext(endpoints);
    // 3 remove other
    endpoints = new LinkedList<Endpoint>();
    subject.onNext(endpoints);
    // 4 add back
    endpoints = new LinkedList<Endpoint>();
    endpoints.add(new Endpoint("localhost", 1234));
    endpoints.add(new Endpoint("localhost", 2468));
    subject.onNext(endpoints);
    subject.onCompleted();
    BlockingObservable<EndpointChange> be = service.deltas().toBlocking();
    // check for two adds
    Iterator<EndpointChange> iter = be.getIterator();
    Assert.assertTrue(iter.hasNext());
    EndpointChange ce1 = iter.next();
    Assert.assertEquals(ce1.getEndpoint().getSlotId(), "localhost:1234");
    Assert.assertEquals(ce1.getType(), EndpointChange.Type.add);
    EndpointChange ce2 = iter.next();
    Assert.assertEquals(ce2.getEndpoint().getSlotId(), "localhost:2468");
    Assert.assertEquals(ce2.getType(), EndpointChange.Type.add);
    // check for complete
    EndpointChange ce3 = iter.next();
    Assert.assertEquals(ce3.getEndpoint().getSlotId(), "localhost:2468");
    Assert.assertEquals(ce3.getType(), EndpointChange.Type.complete);
    // check for complete
    EndpointChange ce4 = iter.next();
    Assert.assertEquals(ce4.getEndpoint().getSlotId(), "localhost:1234");
    Assert.assertEquals(ce4.getType(), EndpointChange.Type.complete);
    // check for add
    EndpointChange ce5 = iter.next();
    Assert.assertEquals(ce5.getEndpoint().getSlotId(), "localhost:1234");
    Assert.assertEquals(ce5.getType(), EndpointChange.Type.add);
    EndpointChange ce6 = iter.next();
    Assert.assertEquals(ce6.getEndpoint().getSlotId(), "localhost:2468");
    Assert.assertEquals(ce6.getType(), EndpointChange.Type.add);
}

Example 22 with Endpoint

use of io.mantisrx.common.network.Endpoint in project mantis by Netflix.

the class ToDeltaEndpointInjector method changes.

private List<EndpointChange> changes(List<Endpoint> previous, List<Endpoint> current) {
    logger.info("Sets to evaluate for differences, current: " + current + " previous: " + previous);
    Map<String, Endpoint> previousSet = new HashMap<>();
    // fill previous
    for (Endpoint endpoint : previous) {
        previousSet.put(uniqueHost(endpoint.getHost(), endpoint.getPort(), endpoint.getSlotId()), endpoint);
    }
    // collect into two buckets: add, complete
    List<EndpointChange> toAdd = new LinkedList<>();
    Set<String> completeCheck = new HashSet<>();
    for (Endpoint endpoint : current) {
        String id = uniqueHost(endpoint.getHost(), endpoint.getPort(), endpoint.getSlotId());
        if (!previousSet.containsKey(id)) {
            EndpointChange ce = new EndpointChange(EndpointChange.Type.add, endpoint);
            toAdd.add(ce);
        } else {
            completeCheck.add(id);
        }
    }
    List<EndpointChange> toComplete = new LinkedList<EndpointChange>();
    // check if need to complete any from current set: set difference
    for (Map.Entry<String, Endpoint> controlledEndpoint : previousSet.entrySet()) {
        if (!completeCheck.contains(controlledEndpoint.getKey())) {
            Endpoint ce = controlledEndpoint.getValue();
            EndpointChange ceToCompletd = new EndpointChange(EndpointChange.Type.complete, ce);
            toComplete.add(ceToCompletd);
        }
    }
    Map<String, EndpointChange> nextSet = new HashMap<>();
    // apply completes
    for (EndpointChange controlledEndpoint : toComplete) {
        nextSet.put(uniqueHost(controlledEndpoint.getEndpoint().getHost(), controlledEndpoint.getEndpoint().getPort(), controlledEndpoint.getEndpoint().getSlotId()), controlledEndpoint);
    }
    // apply adds
    for (EndpointChange controlledEndpoint : toAdd) {
        nextSet.put(uniqueHost(controlledEndpoint.getEndpoint().getHost(), controlledEndpoint.getEndpoint().getPort(), controlledEndpoint.getEndpoint().getSlotId()), controlledEndpoint);
    }
    logger.info("Differences to be applied: " + nextSet);
    return new ArrayList<>(nextSet.values());
}

Example 23 with Endpoint

use of io.mantisrx.common.network.Endpoint in project mantis by Netflix.

the class LocalJobExecutorNetworked method startSink.

@SuppressWarnings({ "rawtypes", "unchecked" })
private static void startSink(StageConfig previousStage, int[] previousStagePorts, StageConfig stage, PortSelector portSelector, SinkHolder sink, Context context, Action0 sinkObservableCompletedCallback, Action0 sinkObservableTerminatedCompletedCallback, Action1<Throwable> sinkObservableErrorCallback, int stageNumber, int workerIndex, int workersAtPreviousStage) {
    if (logger.isDebugEnabled()) {
        StringBuilder portsToString = new StringBuilder();
        for (int previousPort : previousStagePorts) {
            portsToString.append(previousPort + " ");
        }
        logger.debug("Creating sink consumer connecting to publishers on ports " + portsToString);
    }
    Observable<Set<Endpoint>> endpoints = staticEndpoints(previousStagePorts, stageNumber, workerIndex, numPartitions);
    WorkerConsumerRemoteObservable sinkConsumer = new // name=null for local
    WorkerConsumerRemoteObservable(// name=null for local
    null, staticInjector(endpoints));
    StageExecutors.executeSink(sinkConsumer, stage, sink, portSelector, new RxMetrics(), context, sinkObservableTerminatedCompletedCallback, null, null, sinkObservableCompletedCallback, sinkObservableErrorCallback);
}

Example 24 with Endpoint

use of io.mantisrx.common.network.Endpoint in project mantis by Netflix.

the class MasterClientWrapper method getSinkLocations.

public Observable<EndpointChange> getSinkLocations(final String jobId, final int sinkStage, final int forPartition, final int totalPartitions) {
    final ConditionalRetry schedInfoRetry = new ConditionalRetry(masterConnectRetryCounter, "SchedInfoRetry", 10);
    Observable<List<Endpoint>> schedulingUpdates = getMasterClientApi().take(1).flatMap((MantisMasterClientApi mantisMasterClientApi) -> {
        return mantisMasterClientApi.schedulingChanges(jobId).doOnError((Throwable throwable) -> {
            logger.warn(throwable.getMessage());
        }).retryWhen(schedInfoRetry.getRetryLogic()).map((JobSchedulingInfo jobSchedulingInfo) -> {
            logger.info("Got scheduling info for {}", jobId);
            if (logger.isDebugEnabled()) {
                logger.debug("Worker Assignments {}", jobSchedulingInfo.getWorkerAssignments().get(sinkStage));
            }
            return jobSchedulingInfo.getWorkerAssignments().get(sinkStage);
        }).map((WorkerAssignments workerAssignments) -> {
            List<Endpoint> endpoints = new ArrayList<>();
            if (workerAssignments != null) {
                logger.info("job " + jobId + " Creating endpoints conx from " + workerAssignments.getHosts().size() + " worker assignments");
                for (WorkerHost host : workerAssignments.getHosts().values()) {
                    final int workerIndex = host.getWorkerIndex();
                    final int totalFromPartitions = workerAssignments.getNumWorkers();
                    numSinkWorkersSubject.onNext(new JobSinkNumWorkers(jobId, totalFromPartitions));
                    if (usePartition(workerIndex, totalFromPartitions, forPartition, totalPartitions)) {
                        // logger.info("Using partition " + workerIndex);
                        if (host.getState() == MantisJobState.Started) {
                            Endpoint ep = new Endpoint(getWrappedHost(host.getHost(), host.getWorkerNumber()), host.getPort().get(0), // completed callback
                            () -> logger.info("job " + jobId + " WorkerIndex " + workerIndex + " completed"), // error callback
                            t1 -> logger.info("job " + jobId + " WorkerIndex " + workerIndex + " failed"));
                            endpoints.add(ep);
                        }
                    }
                }
            } else {
                logger.info("job " + jobId + " Has no active workers!");
            }
            return endpoints;
        }).doOnError((Throwable throwable) -> {
            logger.error(throwable.getMessage(), throwable);
        });
    });
    return (new ToDeltaEndpointInjector(schedulingUpdates)).deltas();
}

Example 25 with Endpoint

use of io.mantisrx.common.network.Endpoint in project mantis by Netflix.

the class DynamicConnectionSet method addConnection.

public void addConnection(String id, Endpoint toAdd) {
    try {
        activeConnectionsLock.lock();
        if (!currentActiveConnections.containsKey(id)) {
            currentActiveConnections.put(id, new Endpoint(toAdd.getHost(), toAdd.getPort(), toAdd.getSlotId(), toAdd.getCompletedCallback(), toAdd.getErrorCallback()));
            activeConnectionsGauge.increment();
            activeConnectionsSubject.onNext(new HashSet<Endpoint>(currentActiveConnections.values()));
        }
    } finally {
        activeConnectionsLock.unlock();
    }
}