Examples with Endpoint io.mantisrx.common.network.Endpoint used on opensource projects

Example 1 with Endpoint

use of io.mantisrx.common.network.Endpoint in project mantis by Netflix.

the class ConsistentHashTest method emptyNodeThrowsTest.

@Test
public void emptyNodeThrowsTest() {
    List<Endpoint> nodes = new ArrayList<Endpoint>();
    try {
        ConsistentHash<Endpoint> ch = new ConsistentHash<Endpoint>(HashFunctions.ketama(), new EndpointConfiguration(), nodes);
        fail();
    } catch (Exception e) {
    }
}

Example 2 with Endpoint

use of io.mantisrx.common.network.Endpoint in project mantis by Netflix.

the class ConsistentHashTest method oneNodeTest.

@Test
public void oneNodeTest() {
    Endpoint n1 = new Endpoint("host1", 7001);
    List<Endpoint> nodes = new ArrayList<Endpoint>();
    nodes.add(n1);
    ConsistentHash<Endpoint> ch = new ConsistentHash<Endpoint>(HashFunctions.ketama(), new EndpointConfiguration(), nodes);
    int hostHitCountNode1 = 0;
    int nonHitCount = 0;
    int MSG_COUNT = 100000;
    for (int i = 0; i < MSG_COUNT; i++) {
        Endpoint sn = ch.get(("msg:" + i).getBytes());
        if (sn.getHost().equals("host1")) {
            hostHitCountNode1++;
        } else {
            nonHitCount++;
        }
    }
    assertTrue(nonHitCount == 0);
    assertEquals(MSG_COUNT, hostHitCountNode1);
}

Example 3 with Endpoint

use of io.mantisrx.common.network.Endpoint in project mantis by Netflix.

the class MasterClientWrapper method getAllNonJobMasterEndpoints.

private List<Endpoint> getAllNonJobMasterEndpoints(final String jobId, final Map<Integer, WorkerAssignments> workerAssignments) {
    List<Endpoint> endpoints = new ArrayList<>();
    int totalWorkers = 0;
    for (Map.Entry<Integer, WorkerAssignments> workerAssignment : workerAssignments.entrySet()) {
        final Integer stageNum = workerAssignment.getKey();
        // skip workers for stage 0
        if (stageNum == 0) {
            continue;
        }
        final WorkerAssignments assignments = workerAssignment.getValue();
        logger.info("job {} Creating endpoints conx from {} worker assignments for stage {}", jobId, assignments.getHosts().size(), stageNum);
        if (logger.isDebugEnabled()) {
            logger.debug("stage {} hosts: {}", stageNum, assignments.getHosts());
        }
        totalWorkers += assignments.getNumWorkers();
        for (WorkerHost host : assignments.getHosts().values()) {
            final int workerIndex = host.getWorkerIndex();
            if (host.getState() == MantisJobState.Started) {
                logger.info("job " + jobId + ": creating new endpoint for worker number=" + host.getWorkerNumber() + ", index=" + host.getWorkerIndex() + ", host:port=" + host.getHost() + ":" + host.getPort().get(0));
                Endpoint ep = new WorkerEndpoint(getWrappedHost(host.getHost(), host.getWorkerNumber()), host.getPort().get(0), stageNum, host.getMetricsPort(), host.getWorkerIndex(), host.getWorkerNumber(), // completed callback
                new Action0() {

                    @Override
                    public void call() {
                        logger.info("job " + jobId + " WorkerIndex " + workerIndex + " completed");
                    }
                }, // error callback
                new Action1<Throwable>() {

                    @Override
                    public void call(Throwable t1) {
                        logger.info("job " + jobId + " WorkerIndex " + workerIndex + " failed");
                    }
                });
                endpoints.add(ep);
            }
        }
    }
    numWorkersSubject.onNext(new JobNumWorkers(jobId, totalWorkers));
    return endpoints;
}

Example 4 with Endpoint

use of io.mantisrx.common.network.Endpoint in project mantis by Netflix.

the class WorkerExecutionOperationsNetworkStage method connectToObservableAtPreviousStages.

@SuppressWarnings({ "rawtypes" })
private WorkerConsumer connectToObservableAtPreviousStages(Observable<JobSchedulingInfo> selfSchedulingInfo, final String jobId, final int previousStageNum, int numInstanceAtPreviousStage, final StageConfig previousStage, final AtomicBoolean acceptSchedulingChanges, final Observer<Status> jobStatus, final int stageNumToExecute, final int workerIndex, final int workerNumber) {
    logger.info("Watching for scheduling changes");
    // Observable<List<Endpoint>> schedulingUpdates = mantisMasterApi.schedulingChanges(jobId)
    Observable<List<Endpoint>> schedulingUpdates = selfSchedulingInfo.flatMap((Func1<JobSchedulingInfo, Observable<WorkerAssignments>>) schedulingChange -> {
        Map<Integer, WorkerAssignments> assignments = schedulingChange.getWorkerAssignments();
        if (assignments != null && !assignments.isEmpty()) {
            return Observable.from(assignments.values());
        } else {
            return Observable.empty();
        }
    }).filter(assignments -> (assignments.getStage() == previousStageNum) && acceptSchedulingChanges.get()).map(assignments -> {
        List<Endpoint> endpoints = new LinkedList<>();
        for (WorkerHost host : assignments.getHosts().values()) {
            if (host.getState() == MantisJobState.Started) {
                logger.info("Received scheduling update from master, connect request for host: " + host.getHost() + " port: " + host.getPort() + " state: " + host.getState() + " adding: " + connectionsPerEndpoint + " connections to host");
                for (int i = 1; i <= connectionsPerEndpoint; i++) {
                    final String endpointId = "stage_" + stageNumToExecute + "_index_" + Integer.toString(workerIndex) + "_partition_" + i;
                    logger.info("Adding endpoint to endpoint injector to be considered for add, with id: " + endpointId);
                    endpoints.add(new Endpoint(host.getHost(), host.getPort().get(0), endpointId));
                }
            }
        }
        return endpoints;
    }).filter(t1 -> (t1.size() > 0));
    String name = jobId + "_" + previousStageNum;
    return new WorkerConsumerRemoteObservable(name, new ToDeltaEndpointInjector(schedulingUpdates));
}

Example 5 with Endpoint

use of io.mantisrx.common.network.Endpoint in project mantis by Netflix.

the class LocalJobExecutorNetworked method startIntermediate.

@SuppressWarnings({ "rawtypes", "unchecked" })
private static void startIntermediate(int[] previousStagePorts, int port, StageConfig stage, Context context, int workerIndex, int workersAtNextStage, int stageNumber, int workersAtPreviousStage) {
    if (logger.isDebugEnabled()) {
        StringBuilder portsToString = new StringBuilder();
        for (int previousPort : previousStagePorts) {
            portsToString.append(previousPort + " ");
        }
        logger.debug("Creating intermediate consumer connecting to publishers on ports " + portsToString);
    }
    Observable<Set<Endpoint>> endpoints = staticEndpoints(previousStagePorts, stageNumber, workerIndex, numPartitions);
    WorkerConsumerRemoteObservable intermediateConsumer = new // name=null local
    WorkerConsumerRemoteObservable(// name=null local
    null, staticInjector(endpoints));
    logger.debug("Creating intermediate publisher on port " + port);
    WorkerPublisherRemoteObservable intermediatePublisher = new WorkerPublisherRemoteObservable<>(port, null, Observable.just(workersAtNextStage * numPartitions), // name is null for local
    null);
    StageExecutors.executeIntermediate(intermediateConsumer, stage, intermediatePublisher, context);
}