Examples with MasterMonitor io.mantisrx.server.core.master.MasterMonitor used on opensource projects

Example 1 with MasterMonitor

use of io.mantisrx.server.core.master.MasterMonitor in project mantis by Netflix.

the class TestGetMasterMonitor method main.

public static void main(String[] args) {
    try {
        Args.parse(TestGetMasterMonitor.class, args);
    } catch (IllegalArgumentException e) {
        Args.usage(TestGetMasterMonitor.class);
        System.exit(1);
    }
    Properties properties = new Properties();
    System.out.println("propfile=" + propFile);
    try (InputStream inputStream = new FileInputStream(propFile)) {
        properties.load(inputStream);
    } catch (IOException e) {
        e.printStackTrace();
    }
    final AtomicInteger counter = new AtomicInteger();
    final CountDownLatch latch = new CountDownLatch(5);
    StaticPropertiesConfigurationFactory configurationFactory = new StaticPropertiesConfigurationFactory(properties);
    CoreConfiguration config = configurationFactory.getConfig();
    final CuratorService curatorService = new CuratorService(config, null);
    MasterMonitor masterMonitor = curatorService.getMasterMonitor();
    masterMonitor.getMasterObservable().filter(new Func1<MasterDescription, Boolean>() {

        @Override
        public Boolean call(MasterDescription masterDescription) {
            return masterDescription != null;
        }
    }).doOnNext(new Action1<MasterDescription>() {

        @Override
        public void call(MasterDescription masterDescription) {
            System.out.println(counter.incrementAndGet() + ": Got new master: " + masterDescription.toString());
            latch.countDown();
        }
    }).subscribe();
    curatorService.start();
    try {
        latch.await();
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
}

Example 2 with MasterMonitor

use of io.mantisrx.server.core.master.MasterMonitor in project mantis by Netflix.

the class MantisMasterClientApiTest method testScaleStageRequestRetries.

// @Test
// @Ignore
// public void testNamedJobCreate() throws InterruptedException {
// 
// MasterMonitor mockMasterMonitor = mock(MasterMonitor.class);
// final BehaviorSubject<MasterDescription> mdSubject = BehaviorSubject.create();
// when(mockMasterMonitor.getMasterObservable()).thenReturn(mdSubject);
// 
// MantisMasterClientApi mantisMasterClientApi = new MantisMasterClientApi(mockMasterMonitor);
// 
// final int serverPort = 8182;
// final String jobName = "TestCreateJobCluster";
// mdSubject.onNext(new MasterDescription("localhost", "127.0.0.1", serverPort, 7090, 7091, "status", 8900, System.currentTimeMillis()));
// 
// JobDefinition jobDefinition = JobDefinition.newBuilder()
// .setName(jobName)
// .setVersion("0.0.1")
// .setUrl("http://www.example.com")
// .setJobSla(io.mantisrx.master.core.proto.JobSla.newBuilder()
// .setUserProvidedType("")
// .setDurationType(io.mantisrx.master.core.proto.JobSla.MantisJobDurationType.Transient)
// .setSlaType(io.mantisrx.master.core.proto.JobSla.StreamSLAType.Lossy)
// .setMinRuntimeSecs(0)
// .setRuntimeLimitSecs(0))
// .setSchedulingInfo(io.mantisrx.master.core.proto.SchedulingInfo.newBuilder()
// .putStages(1, io.mantisrx.master.core.proto.SchedulingInfo.StageSchedulingInfo.newBuilder()
// .setNumberOfInstances(1)
// .setMachineDefinition(MachineDefinition.newBuilder()
// .setCpuCores(2)
// .setDiskMB(1024)
// .setMemoryMB(2048)
// .setNetworkMbps(64)
// .setNumPorts(1)
// .build())
// .build())
// .build())
// .build();
// io.mantisrx.master.core.proto.JobOwner owner = io.mantisrx.master.core.proto.JobOwner.newBuilder()
// .setName("Test")
// .setContactEmail("test@netflix.com")
// .setDescription("")
// .setRepo("http://www.example.com")
// .build();
// CreateJobClusterRequest req = CreateJobClusterRequest.newBuilder()
// .setJobDefinition(jobDefinition)
// .setOwner(owner)
// .build();
// 
// Observable<Void> testCluster = mantisMasterClientApi.createNamedJob(req);
// final CountDownLatch latch = new CountDownLatch(1);
// 
// testCluster.subscribe((x) -> {
// latch.countDown();
// System.out.println("job cluster create response complete");
// });
// 
// latch.await();
// 
// Observable<JobSubmitResponse> jobSubmitResponseObs = mantisMasterClientApi.submitJob(jobDefinition);
// final CountDownLatch latch2 = new CountDownLatch(1);
// 
// jobSubmitResponseObs.subscribe((x) -> {
// latch2.countDown();
// System.out.println("job submit complete");
// });
// 
// latch2.await();
// }
@Test
public void testScaleStageRequestRetries() throws InterruptedException {
    MasterMonitor mockMasterMonitor = mock(MasterMonitor.class);
    final BehaviorSubject<MasterDescription> mdSubject = BehaviorSubject.create();
    when(mockMasterMonitor.getMasterObservable()).thenReturn(mdSubject);
    MantisMasterClientApi mantisMasterClientApi = new MantisMasterClientApi(mockMasterMonitor);
    final int serverPort = port.incrementAndGet();
    final String jobId = "test-job-id";
    final int stageNum = 1;
    final int numWorkers = 2;
    final String reason = "test reason";
    mdSubject.onNext(new MasterDescription("localhost", "127.0.0.1", serverPort, 7090, 7091, "status", 8900, System.currentTimeMillis()));
    final CountDownLatch retryLatch = new CountDownLatch(2);
    final Func1<Observable<? extends Throwable>, Observable<?>> retryLogic = new Func1<Observable<? extends Throwable>, Observable<?>>() {

        @Override
        public Observable<?> call(Observable<? extends Throwable> attempts) {
            return attempts.zipWith(Observable.range(1, 5), new Func2<Throwable, Integer, Integer>() {

                @Override
                public Integer call(Throwable t1, Integer integer) {
                    return integer;
                }
            }).flatMap(new Func1<Integer, Observable<?>>() {

                @Override
                public Observable<?> call(Integer retryCount) {
                    logger.info(retryCount + " retrying conx after sleeping for 250ms");
                    if (retryCount == 2) {
                        Schedulers.newThread().createWorker().schedule(new Action0() {

                            @Override
                            public void call() {
                                final HttpServer<String, String> httpServer = createHttpServer(serverPort);
                                startedServers.add(httpServer);
                                httpServer.start();
                            }
                        });
                    }
                    retryLatch.countDown();
                    return Observable.timer(250, TimeUnit.MILLISECONDS);
                }
            });
        }
    };
    final Observable<Boolean> resultObs = mantisMasterClientApi.scaleJobStage(jobId, stageNum, numWorkers, reason).retryWhen(retryLogic);
    final CountDownLatch completedLatch = new CountDownLatch(1);
    resultObs.doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            fail("got unexpected error" + throwable.getMessage());
        }
    }).doOnCompleted(new Action0() {

        @Override
        public void call() {
            completedLatch.countDown();
        }
    }).subscribe();
    assertTrue(retryLatch.await(5, TimeUnit.SECONDS));
    assertTrue(completedLatch.await(5, TimeUnit.SECONDS));
}

Example 3 with MasterMonitor

use of io.mantisrx.server.core.master.MasterMonitor in project mantis by Netflix.

the class MantisMasterClientApiTest method testScaleStageRequestRetriesNewMaster.

@Test
public void testScaleStageRequestRetriesNewMaster() throws InterruptedException {
    MasterMonitor mockMasterMonitor = mock(MasterMonitor.class);
    final BehaviorSubject<MasterDescription> mdSubject = BehaviorSubject.create();
    when(mockMasterMonitor.getMasterObservable()).thenReturn(mdSubject);
    MantisMasterClientApi mantisMasterClientApi = new MantisMasterClientApi(mockMasterMonitor);
    final int oldMasterPort = port.incrementAndGet();
    final int newMasterPort = port.incrementAndGet();
    final String jobId = "test-job-id";
    final int stageNum = 1;
    final int numWorkers = 2;
    final String reason = "test reason";
    mdSubject.onNext(new MasterDescription("localhost", "127.0.0.1", oldMasterPort, 7090, 7091, "status", 8900, System.currentTimeMillis()));
    final CountDownLatch retryLatch = new CountDownLatch(3);
    final Func1<Observable<? extends Throwable>, Observable<?>> retryLogic = new Func1<Observable<? extends Throwable>, Observable<?>>() {

        @Override
        public Observable<?> call(Observable<? extends Throwable> attempts) {
            return attempts.zipWith(Observable.range(1, 5), new Func2<Throwable, Integer, Integer>() {

                @Override
                public Integer call(Throwable t1, Integer integer) {
                    return integer;
                }
            }).flatMap(new Func1<Integer, Observable<?>>() {

                @Override
                public Observable<?> call(Integer retryCount) {
                    logger.info(retryCount + " retrying conx after sleeping for 250ms");
                    if (retryCount == 2) {
                        Schedulers.newThread().createWorker().schedule(new Action0() {

                            @Override
                            public void call() {
                                final HttpServer<String, String> httpServer = createHttpServer(newMasterPort);
                                startedServers.add(httpServer);
                                httpServer.start();
                            }
                        });
                    }
                    if (retryCount == 3) {
                        mdSubject.onNext(new MasterDescription("localhost", "127.0.0.1", newMasterPort, 7090, 7091, "status", 8900, System.currentTimeMillis()));
                    }
                    retryLatch.countDown();
                    return Observable.timer(250, TimeUnit.MILLISECONDS);
                }
            });
        }
    };
    final Observable<Boolean> resultObs = mantisMasterClientApi.scaleJobStage(jobId, stageNum, numWorkers, reason).retryWhen(retryLogic);
    final CountDownLatch completedLatch = new CountDownLatch(1);
    resultObs.doOnError(new Action1<Throwable>() {

        @Override
        public void call(Throwable throwable) {
            fail("got unexpected error" + throwable.getMessage());
        }
    }).doOnCompleted(new Action0() {

        @Override
        public void call() {
            completedLatch.countDown();
        }
    }).subscribe();
    assertTrue(retryLatch.await(5, TimeUnit.SECONDS));
    assertTrue(completedLatch.await(5, TimeUnit.SECONDS));
}

Example 4 with MasterMonitor

use of io.mantisrx.server.core.master.MasterMonitor in project mantis by Netflix.

the class LeaderRedirectionFilterTest method testRouteChangesIfNotLeader.

@Test
public void testRouteChangesIfNotLeader() {
    final MasterDescription fakeMasterDesc = new MasterDescription("localhost", "127.0.0.1", 8100, 8100 + 2, 8100 + 4, "api/postjobstatus", 8100 + 6, System.currentTimeMillis());
    MasterMonitor masterMonitor = new LocalMasterMonitor(fakeMasterDesc);
    ILeadershipManager leadershipManager = new LeadershipManagerLocalImpl(fakeMasterDesc);
    // Stop being leader, the filter should redirect so the returned Route is different from the input Route
    leadershipManager.stopBeingLeader();
    LeaderRedirectionFilter filter = new LeaderRedirectionFilter(masterMonitor, leadershipManager);
    Route testRoute = route(path("test", () -> complete("done")));
    Route route = filter.redirectIfNotLeader(testRoute);
    // filter should return input Route if we are current leader
    assertNotEquals(testRoute, route);
}

Example 5 with MasterMonitor

use of io.mantisrx.server.core.master.MasterMonitor in project mantis by Netflix.

the class LeaderRedirectionFilterTest method testRouteUnchangedIfLeader.

@Test
public void testRouteUnchangedIfLeader() {
    // Become leader and make Master monitor return the localhost master, filter should return input Route
    final MasterDescription fakeMasterDesc = new MasterDescription("localhost", "127.0.0.1", 8100, 8100 + 2, 8100 + 4, "api/postjobstatus", 8100 + 6, System.currentTimeMillis());
    MasterMonitor masterMonitor = new LocalMasterMonitor(fakeMasterDesc);
    ILeadershipManager leadershipManager = new LeadershipManagerLocalImpl(fakeMasterDesc);
    leadershipManager.becomeLeader();
    LeaderRedirectionFilter filter = new LeaderRedirectionFilter(masterMonitor, leadershipManager);
    Route testRoute = route(path("test", () -> complete("done")));
    Route route = filter.redirectIfNotLeader(testRoute);
    // leader is not ready by default
    assertNotEquals(testRoute, route);
    // mark leader ready
    leadershipManager.setLeaderReady();
    Route route2 = filter.redirectIfNotLeader(testRoute);
    // leader is not ready by default
    assertEquals(testRoute, route2);
}