Examples with MantisScheduler io.mantisrx.server.master.scheduler.MantisScheduler used on opensource projects

Example 1 with MantisScheduler

use of io.mantisrx.server.master.scheduler.MantisScheduler in project mantis by Netflix.

the class JobStatusRouteTest method setup.

@BeforeClass
public static void setup() throws Exception {
    JobTestHelper.deleteAllFiles();
    JobTestHelper.createDirsIfRequired();
    final CountDownLatch latch = new CountDownLatch(1);
    t = new Thread(() -> {
        try {
            // boot up server using the route as defined below
            final Http http = Http.get(system);
            final ActorMaterializer materializer = ActorMaterializer.create(system);
            final LifecycleEventPublisher lifecycleEventPublisher = new LifecycleEventPublisherImpl(new AuditEventSubscriberLoggingImpl(), new StatusEventSubscriberLoggingImpl(), new WorkerEventSubscriberLoggingImpl());
            ActorRef jobClustersManagerActor = system.actorOf(JobClustersManagerActor.props(new MantisJobStore(new io.mantisrx.server.master.persistence.SimpleCachedFileStorageProvider(true)), lifecycleEventPublisher), "jobClustersManager");
            MantisScheduler fakeScheduler = new FakeMantisScheduler(jobClustersManagerActor);
            jobClustersManagerActor.tell(new JobClusterManagerProto.JobClustersManagerInitialize(fakeScheduler, false), ActorRef.noSender());
            agentsErrorMonitorActor.tell(new AgentsErrorMonitorActor.InitializeAgentsErrorMonitor(fakeScheduler), ActorRef.noSender());
            final JobStatusRouteHandler jobStatusRouteHandler = new JobStatusRouteHandlerAkkaImpl(system, statusEventBrokerActor);
            final JobStatusRoute jobStatusRoute = new JobStatusRoute(jobStatusRouteHandler);
            final Flow<HttpRequest, HttpResponse, NotUsed> routeFlow = jobStatusRoute.createRoute(Function.identity()).flow(system, materializer);
            logger.info("starting test server on port {}", serverPort);
            latch.countDown();
            binding = http.bindAndHandle(routeFlow, ConnectHttp.toHost("localhost", serverPort), materializer);
        } catch (Exception e) {
            logger.info("caught exception", e);
            latch.countDown();
            e.printStackTrace();
        }
    });
    t.setDaemon(true);
    t.start();
    latch.await();
}

Example 2 with MantisScheduler

use of io.mantisrx.server.master.scheduler.MantisScheduler in project mantis by Netflix.

the class AgentClustersRouteTest method setup.

@BeforeClass
public void setup() throws InterruptedException {
    TestHelpers.setupMasterConfig();
    final CountDownLatch latch = new CountDownLatch(1);
    t = new Thread(() -> {
        try {
            // boot up server using the route as defined below
            final Http http = Http.get(system);
            final ActorMaterializer materializer = ActorMaterializer.create(system);
            IMantisStorageProvider storageProvider = new SimpleCachedFileStorageProvider(true);
            final LifecycleEventPublisher lifecycleEventPublisher = new LifecycleEventPublisherImpl(new AuditEventSubscriberLoggingImpl(), new StatusEventSubscriberLoggingImpl(), new WorkerEventSubscriberLoggingImpl());
            ActorRef jobClustersManagerActor = system.actorOf(JobClustersManagerActor.props(new MantisJobStore(storageProvider), lifecycleEventPublisher), "jobClustersManager");
            MantisScheduler fakeScheduler = new FakeMantisScheduler(jobClustersManagerActor);
            jobClustersManagerActor.tell(new JobClusterManagerProto.JobClustersManagerInitialize(fakeScheduler, false), ActorRef.noSender());
            setupDummyAgentClusterAutoScaler();
            final AgentClustersRoute agentClusterV2Route = new AgentClustersRoute(new AgentClusterOperationsImpl(storageProvider, new JobMessageRouterImpl(jobClustersManagerActor), fakeScheduler, lifecycleEventPublisher, "cluster"));
            final Flow<HttpRequest, HttpResponse, NotUsed> routeFlow = agentClusterV2Route.createRoute(Function.identity()).flow(system, materializer);
            logger.info("test server starting on port {}", serverPort);
            latch.countDown();
            binding = http.bindAndHandle(routeFlow, ConnectHttp.toHost("localhost", serverPort), materializer);
        } catch (Exception e) {
            logger.info("caught exception", e);
            latch.countDown();
            e.printStackTrace();
        }
    });
    t.setDaemon(true);
    t.start();
    latch.await();
}

Example 3 with MantisScheduler

use of io.mantisrx.server.master.scheduler.MantisScheduler in project mantis by Netflix.

the class JobRouteTest method setup.

@BeforeClass
public static void setup() throws Exception {
    JobTestHelper.deleteAllFiles();
    JobTestHelper.createDirsIfRequired();
    final CountDownLatch latch = new CountDownLatch(1);
    TestHelpers.setupMasterConfig();
    t = new Thread(() -> {
        try {
            // boot up server using the route as defined below
            final Http http = Http.get(system);
            final ActorMaterializer materializer = ActorMaterializer.create(system);
            // SimpleCachedFileStorageProvider simpleCachedFileStorageProvider = new SimpleCachedFileStorageProvider();
            // new File("/tmp/MantisSpool/namedJobs").mkdirs();
            // IMantisStorageProvider storageProvider = new MantisStorageProviderAdapter(simpleCachedFileStorageProvider);
            final LifecycleEventPublisher lifecycleEventPublisher = new LifecycleEventPublisherImpl(new AuditEventSubscriberLoggingImpl(), new StatusEventSubscriberLoggingImpl(), new WorkerEventSubscriberLoggingImpl());
            ActorRef jobClustersManagerActor = system.actorOf(JobClustersManagerActor.props(new MantisJobStore(new io.mantisrx.server.master.persistence.SimpleCachedFileStorageProvider(true)), lifecycleEventPublisher), "jobClustersManager");
            MantisScheduler fakeScheduler = new FakeMantisScheduler(jobClustersManagerActor);
            jobClustersManagerActor.tell(new JobClusterManagerProto.JobClustersManagerInitialize(fakeScheduler, false), ActorRef.noSender());
            final JobClusterRouteHandler jobClusterRouteHandler = new JobClusterRouteHandlerAkkaImpl(jobClustersManagerActor);
            final JobRouteHandler jobRouteHandler = new JobRouteHandlerAkkaImpl(jobClustersManagerActor);
            MasterDescription masterDescription = new MasterDescription("127.0.0.1", "127.0.0.1", serverPort, serverPort, serverPort, "api/postjobstatus", serverPort, System.currentTimeMillis());
            Duration idleTimeout = system.settings().config().getDuration("akka.http.server.idle-timeout");
            logger.info("idle timeout {} sec ", idleTimeout.getSeconds());
            final JobDiscoveryRouteHandler jobDiscoveryRouteHandler = new JobDiscoveryRouteHandlerAkkaImpl(jobClustersManagerActor, idleTimeout);
            final MasterDescriptionRoute masterDescriptionRoute = new MasterDescriptionRoute(masterDescription);
            final JobRoute v0JobRoute = new JobRoute(jobRouteHandler, system);
            final JobDiscoveryRoute v0JobDiscoveryRoute = new JobDiscoveryRoute(jobDiscoveryRouteHandler);
            final JobClusterRoute v0JobClusterRoute = new JobClusterRoute(jobClusterRouteHandler, jobRouteHandler, system);
            final JobClustersRoute v1JobClusterRoute = new JobClustersRoute(jobClusterRouteHandler, system);
            final JobsRoute v1JobsRoute = new JobsRoute(jobClusterRouteHandler, jobRouteHandler, system);
            final AdminMasterRoute v1AdminMasterRoute = new AdminMasterRoute(masterDescription);
            final JobStatusRouteHandler jobStatusRouteHandler = mock(JobStatusRouteHandler.class);
            when(jobStatusRouteHandler.jobStatus(anyString())).thenReturn(Flow.create());
            final JobStatusRoute v0JobStatusRoute = new JobStatusRoute(jobStatusRouteHandler);
            final AgentClusterOperations mockAgentClusterOps = mock(AgentClusterOperations.class);
            final AgentClusterRoute v0AgentClusterRoute = new AgentClusterRoute(mockAgentClusterOps, system);
            final AgentClustersRoute v1AgentClusterRoute = new AgentClustersRoute(mockAgentClusterOps);
            final JobDiscoveryStreamRoute v1JobDiscoveryStreamRoute = new JobDiscoveryStreamRoute(jobDiscoveryRouteHandler);
            final LastSubmittedJobIdStreamRoute v1LastSubmittedJobIdStreamRoute = new LastSubmittedJobIdStreamRoute(jobDiscoveryRouteHandler);
            final JobStatusStreamRoute v1JobStatusStreamRoute = new JobStatusStreamRoute(jobStatusRouteHandler);
            LocalMasterMonitor localMasterMonitor = new LocalMasterMonitor(masterDescription);
            LeadershipManagerLocalImpl leadershipMgr = new LeadershipManagerLocalImpl(masterDescription);
            leadershipMgr.setLeaderReady();
            LeaderRedirectionFilter leaderRedirectionFilter = new LeaderRedirectionFilter(localMasterMonitor, leadershipMgr);
            final MantisMasterRoute app = new MantisMasterRoute(leaderRedirectionFilter, masterDescriptionRoute, v0JobClusterRoute, v0JobRoute, v0JobDiscoveryRoute, v0JobStatusRoute, v0AgentClusterRoute, v1JobClusterRoute, v1JobsRoute, v1AdminMasterRoute, v1AgentClusterRoute, v1JobDiscoveryStreamRoute, v1LastSubmittedJobIdStreamRoute, v1JobStatusStreamRoute);
            final Flow<HttpRequest, HttpResponse, NotUsed> routeFlow = app.createRoute().flow(system, materializer);
            logger.info("starting test server on port {}", serverPort);
            binding = http.bindAndHandle(routeFlow, ConnectHttp.toHost("localhost", serverPort), materializer);
            latch.countDown();
        } catch (Exception e) {
            logger.info("caught exception", e);
            latch.countDown();
            e.printStackTrace();
        }
    });
    t.setDaemon(true);
    t.start();
    latch.await();
    Thread.sleep(100);
}

Example 4 with MantisScheduler

use of io.mantisrx.server.master.scheduler.MantisScheduler in project mantis by Netflix.

the class JobClusterRouteTest method setup.

@BeforeClass
public static void setup() throws Exception {
    TestHelpers.setupMasterConfig();
    final CountDownLatch latch = new CountDownLatch(1);
    t = new Thread(() -> {
        try {
            // boot up server using the route as defined below
            final Http http = Http.get(system);
            final ActorMaterializer materializer = ActorMaterializer.create(system);
            final LifecycleEventPublisher lifecycleEventPublisher = new LifecycleEventPublisherImpl(new AuditEventSubscriberLoggingImpl(), new StatusEventSubscriberLoggingImpl(), new WorkerEventSubscriberLoggingImpl());
            ActorRef jobClustersManagerActor = system.actorOf(JobClustersManagerActor.props(new MantisJobStore(new SimpleCachedFileStorageProvider(true)), lifecycleEventPublisher), "jobClustersManager");
            MantisScheduler fakeScheduler = new FakeMantisScheduler(jobClustersManagerActor);
            jobClustersManagerActor.tell(new JobClusterManagerProto.JobClustersManagerInitialize(fakeScheduler, false), ActorRef.noSender());
            final JobClusterRouteHandler jobClusterRouteHandler = new JobClusterRouteHandlerAkkaImpl(jobClustersManagerActor);
            final JobRouteHandler jobRouteHandler = new JobRouteHandlerAkkaImpl(jobClustersManagerActor);
            final JobClusterRoute app = new JobClusterRoute(jobClusterRouteHandler, jobRouteHandler, system);
            final Flow<HttpRequest, HttpResponse, NotUsed> routeFlow = app.createRoute(Function.identity()).flow(system, materializer);
            logger.info("starting test server on port {}", serverPort);
            binding = http.bindAndHandle(routeFlow, ConnectHttp.toHost("localhost", serverPort), materializer);
            latch.countDown();
        } catch (Exception e) {
            logger.info("caught exception", e);
            latch.countDown();
            e.printStackTrace();
        }
    });
    t.setDaemon(true);
    t.start();
    latch.await();
}

Example 5 with MantisScheduler

use of io.mantisrx.server.master.scheduler.MantisScheduler in project mantis by Netflix.

the class WorkerRegistryV2Test method testJobScaleDown.

@Test
public void testJobScaleDown() throws Exception {
    WorkerRegistryV2 workerRegistryV2 = new WorkerRegistryV2();
    LifecycleEventPublisher eventPublisher = new LifecycleEventPublisherImpl(new AuditEventSubscriberLoggingImpl(), new StatusEventSubscriberLoggingImpl(), new DummyWorkerEventSubscriberImpl(workerRegistryV2));
    Map<StageScalingPolicy.ScalingReason, StageScalingPolicy.Strategy> smap = new HashMap<>();
    smap.put(StageScalingPolicy.ScalingReason.CPU, new StageScalingPolicy.Strategy(StageScalingPolicy.ScalingReason.CPU, 0.5, 0.75, null));
    smap.put(StageScalingPolicy.ScalingReason.DataDrop, new StageScalingPolicy.Strategy(StageScalingPolicy.ScalingReason.DataDrop, 0.0, 2.0, null));
    SchedulingInfo sInfo = new SchedulingInfo.Builder().numberOfStages(1).multiWorkerScalableStageWithConstraints(2, new MachineDefinition(1.0, 1.0, 1.0, 3), Lists.newArrayList(), Lists.newArrayList(), new StageScalingPolicy(1, 0, 10, 1, 1, 0, smap)).build();
    String clusterName = "testJobScaleDown";
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    MantisJobStore jobStoreMock = mock(MantisJobStore.class);
    ActorRef jobActor = JobTestHelper.submitSingleStageScalableJob(system, probe, clusterName, sInfo, schedulerMock, jobStoreMock, eventPublisher);
    assertEquals(3, workerRegistryV2.getNumRunningWorkers());
    // send scale down request
    jobActor.tell(new JobClusterManagerProto.ScaleStageRequest(clusterName + "-1", 1, 1, "", ""), probe.getRef());
    JobClusterManagerProto.ScaleStageResponse scaleResp = probe.expectMsgClass(JobClusterManagerProto.ScaleStageResponse.class);
    System.out.println("ScaleDownResp " + scaleResp.message);
    assertEquals(SUCCESS, scaleResp.responseCode);
    assertEquals(1, scaleResp.getActualNumWorkers());
    jobActor.tell(new JobClusterManagerProto.GetJobDetailsRequest("user", new JobId(clusterName, 1)), probe.getRef());
    JobClusterManagerProto.GetJobDetailsResponse resp = probe.expectMsgClass(JobClusterManagerProto.GetJobDetailsResponse.class);
    Map<Integer, ? extends IMantisStageMetadata> stageMetadata = resp.getJobMetadata().get().getStageMetadata();
    assertEquals(1, stageMetadata.get(1).getAllWorkers().size());
    int cnt = 0;
    for (int i = 0; i < 50; i++) {
        cnt++;
        if (workerRegistryV2.getNumRunningWorkers() == 2) {
            break;
        }
    }
    assertTrue(cnt < 50);
// assertEquals(2, WorkerRegistryV2.INSTANCE.getNumRunningWorkers());
}