Search in sources :

Example 1 with AgentClusterOperations

use of io.mantisrx.master.vm.AgentClusterOperations in project mantis by Netflix.

the class JobRouteTest method setup.

@BeforeClass
public static void setup() throws Exception {
    JobTestHelper.deleteAllFiles();
    JobTestHelper.createDirsIfRequired();
    final CountDownLatch latch = new CountDownLatch(1);
    TestHelpers.setupMasterConfig();
    t = new Thread(() -> {
        try {
            // boot up server using the route as defined below
            final Http http = Http.get(system);
            final ActorMaterializer materializer = ActorMaterializer.create(system);
            // SimpleCachedFileStorageProvider simpleCachedFileStorageProvider = new SimpleCachedFileStorageProvider();
            // new File("/tmp/MantisSpool/namedJobs").mkdirs();
            // IMantisStorageProvider storageProvider = new MantisStorageProviderAdapter(simpleCachedFileStorageProvider);
            final LifecycleEventPublisher lifecycleEventPublisher = new LifecycleEventPublisherImpl(new AuditEventSubscriberLoggingImpl(), new StatusEventSubscriberLoggingImpl(), new WorkerEventSubscriberLoggingImpl());
            ActorRef jobClustersManagerActor = system.actorOf(JobClustersManagerActor.props(new MantisJobStore(new io.mantisrx.server.master.persistence.SimpleCachedFileStorageProvider(true)), lifecycleEventPublisher), "jobClustersManager");
            MantisScheduler fakeScheduler = new FakeMantisScheduler(jobClustersManagerActor);
            jobClustersManagerActor.tell(new JobClusterManagerProto.JobClustersManagerInitialize(fakeScheduler, false), ActorRef.noSender());
            final JobClusterRouteHandler jobClusterRouteHandler = new JobClusterRouteHandlerAkkaImpl(jobClustersManagerActor);
            final JobRouteHandler jobRouteHandler = new JobRouteHandlerAkkaImpl(jobClustersManagerActor);
            MasterDescription masterDescription = new MasterDescription("127.0.0.1", "127.0.0.1", serverPort, serverPort, serverPort, "api/postjobstatus", serverPort, System.currentTimeMillis());
            Duration idleTimeout = system.settings().config().getDuration("akka.http.server.idle-timeout");
            logger.info("idle timeout {} sec ", idleTimeout.getSeconds());
            final JobDiscoveryRouteHandler jobDiscoveryRouteHandler = new JobDiscoveryRouteHandlerAkkaImpl(jobClustersManagerActor, idleTimeout);
            final MasterDescriptionRoute masterDescriptionRoute = new MasterDescriptionRoute(masterDescription);
            final JobRoute v0JobRoute = new JobRoute(jobRouteHandler, system);
            final JobDiscoveryRoute v0JobDiscoveryRoute = new JobDiscoveryRoute(jobDiscoveryRouteHandler);
            final JobClusterRoute v0JobClusterRoute = new JobClusterRoute(jobClusterRouteHandler, jobRouteHandler, system);
            final JobClustersRoute v1JobClusterRoute = new JobClustersRoute(jobClusterRouteHandler, system);
            final JobsRoute v1JobsRoute = new JobsRoute(jobClusterRouteHandler, jobRouteHandler, system);
            final AdminMasterRoute v1AdminMasterRoute = new AdminMasterRoute(masterDescription);
            final JobStatusRouteHandler jobStatusRouteHandler = mock(JobStatusRouteHandler.class);
            when(jobStatusRouteHandler.jobStatus(anyString())).thenReturn(Flow.create());
            final JobStatusRoute v0JobStatusRoute = new JobStatusRoute(jobStatusRouteHandler);
            final AgentClusterOperations mockAgentClusterOps = mock(AgentClusterOperations.class);
            final AgentClusterRoute v0AgentClusterRoute = new AgentClusterRoute(mockAgentClusterOps, system);
            final AgentClustersRoute v1AgentClusterRoute = new AgentClustersRoute(mockAgentClusterOps);
            final JobDiscoveryStreamRoute v1JobDiscoveryStreamRoute = new JobDiscoveryStreamRoute(jobDiscoveryRouteHandler);
            final LastSubmittedJobIdStreamRoute v1LastSubmittedJobIdStreamRoute = new LastSubmittedJobIdStreamRoute(jobDiscoveryRouteHandler);
            final JobStatusStreamRoute v1JobStatusStreamRoute = new JobStatusStreamRoute(jobStatusRouteHandler);
            LocalMasterMonitor localMasterMonitor = new LocalMasterMonitor(masterDescription);
            LeadershipManagerLocalImpl leadershipMgr = new LeadershipManagerLocalImpl(masterDescription);
            leadershipMgr.setLeaderReady();
            LeaderRedirectionFilter leaderRedirectionFilter = new LeaderRedirectionFilter(localMasterMonitor, leadershipMgr);
            final MantisMasterRoute app = new MantisMasterRoute(leaderRedirectionFilter, masterDescriptionRoute, v0JobClusterRoute, v0JobRoute, v0JobDiscoveryRoute, v0JobStatusRoute, v0AgentClusterRoute, v1JobClusterRoute, v1JobsRoute, v1AdminMasterRoute, v1AgentClusterRoute, v1JobDiscoveryStreamRoute, v1LastSubmittedJobIdStreamRoute, v1JobStatusStreamRoute);
            final Flow<HttpRequest, HttpResponse, NotUsed> routeFlow = app.createRoute().flow(system, materializer);
            logger.info("starting test server on port {}", serverPort);
            binding = http.bindAndHandle(routeFlow, ConnectHttp.toHost("localhost", serverPort), materializer);
            latch.countDown();
        } catch (Exception e) {
            logger.info("caught exception", e);
            latch.countDown();
            e.printStackTrace();
        }
    });
    t.setDaemon(true);
    t.start();
    latch.await();
    Thread.sleep(100);
}
Also used : JobClusterRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobClusterRouteHandler) JobRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobRouteHandlerAkkaImpl) JobRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobRouteHandler) MasterDescription(io.mantisrx.server.core.master.MasterDescription) FakeMantisScheduler(io.mantisrx.master.scheduler.FakeMantisScheduler) StatusEventSubscriberLoggingImpl(io.mantisrx.master.events.StatusEventSubscriberLoggingImpl) ActorRef(akka.actor.ActorRef) JobClusterRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobClusterRouteHandlerAkkaImpl) Http(akka.http.javadsl.Http) ConnectHttp(akka.http.javadsl.ConnectHttp) MantisScheduler(io.mantisrx.server.master.scheduler.MantisScheduler) FakeMantisScheduler(io.mantisrx.master.scheduler.FakeMantisScheduler) JobStatusStreamRoute(io.mantisrx.master.api.akka.route.v1.JobStatusStreamRoute) JobDiscoveryStreamRoute(io.mantisrx.master.api.akka.route.v1.JobDiscoveryStreamRoute) AgentClusterOperations(io.mantisrx.master.vm.AgentClusterOperations) ActorMaterializer(akka.stream.ActorMaterializer) LifecycleEventPublisherImpl(io.mantisrx.master.events.LifecycleEventPublisherImpl) JobStatusRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobStatusRouteHandler) WorkerEventSubscriberLoggingImpl(io.mantisrx.master.events.WorkerEventSubscriberLoggingImpl) AdminMasterRoute(io.mantisrx.master.api.akka.route.v1.AdminMasterRoute) AgentClustersRoute(io.mantisrx.master.api.akka.route.v1.AgentClustersRoute) LastSubmittedJobIdStreamRoute(io.mantisrx.master.api.akka.route.v1.LastSubmittedJobIdStreamRoute) Duration(java.time.Duration) LeaderRedirectionFilter(io.mantisrx.server.master.LeaderRedirectionFilter) LifecycleEventPublisher(io.mantisrx.master.events.LifecycleEventPublisher) MantisMasterRoute(io.mantisrx.master.api.akka.route.MantisMasterRoute) CountDownLatch(java.util.concurrent.CountDownLatch) JobDiscoveryRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandlerAkkaImpl) TimeoutException(java.util.concurrent.TimeoutException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) Flow(akka.stream.javadsl.Flow) AuditEventSubscriberLoggingImpl(io.mantisrx.master.events.AuditEventSubscriberLoggingImpl) JobsRoute(io.mantisrx.master.api.akka.route.v1.JobsRoute) LocalMasterMonitor(io.mantisrx.server.core.master.LocalMasterMonitor) LeadershipManagerLocalImpl(io.mantisrx.server.master.LeadershipManagerLocalImpl) MantisJobStore(io.mantisrx.server.master.persistence.MantisJobStore) JobDiscoveryRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandler) JobClustersRoute(io.mantisrx.master.api.akka.route.v1.JobClustersRoute) BeforeClass(org.testng.annotations.BeforeClass)

Example 2 with AgentClusterOperations

use of io.mantisrx.master.vm.AgentClusterOperations in project mantis by Netflix.

the class JobsRouteTest method setup.

@BeforeClass
public void setup() throws Exception {
    JobTestHelper.deleteAllFiles();
    JobTestHelper.createDirsIfRequired();
    TestHelpers.setupMasterConfig();
    final CountDownLatch latch = new CountDownLatch(1);
    t = new Thread(() -> {
        try {
            // boot up server using the route as defined below
            final LifecycleEventPublisher lifecycleEventPublisher = new LifecycleEventPublisherImpl(new AuditEventSubscriberLoggingImpl(), new StatusEventSubscriberLoggingImpl(), new WorkerEventSubscriberLoggingImpl());
            ActorRef jobClustersManagerActor = system.actorOf(JobClustersManagerActor.props(new MantisJobStore(new io.mantisrx.server.master.persistence.SimpleCachedFileStorageProvider(true)), lifecycleEventPublisher), "jobClustersManager");
            MantisScheduler fakeScheduler = new FakeMantisScheduler(jobClustersManagerActor);
            jobClustersManagerActor.tell(new JobClusterManagerProto.JobClustersManagerInitialize(fakeScheduler, false), ActorRef.noSender());
            final JobClusterRouteHandler jobClusterRouteHandler = new JobClusterRouteHandlerAkkaImpl(jobClustersManagerActor);
            final JobRouteHandler jobRouteHandler = new JobRouteHandlerAkkaImpl(jobClustersManagerActor);
            MasterDescription masterDescription = new MasterDescription("127.0.0.1", "127.0.0.1", SERVER_PORT, SERVER_PORT, SERVER_PORT, "api/postjobstatus", SERVER_PORT, System.currentTimeMillis());
            Duration idleTimeout = system.settings().config().getDuration("akka.http.server.idle-timeout");
            logger.info("idle timeout {} sec ", idleTimeout.getSeconds());
            final AgentClusterOperations mockAgentClusterOps = mock(AgentClusterOperations.class);
            final JobStatusRouteHandler jobStatusRouteHandler = mock(JobStatusRouteHandler.class);
            when(jobStatusRouteHandler.jobStatus(anyString())).thenReturn(Flow.create());
            final JobRoute v0JobRoute = new JobRoute(jobRouteHandler, system);
            JobDiscoveryRouteHandler jobDiscoveryRouteHandler = new JobDiscoveryRouteHandlerAkkaImpl(jobClustersManagerActor, idleTimeout);
            final JobDiscoveryRoute v0JobDiscoveryRoute = new JobDiscoveryRoute(jobDiscoveryRouteHandler);
            final JobClusterRoute v0JobClusterRoute = new JobClusterRoute(jobClusterRouteHandler, jobRouteHandler, system);
            final JobStatusRoute v0JobStatusRoute = new JobStatusRoute(jobStatusRouteHandler);
            final AgentClusterRoute v0AgentClusterRoute = new AgentClusterRoute(mockAgentClusterOps, system);
            final MasterDescriptionRoute v0MasterDescriptionRoute = new MasterDescriptionRoute(masterDescription);
            final JobsRoute v1JobsRoute = new JobsRoute(jobClusterRouteHandler, jobRouteHandler, system);
            final JobClustersRoute v1JobClusterRoute = new JobClustersRoute(jobClusterRouteHandler, system);
            final AgentClustersRoute v1AgentClustersRoute = new AgentClustersRoute(mockAgentClusterOps);
            final JobStatusStreamRoute v1JobStatusStreamRoute = new JobStatusStreamRoute(jobStatusRouteHandler);
            final AdminMasterRoute v1AdminMasterRoute = new AdminMasterRoute(masterDescription);
            final JobDiscoveryStreamRoute v1JobDiscoveryStreamRoute = new JobDiscoveryStreamRoute(jobDiscoveryRouteHandler);
            final LastSubmittedJobIdStreamRoute v1LastSubmittedJobIdStreamRoute = new LastSubmittedJobIdStreamRoute(jobDiscoveryRouteHandler);
            LocalMasterMonitor localMasterMonitor = new LocalMasterMonitor(masterDescription);
            LeadershipManagerLocalImpl leadershipMgr = new LeadershipManagerLocalImpl(masterDescription);
            leadershipMgr.setLeaderReady();
            LeaderRedirectionFilter leaderRedirectionFilter = new LeaderRedirectionFilter(localMasterMonitor, leadershipMgr);
            final MantisMasterRoute app = new MantisMasterRoute(leaderRedirectionFilter, v0MasterDescriptionRoute, v0JobClusterRoute, v0JobRoute, v0JobDiscoveryRoute, v0JobStatusRoute, v0AgentClusterRoute, v1JobClusterRoute, v1JobsRoute, v1AdminMasterRoute, v1AgentClustersRoute, v1JobDiscoveryStreamRoute, v1LastSubmittedJobIdStreamRoute, v1JobStatusStreamRoute);
            final Flow<HttpRequest, HttpResponse, NotUsed> routeFlow = app.createRoute().orElse(v1JobsRoute.createRoute(Function.identity())).flow(system, materializer);
            logger.info("starting test server on port {}", SERVER_PORT);
            latch.countDown();
            binding = http.bindAndHandle(routeFlow, ConnectHttp.toHost("localhost", SERVER_PORT), materializer);
        } catch (Exception e) {
            logger.info("caught exception", e);
            latch.countDown();
            e.printStackTrace();
        }
    });
    t.setDaemon(true);
    t.start();
    latch.await();
}
Also used : JobClusterRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobClusterRouteHandler) AgentClusterRoute(io.mantisrx.master.api.akka.route.v0.AgentClusterRoute) JobRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobRouteHandlerAkkaImpl) MasterDescriptionRoute(io.mantisrx.master.api.akka.route.v0.MasterDescriptionRoute) JobRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobRouteHandler) MasterDescription(io.mantisrx.server.core.master.MasterDescription) FakeMantisScheduler(io.mantisrx.master.scheduler.FakeMantisScheduler) ActorRef(akka.actor.ActorRef) JobDiscoveryRoute(io.mantisrx.master.api.akka.route.v0.JobDiscoveryRoute) JobClusterRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobClusterRouteHandlerAkkaImpl) JobClusterRoute(io.mantisrx.master.api.akka.route.v0.JobClusterRoute) MantisScheduler(io.mantisrx.server.master.scheduler.MantisScheduler) FakeMantisScheduler(io.mantisrx.master.scheduler.FakeMantisScheduler) AgentClusterOperations(io.mantisrx.master.vm.AgentClusterOperations) JobStatusRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobStatusRouteHandler) JobStatusRoute(io.mantisrx.master.api.akka.route.v0.JobStatusRoute) Duration(java.time.Duration) LeaderRedirectionFilter(io.mantisrx.server.master.LeaderRedirectionFilter) MantisMasterRoute(io.mantisrx.master.api.akka.route.MantisMasterRoute) CountDownLatch(java.util.concurrent.CountDownLatch) JobDiscoveryRouteHandlerAkkaImpl(io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandlerAkkaImpl) IOException(java.io.IOException) Flow(akka.stream.javadsl.Flow) JobRoute(io.mantisrx.master.api.akka.route.v0.JobRoute) LocalMasterMonitor(io.mantisrx.server.core.master.LocalMasterMonitor) LeadershipManagerLocalImpl(io.mantisrx.server.master.LeadershipManagerLocalImpl) MantisJobStore(io.mantisrx.server.master.persistence.MantisJobStore) JobDiscoveryRouteHandler(io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandler) BeforeClass(org.testng.annotations.BeforeClass)

Aggregations

ActorRef (akka.actor.ActorRef)2 Flow (akka.stream.javadsl.Flow)2 MantisMasterRoute (io.mantisrx.master.api.akka.route.MantisMasterRoute)2 JobClusterRouteHandler (io.mantisrx.master.api.akka.route.handlers.JobClusterRouteHandler)2 JobClusterRouteHandlerAkkaImpl (io.mantisrx.master.api.akka.route.handlers.JobClusterRouteHandlerAkkaImpl)2 JobDiscoveryRouteHandler (io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandler)2 JobDiscoveryRouteHandlerAkkaImpl (io.mantisrx.master.api.akka.route.handlers.JobDiscoveryRouteHandlerAkkaImpl)2 JobRouteHandler (io.mantisrx.master.api.akka.route.handlers.JobRouteHandler)2 JobRouteHandlerAkkaImpl (io.mantisrx.master.api.akka.route.handlers.JobRouteHandlerAkkaImpl)2 JobStatusRouteHandler (io.mantisrx.master.api.akka.route.handlers.JobStatusRouteHandler)2 FakeMantisScheduler (io.mantisrx.master.scheduler.FakeMantisScheduler)2 AgentClusterOperations (io.mantisrx.master.vm.AgentClusterOperations)2 LocalMasterMonitor (io.mantisrx.server.core.master.LocalMasterMonitor)2 MasterDescription (io.mantisrx.server.core.master.MasterDescription)2 LeaderRedirectionFilter (io.mantisrx.server.master.LeaderRedirectionFilter)2 LeadershipManagerLocalImpl (io.mantisrx.server.master.LeadershipManagerLocalImpl)2 MantisJobStore (io.mantisrx.server.master.persistence.MantisJobStore)2 MantisScheduler (io.mantisrx.server.master.scheduler.MantisScheduler)2 IOException (java.io.IOException)2 Duration (java.time.Duration)2