Examples with MantisStorageProviderAdapter io.mantisrx.server.master.persistence.MantisStorageProviderAdapter used on opensource projects

Example 1 with MantisStorageProviderAdapter

use of io.mantisrx.server.master.persistence.MantisStorageProviderAdapter in project mantis by Netflix.

the class MantisMasterAPI method main.

public static void main(String[] args) throws Exception {
    // boot up server using the route as defined below
    int port = 8182;
    TestHelpers.setupMasterConfig();
    ActorSystem system = ActorSystem.create("MantisMasterAPI");
    final ActorRef actor = system.actorOf(Props.create(DeadLetterActor.class));
    system.eventStream().subscribe(actor, DeadLetter.class);
    final Http http = Http.get(system);
    final ActorMaterializer materializer = ActorMaterializer.create(system);
    final AuditEventSubscriber auditEventSubscriber = new AuditEventSubscriberLoggingImpl();
    ActorRef auditEventBrokerActor = system.actorOf(AuditEventBrokerActor.props(auditEventSubscriber), "AuditEventBroker");
    final AuditEventSubscriber auditEventSubscriberAkka = new AuditEventSubscriberAkkaImpl(auditEventBrokerActor);
    final LifecycleEventPublisher lifecycleEventPublisher = new LifecycleEventPublisherImpl(auditEventSubscriberAkka, new StatusEventSubscriberLoggingImpl(), new WorkerEventSubscriberLoggingImpl());
    IMantisStorageProvider storageProvider = new MantisStorageProviderAdapter(new SimpleCachedFileStorageProvider(), lifecycleEventPublisher);
    ActorRef jobClustersManager = system.actorOf(JobClustersManagerActor.props(new MantisJobStore(storageProvider), lifecycleEventPublisher), "JobClustersManager");
    final FakeMantisScheduler fakeScheduler = new FakeMantisScheduler(jobClustersManager);
    jobClustersManager.tell(new JobClusterManagerProto.JobClustersManagerInitialize(fakeScheduler, true), ActorRef.noSender());
    // Schedulers.newThread().createWorker().schedulePeriodically(() -> jobClustersManager.tell(new NullPointerException(), ActorRef.noSender()),0, 100, TimeUnit.SECONDS);
    setupDummyAgentClusterAutoScaler();
    final JobClusterRouteHandler jobClusterRouteHandler = new JobClusterRouteHandlerAkkaImpl(jobClustersManager);
    final JobRouteHandler jobRouteHandler = new JobRouteHandlerAkkaImpl(jobClustersManager);
    MasterDescription masterDescription = new MasterDescription("localhost", "127.0.0.1", port, port + 2, port + 4, "api/postjobstatus", port + 6, System.currentTimeMillis());
    final MasterDescriptionRoute masterDescriptionRoute = new MasterDescriptionRoute(masterDescription);
    Duration idleTimeout = system.settings().config().getDuration("akka.http.server.idle-timeout");
    logger.info("idle timeout {} sec ", idleTimeout.getSeconds());
    ActorRef agentsErrorMonitorActor = system.actorOf(AgentsErrorMonitorActor.props(), "AgentsErrorMonitor");
    ActorRef statusEventBrokerActor = system.actorOf(StatusEventBrokerActor.props(agentsErrorMonitorActor), "StatusEventBroker");
    agentsErrorMonitorActor.tell(new AgentsErrorMonitorActor.InitializeAgentsErrorMonitor(fakeScheduler), ActorRef.noSender());
    final JobStatusRouteHandler jobStatusRouteHandler = new JobStatusRouteHandlerAkkaImpl(system, statusEventBrokerActor);
    final AgentClusterOperationsImpl agentClusterOperations = new AgentClusterOperationsImpl(storageProvider, new JobMessageRouterImpl(jobClustersManager), fakeScheduler, lifecycleEventPublisher, "cluster");
    final JobDiscoveryRouteHandler jobDiscoveryRouteHandler = new JobDiscoveryRouteHandlerAkkaImpl(jobClustersManager, idleTimeout);
    final JobRoute v0JobRoute = new JobRoute(jobRouteHandler, system);
    final JobDiscoveryRoute v0JobDiscoveryRoute = new JobDiscoveryRoute(jobDiscoveryRouteHandler);
    final JobClusterRoute v0JobClusterRoute = new JobClusterRoute(jobClusterRouteHandler, jobRouteHandler, system);
    final JobStatusRoute v0JobStatusRoute = new JobStatusRoute(jobStatusRouteHandler);
    final AgentClusterRoute v0AgentClusterRoute = new AgentClusterRoute(agentClusterOperations, system);
    final JobClustersRoute v1JobClustersRoute = new JobClustersRoute(jobClusterRouteHandler, system);
    final JobsRoute v1JobsRoute = new JobsRoute(jobClusterRouteHandler, jobRouteHandler, system);
    final AdminMasterRoute v1AdminMasterRoute = new AdminMasterRoute(masterDescription);
    final AgentClustersRoute v1AgentClustersRoute = new AgentClustersRoute(agentClusterOperations);
    final JobDiscoveryStreamRoute v1JobDiscoveryStreamRoute = new JobDiscoveryStreamRoute(jobDiscoveryRouteHandler);
    final LastSubmittedJobIdStreamRoute v1LastSubmittedJobIdStreamRoute = new LastSubmittedJobIdStreamRoute(jobDiscoveryRouteHandler);
    final JobStatusStreamRoute v1JobStatusStreamRoute = new JobStatusStreamRoute(jobStatusRouteHandler);
    LocalMasterMonitor localMasterMonitor = new LocalMasterMonitor(masterDescription);
    LeadershipManagerLocalImpl leadershipMgr = new LeadershipManagerLocalImpl(masterDescription);
    leadershipMgr.setLeaderReady();
    LeaderRedirectionFilter leaderRedirectionFilter = new LeaderRedirectionFilter(localMasterMonitor, leadershipMgr);
    final MantisMasterRoute app = new MantisMasterRoute(leaderRedirectionFilter, masterDescriptionRoute, v0JobClusterRoute, v0JobRoute, v0JobDiscoveryRoute, v0JobStatusRoute, v0AgentClusterRoute, v1JobClustersRoute, v1JobsRoute, v1AdminMasterRoute, v1AgentClustersRoute, v1JobDiscoveryStreamRoute, v1LastSubmittedJobIdStreamRoute, v1JobStatusStreamRoute);
    final Flow<HttpRequest, HttpResponse, NotUsed> routeFlow = app.createRoute().flow(system, materializer);
    final CompletionStage<ServerBinding> binding = http.bindAndHandle(routeFlow, ConnectHttp.toHost("localhost", port), materializer);
    binding.exceptionally(failure -> {
        System.err.println("Something very bad happened! " + failure.getMessage());
        system.terminate();
        return null;
    });
    // Schedulers.newThread().createWorker().schedule(() -> leadershipMgr.stopBeingLeader(), 10, TimeUnit.SECONDS);
    System.out.println("Server online at http://localhost:" + port + "/\nPress RETURN to stop...");
    // let it run until user presses return
    System.in.read();
    binding.thenCompose(// trigger unbinding from the port
    ServerBinding::unbind).thenAccept(// and shutdown when done
    unbound -> system.terminate());
}

Example 2 with MantisStorageProviderAdapter

use of io.mantisrx.server.master.persistence.MantisStorageProviderAdapter in project mantis by Netflix.

the class JobClusterTest method setup.

@BeforeClass
public static void setup() {
    Config config = ConfigFactory.parseString("akka {\n" + "  loggers = [\"akka.testkit.TestEventListener\"]\n" + "  loglevel = \"WARNING\"\n" + "  stdout-loglevel = \"WARNING\"\n" + "  test.single-expect-default = 1000 millis\n" + "}\n");
    system = ActorSystem.create("JobClusterTest", config.withFallback(ConfigFactory.load()));
    JobTestHelper.createDirsIfRequired();
    TestHelpers.setupMasterConfig();
    storageProvider = new MantisStorageProviderAdapter(new io.mantisrx.server.master.store.SimpleCachedFileStorageProvider(), eventPublisher);
    jobStore = new MantisJobStore(storageProvider);
}

Example 3 with MantisStorageProviderAdapter

use of io.mantisrx.server.master.persistence.MantisStorageProviderAdapter in project mantis by Netflix.

the class JobClusterManagerTest method testBootStrapJobClustersAndJobs.

@Test
public void testBootStrapJobClustersAndJobs() {
    TestKit probe = new TestKit(system);
    JobTestHelper.deleteAllFiles();
    MantisJobStore jobStore = new MantisJobStore(new MantisStorageProviderAdapter(new io.mantisrx.server.master.store.SimpleCachedFileStorageProvider(), eventPublisher));
    MantisJobStore jobStoreSpied = Mockito.spy(jobStore);
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    ActorRef jobClusterManagerActor = system.actorOf(JobClustersManagerActor.props(jobStoreSpied, eventPublisher));
    jobClusterManagerActor.tell(new JobClusterManagerProto.JobClustersManagerInitialize(schedulerMock, false), probe.getRef());
    JobClustersManagerInitializeResponse iResponse = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), JobClustersManagerInitializeResponse.class);
    List<String> clusterNames = Lists.newArrayList("testBootStrapJobClustersAndJobs1", "testBootStrapJobClustersAndJobs2", "testBootStrapJobClustersAndJobs3");
    String clusterWithNoJob = "testBootStrapJobClusterWithNoJob";
    createJobClusterAndAssert(jobClusterManagerActor, clusterWithNoJob);
    WorkerMigrationConfig migrationConfig = new WorkerMigrationConfig(MigrationStrategyEnum.PERCENTAGE, "{\"percentToMove\":60, \"intervalMs\":30000}");
    // Create 3 clusters and submit 1 job each
    for (String cluster : clusterNames) {
        createJobClusterAndAssert(jobClusterManagerActor, cluster, migrationConfig);
        submitJobAndAssert(jobClusterManagerActor, cluster);
        if (cluster.equals("testBootStrapJobClustersAndJobs1")) {
            // send worker events for job 1 so it goes to started state
            String jobId = "testBootStrapJobClustersAndJobs1-1";
            WorkerId workerId = new WorkerId(jobId, 0, 1);
            WorkerEvent launchedEvent = new WorkerLaunched(workerId, 0, "host1", "vm1", empty(), new WorkerPorts(Lists.newArrayList(8000, 9000, 9010, 9020, 9030)));
            jobClusterManagerActor.tell(launchedEvent, probe.getRef());
            WorkerEvent startInitEvent = new WorkerStatus(new Status(workerId.getJobId(), 1, workerId.getWorkerIndex(), workerId.getWorkerNum(), TYPE.INFO, "test START_INIT", MantisJobState.StartInitiated));
            jobClusterManagerActor.tell(startInitEvent, probe.getRef());
            WorkerEvent heartBeat = new WorkerHeartbeat(new Status(jobId, 1, workerId.getWorkerIndex(), workerId.getWorkerNum(), TYPE.HEARTBEAT, "", MantisJobState.Started));
            jobClusterManagerActor.tell(heartBeat, probe.getRef());
            // get Job status
            jobClusterManagerActor.tell(new GetJobDetailsRequest("user", JobId.fromId(jobId).get()), probe.getRef());
            GetJobDetailsResponse resp2 = probe.expectMsgClass(GetJobDetailsResponse.class);
            // Ensure its launched
            assertEquals(SUCCESS, resp2.responseCode);
            assertEquals(JobState.Launched, resp2.getJobMetadata().get().getState());
        }
    }
    // kill 1 of the jobs to test archive path
    JobClusterManagerProto.KillJobRequest killRequest = new JobClusterManagerProto.KillJobRequest("testBootStrapJobClustersAndJobs2-1", JobCompletedReason.Killed.toString(), "njoshi");
    jobClusterManagerActor.tell(killRequest, probe.getRef());
    JobClusterManagerProto.KillJobResponse killJobResponse = probe.expectMsgClass(JobClusterManagerProto.KillJobResponse.class);
    assertEquals(SUCCESS, killJobResponse.responseCode);
    JobTestHelper.sendWorkerTerminatedEvent(probe, jobClusterManagerActor, "testBootStrapJobClustersAndJobs2-1", new WorkerId("testBootStrapJobClustersAndJobs2-1", 0, 1));
    try {
        Thread.sleep(500);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    // Stop job cluster Manager Actor
    system.stop(jobClusterManagerActor);
    // create new instance
    jobClusterManagerActor = system.actorOf(JobClustersManagerActor.props(jobStoreSpied, eventPublisher));
    // initialize it
    jobClusterManagerActor.tell(new JobClusterManagerProto.JobClustersManagerInitialize(schedulerMock, true), probe.getRef());
    JobClustersManagerInitializeResponse initializeResponse = probe.expectMsgClass(JobClustersManagerInitializeResponse.class);
    // probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES),JobClusterManagerProto.JobClustersManagerInitializeResponse.class);
    // probe.expectMsgClass(JobClusterManagerProto.JobClustersManagerInitializeResponse.class);
    assertEquals(SUCCESS, initializeResponse.responseCode);
    // Get Cluster Config
    jobClusterManagerActor.tell(new GetJobClusterRequest("testBootStrapJobClustersAndJobs1"), probe.getRef());
    GetJobClusterResponse clusterResponse = probe.expectMsgClass(GetJobClusterResponse.class);
    assertEquals(SUCCESS, clusterResponse.responseCode);
    assertTrue(clusterResponse.getJobCluster().isPresent());
    WorkerMigrationConfig mConfig = clusterResponse.getJobCluster().get().getMigrationConfig();
    assertEquals(migrationConfig.getStrategy(), mConfig.getStrategy());
    assertEquals(migrationConfig.getConfigString(), migrationConfig.getConfigString());
    // get Job status
    jobClusterManagerActor.tell(new GetJobDetailsRequest("user", JobId.fromId("testBootStrapJobClustersAndJobs1-1").get()), probe.getRef());
    GetJobDetailsResponse resp2 = probe.expectMsgClass(GetJobDetailsResponse.class);
    // Ensure its launched
    System.out.println("Resp2 -> " + resp2.message);
    assertEquals(SUCCESS, resp2.responseCode);
    assertEquals(JobState.Launched, resp2.getJobMetadata().get().getState());
    // 1 jobs should be in completed state
    jobClusterManagerActor.tell(new GetJobDetailsRequest("user", JobId.fromId("testBootStrapJobClustersAndJobs2-1").get()), probe.getRef());
    resp2 = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), GetJobDetailsResponse.class);
    // Ensure its completed
    assertEquals(SUCCESS, resp2.responseCode);
    assertEquals(JobState.Completed, resp2.getJobMetadata().get().getState());
    jobClusterManagerActor.tell(new GetJobDetailsRequest("user", JobId.fromId("testBootStrapJobClustersAndJobs3-1").get()), probe.getRef());
    resp2 = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), GetJobDetailsResponse.class);
    // Ensure its Accepted
    assertEquals(SUCCESS, resp2.responseCode);
    assertEquals(JobState.Accepted, resp2.getJobMetadata().get().getState());
    try {
        Optional<JobWorker> workerByIndex = resp2.getJobMetadata().get().getWorkerByIndex(1, 0);
        assertTrue(workerByIndex.isPresent());
        Optional<IMantisStageMetadata> stageMetadata = resp2.getJobMetadata().get().getStageMetadata(1);
        assertTrue(stageMetadata.isPresent());
        JobWorker workerByIndex1 = stageMetadata.get().getWorkerByIndex(0);
        System.out.println("Got worker by index : " + workerByIndex1);
        Optional<JobWorker> worker = resp2.getJobMetadata().get().getWorkerByNumber(1);
        assertTrue(worker.isPresent());
    } catch (io.mantisrx.server.master.persistence.exceptions.InvalidJobException e) {
        e.printStackTrace();
    }
    jobClusterManagerActor.tell(new GetLastSubmittedJobIdStreamRequest("testBootStrapJobClustersAndJobs1"), probe.getRef());
    GetLastSubmittedJobIdStreamResponse lastSubmittedJobIdStreamResponse = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), GetLastSubmittedJobIdStreamResponse.class);
    lastSubmittedJobIdStreamResponse.getjobIdBehaviorSubject().get().take(1).toBlocking().subscribe((jId) -> {
        assertEquals(new JobId("testBootStrapJobClustersAndJobs1", 1), jId);
    });
    jobClusterManagerActor.tell(new GetJobClusterRequest(clusterWithNoJob), probe.getRef());
    GetJobClusterResponse jobClusterResponse = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), GetJobClusterResponse.class);
    assertEquals(SUCCESS, jobClusterResponse.responseCode);
    assertTrue(jobClusterResponse.getJobCluster().isPresent());
    assertEquals(clusterWithNoJob, jobClusterResponse.getJobCluster().get().getName());
    // 1 running worker
    verify(schedulerMock, timeout(100_1000).times(1)).initializeRunningWorker(any(), any());
    // 2 worker schedule requests
    verify(schedulerMock, timeout(100_000).times(4)).scheduleWorker(any());
    try {
        Mockito.verify(jobStoreSpied).loadAllArchivedJobsAsync();
        Mockito.verify(jobStoreSpied).loadAllActiveJobs();
        Mockito.verify(jobStoreSpied).loadAllCompletedJobs();
        Mockito.verify(jobStoreSpied).archiveWorker(any());
        Mockito.verify(jobStoreSpied).archiveJob(any());
    } catch (IOException e) {
        e.printStackTrace();
        fail();
    }
}

Example 4 with MantisStorageProviderAdapter

use of io.mantisrx.server.master.persistence.MantisStorageProviderAdapter in project mantis by Netflix.

the class JobClusterManagerTest method testBootStrapJobClustersAndJobs1.

@Test
public void testBootStrapJobClustersAndJobs1() {
    TestKit probe = new TestKit(system);
    JobTestHelper.deleteAllFiles();
    MantisJobStore jobStore = new MantisJobStore(new MantisStorageProviderAdapter(new io.mantisrx.server.master.store.SimpleCachedFileStorageProvider(), eventPublisher));
    MantisJobStore jobStoreSpied = Mockito.spy(jobStore);
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    ActorRef jobClusterManagerActor = system.actorOf(JobClustersManagerActor.props(jobStoreSpied, eventPublisher));
    jobClusterManagerActor.tell(new JobClusterManagerProto.JobClustersManagerInitialize(schedulerMock, true), probe.getRef());
    JobClustersManagerInitializeResponse iResponse = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), JobClustersManagerInitializeResponse.class);
    // List<String> clusterNames = Lists.newArrayList("testBootStrapJobClustersAndJobs1");
    String clusterWithNoJob = "testBootStrapJobClusterWithNoJob";
    createJobClusterAndAssert(jobClusterManagerActor, clusterWithNoJob);
    // kill 1 of the jobs to test archive path
    // Stop job cluster Manager Actor
    system.stop(jobClusterManagerActor);
    // create new instance
    jobClusterManagerActor = system.actorOf(JobClustersManagerActor.props(jobStore, eventPublisher));
    // initialize it
    jobClusterManagerActor.tell(new JobClusterManagerProto.JobClustersManagerInitialize(schedulerMock, true), probe.getRef());
    // JobClusterManagerProto.JobClustersManagerInitializeResponse initializeResponse = probe.expectMsgClass(JobClusterManagerProto.JobClustersManagerInitializeResponse.class);
    JobClustersManagerInitializeResponse initializeResponse = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), JobClustersManagerInitializeResponse.class);
    assertEquals(SUCCESS, initializeResponse.responseCode);
    jobClusterManagerActor.tell(new GetJobClusterRequest(clusterWithNoJob), probe.getRef());
    GetJobClusterResponse jobClusterResponse = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), GetJobClusterResponse.class);
    assertEquals(SUCCESS, jobClusterResponse.responseCode);
    assertTrue(jobClusterResponse.getJobCluster().isPresent());
    assertEquals(clusterWithNoJob, jobClusterResponse.getJobCluster().get().getName());
    try {
        // Mockito.verify(jobStoreSpied).loadAllArchivedJobsAsync();
        Mockito.verify(jobStoreSpied).loadAllJobClusters();
        Mockito.verify(jobStoreSpied).loadAllActiveJobs();
        Mockito.verify(jobStoreSpied).loadAllCompletedJobs();
    // Mockito.verify(jobStoreSpied).archiveWorker(any());
    // Mockito.verify(jobStoreSpied).archiveJob(any());
    } catch (IOException e) {
        e.printStackTrace();
        fail();
    }
}

Example 5 with MantisStorageProviderAdapter

use of io.mantisrx.server.master.persistence.MantisStorageProviderAdapter in project mantis by Netflix.

the class JobClusterManagerTest method testBootStrapJobClustersAndJobsNegativeTest.

@Test
public void testBootStrapJobClustersAndJobsNegativeTest() throws IOException {
    TestKit probe = new TestKit(system);
    JobTestHelper.deleteAllFiles();
    MantisStorageProviderAdapter storageProviderAdapter = mock(MantisStorageProviderAdapter.class);
    when(storageProviderAdapter.loadAllJobClusters()).thenThrow(new IOException("StorageException"));
    MantisJobStore jobStore = new MantisJobStore(storageProviderAdapter);
    MantisJobStore jobStoreSpied = Mockito.spy(jobStore);
    MantisScheduler schedulerMock = mock(MantisScheduler.class);
    ActorRef jobClusterManagerActor = system.actorOf(JobClustersManagerActor.props(jobStoreSpied, eventPublisher));
    jobClusterManagerActor.tell(new JobClusterManagerProto.JobClustersManagerInitialize(schedulerMock, true), probe.getRef());
    JobClustersManagerInitializeResponse iResponse = probe.expectMsgClass(Duration.of(10, ChronoUnit.MINUTES), JobClustersManagerInitializeResponse.class);
    assertEquals(BaseResponse.ResponseCode.SERVER_ERROR, iResponse.responseCode);
}