Examples with MantisJobMetadataImpl io.mantisrx.master.jobcluster.job.MantisJobMetadataImpl used on opensource projects

Example 1 with MantisJobMetadataImpl

use of io.mantisrx.master.jobcluster.job.MantisJobMetadataImpl in project mantis by Netflix.

the class JobClusterActor method initRunningJobs.

/**
 * Iterate through list of jobs in Active jobs table.
 * if a Job is completed move it completed table
 * else bootstrap the job (create actor, send init request)
 * Finally setup sla enforcement
 * @param initReq
 * @param sender
 */
private void initRunningJobs(JobClusterProto.InitializeJobClusterRequest initReq, ActorRef sender) {
    List<CompletedJob> completedJobsList = initReq.completedJobsList;
    List<IMantisJobMetadata> jobList = initReq.jobList;
    logger.info("In _initJobs for cluster {}: {} activeJobs and {} completedJobs", name, jobList.size(), completedJobsList.size());
    if (logger.isDebugEnabled()) {
        logger.debug("In _initJobs for cluster {} activeJobs -> {} and completedJobs -> {}", name, jobList, completedJobsList);
    }
    Observable.from(jobList).flatMap((jobMeta) -> {
        if (JobState.isTerminalState(jobMeta.getState())) {
            jobManager.persistToCompletedJobAndArchiveJobTables(jobMeta);
            return Observable.empty();
        } else {
            if (jobMeta.getSchedulingInfo() == null) {
                logger.error("Scheduling info is null for active job {} in cluster {}." + "Skipping bootstrap ", jobMeta.getJobId(), name);
                return Observable.empty();
            } else {
                return Observable.just(jobMeta);
            }
        }
    }).flatMap((jobMeta) -> jobManager.bootstrapJob((MantisJobMetadataImpl) jobMeta, this.jobClusterMetadata)).subscribe((jobInited) -> {
        logger.info("Job Id {} initialized with code {}", jobInited.jobId, jobInited.responseCode);
    }, (error) -> logger.warn("Exception initializing jobs {}", error.getMessage()), () -> {
        if (initReq.jobList.size() > 0) {
            JobId lastJobId = new JobId(this.name, initReq.lastJobNumber);
            this.jobIdSubmissionSubject.onNext(lastJobId);
        }
        setBookkeepingTimer(BOOKKEEPING_INTERVAL_SECS);
        getContext().become(initializedBehavior);
        logger.info("Job Cluster {} initialized", this.name);
        sender.tell(new JobClusterProto.InitializeJobClusterResponse(initReq.requestId, SUCCESS, String.format("JobCluster %s initialized successfully", initReq.jobClusterDefinition.getName()), initReq.jobClusterDefinition.getName(), initReq.requestor), getSelf());
    });
}

Example 2 with MantisJobMetadataImpl

use of io.mantisrx.master.jobcluster.job.MantisJobMetadataImpl in project mantis by Netflix.

the class NoOpMantisJobOperations method convertMantisJobWriteableToMantisJobMetadata.

// TODO job specific migration config is not supported, migration config will be at cluster level
public static IMantisJobMetadata convertMantisJobWriteableToMantisJobMetadata(MantisJobMetadata archJob, LifecycleEventPublisher eventPublisher, boolean isArchived) throws Exception {
    if (logger.isTraceEnabled()) {
        logger.trace("DataFormatAdapter:Converting {}", archJob);
    }
    // convert stages to new format
    List<IMantisStageMetadata> convertedStageList = new ArrayList<>();
    for (MantisStageMetadata stageMeta : ((MantisJobMetadataWritable) archJob).getStageMetadata()) {
        // if this is an archived job then add workerIndex may fail as there maybe multiple workers related to a given index so skip adding workers to stage
        boolean skipAddingWorkers = false;
        if (isArchived) {
            skipAddingWorkers = true;
        }
        convertedStageList.add(convertMantisStageMetadataWriteableToMantisStageMetadata(stageMeta, eventPublisher, skipAddingWorkers));
    }
    // generate SchedulingInfo
    SchedulingInfo schedulingInfo = generateSchedulingInfo(convertedStageList);
    URL jarUrl = archJob.getJarUrl();
    Optional<String> artifactName = extractArtifactName(jarUrl);
    // generate job defn
    JobDefinition jobDefn = new JobDefinition(archJob.getName(), archJob.getUser(), artifactName.orElse(""), null, archJob.getParameters(), archJob.getSla(), archJob.getSubscriptionTimeoutSecs(), schedulingInfo, archJob.getNumStages(), archJob.getLabels(), null);
    Optional<JobId> jIdOp = JobId.fromId(archJob.getJobId());
    if (!jIdOp.isPresent()) {
        throw new IllegalArgumentException("Invalid JobId " + archJob.getJobId());
    }
    // generate job meta
    MantisJobMetadataImpl mantisJobMetadata = new MantisJobMetadataImpl(jIdOp.get(), archJob.getSubmittedAt(), archJob.getStartedAt(), jobDefn, convertMantisJobStateToJobState(archJob.getState()), archJob.getNextWorkerNumberToUse());
    // add the stages
    for (IMantisStageMetadata stageMetadata : convertedStageList) {
        mantisJobMetadata.addJobStageIfAbsent(stageMetadata);
    }
    if (logger.isTraceEnabled()) {
        logger.trace("DataFormatAdapter:Completed conversion to IMantisJobMetadata {}", mantisJobMetadata);
    }
    return mantisJobMetadata;
}

Example 3 with MantisJobMetadataImpl

use of io.mantisrx.master.jobcluster.job.MantisJobMetadataImpl in project mantis by Netflix.

the class SimpleCachedFileStorageProviderTest method testCreateJob.

@Test
public void testCreateJob() {
    String clusterName = "testCreateJob";
    SimpleCachedFileStorageProvider sProvider = new SimpleCachedFileStorageProvider();
    IJobClusterDefinition jobClusterDefn = JobTestHelper.generateJobClusterDefinition(clusterName);
    JobDefinition jobDefinition;
    try {
        jobDefinition = JobTestHelper.generateJobDefinition(clusterName);
        JobId jobId = JobId.fromId(clusterName + "-1").get();
        IMantisJobMetadata mantisJobMetaData = new MantisJobMetadataImpl.Builder().withJobId(jobId).withSubmittedAt(Instant.now()).withJobState(JobState.Accepted).withNextWorkerNumToUse(1).withJobDefinition(jobDefinition).build();
        sProvider.storeNewJob(mantisJobMetaData);
        SchedulingInfo schedInfo = jobDefinition.getSchedulingInfo();
        int numStages = schedInfo.getStages().size();
        for (int s = 1; s <= numStages; s++) {
            StageSchedulingInfo stage = schedInfo.getStages().get(s);
            IMantisStageMetadata msmd = new MantisStageMetadataImpl.Builder().withJobId(jobId).withStageNum(s).withNumStages(1).withMachineDefinition(stage.getMachineDefinition()).withNumWorkers(stage.getNumberOfInstances()).withHardConstraints(stage.getHardConstraints()).withSoftConstraints(stage.getSoftConstraints()).withScalingPolicy(stage.getScalingPolicy()).isScalable(stage.getScalable()).build();
            ((MantisJobMetadataImpl) mantisJobMetaData).addJobStageIfAbsent(msmd);
            sProvider.updateMantisStage(msmd);
            for (int w = 0; w < stage.getNumberOfInstances(); w++) {
                JobWorker mwmd = new JobWorker.Builder().withJobId(jobId).withWorkerIndex(w).withWorkerNumber(1).withNumberOfPorts(stage.getMachineDefinition().getNumPorts() + MANTIS_SYSTEM_ALLOCATED_NUM_PORTS).withStageNum(w + 1).withLifecycleEventsPublisher(eventPublisher).build();
                ((MantisJobMetadataImpl) mantisJobMetaData).addWorkerMetadata(1, mwmd);
                sProvider.storeWorker(mwmd.getMetadata());
            }
        }
        Optional<IMantisJobMetadata> loadedJobMetaOp = sProvider.loadActiveJob(jobId.getId());
        assertTrue(loadedJobMetaOp.isPresent());
        IMantisJobMetadata loadedJobMeta = loadedJobMetaOp.get();
        System.out.println("Original Job -> " + mantisJobMetaData);
        System.out.println("Loaded Job ->" + loadedJobMeta);
        isEqual(mantisJobMetaData, loadedJobMeta);
    } catch (Exception e) {
        e.printStackTrace();
        fail();
    }
}

Example 4 with MantisJobMetadataImpl

use of io.mantisrx.master.jobcluster.job.MantisJobMetadataImpl in project mantis by Netflix.

the class DataFormatAdapterTest method convertMantisJobWriteableTest.

@Test
public void convertMantisJobWriteableTest() throws Exception {
    String artifactName = "artifact";
    String version = "1.0.0";
    String clusterName = "myCluster";
    List<Label> labels = new ArrayList<>();
    Label label = new Label("myLable", "myVal");
    labels.add(label);
    List<Parameter> params = new ArrayList<>();
    Parameter param = new Parameter("myparam", "myval");
    params.add(param);
    long subTimeout = 1000;
    JobSla jobSla = new JobSla(100, 10, JobSla.StreamSLAType.Lossy, MantisJobDurationType.Perpetual, "userType");
    JobDefinition jobDefn = new JobDefinition.Builder().withArtifactName(artifactName).withName(clusterName).withLabels(labels).withParameters(params).withSchedulingInfo(DEFAULT_SCHED_INFO).withUser("user").withJobSla(jobSla).withSubscriptionTimeoutSecs(subTimeout).withNumberOfStages(DEFAULT_SCHED_INFO.getStages().size()).build();
    JobId jobId = new JobId(clusterName, 1);
    long currTime = System.currentTimeMillis();
    Instant startedAt = Instant.ofEpochMilli(currTime);
    Instant endedAt = startedAt.plusSeconds(5);
    Instant submittedAt = startedAt.minusSeconds(5);
    IMantisJobMetadata jobmeta = new MantisJobMetadataImpl.Builder().withJobDefinition(jobDefn).withJobId(jobId).withNextWorkerNumToUse(2).withSubmittedAt(submittedAt).withJobState(JobState.Launched).build();
    IMantisWorkerMetadata workerMetadata = new MantisWorkerMetadataImpl(0, 1, jobId.getId(), 1, 3, new WorkerPorts(Lists.newArrayList(8000, 9000, 9010, 9020, 9030)), WorkerState.Started, "slave", "slaveId", startedAt.toEpochMilli(), startedAt.toEpochMilli(), startedAt.toEpochMilli(), startedAt.toEpochMilli(), -1, JobCompletedReason.Normal, 0, 0, of("cluster"));
    ((MantisJobMetadataImpl) jobmeta).addJobStageIfAbsent(new MantisStageMetadataImpl.Builder().withNumStages(1).withStageNum(1).withNumWorkers(1).withJobId(jobId).withHardConstraints(Lists.newArrayList()).withSoftConstraints(Lists.newArrayList()).withMachineDefinition(DEFAULT_MACHINE_DEFINITION).build());
    ((MantisJobMetadataImpl) jobmeta).addWorkerMetadata(1, new JobWorker(workerMetadata, eventPublisher));
    MantisJobMetadata oldFormat = DataFormatAdapter.convertMantisJobMetadataToMantisJobMetadataWriteable(jobmeta);
    System.out.println("oldForamt -> " + oldFormat);
    assertEquals(jobId.getId(), oldFormat.getJobId());
    assertEquals(label, oldFormat.getLabels().get(0));
    assertEquals(param, oldFormat.getParameters().get(0));
    assertEquals(clusterName, oldFormat.getName());
    assertEquals(jobSla, oldFormat.getSla());
    assertEquals(1, oldFormat.getNumStages());
    assertEquals(subTimeout, oldFormat.getSubscriptionTimeoutSecs());
    assertEquals(2, oldFormat.getNextWorkerNumberToUse());
    assertEquals("http://" + artifactName, oldFormat.getJarUrl().toString());
    assertEquals(MantisJobState.Launched, oldFormat.getState());
    assertEquals(submittedAt.toEpochMilli(), oldFormat.getSubmittedAt());
    assertEquals("user", oldFormat.getUser());
    IMantisJobMetadata reconverted = DataFormatAdapter.convertMantisJobWriteableToMantisJobMetadata(oldFormat, eventPublisher);
    System.out.println("newForamt -> " + reconverted);
    // assertEquals(jobmeta, reconverted);
    // assertTrue(jobmeta.equals(reconverted));
    assertEquals(jobmeta.getArtifactName(), reconverted.getArtifactName());
    assertEquals(jobmeta.getClusterName(), reconverted.getClusterName());
    System.out.println("expected Jobdef " + jobmeta.getJobDefinition());
    System.out.println("actual   Jobdef " + reconverted.getJobDefinition());
    assertEquals(jobmeta.getJobDefinition(), reconverted.getJobDefinition());
    assertEquals(jobmeta.getJobId(), reconverted.getJobId());
    assertEquals(jobmeta.getJobJarUrl(), reconverted.getJobJarUrl());
    assertEquals(jobmeta.getLabels().get(0), reconverted.getLabels().get(0));
    assertEquals(jobmeta.getParameters().get(0), reconverted.getParameters().get(0));
    assertEquals(jobmeta.getMinRuntimeSecs(), reconverted.getMinRuntimeSecs());
    assertEquals(jobmeta.getNextWorkerNumberToUse(), reconverted.getNextWorkerNumberToUse());
    assertEquals(jobmeta.getSla().get(), reconverted.getSla().get());
    assertEquals(jobmeta.getSubmittedAtInstant(), reconverted.getSubmittedAtInstant());
    assertEquals(jobmeta.getState(), reconverted.getState());
    assertEquals(jobmeta.getSubscriptionTimeoutSecs(), reconverted.getSubscriptionTimeoutSecs());
    assertEquals(jobmeta.getTotalStages(), reconverted.getTotalStages());
    assertEquals(jobmeta.getUser(), reconverted.getUser());
// assertEquals(jobmeta.getSchedulingInfo(), reconverted.getSchedulingInfo());
}

Example 5 with MantisJobMetadataImpl

use of io.mantisrx.master.jobcluster.job.MantisJobMetadataImpl in project mantis by Netflix.

the class SimpleCachedFileStorageProvider method loadJob.

private Optional<IMantisJobMetadata> loadJob(String dir, String jobId) throws IOException {
    File jobFile = new File(getJobFileName(dir, jobId));
    IMantisJobMetadata job = null;
    if (jobFile.exists()) {
        try (FileInputStream fis = new FileInputStream(jobFile)) {
            job = mapper.readValue(fis, MantisJobMetadataImpl.class);
        }
        for (IMantisStageMetadata stage : readStagesFor(new File(dir), jobId)) ((MantisJobMetadataImpl) job).addJobStageIfAbsent(stage);
        for (IMantisWorkerMetadata worker : readWorkersFor(new File(dir), jobId)) {
            try {
                JobWorker jobWorker = new JobWorker.Builder().from(worker).withLifecycleEventsPublisher(eventPublisher).build();
                ((MantisJobMetadataImpl) job).addWorkerMetadata(worker.getStageNum(), jobWorker);
            } catch (InvalidJobException e) {
                logger.warn("Unexpected error adding worker index=" + worker.getWorkerIndex() + ", number=" + worker.getWorkerNumber() + " for job " + jobId + ": " + e.getMessage(), e);
            }
        }
    }
    return Optional.ofNullable(job);
}