Examples with Metadata io.mantisrx.runtime.Metadata used on opensource projects

Example 1 with Metadata

use of io.mantisrx.runtime.Metadata in project mantis by Netflix.

the class CreateJobDescriptorFile method execute.

@SuppressWarnings("unchecked")
@Override
public void execute() throws CommandException {
    Metadata jobMetadata = job.getMetadata();
    // create stage info
    Map<Integer, StageInfo> stagesInfo = new HashMap<>();
    List<StageConfig<?, ?>> stages = job.getStages();
    int numStages = 0;
    for (StageConfig<?, ?> stage : stages) {
        stagesInfo.put(numStages, new StageInfo(numStages, stage.getDescription()));
        numStages++;
    }
    // create parameter info
    final Map<String, ParameterInfo> parameterInfo = createParameterInfo(job.getParameterDefinitions());
    final Map<String, ParameterInfo> systemParameterInfo = createParameterInfo(ParameterUtils.getSystemParameters());
    final int totalNumStages = numStages;
    final Map<String, ParameterInfo> sysParams = systemParameterInfo.entrySet().stream().filter(sysParam -> {
        for (int stageNum = totalNumStages + 1; stageNum <= ParameterUtils.MAX_NUM_STAGES_FOR_JVM_OPTS_OVERRIDE; stageNum++) {
            if (sysParam.getKey().equals(String.format(ParameterUtils.PER_STAGE_JVM_OPTS_FORMAT, stageNum))) {
                return false;
            }
        }
        return true;
    }).collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue()));
    parameterInfo.putAll(sysParams);
    // create source/sink info
    Metadata sourceMetadata = job.getSource().getMetadata();
    MetadataInfo sourceMetadataInfo = toMetaDataInfo(sourceMetadata);
    Metadata sinkMetadata = job.getSink().getMetadata();
    MetadataInfo sinkMetadataInfo = toMetaDataInfo(sinkMetadata);
    JobInfo jobInfo = new JobInfo(jobMetadata.getName(), jobMetadata.getDescription(), numStages, parameterInfo, sourceMetadataInfo, sinkMetadataInfo, stagesInfo);
    JobDescriptor jobDescriptor = new JobDescriptor(jobInfo, project, version, System.currentTimeMillis(), readyForJobMaster);
    try {
        mapper.writeValue(descriptorFile, jobDescriptor);
    } catch (IOException e) {
        throw new DescriptorException(e);
    }
}

Example 2 with Metadata

use of io.mantisrx.runtime.Metadata in project mantis by Netflix.

the class PushRequestEventSourceJob method getJobInstance.

@Override
public Job<String> getJobInstance() {
    String jobId = Utils.getEnvVariable("JOB_ID", "PushRequestEventSourceJobLocal-1");
    String mantisClientId = MANTIS_CLIENT_ID + "_" + jobId;
    QueryRegistry queryRegistry = new QueryRegistry.Builder().withClientIdPrefix(mantisClientId).build();
    String customPortName = "MANTIS_WORKER_CUSTOM_PORT";
    String consolePort = Utils.getEnvVariable(customPortName, "9090");
    int port = 9090;
    if (consolePort != null && !consolePort.isEmpty()) {
        port = Integer.parseInt(consolePort);
    }
    return MantisJob.source(new PushHttpSource(queryRegistry, port)).stage(new EchoStage(), EchoStage.config()).sink(new SourceSink(new RequestPreProcessor(queryRegistry), new RequestPostProcessor(queryRegistry), mantisClientId)).parameterDefinition(new IntParameter().name(MantisSourceJobConstants.ECHO_STAGE_BUFFER_MILLIS).description("millis to buffer events before processing").validator(Validators.range(100, 1000)).defaultValue(250).build()).metadata(new Metadata.Builder().name("PushRequestEventSourceJob").description("Fetches request events from any source in a distributed manner. " + "The output is served via HTTP server using SSE protocol.").build()).create();
}

Example 3 with Metadata

use of io.mantisrx.runtime.Metadata in project mantis by Netflix.

the class TestJobParameterized method getJobInstance.

@Override
public Job<Integer> getJobInstance() {
    return MantisJob.<Integer>source(new Source<Integer>() {

        @Override
        public Observable<Observable<Integer>> call(Context context, Index index) {
            Integer start = (Integer) context.getParameters().get("start-range");
            Integer end = (Integer) context.getParameters().get("end-range");
            return Observable.just(Observable.range(start, end));
        }
    }).stage(new ScalarComputation<Integer, Integer>() {

        @Override
        public Observable<Integer> call(Context context, Observable<Integer> t1) {
            final Integer scale = (Integer) context.getParameters().get("scale-by");
            return t1.map(new Func1<Integer, Integer>() {

                @Override
                public Integer call(Integer t1) {
                    return t1 * scale;
                }
            });
        }
    }, new ScalarToScalar.Config<Integer, Integer>().codec(Codecs.integer())).sink(new Sink<Integer>() {

        @Override
        public void init(Context context) {
            System.out.println("sink init called");
        }

        @Override
        public void call(Context context, PortRequest p, Observable<Integer> o) {
            final String message = (String) context.getParameters().get("sink-message");
            o.toBlocking().forEach(new Action1<Integer>() {

                @Override
                public void call(Integer t1) {
                    System.out.println(message + t1);
                    itemsWritten.add(t1);
                }
            });
        }
    }).metadata(new Metadata.Builder().name("test job").description("showcase parameters").build()).parameterDefinition(new IntParameter().name("start-range").validator(Validators.range(0, 100)).build()).parameterDefinition(new IntParameter().name("end-range").validator(Validators.range(100, 1000)).build()).parameterDefinition(new IntParameter().name("scale-by").description("scale each value from the range").validator(Validators.range(1, 10)).build()).parameterDefinition(new StringParameter().name("sink-message").defaultValue("hello: ").description("concat with results").validator(Validators.notNullOrEmpty()).build()).create();
}