Examples with RunId org.apache.twill.api.RunId used on opensource projects

Example 36 with RunId

use of org.apache.twill.api.RunId in project cdap by caskdata.

the class WorkflowStatsSLAHttpHandlerTest method setupRuns.

/*
   * This helper is used only for the details and compare endpoints and not the statistics endpoint because
   * the statistics endpoint needs to handle number of spark runs differently and also have tests for a
   * specific run's spark job.
   */
private List<RunId> setupRuns(WorkflowId workflowProgram, ProgramId mapreduceProgram, ProgramId sparkProgram, Store store, int count) throws Exception {
    List<RunId> runIdList = new ArrayList<>();
    long startTime = System.currentTimeMillis();
    long currentTimeMillis;
    for (int i = 0; i < count; i++) {
        // work-flow runs every 5 minutes
        currentTimeMillis = startTime + (i * TimeUnit.MINUTES.toMillis(5));
        RunId workflowRunId = RunIds.generate(currentTimeMillis);
        runIdList.add(workflowRunId);
        store.setStart(workflowProgram, workflowRunId.getId(), RunIds.getTime(workflowRunId, TimeUnit.SECONDS));
        // MR job starts 2 seconds after workflow started
        RunId mapreduceRunid = RunIds.generate(currentTimeMillis + TimeUnit.SECONDS.toMillis(2));
        Map<String, String> systemArgs = ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NODE_ID, mapreduceProgram.getProgram(), ProgramOptionConstants.WORKFLOW_NAME, workflowProgram.getProgram(), ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId.getId());
        store.setStart(mapreduceProgram, mapreduceRunid.getId(), RunIds.getTime(mapreduceRunid, TimeUnit.SECONDS), null, ImmutableMap.<String, String>of(), systemArgs);
        store.setStop(mapreduceProgram, mapreduceRunid.getId(), // map-reduce job ran for 17 seconds
        TimeUnit.MILLISECONDS.toSeconds(currentTimeMillis) + 19, ProgramRunStatus.COMPLETED);
        Map<String, String> mapTypeContext = ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, mapreduceProgram.getNamespace(), Constants.Metrics.Tag.APP, mapreduceProgram.getApplication(), Constants.Metrics.Tag.MAPREDUCE, mapreduceProgram.getProgram(), Constants.Metrics.Tag.RUN_ID, mapreduceRunid.toString(), Constants.Metrics.Tag.MR_TASK_TYPE, MapReduceMetrics.TaskType.Mapper.getId());
        metricStore.add(new MetricValues(mapTypeContext, MapReduceMetrics.METRIC_INPUT_RECORDS, 10, 38L, MetricType.GAUGE));
        // spark starts 20 seconds after workflow starts
        systemArgs = ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NODE_ID, sparkProgram.getProgram(), ProgramOptionConstants.WORKFLOW_NAME, workflowProgram.getProgram(), ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId.getId());
        RunId sparkRunid = RunIds.generate(currentTimeMillis + TimeUnit.SECONDS.toMillis(20));
        store.setStart(sparkProgram, sparkRunid.getId(), RunIds.getTime(sparkRunid, TimeUnit.SECONDS), null, ImmutableMap.<String, String>of(), systemArgs);
        // spark job runs for 38 seconds
        long stopTime = TimeUnit.MILLISECONDS.toSeconds(currentTimeMillis) + 58;
        store.setStop(sparkProgram, sparkRunid.getId(), stopTime, ProgramRunStatus.COMPLETED);
        // workflow ran for 1 minute
        long workflowStopTime = TimeUnit.MILLISECONDS.toSeconds(currentTimeMillis) + 60;
        store.setStop(workflowProgram, workflowRunId.getId(), workflowStopTime, ProgramRunStatus.COMPLETED);
    }
    return runIdList;
}

Example 37 with RunId

use of org.apache.twill.api.RunId in project cdap by caskdata.

the class AbstractProgramRuntimeService method run.

@Override
public final RuntimeInfo run(ProgramDescriptor programDescriptor, ProgramOptions options) {
    ProgramId programId = programDescriptor.getProgramId();
    ProgramRunner runner = programRunnerFactory.create(programId.getType());
    RunId runId = RunIds.generate();
    File tempDir = createTempDirectory(programId, runId);
    Runnable cleanUpTask = createCleanupTask(tempDir, runner);
    try {
        // Get the artifact details and save it into the program options.
        ArtifactId artifactId = programDescriptor.getArtifactId();
        ArtifactDetail artifactDetail = getArtifactDetail(artifactId);
        ProgramOptions runtimeProgramOptions = updateProgramOptions(programId, options, runId);
        // Take a snapshot of all the plugin artifacts used by the program
        ProgramOptions optionsWithPlugins = createPluginSnapshot(runtimeProgramOptions, programId, tempDir, programDescriptor.getApplicationSpecification());
        // Create and run the program
        Program executableProgram = createProgram(cConf, runner, programDescriptor, artifactDetail, tempDir);
        cleanUpTask = createCleanupTask(cleanUpTask, executableProgram);
        RuntimeInfo runtimeInfo = createRuntimeInfo(runner.run(executableProgram, optionsWithPlugins), programId);
        monitorProgram(runtimeInfo, cleanUpTask);
        return runtimeInfo;
    } catch (Exception e) {
        cleanUpTask.run();
        LOG.error("Exception while trying to run program", e);
        throw Throwables.propagate(e);
    }
}

Example 38 with RunId

use of org.apache.twill.api.RunId in project cdap by caskdata.

the class InMemoryServiceProgramRunner method run.

@Override
public ProgramController run(Program program, ProgramOptions options) {
    // Extract and verify parameters
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    Preconditions.checkNotNull(appSpec, "Missing application specification.");
    ProgramType processorType = program.getType();
    Preconditions.checkNotNull(processorType, "Missing processor type.");
    Preconditions.checkArgument(processorType == ProgramType.SERVICE, "Only SERVICE process type is supported.");
    ServiceSpecification serviceSpec = appSpec.getServices().get(program.getName());
    Preconditions.checkNotNull(serviceSpec, "Missing ServiceSpecification for %s", program.getName());
    //RunId for the service
    RunId runId = RunIds.generate();
    return startAll(program, options, runId, serviceSpec.getInstances());
}

Example 39 with RunId

use of org.apache.twill.api.RunId in project cdap by caskdata.

the class InMemoryWorkerRunner method run.

@Override
public ProgramController run(Program program, ProgramOptions options) {
    // Extract and verify parameters
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    Preconditions.checkNotNull(appSpec, "Missing application specification.");
    ProgramType type = program.getType();
    Preconditions.checkNotNull(type, "Missing processor type.");
    Preconditions.checkArgument(type == ProgramType.WORKER, "Only WORKER process type is supported.");
    WorkerSpecification workerSpec = appSpec.getWorkers().get(program.getName());
    Preconditions.checkNotNull(workerSpec, "Missing WorkerSpecification for %s", program.getName());
    String instances = options.getArguments().getOption(ProgramOptionConstants.INSTANCES, String.valueOf(workerSpec.getInstances()));
    WorkerSpecification newWorkerSpec = new WorkerSpecification(workerSpec.getClassName(), workerSpec.getName(), workerSpec.getDescription(), workerSpec.getProperties(), workerSpec.getDatasets(), workerSpec.getResources(), Integer.valueOf(instances));
    //RunId for worker
    RunId runId = ProgramRunners.getRunId(options);
    return startAll(program, options, runId, newWorkerSpec.getInstances());
}

Example 40 with RunId

use of org.apache.twill.api.RunId in project cdap by caskdata.

the class AppMetadataStoreTest method testOldRunRecordFormat.

@Test
public void testOldRunRecordFormat() throws Exception {
    DatasetId storeTable = NamespaceId.DEFAULT.dataset("testOldRunRecordFormat");
    datasetFramework.addInstance(Table.class.getName(), storeTable, DatasetProperties.EMPTY);
    Table table = datasetFramework.getDataset(storeTable, ImmutableMap.<String, String>of(), null);
    Assert.assertNotNull(table);
    final AppMetadataStore metadataStoreDataset = new AppMetadataStore(table, cConf, new AtomicBoolean(false));
    TransactionExecutor txnl = txExecutorFactory.createExecutor(Collections.singleton((TransactionAware) metadataStoreDataset));
    ApplicationId application = NamespaceId.DEFAULT.app("app");
    final ProgramId program = application.program(ProgramType.values()[ProgramType.values().length - 1], "program");
    final RunId runId = RunIds.generate();
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            metadataStoreDataset.recordProgramStartOldFormat(program, runId.getId(), RunIds.getTime(runId, TimeUnit.SECONDS), null, null, null);
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Set<RunId> runIds = metadataStoreDataset.getRunningInRange(0, Long.MAX_VALUE);
            Assert.assertEquals(1, runIds.size());
            RunRecordMeta meta = metadataStoreDataset.getRun(program, runIds.iterator().next().getId());
            Assert.assertNotNull(meta);
            Assert.assertEquals(runId.getId(), meta.getPid());
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            metadataStoreDataset.recordProgramStopOldFormat(program, runId.getId(), TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()), ProgramRunStatus.COMPLETED, null);
            Map<ProgramRunId, RunRecordMeta> runRecordMap = metadataStoreDataset.getRuns(program, ProgramRunStatus.COMPLETED, 0, Long.MAX_VALUE, Integer.MAX_VALUE, null);
            Assert.assertEquals(1, runRecordMap.size());
            ProgramRunId programRunId = runRecordMap.keySet().iterator().next();
            Assert.assertEquals(program, programRunId.getParent());
            Assert.assertEquals(runId.getId(), programRunId.getRun());
        }
    });
}