Examples with RunId org.apache.twill.api.RunId used on opensource projects

Example 6 with RunId

use of org.apache.twill.api.RunId in project cdap by caskdata.

the class MockLogReader method generateWorkflowLogs.

/**
   * Generate Workflow logs.
   */
private void generateWorkflowLogs() {
    ProgramId workflowId = SOME_WORKFLOW_APP.workflow(SOME_WORKFLOW);
    long currentTime = TimeUnit.SECONDS.toMillis(10);
    RunId workflowRunId = RunIds.generate();
    store.setStart(workflowId, workflowRunId.getId(), currentTime);
    runRecordMap.put(workflowId, store.getRun(workflowId, workflowRunId.getId()));
    WorkflowLoggingContext wfLoggingContext = new WorkflowLoggingContext(workflowId.getNamespace(), workflowId.getApplication(), workflowId.getProgram(), workflowRunId.getId());
    generateWorkflowRunLogs(wfLoggingContext);
    // Generate logs for MapReduce program started by above Workflow run
    ProgramId mapReduceId = SOME_WORKFLOW_APP.mr(SOME_MAPREDUCE);
    currentTime = TimeUnit.SECONDS.toMillis(20);
    RunId mapReduceRunId = RunIds.generate();
    Map<String, String> systemArgs = ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NODE_ID, SOME_MAPREDUCE, ProgramOptionConstants.WORKFLOW_NAME, SOME_WORKFLOW, ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId.getId());
    store.setStart(mapReduceId, mapReduceRunId.getId(), currentTime, null, new HashMap<String, String>(), systemArgs);
    runRecordMap.put(mapReduceId, store.getRun(mapReduceId, mapReduceRunId.getId()));
    WorkflowProgramLoggingContext context = new WorkflowProgramLoggingContext(workflowId.getNamespace(), workflowId.getApplication(), workflowId.getProgram(), workflowRunId.getId(), ProgramType.MAPREDUCE, SOME_MAPREDUCE, mapReduceRunId.getId());
    generateWorkflowRunLogs(context);
    // Generate logs for Spark program started by Workflow run above
    ProgramId sparkId = SOME_WORKFLOW_APP.spark(SOME_SPARK);
    currentTime = TimeUnit.SECONDS.toMillis(40);
    RunId sparkRunId = RunIds.generate();
    systemArgs = ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NODE_ID, SOME_SPARK, ProgramOptionConstants.WORKFLOW_NAME, SOME_WORKFLOW, ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId.getId());
    store.setStart(sparkId, sparkRunId.getId(), currentTime, null, new HashMap<String, String>(), systemArgs);
    runRecordMap.put(sparkId, store.getRun(sparkId, sparkRunId.getId()));
    context = new WorkflowProgramLoggingContext(workflowId.getNamespace(), workflowId.getApplication(), workflowId.getProgram(), workflowRunId.getId(), ProgramType.SPARK, SOME_SPARK, sparkRunId.getId());
    generateWorkflowRunLogs(context);
    // Generate some more logs for Workflow
    generateWorkflowRunLogs(wfLoggingContext);
}

Example 7 with RunId

use of org.apache.twill.api.RunId in project cdap by caskdata.

the class MockLogReader method generateLogs.

/**
   * This method is used to generate the logs for program which are used for testing.
   * Single call to this method would add {@link #MAX} number of events.
   * First 20 events are generated without {@link ApplicationLoggingContext#TAG_RUN_ID} tag.
   * For next 40 events, alternate event is tagged with {@code ApplicationLoggingContext#TAG_RUN_ID}.
   * Last 20 events are not tagged with {@code ApplicationLoggingContext#TAG_RUN_ID}.
   * All events are alternately marked as {@link Level#ERROR} and {@link Level#WARN}.
   * All events are alternately tagged with "plugin", "program" and "system" as value of MDC property ".origin"
   * All events are alternately tagged with "lifecycle" as value of MDC property "MDC:eventType
   */
private void generateLogs(LoggingContext loggingContext, ProgramId programId, ProgramRunStatus runStatus) throws InterruptedException {
    // All possible values of " MDC property ".origin
    String[] origins = { "plugin", "program", "system" };
    String entityId = LoggingContextHelper.getEntityId(loggingContext).getValue();
    StackTraceElement stackTraceElementNative = new StackTraceElement("co.cask.Test", "testMethod", null, -2);
    RunId runId = null;
    Long stopTs = null;
    for (int i = 0; i < MAX; ++i) {
        // Setup run id for event with ids >= 20
        if (i == 20) {
            runId = RunIds.generate(TimeUnit.SECONDS.toMillis(getMockTimeSecs(i)));
        } else if (i == 60 && runStatus != ProgramRunStatus.RUNNING && runStatus != ProgramRunStatus.SUSPENDED) {
            // Record stop time for run for 60th event, but still continue to record run in the other logging events.
            stopTs = getMockTimeSecs(i);
        }
        LoggingEvent event = new LoggingEvent("co.cask.Test", (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME), i % 2 == 0 ? Level.ERROR : Level.WARN, entityId + "<img>-" + i, null, null);
        event.setTimeStamp(TimeUnit.SECONDS.toMillis(getMockTimeSecs(i)));
        // Add runid to logging context
        Map<String, String> tagMap = Maps.newHashMap(Maps.transformValues(loggingContext.getSystemTagsMap(), TAG_TO_STRING_FUNCTION));
        if (runId != null && stopTs == null && i % 2 == 0) {
            tagMap.put(ApplicationLoggingContext.TAG_RUN_ID, runId.getId());
        }
        // Determine the value of ".origin" property by (i % 3)
        tagMap.put(".origin", origins[i % 3]);
        if (i % 2 == 0) {
            tagMap.put("MDC:eventType", "lifecycle");
        }
        if (i == 30) {
            event.setCallerData(new StackTraceElement[] { stackTraceElementNative });
        }
        event.setMDCPropertyMap(tagMap);
        logEvents.add(new LogEvent(event, new LogOffset(i, i)));
    }
    long startTs = RunIds.getTime(runId, TimeUnit.SECONDS);
    if (programId != null) {
        //noinspection ConstantConditions
        runRecordMap.put(programId, new RunRecord(runId.getId(), startTs, stopTs, runStatus, null));
        store.setStart(programId, runId.getId(), startTs);
        if (stopTs != null) {
            store.setStop(programId, runId.getId(), stopTs, runStatus);
        }
    }
}

Example 8 with RunId

use of org.apache.twill.api.RunId in project cdap by caskdata.

the class FlowProgramRunner method run.

@Override
public ProgramController run(Program program, ProgramOptions options) {
    // Extract and verify parameters
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    Preconditions.checkNotNull(appSpec, "Missing application specification.");
    ProgramType processorType = program.getType();
    Preconditions.checkNotNull(processorType, "Missing processor type.");
    Preconditions.checkArgument(processorType == ProgramType.FLOW, "Only FLOW process type is supported.");
    FlowSpecification flowSpec = appSpec.getFlows().get(program.getName());
    Preconditions.checkNotNull(flowSpec, "Missing FlowSpecification for %s", program.getName());
    try {
        // Launch flowlet program runners
        RunId runId = ProgramRunners.getRunId(options);
        Multimap<String, QueueName> consumerQueues = FlowUtils.configureQueue(program, flowSpec, streamAdmin, queueAdmin, txExecutorFactory);
        final Table<String, Integer, ProgramController> flowlets = createFlowlets(program, options, flowSpec);
        return new FlowProgramController(flowlets, program, options, flowSpec, consumerQueues);
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 9 with RunId

use of org.apache.twill.api.RunId in project cdap by caskdata.

the class DistributedProgramRuntimeService method list.

@Override
public synchronized Map<RunId, RuntimeInfo> list(ProgramType type) {
    Map<RunId, RuntimeInfo> result = Maps.newHashMap();
    result.putAll(super.list(type));
    // Table holds the Twill RunId and TwillController associated with the program matching the input type
    Table<ProgramId, RunId, TwillController> twillProgramInfo = HashBasedTable.create();
    // Goes through all live application and fill the twillProgramInfo table
    for (TwillRunner.LiveInfo liveInfo : twillRunner.lookupLive()) {
        String appName = liveInfo.getApplicationName();
        ProgramId programId = TwillAppNames.fromTwillAppName(appName, false);
        if (programId == null) {
            continue;
        }
        if (!type.equals(programId.getType())) {
            continue;
        }
        for (TwillController controller : liveInfo.getControllers()) {
            RunId twillRunId = controller.getRunId();
            if (isTwillRunIdCached(twillRunId)) {
                continue;
            }
            twillProgramInfo.put(programId, twillRunId, controller);
        }
    }
    if (twillProgramInfo.isEmpty()) {
        return ImmutableMap.copyOf(result);
    }
    final Set<RunId> twillRunIds = twillProgramInfo.columnKeySet();
    Collection<RunRecordMeta> activeRunRecords = store.getRuns(ProgramRunStatus.RUNNING, new Predicate<RunRecordMeta>() {

        @Override
        public boolean apply(RunRecordMeta record) {
            return record.getTwillRunId() != null && twillRunIds.contains(org.apache.twill.internal.RunIds.fromString(record.getTwillRunId()));
        }
    }).values();
    for (RunRecordMeta record : activeRunRecords) {
        String twillRunId = record.getTwillRunId();
        if (twillRunId == null) {
            // This is unexpected. Just log and ignore the run record
            LOG.warn("No twill runId for in run record {}.", record);
            continue;
        }
        RunId twillRunIdFromRecord = org.apache.twill.internal.RunIds.fromString(twillRunId);
        // Get the CDAP RunId from RunRecord
        RunId runId = RunIds.fromString(record.getPid());
        // Get the Program and TwillController for the current twillRunId
        Map<ProgramId, TwillController> mapForTwillId = twillProgramInfo.columnMap().get(twillRunIdFromRecord);
        Map.Entry<ProgramId, TwillController> entry = mapForTwillId.entrySet().iterator().next();
        // Create RuntimeInfo for the current Twill RunId
        RuntimeInfo runtimeInfo = createRuntimeInfo(entry.getKey(), entry.getValue(), runId);
        if (runtimeInfo != null) {
            result.put(runId, runtimeInfo);
            updateRuntimeInfo(type, runId, runtimeInfo);
        } else {
            LOG.warn("Unable to find program {} {}", type, entry.getKey());
        }
    }
    return ImmutableMap.copyOf(result);
}

Example 10 with RunId

use of org.apache.twill.api.RunId in project cdap by caskdata.

the class DistributedProgramRuntimeService method lookup.

@Override
public synchronized RuntimeInfo lookup(ProgramId programId, final RunId runId) {
    RuntimeInfo runtimeInfo = super.lookup(programId, runId);
    if (runtimeInfo != null) {
        return runtimeInfo;
    }
    // Goes through all live application and fill the twillProgramInfo table
    for (TwillRunner.LiveInfo liveInfo : twillRunner.lookupLive()) {
        String appName = liveInfo.getApplicationName();
        ProgramId id = TwillAppNames.fromTwillAppName(appName, false);
        if (id == null) {
            continue;
        }
        if (!id.equals(programId)) {
            continue;
        }
        // Program matched
        RunRecordMeta record = store.getRun(programId, runId.getId());
        if (record == null) {
            return null;
        }
        if (record.getTwillRunId() == null) {
            LOG.warn("Twill RunId does not exist for the program {}, runId {}", programId, runId.getId());
            return null;
        }
        RunId twillRunIdFromRecord = org.apache.twill.internal.RunIds.fromString(record.getTwillRunId());
        for (TwillController controller : liveInfo.getControllers()) {
            RunId twillRunId = controller.getRunId();
            if (!twillRunId.equals(twillRunIdFromRecord)) {
                continue;
            }
            runtimeInfo = createRuntimeInfo(programId, controller, runId);
            if (runtimeInfo != null) {
                updateRuntimeInfo(programId.getType(), runId, runtimeInfo);
            } else {
                LOG.warn("Unable to find program for runId {}", runId);
            }
            return runtimeInfo;
        }
    }
    return null;
}