Examples with RunRecord co.cask.cdap.proto.RunRecord used on opensource projects

Example 6 with RunRecord

use of co.cask.cdap.proto.RunRecord in project cdap by caskdata.

the class DataPipelineTest method testPipelineWithAllActions.

@Test
public void testPipelineWithAllActions() throws Exception {
    String actionTable = "actionTable";
    String action1RowKey = "action1.row";
    String action1ColumnKey = "action1.column";
    String action1Value = "action1.value";
    String action2RowKey = "action2.row";
    String action2ColumnKey = "action2.column";
    String action2Value = "action2.value";
    String action3RowKey = "action3.row";
    String action3ColumnKey = "action3.column";
    String action3Value = "action3.value";
    ETLBatchConfig etlConfig = ETLBatchConfig.builder("* * * * *").addStage(new ETLStage("action1", MockAction.getPlugin(actionTable, action1RowKey, action1ColumnKey, action1Value))).addStage(new ETLStage("action2", MockAction.getPlugin(actionTable, action2RowKey, action2ColumnKey, action2Value))).addStage(new ETLStage("action3", MockAction.getPlugin(actionTable, action3RowKey, action3ColumnKey, action3Value))).addConnection("action1", "action2").addConnection("action1", "action3").setEngine(Engine.MAPREDUCE).build();
    AppRequest<ETLBatchConfig> appRequest = new AppRequest<>(APP_ARTIFACT, etlConfig);
    ApplicationId appId = NamespaceId.DEFAULT.app("MyActionOnlyApp");
    ApplicationManager appManager = deployApplication(appId.toId(), appRequest);
    WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME);
    workflowManager.start();
    workflowManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES);
    DataSetManager<Table> actionTableDS = getDataset(actionTable);
    Assert.assertEquals(action1Value, MockAction.readOutput(actionTableDS, action1RowKey, action1ColumnKey));
    Assert.assertEquals(action2Value, MockAction.readOutput(actionTableDS, action2RowKey, action2ColumnKey));
    Assert.assertEquals(action3Value, MockAction.readOutput(actionTableDS, action3RowKey, action3ColumnKey));
    List<RunRecord> history = workflowManager.getHistory(ProgramRunStatus.COMPLETED);
    Assert.assertEquals(1, history.size());
    String runId = history.get(0).getPid();
    WorkflowTokenDetail tokenDetail = workflowManager.getToken(runId, WorkflowToken.Scope.USER, action1RowKey + action1ColumnKey);
    validateToken(tokenDetail, action1RowKey + action1ColumnKey, action1Value);
    tokenDetail = workflowManager.getToken(runId, WorkflowToken.Scope.USER, action2RowKey + action2ColumnKey);
    validateToken(tokenDetail, action2RowKey + action2ColumnKey, action2Value);
    tokenDetail = workflowManager.getToken(runId, WorkflowToken.Scope.USER, action3RowKey + action3ColumnKey);
    validateToken(tokenDetail, action3RowKey + action3ColumnKey, action3Value);
}

Example 7 with RunRecord

use of co.cask.cdap.proto.RunRecord in project cdap by caskdata.

the class DataPipelineTest method testExternalDatasetTracking.

private void testExternalDatasetTracking(Engine engine, boolean backwardsCompatible) throws Exception {
    String suffix = engine.name() + (backwardsCompatible ? "-bc" : "");
    // Define input/output datasets
    String expectedExternalDatasetInput = "fileInput-" + suffix;
    String expectedExternalDatasetOutput = "fileOutput-" + suffix;
    // Define input/output directories
    File inputDir = TMP_FOLDER.newFolder("input-" + suffix);
    String inputFile = "input-file1.txt";
    File outputDir = TMP_FOLDER.newFolder("output-" + suffix);
    File outputSubDir1 = new File(outputDir, "subdir1");
    File outputSubDir2 = new File(outputDir, "subdir2");
    if (!backwardsCompatible) {
        // Assert that there are no external datasets
        Assert.assertNull(getDataset(NamespaceId.DEFAULT.dataset(expectedExternalDatasetInput)).get());
        Assert.assertNull(getDataset(NamespaceId.DEFAULT.dataset(expectedExternalDatasetOutput)).get());
    }
    ETLBatchConfig.Builder builder = ETLBatchConfig.builder("* * * * *");
    ETLBatchConfig etlConfig = builder.setEngine(engine).addStage(new ETLStage("source", MockExternalSource.getPlugin(expectedExternalDatasetInput, inputDir.getAbsolutePath()))).addStage(new ETLStage("sink1", MockExternalSink.getPlugin(backwardsCompatible ? null : expectedExternalDatasetOutput, "dir1", outputSubDir1.getAbsolutePath()))).addStage(new ETLStage("sink2", MockExternalSink.getPlugin(backwardsCompatible ? null : expectedExternalDatasetOutput, "dir2", outputSubDir2.getAbsolutePath()))).addConnection("source", "sink1").addConnection("source", "sink2").build();
    AppRequest<ETLBatchConfig> appRequest = new AppRequest<>(APP_ARTIFACT, etlConfig);
    ApplicationId appId = NamespaceId.DEFAULT.app("ExternalDatasetApp-" + suffix);
    ApplicationManager appManager = deployApplication(appId.toId(), appRequest);
    Schema schema = Schema.recordOf("testRecord", Schema.Field.of("name", Schema.of(Schema.Type.STRING)));
    StructuredRecord recordSamuel = StructuredRecord.builder(schema).set("name", "samuel").build();
    StructuredRecord recordBob = StructuredRecord.builder(schema).set("name", "bob").build();
    StructuredRecord recordJane = StructuredRecord.builder(schema).set("name", "jane").build();
    ImmutableList<StructuredRecord> allInput = ImmutableList.of(recordSamuel, recordBob, recordJane);
    // Create input files
    MockExternalSource.writeInput(new File(inputDir, inputFile).getAbsolutePath(), allInput);
    WorkflowManager workflowManager = appManager.getWorkflowManager(SmartWorkflow.NAME);
    workflowManager.start();
    workflowManager.waitForRun(ProgramRunStatus.COMPLETED, 5, TimeUnit.MINUTES);
    List<RunRecord> history = workflowManager.getHistory();
    // there should be only one completed run
    Assert.assertEquals(1, history.size());
    Assert.assertEquals(ProgramRunStatus.COMPLETED, history.get(0).getStatus());
    // Assert output
    Assert.assertEquals(allInput, MockExternalSink.readOutput(outputSubDir1.getAbsolutePath()));
    Assert.assertEquals(allInput, MockExternalSink.readOutput(outputSubDir2.getAbsolutePath()));
    if (!backwardsCompatible) {
        // Assert that external datasets got created
        Assert.assertNotNull(getDataset(NamespaceId.DEFAULT.dataset(expectedExternalDatasetInput)).get());
        Assert.assertNotNull(getDataset(NamespaceId.DEFAULT.dataset(expectedExternalDatasetOutput)).get());
    }
}

Example 8 with RunRecord

use of co.cask.cdap.proto.RunRecord in project cdap by caskdata.

the class GetProgramRunsCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream output) throws Exception {
    String[] programIdParts = arguments.get(elementType.getArgumentName().toString()).split("\\.");
    String appId = programIdParts[0];
    long currentTime = System.currentTimeMillis();
    long startTime = getTimestamp(arguments.getOptional(ArgumentName.START_TIME.toString(), "min"), currentTime);
    long endTime = getTimestamp(arguments.getOptional(ArgumentName.END_TIME.toString(), "max"), currentTime);
    int limit = arguments.getIntOptional(ArgumentName.LIMIT.toString(), Integer.MAX_VALUE);
    List<RunRecord> records;
    if (elementType.getProgramType() != null) {
        if (programIdParts.length < 2) {
            throw new CommandInputError(this);
        }
        String programName = programIdParts[1];
        ProgramId programId = cliConfig.getCurrentNamespace().app(appId).program(elementType.getProgramType(), programName);
        if (arguments.hasArgument(ArgumentName.RUN_STATUS.toString())) {
            records = programClient.getProgramRuns(programId, arguments.get(ArgumentName.RUN_STATUS.toString()), startTime, endTime, limit);
        } else {
            records = programClient.getAllProgramRuns(programId, startTime, endTime, limit);
        }
    } else {
        throw new IllegalArgumentException("Unrecognized program element type for history: " + elementType);
    }
    Table table = Table.builder().setHeader("pid", "end status", "start", "stop").setRows(records, new RowMaker<RunRecord>() {

        @Override
        public List<?> makeRow(RunRecord object) {
            return Lists.newArrayList(object.getPid(), object.getStatus(), object.getStartTs(), object.getStatus().name().equals("RUNNING") ? "" : object.getStopTs());
        }
    }).build();
    cliConfig.getTableRenderer().render(cliConfig, output, table);
}

Example 9 with RunRecord

use of co.cask.cdap.proto.RunRecord in project cdap by caskdata.

the class MockLogReader method generateLogs.

/**
   * This method is used to generate the logs for program which are used for testing.
   * Single call to this method would add {@link #MAX} number of events.
   * First 20 events are generated without {@link ApplicationLoggingContext#TAG_RUN_ID} tag.
   * For next 40 events, alternate event is tagged with {@code ApplicationLoggingContext#TAG_RUN_ID}.
   * Last 20 events are not tagged with {@code ApplicationLoggingContext#TAG_RUN_ID}.
   * All events are alternately marked as {@link Level#ERROR} and {@link Level#WARN}.
   * All events are alternately tagged with "plugin", "program" and "system" as value of MDC property ".origin"
   * All events are alternately tagged with "lifecycle" as value of MDC property "MDC:eventType
   */
private void generateLogs(LoggingContext loggingContext, ProgramId programId, ProgramRunStatus runStatus) throws InterruptedException {
    // All possible values of " MDC property ".origin
    String[] origins = { "plugin", "program", "system" };
    String entityId = LoggingContextHelper.getEntityId(loggingContext).getValue();
    StackTraceElement stackTraceElementNative = new StackTraceElement("co.cask.Test", "testMethod", null, -2);
    RunId runId = null;
    Long stopTs = null;
    for (int i = 0; i < MAX; ++i) {
        // Setup run id for event with ids >= 20
        if (i == 20) {
            runId = RunIds.generate(TimeUnit.SECONDS.toMillis(getMockTimeSecs(i)));
        } else if (i == 60 && runStatus != ProgramRunStatus.RUNNING && runStatus != ProgramRunStatus.SUSPENDED) {
            // Record stop time for run for 60th event, but still continue to record run in the other logging events.
            stopTs = getMockTimeSecs(i);
        }
        LoggingEvent event = new LoggingEvent("co.cask.Test", (ch.qos.logback.classic.Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME), i % 2 == 0 ? Level.ERROR : Level.WARN, entityId + "<img>-" + i, null, null);
        event.setTimeStamp(TimeUnit.SECONDS.toMillis(getMockTimeSecs(i)));
        // Add runid to logging context
        Map<String, String> tagMap = Maps.newHashMap(Maps.transformValues(loggingContext.getSystemTagsMap(), TAG_TO_STRING_FUNCTION));
        if (runId != null && stopTs == null && i % 2 == 0) {
            tagMap.put(ApplicationLoggingContext.TAG_RUN_ID, runId.getId());
        }
        // Determine the value of ".origin" property by (i % 3)
        tagMap.put(".origin", origins[i % 3]);
        if (i % 2 == 0) {
            tagMap.put("MDC:eventType", "lifecycle");
        }
        if (i == 30) {
            event.setCallerData(new StackTraceElement[] { stackTraceElementNative });
        }
        event.setMDCPropertyMap(tagMap);
        logEvents.add(new LogEvent(event, new LogOffset(i, i)));
    }
    long startTs = RunIds.getTime(runId, TimeUnit.SECONDS);
    if (programId != null) {
        //noinspection ConstantConditions
        runRecordMap.put(programId, new RunRecord(runId.getId(), startTs, stopTs, runStatus, null));
        store.setStart(programId, runId.getId(), startTs);
        if (stopTs != null) {
            store.setStop(programId, runId.getId(), stopTs, runStatus);
        }
    }
}

Example 10 with RunRecord

use of co.cask.cdap.proto.RunRecord in project cdap by caskdata.

the class LogHandlerTestRun method testPrevRunId.

private void testPrevRunId(String appId, String entityType, String entityId, String namespace, String format, List<String> suppress) throws Exception {
    ProgramId programId = new NamespaceId(namespace).app(appId).program(ProgramType.valueOfCategoryName(entityType), entityId);
    RunRecord runRecord = mockLogReader.getRunRecord(programId);
    int expectedEvents = 20;
    if (runRecord.getStatus() == ProgramRunStatus.RUNNING || runRecord.getStatus() == ProgramRunStatus.SUSPENDED) {
        expectedEvents = 30;
    }
    String prevRunIdUrl;
    if (suppress.isEmpty()) {
        prevRunIdUrl = String.format("apps/%s/%s/%s/runs/%s/logs/prev?format=%s&max=100", appId, entityType, entityId, runRecord.getPid(), format);
    } else {
        String fieldsToSuppress = getSuppressStr(suppress);
        prevRunIdUrl = String.format("apps/%s/%s/%s/runs/%s/logs/prev?format=%s&max=100&suppress=%s", appId, entityType, entityId, runRecord.getPid(), format, fieldsToSuppress);
    }
    HttpResponse response = doGet(getVersionedAPIPath(prevRunIdUrl, namespace));
    verifyLogs(response, entityId, format, true, false, true, expectedEvents, 20, suppress);
}