Search in sources :

Example 11 with ProgramRunId

use of io.cdap.cdap.proto.id.ProgramRunId in project cdap by caskdata.

the class AppMetadataStoreTest method testGetRuns.

@Test
public void testGetRuns() throws Exception {
    // Add some run records
    final Set<String> expected = new TreeSet<>();
    final Set<String> expectedHalf = new TreeSet<>();
    final Set<ProgramRunId> programRunIdSet = new HashSet<>();
    final Set<ProgramRunId> programRunIdSetHalf = new HashSet<>();
    for (int i = 0; i < 100; ++i) {
        ApplicationId application = NamespaceId.DEFAULT.app("app");
        final ProgramId program = application.program(ProgramType.SERVICE, "program");
        final RunId runId = RunIds.generate(runIdTime.incrementAndGet());
        expected.add(runId.toString());
        final int index = i;
        // Add every other runId
        if ((i % 2) == 0) {
            expectedHalf.add(runId.toString());
        }
        ProgramRunId programRunId = program.run(runId);
        programRunIdSet.add(programRunId);
        // Add every other programRunId
        if ((i % 2) == 0) {
            programRunIdSetHalf.add(programRunId);
        }
        // A sourceId to keep incrementing for each call of app meta data store persisting
        TransactionRunners.run(transactionRunner, context -> {
            AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
            // Start the program and stop it
            recordProvisionAndStart(programRunId, metadataStoreDataset);
            metadataStoreDataset.recordProgramRunning(programRunId, RunIds.getTime(runId, TimeUnit.SECONDS), null, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
            metadataStoreDataset.recordProgramStop(programRunId, RunIds.getTime(runId, TimeUnit.SECONDS), ProgramRunStatus.values()[index % ProgramRunStatus.values().length], null, AppFabricTestHelper.createSourceId(sourceId.incrementAndGet()));
        });
    }
    TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
        Map<ProgramRunId, RunRecordDetail> runMap = metadataStoreDataset.getRuns(programRunIdSet);
        Set<String> actual = new TreeSet<>();
        for (Map.Entry<ProgramRunId, RunRecordDetail> entry : runMap.entrySet()) {
            actual.add(entry.getValue().getPid());
        }
        Assert.assertEquals(expected, actual);
        Map<ProgramRunId, RunRecordDetail> runMapHalf = metadataStoreDataset.getRuns(programRunIdSetHalf);
        Set<String> actualHalf = new TreeSet<>();
        for (Map.Entry<ProgramRunId, RunRecordDetail> entry : runMapHalf.entrySet()) {
            actualHalf.add(entry.getValue().getPid());
        }
        Assert.assertEquals(expectedHalf, actualHalf);
    });
}
Also used : ProgramId(io.cdap.cdap.proto.id.ProgramId) TreeSet(java.util.TreeSet) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) RunId(org.apache.twill.api.RunId) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 12 with ProgramRunId

use of io.cdap.cdap.proto.id.ProgramRunId in project cdap by caskdata.

the class DefaultPreviewStoreTest method testPreviewInfo.

@Test
public void testPreviewInfo() throws IOException {
    // test non existing preview
    ApplicationId nonexist = new ApplicationId("ns1", "nonexist");
    Assert.assertNull(store.getProgramRunId(nonexist));
    Assert.assertNull(store.getPreviewStatus(nonexist));
    // test put and get
    ApplicationId applicationId = new ApplicationId("ns1", "app1");
    ProgramRunId runId = new ProgramRunId("ns1", "app1", ProgramType.WORKFLOW, "test", RunIds.generate().getId());
    PreviewStatus status = new PreviewStatus(PreviewStatus.Status.COMPLETED, System.currentTimeMillis(), null, 0L, System.currentTimeMillis());
    store.setProgramId(runId);
    store.setPreviewStatus(applicationId, status);
    Assert.assertEquals(runId, store.getProgramRunId(applicationId));
    Assert.assertEquals(status, store.getPreviewStatus(applicationId));
}
Also used : PreviewStatus(io.cdap.cdap.app.preview.PreviewStatus) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) Test(org.junit.Test)

Example 13 with ProgramRunId

use of io.cdap.cdap.proto.id.ProgramRunId in project cdap by caskdata.

the class DeleteWorkflowLocalDatasetsCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream printStream) throws Exception {
    String[] programIdParts = arguments.get(elementType.getArgumentName().toString()).split("\\.");
    if (programIdParts.length < 2) {
        throw new CommandInputError(this);
    }
    ProgramRunId programRunId = new ProgramRunId(cliConfig.getCurrentNamespace().getNamespace(), programIdParts[0], ProgramType.WORKFLOW, programIdParts[1], arguments.get(ArgumentName.RUN_ID.toString()));
    workflowClient.deleteWorkflowLocalDatasets(programRunId);
    printStream.printf("Successfully deleted local datasets associated with the workflow run.");
}
Also used : CommandInputError(io.cdap.cdap.cli.exception.CommandInputError) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId)

Example 14 with ProgramRunId

use of io.cdap.cdap.proto.id.ProgramRunId in project cdap by caskdata.

the class GetWorkflowStateCommand method perform.

@Override
public void perform(Arguments arguments, PrintStream printStream) throws Exception {
    String[] programIdParts = arguments.get(elementType.getArgumentName().toString()).split("\\.");
    if (programIdParts.length < 2) {
        throw new CommandInputError(this);
    }
    ProgramRunId programRunId = cliConfig.getCurrentNamespace().app(programIdParts[0]).workflow(programIdParts[1]).run(arguments.get(ArgumentName.RUN_ID.toString()));
    Table table = getWorkflowNodeStates(programRunId);
    cliConfig.getTableRenderer().render(cliConfig, printStream, table);
}
Also used : CommandInputError(io.cdap.cdap.cli.exception.CommandInputError) Table(io.cdap.cdap.cli.util.table.Table) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId)

Example 15 with ProgramRunId

use of io.cdap.cdap.proto.id.ProgramRunId in project cdap by caskdata.

the class LineageAdmin method filterAndAddRelations.

/**
 * Filter the relations based on the rollUp flag, if set to true, the method will replace the inner program with
 * the workflow using the map and ignore the local datasets relations. The local dataset always ends with the run
 * id of the workflow. The set of filtered local datasets is returned
 */
private Set<DatasetId> filterAndAddRelations(boolean rollUpWorkflow, Multimap<RelationKey, Relation> relations, Map<ProgramRunId, ProgramRunId> programWorkflowMap, Set<Relation> relationss) {
    Set<DatasetId> localDatasets = new HashSet<>();
    for (Relation relation : relationss) {
        if (rollUpWorkflow && programWorkflowMap.containsKey(relation.getProgramRunId())) {
            ProgramRunId workflowId = programWorkflowMap.get(relation.getProgramRunId());
            // skip the relation for local datasets, local datasets always end with the workflow run id
            DatasetId data = (DatasetId) relation.getData();
            if (data.getDataset().endsWith(workflowId.getRun())) {
                localDatasets.add(data);
                continue;
            }
            relation = new Relation(data, workflowId.getParent(), relation.getAccess(), RunIds.fromString(workflowId.getRun()));
        }
        relations.put(new RelationKey(relation), relation);
    }
    return localDatasets;
}
Also used : Relation(io.cdap.cdap.data2.metadata.lineage.Relation) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) DatasetId(io.cdap.cdap.proto.id.DatasetId) HashSet(java.util.HashSet)

Aggregations

ProgramRunId (io.cdap.cdap.proto.id.ProgramRunId)316 Test (org.junit.Test)170 ProgramId (io.cdap.cdap.proto.id.ProgramId)124 RunId (org.apache.twill.api.RunId)76 ApplicationId (io.cdap.cdap.proto.id.ApplicationId)74 ArrayList (java.util.ArrayList)58 RunRecordDetail (io.cdap.cdap.internal.app.store.RunRecordDetail)50 HashMap (java.util.HashMap)50 ArtifactId (io.cdap.cdap.api.artifact.ArtifactId)48 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)48 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)48 ApplicationSpecification (io.cdap.cdap.api.app.ApplicationSpecification)46 IOException (java.io.IOException)44 HashSet (java.util.HashSet)42 ProgramOptions (io.cdap.cdap.app.runtime.ProgramOptions)40 ProgramRunStatus (io.cdap.cdap.proto.ProgramRunStatus)38 Map (java.util.Map)38 SimpleProgramOptions (io.cdap.cdap.internal.app.runtime.SimpleProgramOptions)36 Injector (com.google.inject.Injector)30 TimeUnit (java.util.concurrent.TimeUnit)30