Examples with Store io.cdap.cdap.app.store.Store used on opensource projects

Example 11 with Store

use of io.cdap.cdap.app.store.Store in project cdap by caskdata.

the class MetricsSuiteTestBase method beforeClass.

@BeforeClass
public static void beforeClass() throws Exception {
    if (nestedStartCount++ > 0) {
        return;
    }
    conf = CConfiguration.create();
    conf.set(Constants.CFG_LOCAL_DATA_DIR, TEMP_FOLDER.newFolder().getAbsolutePath());
    conf.set(Constants.Metrics.ADDRESS, InetAddress.getLoopbackAddress().getHostAddress());
    conf.setBoolean(Constants.Dangerous.UNRECOVERABLE_RESET, true);
    conf.setBoolean(Constants.Metrics.CONFIG_AUTHENTICATION_REQUIRED, true);
    conf.set(Constants.Metrics.CLUSTER_NAME, CLUSTER);
    Injector injector = startMetricsService(conf);
    store = injector.getInstance(Store.class);
    locationFactory = injector.getInstance(LocationFactory.class);
    metricStore = injector.getInstance(MetricStore.class);
}

Example 12 with Store

use of io.cdap.cdap.app.store.Store in project cdap by caskdata.

the class LineageAdmin method computeWorkflowInnerPrograms.

/**
 * Compute the inner programs and program runs based on the program relations and add them to the collections.
 *
 * @param toVisitPrograms the collection of next to visit programs
 * @param programWorkflowMap the program workflow run id map
 * @param programRelations the program relations of the dataset
 */
private void computeWorkflowInnerPrograms(Set<ProgramId> toVisitPrograms, Map<ProgramRunId, ProgramRunId> programWorkflowMap, Set<Relation> programRelations) {
    // Step 1 walk through the program relations, filter out the possible mapreduce and spark programs that
    // could be in the workflow, and get the appSpec for the program, to determine what other programs
    // are in the workflow
    Map<ApplicationId, ApplicationSpecification> appSpecs = new HashMap<>();
    Set<ProgramRunId> possibleInnerPrograms = new HashSet<>();
    programRelations.forEach(relation -> {
        ProgramType type = relation.getProgram().getType();
        if (type.equals(ProgramType.MAPREDUCE) || type.equals(ProgramType.SPARK)) {
            possibleInnerPrograms.add(relation.getProgramRunId());
            appSpecs.computeIfAbsent(relation.getProgram().getParent(), store::getApplication);
        }
    });
    // Step 2, get the run record for all the possible inner programs, the run record contains the
    // workflow information, fetch the workflow id and add them to the map
    Map<ProgramRunId, RunRecordDetail> runRecords = store.getRuns(possibleInnerPrograms);
    Set<ProgramRunId> workflowRunIds = new HashSet<>();
    runRecords.entrySet().stream().filter(e -> e.getValue() != null).forEach(entry -> {
        ProgramRunId programRunId = entry.getKey();
        RunRecordDetail runRecord = entry.getValue();
        if (runRecord.getSystemArgs().containsKey(ProgramOptionConstants.WORKFLOW_RUN_ID)) {
            ProgramRunId wfRunId = extractWorkflowRunId(programRunId, runRecord);
            programWorkflowMap.put(programRunId, wfRunId);
            workflowRunIds.add(wfRunId);
        }
    });
    // Step 3, fetch run records of the workflow, the properties of the workflow run record has all
    // the inner program run ids, compare them with the app spec to get the type of the program
    runRecords = store.getRuns(workflowRunIds);
    runRecords.entrySet().stream().filter(e -> e.getValue() != null).forEach(entry -> {
        ProgramRunId programRunId = entry.getKey();
        RunRecordDetail runRecord = entry.getValue();
        extractAndAddInnerPrograms(toVisitPrograms, programWorkflowMap, appSpecs, programRunId, runRecord);
    });
}

Example 13 with Store

use of io.cdap.cdap.app.store.Store in project cdap by caskdata.

the class LineageAdminTest method testBranchLineage.

@Test
public void testBranchLineage() {
    // Lineage for:
    // 
    // ->D4        -> D5 -> P3 -> D6
    // |           |
    // |           |
    // D1 -> P1 -> D2 -> P2 -> D3
    // |     |           |
    // |     |           |
    // S1 -->|     ---------------> P4 -> D7
    TransactionRunner transactionRunner = getInjector().getInstance(TransactionRunner.class);
    LineageStoreReader lineageReader = new DefaultLineageStoreReader(transactionRunner);
    LineageWriter lineageWriter = new BasicLineageWriter(transactionRunner);
    Store store = getInjector().getInstance(Store.class);
    LineageAdmin lineageAdmin = new LineageAdmin(lineageReader, store);
    // Add accesses
    addRuns(store, run1, run2, run3, run4, run5);
    // It is okay to use current time here since access time is ignore during assertions
    lineageWriter.addAccess(run1, dataset1, AccessType.READ);
    lineageWriter.addAccess(run1, dataset2, AccessType.WRITE);
    lineageWriter.addAccess(run1, dataset4, AccessType.WRITE);
    lineageWriter.addAccess(run2, dataset2, AccessType.READ);
    lineageWriter.addAccess(run2, dataset3, AccessType.WRITE);
    lineageWriter.addAccess(run2, dataset5, AccessType.WRITE);
    lineageWriter.addAccess(run3, dataset5, AccessType.READ, null);
    lineageWriter.addAccess(run3, dataset6, AccessType.WRITE, null);
    lineageWriter.addAccess(run4, dataset2, AccessType.READ, null);
    lineageWriter.addAccess(run4, dataset3, AccessType.READ, null);
    lineageWriter.addAccess(run4, dataset7, AccessType.WRITE, null);
    Lineage expectedLineage = new Lineage(ImmutableSet.of(new Relation(dataset1, program1, AccessType.READ, twillRunId(run1)), new Relation(dataset2, program1, AccessType.WRITE, twillRunId(run1)), new Relation(dataset4, program1, AccessType.WRITE, twillRunId(run1)), new Relation(dataset2, program2, AccessType.READ, twillRunId(run2)), new Relation(dataset3, program2, AccessType.WRITE, twillRunId(run2)), new Relation(dataset5, program2, AccessType.WRITE, twillRunId(run2)), new Relation(dataset5, program3, AccessType.READ, twillRunId(run3)), new Relation(dataset6, program3, AccessType.WRITE, twillRunId(run3)), new Relation(dataset2, program4, AccessType.READ, twillRunId(run4)), new Relation(dataset3, program4, AccessType.READ, twillRunId(run4)), new Relation(dataset7, program4, AccessType.WRITE, twillRunId(run4))));
    // Lineage for D7
    Assert.assertEquals(expectedLineage, lineageAdmin.computeLineage(dataset7, 500, 20000, 100));
    // Lineage for D6
    Assert.assertEquals(expectedLineage, lineageAdmin.computeLineage(dataset6, 500, 20000, 100));
    // Lineage for D3
    Assert.assertEquals(expectedLineage, lineageAdmin.computeLineage(dataset3, 500, 20000, 100));
}

Example 14 with Store

use of io.cdap.cdap.app.store.Store in project cdap by caskdata.

the class LineageAdminTest method testLocalDatasetsInWorkflow.

@Test
public void testLocalDatasetsInWorkflow() throws Exception {
    TransactionRunner transactionRunner = getInjector().getInstance(TransactionRunner.class);
    LineageStoreReader lineageReader = new DefaultLineageStoreReader(transactionRunner);
    LineageWriter lineageWriter = new BasicLineageWriter(transactionRunner);
    ApplicationId testApp = NamespaceId.DEFAULT.app("testLocalDatasets");
    ProgramId workflowId = testApp.workflow("wf1");
    // if the spark and mr job are inner jobs of workflow, they should be in the same app
    ProgramId mrId1 = testApp.mr("mr1");
    ProgramId mrId2 = testApp.mr("mr2");
    ProgramId sparkId = testApp.spark("spark1");
    ImmutableList<WorkflowNode> nodes = ImmutableList.of(new WorkflowActionNode("mr1", new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "mr1")), new WorkflowActionNode("mr2", new ScheduleProgramInfo(SchedulableProgramType.MAPREDUCE, "mr2")), new WorkflowActionNode("spark1", new ScheduleProgramInfo(SchedulableProgramType.SPARK, "spark1")));
    WorkflowSpecification wfSpec = new WorkflowSpecification("test", "wf1", "", Collections.emptyMap(), nodes, Collections.emptyMap(), Collections.emptyMap());
    ApplicationSpecification appSpec = new DefaultApplicationSpecification("testLocalDatasets", ProjectInfo.getVersion().toString(), "dummy app", null, NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), ImmutableMap.of(workflowId.getProgram(), wfSpec), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    Store store = getInjector().getInstance(Store.class);
    store.addApplication(testApp, appSpec);
    LineageAdmin lineageAdmin = new LineageAdmin(lineageReader, store);
    // Add accesses for D1 -|
    // |-> MR1 -> LOCAL1 -> MR2 -> LOCAL2 -> SPARK -> D3
    // D2 -|
    // P1 and P2 are inner programs of the workflow
    // We need to use current time here as metadata store stores access time using current time
    ProgramRunId mr1Run = mrId1.run(RunIds.generate(System.currentTimeMillis()).getId());
    ProgramRunId mr2Run = mrId2.run((RunIds.generate(System.currentTimeMillis()).getId()));
    ProgramRunId sparkRun = sparkId.run(RunIds.generate(System.currentTimeMillis()).getId());
    ProgramRunId workflow = workflowId.run(RunIds.generate(System.currentTimeMillis()).getId());
    // local datasets always end with workflow run id
    DatasetId localDataset1 = NamespaceId.DEFAULT.dataset("localDataset1" + workflow.getRun());
    DatasetId localDataset2 = NamespaceId.DEFAULT.dataset("localDataset2" + workflow.getRun());
    addRuns(store, workflow);
    // only mr and spark can be inner programs
    addWorkflowRuns(store, workflow.getProgram(), workflow.getRun(), mr1Run, mr2Run, sparkRun);
    lineageWriter.addAccess(mr1Run, dataset1, AccessType.READ);
    lineageWriter.addAccess(mr1Run, dataset2, AccessType.READ);
    lineageWriter.addAccess(mr1Run, localDataset1, AccessType.WRITE);
    lineageWriter.addAccess(mr2Run, localDataset1, AccessType.READ);
    lineageWriter.addAccess(mr2Run, localDataset2, AccessType.WRITE);
    lineageWriter.addAccess(sparkRun, localDataset2, AccessType.READ);
    lineageWriter.addAccess(sparkRun, dataset3, AccessType.WRITE);
    // compute the lineage without roll up, the local datasets and inner program should not roll up
    Lineage expectedLineage = new Lineage(ImmutableSet.of(new Relation(dataset1, mrId1, AccessType.READ, twillRunId(mr1Run)), new Relation(dataset2, mrId1, AccessType.READ, twillRunId(mr1Run)), new Relation(localDataset1, mrId1, AccessType.WRITE, twillRunId(mr1Run)), new Relation(localDataset1, mrId2, AccessType.READ, twillRunId(mr2Run)), new Relation(localDataset2, mrId2, AccessType.WRITE, twillRunId(mr2Run)), new Relation(localDataset2, sparkId, AccessType.READ, twillRunId(sparkRun)), new Relation(dataset3, sparkId, AccessType.WRITE, twillRunId(sparkRun))));
    Lineage resultLineage = lineageAdmin.computeLineage(dataset1, 500, System.currentTimeMillis() + 10000, 100, null);
    // Lineage for D1
    Assert.assertEquals(expectedLineage, resultLineage);
    // D3 should have same lineage for all levels
    resultLineage = lineageAdmin.computeLineage(dataset3, 500, System.currentTimeMillis() + 10000, 100, null);
    Assert.assertEquals(expectedLineage, resultLineage);
    // if only query for one level with no roll up, the roll up should not happen and the inner program and local
    // dataset should get returned
    expectedLineage = new Lineage(ImmutableSet.of(new Relation(dataset3, sparkId, AccessType.WRITE, twillRunId(sparkRun)), new Relation(localDataset2, sparkId, AccessType.READ, twillRunId(sparkRun))));
    resultLineage = lineageAdmin.computeLineage(dataset3, 500, System.currentTimeMillis() + 10000, 1, null);
    Assert.assertEquals(expectedLineage, resultLineage);
    // query for roll up the workflow, all the inner program and local datasets should not be in the result,
    // the entire workflow information should get returned
    expectedLineage = new Lineage(ImmutableSet.of(new Relation(dataset1, workflowId, AccessType.READ, twillRunId(workflow)), new Relation(dataset2, workflowId, AccessType.READ, twillRunId(workflow)), new Relation(dataset3, workflowId, AccessType.WRITE, twillRunId(workflow))));
    // D1, D2, D3 should give same result
    resultLineage = lineageAdmin.computeLineage(dataset1, 500, System.currentTimeMillis() + 10000, 1, "workflow");
    Assert.assertEquals(expectedLineage, resultLineage);
    resultLineage = lineageAdmin.computeLineage(dataset2, 500, System.currentTimeMillis() + 10000, 1, "workflow");
    Assert.assertEquals(expectedLineage, resultLineage);
    resultLineage = lineageAdmin.computeLineage(dataset3, 500, System.currentTimeMillis() + 10000, 1, "workflow");
    Assert.assertEquals(expectedLineage, resultLineage);
}

Example 15 with Store

use of io.cdap.cdap.app.store.Store in project cdap by caskdata.

the class LineageAdminTest method testSimpleLoopLineage.

@Test
public void testSimpleLoopLineage() {
    // Lineage for D1 -> P1 -> D2 -> P2 -> D3 -> P3 -> D4
    // |                 |
    // |                 V
    // |<-----------------
    // 
    TransactionRunner transactionRunner = getInjector().getInstance(TransactionRunner.class);
    LineageStoreReader lineageReader = new DefaultLineageStoreReader(transactionRunner);
    LineageWriter lineageWriter = new BasicLineageWriter(transactionRunner);
    Store store = getInjector().getInstance(Store.class);
    LineageAdmin lineageAdmin = new LineageAdmin(lineageReader, store);
    // Add access
    addRuns(store, run1, run2, run3, run4, run5);
    // It is okay to use current time here since access time is ignore during assertions
    lineageWriter.addAccess(run1, dataset1, AccessType.READ);
    lineageWriter.addAccess(run1, dataset2, AccessType.WRITE);
    lineageWriter.addAccess(run2, dataset2, AccessType.READ);
    lineageWriter.addAccess(run2, dataset1, AccessType.WRITE);
    lineageWriter.addAccess(run2, dataset3, AccessType.WRITE);
    lineageWriter.addAccess(run3, dataset3, AccessType.READ, null);
    lineageWriter.addAccess(run3, dataset4, AccessType.WRITE, null);
    Lineage expectedLineage = new Lineage(ImmutableSet.of(new Relation(dataset2, program1, AccessType.WRITE, twillRunId(run1)), new Relation(dataset1, program1, AccessType.READ, twillRunId(run1)), new Relation(dataset1, program2, AccessType.WRITE, twillRunId(run2)), new Relation(dataset2, program2, AccessType.READ, twillRunId(run2)), new Relation(dataset3, program2, AccessType.WRITE, twillRunId(run2)), new Relation(dataset4, program3, AccessType.WRITE, twillRunId(run3)), new Relation(dataset3, program3, AccessType.READ, twillRunId(run3))));
    // Lineage for D1
    Assert.assertEquals(expectedLineage, lineageAdmin.computeLineage(dataset1, 500, 20000, 100));
    // Lineage for D2
    Assert.assertEquals(expectedLineage, lineageAdmin.computeLineage(dataset2, 500, 20000, 100));
    // Lineage for D1 for one level D1 -> P1 -> D2 -> P2 -> D3
    // |                 |
    // |                 V
    // |<-----------------
    // 
    Lineage oneLevelLineage = lineageAdmin.computeLineage(dataset1, 500, 20000, 1);
    Assert.assertEquals(ImmutableSet.of(new Relation(dataset2, program1, AccessType.WRITE, twillRunId(run1)), new Relation(dataset1, program1, AccessType.READ, twillRunId(run1)), new Relation(dataset1, program2, AccessType.WRITE, twillRunId(run2)), new Relation(dataset2, program2, AccessType.READ, twillRunId(run2)), new Relation(dataset3, program2, AccessType.WRITE, twillRunId(run2))), oneLevelLineage.getRelations());
}