Examples with DatasetId co.cask.cdap.proto.id.DatasetId used on opensource projects

Example 86 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class SchemaIndexerTest method testComplexRecord.

@Test
public void testComplexRecord() throws Exception {
    Schema complexSchema = Schema.recordOf("record1", Schema.Field.of("map1", Schema.mapOf(Schema.recordOf("record21", // String x
    Schema.Field.of("x", Schema.of(Schema.Type.STRING)), // String[] y
    Schema.Field.of("y", Schema.arrayOf(Schema.of(Schema.Type.STRING))), // Map<byte[],double> z
    Schema.Field.of("z", Schema.mapOf(Schema.of(Schema.Type.BYTES), Schema.of(Schema.Type.DOUBLE)))), Schema.arrayOf(Schema.recordOf("record22", Schema.Field.of("a", // Map<array<byte[]>, Map<boolean,byte[]> a
    Schema.mapOf(Schema.arrayOf(Schema.of(Schema.Type.BYTES)), Schema.mapOf(Schema.of(Schema.Type.BOOLEAN), Schema.of(Schema.Type.BYTES)))))))), Schema.Field.of("i", Schema.nullableOf(Schema.of(Schema.Type.INT))), Schema.Field.of("j", Schema.unionOf(Schema.of(Schema.Type.INT), Schema.of(Schema.Type.LONG), Schema.of(Schema.Type.NULL))));
    Schema anotherComplexSchema = Schema.arrayOf(Schema.of(Schema.Type.STRING));
    Schema superComplexSchema = Schema.unionOf(complexSchema, anotherComplexSchema, Schema.of(Schema.Type.NULL));
    Set<String> expected = ImmutableSet.of("map1", "map1:MAP", "record21", "record21:RECORD", "x", "x:STRING", "y", "y:ARRAY", "z", "z:MAP", "record22", "record22:RECORD", "a", "a:MAP", "i", "i:INT", "j", "j:UNION", "record1", "record1:RECORD");
    SchemaIndexer indexer = new SchemaIndexer();
    DatasetId datasetInstance = new DatasetId("ns1", "ds1");
    Set<String> actual = indexer.getIndexes(new MetadataEntry(datasetInstance, KEY, superComplexSchema.toString()));
    Assert.assertEquals(addKeyPrefix(expected), actual);
}

Example 87 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class LineageDatasetTest method testMultipleRelations.

@Test
public void testMultipleRelations() throws Exception {
    final LineageDataset lineageDataset = getLineageDataset("testMultipleRelations");
    Assert.assertNotNull(lineageDataset);
    TransactionExecutor txnl = dsFrameworkUtil.newInMemoryTransactionExecutor((TransactionAware) lineageDataset);
    final RunId runId1 = RunIds.generate(10000);
    final RunId runId2 = RunIds.generate(20000);
    final RunId runId3 = RunIds.generate(30000);
    final RunId runId4 = RunIds.generate(40000);
    final DatasetId datasetInstance1 = NamespaceId.DEFAULT.dataset("dataset1");
    final DatasetId datasetInstance2 = NamespaceId.DEFAULT.dataset("dataset2");
    final StreamId stream1 = NamespaceId.DEFAULT.stream("stream1");
    final StreamId stream2 = NamespaceId.DEFAULT.stream("stream2");
    final ProgramId program1 = NamespaceId.DEFAULT.app("app1").flow("flow1");
    final FlowletId flowlet1 = program1.flowlet("flowlet1");
    final ProgramId program2 = NamespaceId.DEFAULT.app("app2").worker("worker2");
    final ProgramId program3 = NamespaceId.DEFAULT.app("app3").service("service3");
    final ProgramRunId run11 = program1.run(runId1.getId());
    final ProgramRunId run22 = program2.run(runId2.getId());
    final ProgramRunId run23 = program2.run(runId3.getId());
    final ProgramRunId run34 = program3.run(runId4.getId());
    final long now = System.currentTimeMillis();
    final long run11Data1AccessTime = now;
    final long run22Data2AccessTime = now + 1;
    final long run22Stream1AccessTime = now + 2;
    final long run23Stream2AccessTime = now + 1;
    final long run23Data2AccessTime = now + 3;
    //noinspection UnnecessaryLocalVariable
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            lineageDataset.addAccess(run11, datasetInstance1, AccessType.READ, run11Data1AccessTime, flowlet1);
            lineageDataset.addAccess(run22, datasetInstance2, AccessType.WRITE, run22Data2AccessTime);
            lineageDataset.addAccess(run22, stream1, AccessType.READ, run22Stream1AccessTime);
            lineageDataset.addAccess(run23, stream2, AccessType.READ, run23Stream2AccessTime);
            lineageDataset.addAccess(run23, datasetInstance2, AccessType.WRITE, run23Data2AccessTime);
            lineageDataset.addAccess(run34, datasetInstance2, AccessType.READ_WRITE, System.currentTimeMillis());
            lineageDataset.addAccess(run34, stream2, AccessType.UNKNOWN, System.currentTimeMillis());
        }
    });
    txnl.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Assert.assertEquals(ImmutableSet.of(new Relation(datasetInstance1, program1, AccessType.READ, runId1, ImmutableSet.of(flowlet1))), lineageDataset.getRelations(datasetInstance1, 0, 100000, Predicates.<Relation>alwaysTrue()));
            Assert.assertEquals(ImmutableSet.of(new Relation(datasetInstance2, program2, AccessType.WRITE, runId2), new Relation(datasetInstance2, program2, AccessType.WRITE, runId3), new Relation(datasetInstance2, program3, AccessType.READ_WRITE, runId4)), lineageDataset.getRelations(datasetInstance2, 0, 100000, Predicates.<Relation>alwaysTrue()));
            Assert.assertEquals(ImmutableSet.of(new Relation(stream1, program2, AccessType.READ, runId2)), lineageDataset.getRelations(stream1, 0, 100000, Predicates.<Relation>alwaysTrue()));
            Assert.assertEquals(ImmutableSet.of(new Relation(stream2, program2, AccessType.READ, runId3), new Relation(stream2, program3, AccessType.UNKNOWN, runId4)), lineageDataset.getRelations(stream2, 0, 100000, Predicates.<Relation>alwaysTrue()));
            Assert.assertEquals(ImmutableSet.of(new Relation(datasetInstance2, program2, AccessType.WRITE, runId2), new Relation(stream1, program2, AccessType.READ, runId2), new Relation(datasetInstance2, program2, AccessType.WRITE, runId3), new Relation(stream2, program2, AccessType.READ, runId3)), lineageDataset.getRelations(program2, 0, 100000, Predicates.<Relation>alwaysTrue()));
            // Reduced time range
            Assert.assertEquals(ImmutableSet.of(new Relation(datasetInstance2, program2, AccessType.WRITE, runId2), new Relation(datasetInstance2, program2, AccessType.WRITE, runId3)), lineageDataset.getRelations(datasetInstance2, 0, 35000, Predicates.<Relation>alwaysTrue()));
            Assert.assertEquals(toSet(program1, datasetInstance1), lineageDataset.getEntitiesForRun(run11));
            Assert.assertEquals(ImmutableList.of(run11Data1AccessTime), lineageDataset.getAccessTimesForRun(run11));
            Assert.assertEquals(toSet(program2, datasetInstance2, stream1), lineageDataset.getEntitiesForRun(run22));
            Assert.assertEquals(ImmutableList.of(run22Data2AccessTime, run22Stream1AccessTime), lineageDataset.getAccessTimesForRun(run22));
            Assert.assertEquals(toSet(program2, datasetInstance2, stream2), lineageDataset.getEntitiesForRun(run23));
            Assert.assertEquals(ImmutableList.of(run23Data2AccessTime, run23Stream2AccessTime), lineageDataset.getAccessTimesForRun(run23));
            Assert.assertEquals(toSet(program3, datasetInstance2, stream2), lineageDataset.getEntitiesForRun(run34));
        }
    });
}

Example 88 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class DatasetClientTestRun method testSystemTypes.

@Test
public void testSystemTypes() throws Exception {
    // Tests that a dataset can be created in a namespace, even if the type does not exist in that namespace.
    // The dataset type is being resolved from the system namespace.
    DatasetTypeId type = TEST_NAMESPACE.datasetType(Table.class.getName());
    DatasetId instance = TEST_NAMESPACE.dataset("tableTypeDataset");
    Assert.assertFalse(typeClient.exists(type));
    Assert.assertFalse(datasetClient.exists(instance));
    datasetClient.create(instance, Table.class.getName());
    Assert.assertTrue(datasetClient.exists(instance));
}

Example 89 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class DatasetClientTestRun method testAll.

@Test
public void testAll() throws Exception {
    DatasetModuleId module = TEST_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME);
    DatasetTypeId type = TEST_NAMESPACE.datasetType(StandaloneDataset.class.getName());
    DatasetModuleId moduleInOtherNamespace = OTHER_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME);
    DatasetTypeId typeInOtherNamespace = OTHER_NAMESPACE.datasetType(StandaloneDataset.class.getName());
    int numBaseModules = moduleClient.list(TEST_NAMESPACE).size();
    int numBaseTypes = typeClient.list(TEST_NAMESPACE).size();
    LOG.info("Adding Dataset module");
    File moduleJarFile = createAppJarFile(StandaloneDatasetModule.class);
    moduleClient.add(TEST_NAMESPACE.datasetModule(StandaloneDatasetModule.NAME), StandaloneDatasetModule.class.getName(), moduleJarFile);
    Assert.assertEquals(numBaseModules + 1, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes + 2, typeClient.list(TEST_NAMESPACE).size());
    LOG.info("Checking that the new Dataset module exists");
    DatasetModuleMeta datasetModuleMeta = moduleClient.get(module);
    Assert.assertNotNull(datasetModuleMeta);
    Assert.assertEquals(StandaloneDatasetModule.NAME, datasetModuleMeta.getName());
    LOG.info("Checking that the new Dataset module does not exist in a different namespace");
    try {
        moduleClient.get(moduleInOtherNamespace);
        Assert.fail("datasetModule found in namespace other than one in which it was expected");
    } catch (DatasetModuleNotFoundException expected) {
    // expected
    }
    LOG.info("Checking that the new Dataset type exists");
    DatasetTypeMeta datasetTypeMeta = typeClient.get(type);
    Assert.assertNotNull(datasetTypeMeta);
    Assert.assertEquals(type.getType(), datasetTypeMeta.getName());
    datasetTypeMeta = typeClient.get(type);
    Assert.assertNotNull(datasetTypeMeta);
    Assert.assertEquals(StandaloneDataset.class.getName(), datasetTypeMeta.getName());
    LOG.info("Checking that the new Dataset type does not exist in a different namespace");
    try {
        typeClient.get(typeInOtherNamespace);
        Assert.fail("datasetType found in namespace other than one in which it was expected");
    } catch (DatasetTypeNotFoundException expected) {
    // expected
    }
    LOG.info("Creating, truncating, and deleting dataset of new Dataset type");
    // Before creating dataset, there are some system datasets already exist
    int numBaseDataset = datasetClient.list(TEST_NAMESPACE).size();
    DatasetId instance = TEST_NAMESPACE.dataset("testDataset");
    String description = "test description";
    datasetClient.create(instance, new DatasetInstanceConfiguration(StandaloneDataset.TYPE_NAME, Collections.<String, String>emptyMap(), description, null));
    Assert.assertEquals(numBaseDataset + 1, datasetClient.list(TEST_NAMESPACE).size());
    // Assert dataset summary for the newly created dataset
    DatasetSpecificationSummary expectedSpec = new DatasetSpecificationSummary(instance.getDataset(), StandaloneDataset.TYPE_NAME, description, Collections.<String, String>emptyMap());
    Assert.assertEquals(expectedSpec, getSpecForDataset(instance, datasetClient.list(TEST_NAMESPACE)));
    datasetClient.truncate(instance);
    DatasetMeta metaBefore = datasetClient.get(instance);
    Assert.assertEquals(0, metaBefore.getSpec().getProperties().size());
    datasetClient.update(instance, ImmutableMap.of("sdf", "foo", "abc", "123"));
    DatasetMeta metaAfter = datasetClient.get(instance);
    Assert.assertEquals(2, metaAfter.getSpec().getProperties().size());
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("sdf"));
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("abc"));
    Assert.assertEquals("foo", metaAfter.getSpec().getProperties().get("sdf"));
    Assert.assertEquals("123", metaAfter.getSpec().getProperties().get("abc"));
    datasetClient.updateExisting(instance, ImmutableMap.of("sdf", "fzz"));
    metaAfter = datasetClient.get(instance);
    Assert.assertEquals(2, metaAfter.getSpec().getProperties().size());
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("sdf"));
    Assert.assertTrue(metaAfter.getSpec().getProperties().containsKey("abc"));
    Assert.assertEquals("fzz", metaAfter.getSpec().getProperties().get("sdf"));
    Assert.assertEquals("123", metaAfter.getSpec().getProperties().get("abc"));
    datasetClient.delete(instance);
    Assert.assertEquals(numBaseDataset, datasetClient.list(TEST_NAMESPACE).size());
    LOG.info("Creating and deleting multiple Datasets");
    for (int i = 1; i <= 3; i++) {
        datasetClient.create(TEST_NAMESPACE.dataset("testDataset" + i), StandaloneDataset.TYPE_NAME);
    }
    Assert.assertEquals(numBaseDataset + 3, datasetClient.list(TEST_NAMESPACE).size());
    for (int i = 1; i <= 3; i++) {
        datasetClient.delete(TEST_NAMESPACE.dataset("testDataset" + i));
    }
    Assert.assertEquals(numBaseDataset, datasetClient.list(TEST_NAMESPACE).size());
    LOG.info("Deleting Dataset module");
    moduleClient.delete(module);
    Assert.assertEquals(numBaseModules, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes, typeClient.list(TEST_NAMESPACE).size());
    LOG.info("Adding Dataset module and then deleting all Dataset modules");
    moduleClient.add(TEST_NAMESPACE.datasetModule("testModule1"), StandaloneDatasetModule.class.getName(), moduleJarFile);
    Assert.assertEquals(numBaseModules + 1, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes + 2, typeClient.list(TEST_NAMESPACE).size());
    moduleClient.deleteAll(TEST_NAMESPACE);
    Assert.assertEquals(numBaseModules, moduleClient.list(TEST_NAMESPACE).size());
    Assert.assertEquals(numBaseTypes, typeClient.list(TEST_NAMESPACE).size());
}

Example 90 with DatasetId

use of co.cask.cdap.proto.id.DatasetId in project cdap by caskdata.

the class LineageTestRun method testAllProgramsLineage.

@Test
public void testAllProgramsLineage() throws Exception {
    NamespaceId namespace = new NamespaceId("testAllProgramsLineage");
    ApplicationId app = namespace.app(AllProgramsApp.NAME);
    ProgramId flow = app.flow(AllProgramsApp.NoOpFlow.NAME);
    ProgramId mapreduce = app.mr(AllProgramsApp.NoOpMR.NAME);
    ProgramId mapreduce2 = app.mr(AllProgramsApp.NoOpMR2.NAME);
    ProgramId spark = app.spark(AllProgramsApp.NoOpSpark.NAME);
    ProgramId service = app.service(AllProgramsApp.NoOpService.NAME);
    ProgramId worker = app.worker(AllProgramsApp.NoOpWorker.NAME);
    ProgramId workflow = app.workflow(AllProgramsApp.NoOpWorkflow.NAME);
    DatasetId dataset = namespace.dataset(AllProgramsApp.DATASET_NAME);
    DatasetId dataset2 = namespace.dataset(AllProgramsApp.DATASET_NAME2);
    DatasetId dataset3 = namespace.dataset(AllProgramsApp.DATASET_NAME3);
    StreamId stream = namespace.stream(AllProgramsApp.STREAM_NAME);
    namespaceClient.create(new NamespaceMeta.Builder().setName(namespace.getNamespace()).build());
    try {
        appClient.deploy(namespace, createAppJarFile(AllProgramsApp.class));
        // Add metadata
        ImmutableSet<String> sparkTags = ImmutableSet.of("spark-tag1", "spark-tag2");
        addTags(spark, sparkTags);
        Assert.assertEquals(sparkTags, getTags(spark, MetadataScope.USER));
        ImmutableSet<String> workerTags = ImmutableSet.of("worker-tag1");
        addTags(worker, workerTags);
        Assert.assertEquals(workerTags, getTags(worker, MetadataScope.USER));
        ImmutableMap<String, String> datasetProperties = ImmutableMap.of("data-key1", "data-value1");
        addProperties(dataset, datasetProperties);
        Assert.assertEquals(datasetProperties, getProperties(dataset, MetadataScope.USER));
        // Start all programs
        RunId flowRunId = runAndWait(flow);
        RunId mrRunId = runAndWait(mapreduce);
        RunId mrRunId2 = runAndWait(mapreduce2);
        RunId sparkRunId = runAndWait(spark);
        runAndWait(workflow);
        RunId workflowMrRunId = getRunId(mapreduce, mrRunId);
        RunId serviceRunId = runAndWait(service);
        // Worker makes a call to service to make it access datasets,
        // hence need to make sure service starts before worker, and stops after it.
        RunId workerRunId = runAndWait(worker);
        // Wait for programs to finish
        waitForStop(flow, true);
        waitForStop(mapreduce, false);
        waitForStop(mapreduce2, false);
        waitForStop(spark, false);
        waitForStop(workflow, false);
        waitForStop(worker, false);
        waitForStop(service, true);
        long now = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
        long oneHour = TimeUnit.HOURS.toSeconds(1);
        // Fetch dataset lineage
        LineageRecord lineage = fetchLineage(dataset, now - oneHour, now + oneHour, toSet(CollapseType.ACCESS), 10);
        // dataset is accessed by all programs
        LineageRecord expected = LineageSerializer.toLineageRecord(now - oneHour, now + oneHour, new Lineage(ImmutableSet.of(// Dataset access
        new Relation(dataset, flow, AccessType.UNKNOWN, flowRunId, toSet(flow.flowlet(AllProgramsApp.A.NAME))), new Relation(dataset, mapreduce, AccessType.WRITE, mrRunId), new Relation(dataset, mapreduce2, AccessType.WRITE, mrRunId2), new Relation(dataset2, mapreduce2, AccessType.READ, mrRunId2), new Relation(dataset, spark, AccessType.READ, sparkRunId), new Relation(dataset2, spark, AccessType.WRITE, sparkRunId), new Relation(dataset3, spark, AccessType.READ, sparkRunId), new Relation(dataset3, spark, AccessType.WRITE, sparkRunId), new Relation(dataset, mapreduce, AccessType.WRITE, workflowMrRunId), new Relation(dataset, service, AccessType.WRITE, serviceRunId), new Relation(dataset, worker, AccessType.WRITE, workerRunId), // Stream access
        new Relation(stream, flow, AccessType.READ, flowRunId, ImmutableSet.of(flow.flowlet(AllProgramsApp.A.NAME))), new Relation(stream, mapreduce, AccessType.READ, mrRunId), new Relation(stream, spark, AccessType.READ, sparkRunId), new Relation(stream, mapreduce, AccessType.READ, workflowMrRunId), new Relation(stream, worker, AccessType.WRITE, workerRunId))), toSet(CollapseType.ACCESS));
        Assert.assertEquals(expected, lineage);
        // Fetch stream lineage
        lineage = fetchLineage(stream, now - oneHour, now + oneHour, toSet(CollapseType.ACCESS), 10);
        // stream too is accessed by all programs
        Assert.assertEquals(expected, lineage);
        // Assert metadata
        // Id.Flow needs conversion to Id.Program JIRA - CDAP-3658
        Assert.assertEquals(toSet(new MetadataRecord(app, MetadataScope.USER, emptyMap(), emptySet()), new MetadataRecord(flow, MetadataScope.USER, emptyMap(), emptySet()), new MetadataRecord(dataset, MetadataScope.USER, datasetProperties, emptySet()), new MetadataRecord(stream, MetadataScope.USER, emptyMap(), emptySet())), fetchRunMetadata(flow.run(flowRunId.getId())));
        // Id.Worker needs conversion to Id.Program JIRA - CDAP-3658
        ProgramId programForWorker = new ProgramId(worker.getNamespace(), worker.getApplication(), worker.getType(), worker.getEntityName());
        Assert.assertEquals(toSet(new MetadataRecord(app, MetadataScope.USER, emptyMap(), emptySet()), new MetadataRecord(programForWorker, MetadataScope.USER, emptyMap(), workerTags), new MetadataRecord(dataset, MetadataScope.USER, datasetProperties, emptySet()), new MetadataRecord(stream, MetadataScope.USER, emptyMap(), emptySet())), fetchRunMetadata(worker.run(workerRunId.getId())));
        // Id.Spark needs conversion to Id.Program JIRA - CDAP-3658
        ProgramId programForSpark = new ProgramId(spark.getNamespace(), spark.getApplication(), spark.getType(), spark.getEntityName());
        Assert.assertEquals(toSet(new MetadataRecord(app, MetadataScope.USER, emptyMap(), emptySet()), new MetadataRecord(programForSpark, MetadataScope.USER, emptyMap(), sparkTags), new MetadataRecord(dataset, MetadataScope.USER, datasetProperties, emptySet()), new MetadataRecord(dataset2, MetadataScope.USER, emptyMap(), emptySet()), new MetadataRecord(dataset3, MetadataScope.USER, emptyMap(), emptySet()), new MetadataRecord(stream, MetadataScope.USER, emptyMap(), emptySet())), fetchRunMetadata(spark.run(sparkRunId.getId())));
    } finally {
        namespaceClient.delete(namespace);
    }
}