use of co.cask.cdap.data2.metadata.dataset.Metadata in project cdap by caskdata.
the class DefaultArtifactRepository method writeSystemMetadata.
private void writeSystemMetadata(co.cask.cdap.proto.id.ArtifactId artifactId, ArtifactInfo artifactInfo) {
// add system metadata for artifacts
ArtifactSystemMetadataWriter writer = new ArtifactSystemMetadataWriter(metadataStore, artifactId, artifactInfo);
writer.write();
}
use of co.cask.cdap.data2.metadata.dataset.Metadata in project cdap by caskdata.
the class MetadataHttpHandler method searchMetadata.
@GET
@Path("/namespaces/{namespace-id}/metadata/search")
public void searchMetadata(HttpRequest request, HttpResponder responder, @PathParam("namespace-id") String namespaceId, @QueryParam("query") String searchQuery, @QueryParam("target") List<String> targets, @QueryParam("sort") @DefaultValue("") String sort, @QueryParam("offset") @DefaultValue("0") int offset, // 2147483647 is Integer.MAX_VALUE
@QueryParam("limit") @DefaultValue("2147483647") int limit, @QueryParam("numCursors") @DefaultValue("0") int numCursors, @QueryParam("cursor") @DefaultValue("") String cursor, @QueryParam("showHidden") @DefaultValue("false") boolean showHidden, @Nullable @QueryParam("entityScope") String entityScope) throws Exception {
if (searchQuery == null || searchQuery.isEmpty()) {
throw new BadRequestException("query is not specified");
}
Set<EntityTypeSimpleName> types = Collections.emptySet();
if (targets != null) {
types = ImmutableSet.copyOf(Iterables.transform(targets, STRING_TO_TARGET_TYPE));
}
SortInfo sortInfo = SortInfo.of(URLDecoder.decode(sort, "UTF-8"));
if (SortInfo.DEFAULT.equals(sortInfo)) {
if (!(cursor.isEmpty()) || 0 != numCursors) {
throw new BadRequestException("Cursors are not supported when sort info is not specified.");
}
}
try {
MetadataSearchResponse response = metadataAdmin.search(namespaceId, URLDecoder.decode(searchQuery, "UTF-8"), types, sortInfo, offset, limit, numCursors, cursor, showHidden, validateEntityScope(entityScope));
responder.sendJson(HttpResponseStatus.OK, GSON.toJson(response, MetadataSearchResponse.class));
} catch (Exception e) {
// if MetadataDataset throws an exception, it gets wrapped
if (Throwables.getRootCause(e) instanceof BadRequestException) {
throw new BadRequestException(e.getMessage(), e);
}
throw e;
}
}
use of co.cask.cdap.data2.metadata.dataset.Metadata in project cdap by caskdata.
the class LineageAdminTest method testWorkflowLineage.
@Test
public void testWorkflowLineage() throws Exception {
// Lineage for D3 -> P2 -> D2 -> P1 -> D1
LineageStore lineageStore = new LineageStore(getTxExecFactory(), getDatasetFramework(), NamespaceId.DEFAULT.dataset("testWorkflowLineage"));
Store store = getInjector().getInstance(Store.class);
MetadataStore metadataStore = getInjector().getInstance(MetadataStore.class);
LineageAdmin lineageAdmin = new LineageAdmin(lineageStore, store, metadataStore, new NoOpEntityExistenceVerifier());
// Define metadata
MetadataRecord run1AppMeta = new MetadataRecord(program1.getParent(), MetadataScope.USER, toMap("pk1", "pk1"), toSet("pt1"));
MetadataRecord run1ProgramMeta = new MetadataRecord(program1, MetadataScope.USER, toMap("pk1", "pk1"), toSet("pt1"));
MetadataRecord run1Data1Meta = new MetadataRecord(dataset1, MetadataScope.USER, toMap("dk1", "dk1"), toSet("dt1"));
MetadataRecord run1Data2Meta = new MetadataRecord(dataset2, MetadataScope.USER, toMap("dk2", "dk2"), toSet("dt2"));
// Add metadata
metadataStore.setProperties(MetadataScope.USER, program1.getParent(), run1AppMeta.getProperties());
// noinspection ToArrayCallWithZeroLengthArrayArgument
metadataStore.addTags(MetadataScope.USER, program1.getParent(), run1AppMeta.getTags().toArray(new String[0]));
metadataStore.setProperties(MetadataScope.USER, program1, run1ProgramMeta.getProperties());
// noinspection ToArrayCallWithZeroLengthArrayArgument
metadataStore.addTags(MetadataScope.USER, program1, run1ProgramMeta.getTags().toArray(new String[0]));
metadataStore.setProperties(MetadataScope.USER, dataset1, run1Data1Meta.getProperties());
// noinspection ToArrayCallWithZeroLengthArrayArgument
metadataStore.addTags(MetadataScope.USER, dataset1, run1Data1Meta.getTags().toArray(new String[0]));
metadataStore.setProperties(MetadataScope.USER, dataset2, run1Data2Meta.getProperties());
// noinspection ToArrayCallWithZeroLengthArrayArgument
metadataStore.addTags(MetadataScope.USER, dataset2, run1Data2Meta.getTags().toArray(new String[0]));
// Add accesses for D3 -> P2 -> D2 -> P1 -> D1 <-> P3
// We need to use current time here as metadata store stores access time using current time
ProgramRunId run1 = program1.run(RunIds.generate(System.currentTimeMillis()).getId());
ProgramRunId run2 = program2.run(RunIds.generate(System.currentTimeMillis()).getId());
ProgramRunId run3 = program3.run(RunIds.generate(System.currentTimeMillis()).getId());
ProgramRunId workflow = program6.run(RunIds.generate(System.currentTimeMillis()).getId());
ProgramRunId run5 = program5.run(RunIds.generate(System.currentTimeMillis()).getId());
addWorkflowRuns(store, workflow.getProgram(), workflow.getRun(), run1, run2, run3);
addRuns(store, workflow);
addRuns(store, run5);
// It is okay to use current time here since access time is ignore during assertions
lineageStore.addAccess(run1, dataset1, AccessType.WRITE, System.currentTimeMillis(), flowlet1);
lineageStore.addAccess(run1, dataset1, AccessType.WRITE, System.currentTimeMillis(), flowlet1);
lineageStore.addAccess(run1, dataset2, AccessType.READ, System.currentTimeMillis(), flowlet1);
lineageStore.addAccess(run2, dataset2, AccessType.WRITE, System.currentTimeMillis(), flowlet2);
lineageStore.addAccess(run2, dataset3, AccessType.READ, System.currentTimeMillis(), flowlet2);
lineageStore.addAccess(run3, dataset1, AccessType.UNKNOWN, System.currentTimeMillis());
lineageStore.addAccess(run5, dataset1, AccessType.READ, System.currentTimeMillis());
// The UNKNOWN access type will get filtered out if there is READ/WRITE. It will be preserved if it is the
// only access type
Lineage expectedLineage = new Lineage(ImmutableSet.of(new Relation(dataset1, program6, AccessType.WRITE, twillRunId(workflow)), new Relation(dataset2, program6, AccessType.READ, twillRunId(workflow)), new Relation(dataset2, program6, AccessType.WRITE, twillRunId(workflow)), new Relation(dataset3, program6, AccessType.READ, twillRunId(workflow)), new Relation(dataset1, program6, AccessType.UNKNOWN, twillRunId(workflow)), new Relation(dataset1, program5, AccessType.READ, twillRunId(run5))));
Lineage resultLineage = lineageAdmin.computeLineage(dataset1, 500, System.currentTimeMillis() + 10000, 100, "workflow");
// Lineage for D1
Assert.assertEquals(expectedLineage, resultLineage);
resultLineage = lineageAdmin.computeLineage(dataset2, 500, System.currentTimeMillis() + 10000, 100, "workflow");
// Lineage for D2
Assert.assertEquals(expectedLineage, resultLineage);
// Lineage for D1 for one level should be D2 -> P1 -> D1 <-> P3
Lineage oneLevelLineage = lineageAdmin.computeLineage(dataset1, 500, System.currentTimeMillis() + 10000, 1, "workflow");
Assert.assertEquals(ImmutableSet.of(new Relation(dataset1, program6, AccessType.WRITE, twillRunId(workflow)), new Relation(dataset2, program6, AccessType.READ, twillRunId(workflow)), new Relation(dataset1, program5, AccessType.READ, twillRunId(run5)), new Relation(dataset1, program6, AccessType.UNKNOWN, twillRunId(workflow))), oneLevelLineage.getRelations());
// Run tests without workflow parameter
expectedLineage = new Lineage(ImmutableSet.of(new Relation(dataset1, program1, AccessType.WRITE, twillRunId(run1), toSet(flowlet1)), new Relation(dataset2, program1, AccessType.READ, twillRunId(run1), toSet(flowlet1)), new Relation(dataset2, program2, AccessType.WRITE, twillRunId(run2), toSet(flowlet2)), new Relation(dataset3, program2, AccessType.READ, twillRunId(run2), toSet(flowlet2)), new Relation(dataset1, program3, AccessType.UNKNOWN, twillRunId(run3)), new Relation(dataset1, program5, AccessType.READ, twillRunId(run5))));
resultLineage = lineageAdmin.computeLineage(dataset1, 500, System.currentTimeMillis() + 10000, 100, null);
// Lineage for D1
Assert.assertEquals(expectedLineage, resultLineage);
resultLineage = lineageAdmin.computeLineage(dataset2, 500, System.currentTimeMillis() + 10000, 100, null);
// Lineage for D2
Assert.assertEquals(expectedLineage, resultLineage);
// Lineage for D1 for one level should be D2 -> P1 -> D1 <-> P3
oneLevelLineage = lineageAdmin.computeLineage(dataset1, 500, System.currentTimeMillis() + 10000, 1, null);
Assert.assertEquals(ImmutableSet.of(new Relation(dataset1, program1, AccessType.WRITE, twillRunId(run1), toSet(flowlet1)), new Relation(dataset2, program1, AccessType.READ, twillRunId(run1), toSet(flowlet1)), new Relation(dataset1, program5, AccessType.READ, twillRunId(run5)), new Relation(dataset1, program3, AccessType.UNKNOWN, twillRunId(run3))), oneLevelLineage.getRelations());
// Assert metadata
Assert.assertEquals(toSet(run1AppMeta, run1ProgramMeta, run1Data1Meta, run1Data2Meta), lineageAdmin.getMetadataForRun(run1));
// Assert that in a different namespace both lineage and metadata should be empty
NamespaceId customNamespace = new NamespaceId("custom_namespace");
DatasetId customDataset1 = customNamespace.dataset(dataset1.getEntityName());
ProgramRunId customRun1 = customNamespace.app(program1.getApplication()).program(program1.getType(), program1.getEntityName()).run(run1.getEntityName());
Assert.assertEquals(new Lineage(ImmutableSet.<Relation>of()), lineageAdmin.computeLineage(customDataset1, 500, System.currentTimeMillis() + 10000, 100));
Assert.assertEquals(ImmutableSet.<MetadataRecord>of(), lineageAdmin.getMetadataForRun(customRun1));
}
use of co.cask.cdap.data2.metadata.dataset.Metadata in project cdap by caskdata.
the class LineageAdminTest method testSimpleLineage.
@Test
public void testSimpleLineage() throws Exception {
// Lineage for D3 -> P2 -> D2 -> P1 -> D1
LineageStore lineageStore = new LineageStore(getTxExecFactory(), getDatasetFramework(), NamespaceId.DEFAULT.dataset("testSimpleLineage"));
Store store = getInjector().getInstance(Store.class);
MetadataStore metadataStore = getInjector().getInstance(MetadataStore.class);
LineageAdmin lineageAdmin = new LineageAdmin(lineageStore, store, metadataStore, new NoOpEntityExistenceVerifier());
// Define metadata
MetadataRecord run1AppMeta = new MetadataRecord(program1.getParent(), MetadataScope.USER, toMap("pk1", "pk1"), toSet("pt1"));
MetadataRecord run1ProgramMeta = new MetadataRecord(program1, MetadataScope.USER, toMap("pk1", "pk1"), toSet("pt1"));
MetadataRecord run1Data1Meta = new MetadataRecord(dataset1, MetadataScope.USER, toMap("dk1", "dk1"), toSet("dt1"));
MetadataRecord run1Data2Meta = new MetadataRecord(dataset2, MetadataScope.USER, toMap("dk2", "dk2"), toSet("dt2"));
// Add metadata
metadataStore.setProperties(MetadataScope.USER, program1.getParent(), run1AppMeta.getProperties());
// noinspection ToArrayCallWithZeroLengthArrayArgument
metadataStore.addTags(MetadataScope.USER, program1.getParent(), run1AppMeta.getTags().toArray(new String[0]));
metadataStore.setProperties(MetadataScope.USER, program1, run1ProgramMeta.getProperties());
// noinspection ToArrayCallWithZeroLengthArrayArgument
metadataStore.addTags(MetadataScope.USER, program1, run1ProgramMeta.getTags().toArray(new String[0]));
metadataStore.setProperties(MetadataScope.USER, dataset1, run1Data1Meta.getProperties());
// noinspection ToArrayCallWithZeroLengthArrayArgument
metadataStore.addTags(MetadataScope.USER, dataset1, run1Data1Meta.getTags().toArray(new String[0]));
metadataStore.setProperties(MetadataScope.USER, dataset2, run1Data2Meta.getProperties());
// noinspection ToArrayCallWithZeroLengthArrayArgument
metadataStore.addTags(MetadataScope.USER, dataset2, run1Data2Meta.getTags().toArray(new String[0]));
// Add accesses for D3 -> P2 -> D2 -> P1 -> D1 <-> P3
// We need to use current time here as metadata store stores access time using current time
ProgramRunId run1 = program1.run(RunIds.generate(System.currentTimeMillis()).getId());
ProgramRunId run2 = program2.run(RunIds.generate(System.currentTimeMillis()).getId());
ProgramRunId run3 = program3.run(RunIds.generate(System.currentTimeMillis()).getId());
addRuns(store, run1, run2, run3);
// It is okay to use current time here since access time is ignore during assertions
lineageStore.addAccess(run1, dataset1, AccessType.UNKNOWN, System.currentTimeMillis(), flowlet1);
lineageStore.addAccess(run1, dataset1, AccessType.WRITE, System.currentTimeMillis(), flowlet1);
lineageStore.addAccess(run1, dataset2, AccessType.READ, System.currentTimeMillis(), flowlet1);
lineageStore.addAccess(run2, dataset2, AccessType.WRITE, System.currentTimeMillis(), flowlet2);
lineageStore.addAccess(run2, dataset3, AccessType.READ, System.currentTimeMillis(), flowlet2);
lineageStore.addAccess(run3, dataset1, AccessType.UNKNOWN, System.currentTimeMillis());
// The UNKNOWN access type will get filtered out if there is READ/WRITE. It will be preserved if it is the
// only access type
Lineage expectedLineage = new Lineage(ImmutableSet.of(new Relation(dataset1, program1, AccessType.WRITE, twillRunId(run1), toSet(flowlet1)), new Relation(dataset2, program1, AccessType.READ, twillRunId(run1), toSet(flowlet1)), new Relation(dataset2, program2, AccessType.WRITE, twillRunId(run2), toSet(flowlet2)), new Relation(dataset3, program2, AccessType.READ, twillRunId(run2), toSet(flowlet2)), new Relation(dataset1, program3, AccessType.UNKNOWN, twillRunId(run3))));
// Lineage for D1
Assert.assertEquals(expectedLineage, lineageAdmin.computeLineage(dataset1, 500, System.currentTimeMillis() + 10000, 100));
// Lineage for D2
Assert.assertEquals(expectedLineage, lineageAdmin.computeLineage(dataset2, 500, System.currentTimeMillis() + 10000, 100));
// Lineage for D1 for one level should be D2 -> P1 -> D1 <-> P3
Lineage oneLevelLineage = lineageAdmin.computeLineage(dataset1, 500, System.currentTimeMillis() + 10000, 1);
Assert.assertEquals(ImmutableSet.of(new Relation(dataset1, program1, AccessType.WRITE, twillRunId(run1), toSet(flowlet1)), new Relation(dataset2, program1, AccessType.READ, twillRunId(run1), toSet(flowlet1)), new Relation(dataset1, program3, AccessType.UNKNOWN, twillRunId(run3))), oneLevelLineage.getRelations());
// Assert metadata
Assert.assertEquals(toSet(run1AppMeta, run1ProgramMeta, run1Data1Meta, run1Data2Meta), lineageAdmin.getMetadataForRun(run1));
// Assert that in a different namespace both lineage and metadata should be empty
NamespaceId customNamespace = new NamespaceId("custom_namespace");
DatasetId customDataset1 = customNamespace.dataset(dataset1.getEntityName());
ProgramRunId customRun1 = customNamespace.app(program1.getApplication()).program(program1.getType(), program1.getEntityName()).run(run1.getEntityName());
Assert.assertEquals(new Lineage(ImmutableSet.<Relation>of()), lineageAdmin.computeLineage(customDataset1, 500, System.currentTimeMillis() + 10000, 100));
Assert.assertEquals(ImmutableSet.<MetadataRecord>of(), lineageAdmin.getMetadataForRun(customRun1));
}
Aggregations