Examples with LineageWriter co.cask.cdap.data2.metadata.writer.LineageWriter used on opensource projects

Example 1 with LineageWriter

use of co.cask.cdap.data2.metadata.writer.LineageWriter in project cdap by caskdata.

the class BasicLineageWriterTest method testWrites.

@Test
public void testWrites() throws Exception {
    Injector injector = getInjector();
    MetadataStore metadataStore = injector.getInstance(MetadataStore.class);
    LineageStore lineageStore = injector.getInstance(LineageStore.class);
    LineageWriter lineageWriter = new BasicLineageWriter(lineageStore);
    // Define entities
    ProgramId program = new ProgramId(NamespaceId.DEFAULT.getNamespace(), "app", ProgramType.FLOW, "flow");
    StreamId stream = new StreamId(NamespaceId.DEFAULT.getNamespace(), "stream");
    ProgramRunId run1 = new ProgramRunId(program.getNamespace(), program.getApplication(), program.getType(), program.getEntityName(), RunIds.generate(10000).getId());
    ProgramRunId run2 = new ProgramRunId(program.getNamespace(), program.getApplication(), program.getType(), program.getEntityName(), RunIds.generate(20000).getId());
    // Tag stream
    metadataStore.addTags(MetadataScope.USER, stream, "stag1", "stag2");
    // Write access for run1
    lineageWriter.addAccess(run1, stream, AccessType.READ);
    Assert.assertEquals(ImmutableSet.of(program, stream), lineageStore.getEntitiesForRun(run1));
    // Record time to verify duplicate writes.
    long beforeSecondTag = System.currentTimeMillis();
    // Wait for next millisecond, since access time is stored in milliseconds.
    TimeUnit.MILLISECONDS.sleep(1);
    // Add another tag to stream
    metadataStore.addTags(MetadataScope.USER, stream, "stag3");
    // Write access for run1 again
    lineageWriter.addAccess(run1, stream, AccessType.READ);
    // The write should be no-op, and access time for run1 should not be updated
    Assert.assertTrue(lineageStore.getAccessTimesForRun(run1).get(0) < beforeSecondTag);
    // However, you can write access for another run
    lineageWriter.addAccess(run2, stream, AccessType.READ);
    // Assert new access time is written
    Assert.assertTrue(lineageStore.getAccessTimesForRun(run2).get(0) >= beforeSecondTag);
}