Search in sources :

Example 1 with UsageWriter

use of io.cdap.cdap.data2.registry.UsageWriter in project cdap by caskdata.

the class MetadataSubscriberServiceTest method testSubscriber.

@Test
public void testSubscriber() throws InterruptedException, ExecutionException, TimeoutException {
    LineageStoreReader lineageReader = getInjector().getInstance(LineageStoreReader.class);
    ProgramRunId run1 = service1.run(RunIds.generate());
    // Try to read lineage, which should be empty since we haven't start the MetadataSubscriberService yet.
    Set<NamespacedEntityId> entities = lineageReader.getEntitiesForRun(run1);
    Assert.assertTrue(entities.isEmpty());
    // Write out some lineage information
    LineageWriter lineageWriter = getInjector().getInstance(MessagingLineageWriter.class);
    lineageWriter.addAccess(run1, dataset1, AccessType.READ);
    lineageWriter.addAccess(run1, dataset2, AccessType.WRITE);
    // Write the field level lineage
    FieldLineageWriter fieldLineageWriter = getInjector().getInstance(MessagingLineageWriter.class);
    ProgramRunId spark1Run1 = spark1.run(RunIds.generate(100));
    ReadOperation read = new ReadOperation("read", "some read", EndPoint.of("ns", "endpoint1"), "offset", "body");
    TransformOperation parse = new TransformOperation("parse", "parse body", Collections.singletonList(InputField.of("read", "body")), "name", "address");
    WriteOperation write = new WriteOperation("write", "write data", EndPoint.of("ns", "endpoint2"), Arrays.asList(InputField.of("read", "offset"), InputField.of("parse", "name"), InputField.of("parse", "address")));
    List<Operation> operations = new ArrayList<>();
    operations.add(read);
    operations.add(write);
    operations.add(parse);
    FieldLineageInfo info1 = new FieldLineageInfo(operations);
    fieldLineageWriter.write(spark1Run1, info1);
    ProgramRunId spark1Run2 = spark1.run(RunIds.generate(200));
    fieldLineageWriter.write(spark1Run2, info1);
    List<Operation> operations2 = new ArrayList<>();
    operations2.add(read);
    operations2.add(parse);
    TransformOperation normalize = new TransformOperation("normalize", "normalize address", Collections.singletonList(InputField.of("parse", "address")), "address");
    operations2.add(normalize);
    WriteOperation anotherWrite = new WriteOperation("anotherwrite", "write data", EndPoint.of("ns", "endpoint2"), Arrays.asList(InputField.of("read", "offset"), InputField.of("parse", "name"), InputField.of("normalize", "address")));
    operations2.add(anotherWrite);
    FieldLineageInfo info2 = new FieldLineageInfo(operations2);
    ProgramRunId spark1Run3 = spark1.run(RunIds.generate(300));
    fieldLineageWriter.write(spark1Run3, info2);
    // Emit some usages
    UsageWriter usageWriter = getInjector().getInstance(MessagingUsageWriter.class);
    usageWriter.register(spark1, dataset1);
    usageWriter.registerAll(Collections.singleton(spark1), dataset3);
    // Verifies lineage has been written
    Set<NamespacedEntityId> expectedLineage = new HashSet<>(Arrays.asList(run1.getParent(), dataset1, dataset2));
    Tasks.waitFor(true, () -> expectedLineage.equals(lineageReader.getEntitiesForRun(run1)), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // There shouldn't be any lineage for the "spark1" program, as only usage has been emitted.
    Assert.assertTrue(lineageReader.getRelations(spark1, 0L, Long.MAX_VALUE, x -> true).isEmpty());
    FieldLineageReader fieldLineageReader = getInjector().getInstance(FieldLineageReader.class);
    Set<Operation> expectedOperations = new HashSet<>();
    expectedOperations.add(read);
    expectedOperations.add(anotherWrite);
    List<ProgramRunOperations> expected = new ArrayList<>();
    // Descending order of program execution
    expected.add(new ProgramRunOperations(Collections.singleton(spark1Run3), expectedOperations));
    expectedOperations = new HashSet<>();
    expectedOperations.add(read);
    expectedOperations.add(write);
    expected.add(new ProgramRunOperations(new HashSet<>(Arrays.asList(spark1Run1, spark1Run2)), expectedOperations));
    EndPointField endPointField = new EndPointField(EndPoint.of("ns", "endpoint2"), "offset");
    Tasks.waitFor(expected, () -> fieldLineageReader.getIncomingOperations(endPointField, 1L, Long.MAX_VALUE - 1), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
    // Verifies usage has been written
    Set<EntityId> expectedUsage = new HashSet<>(Arrays.asList(dataset1, dataset3));
    UsageRegistry usageRegistry = getInjector().getInstance(UsageRegistry.class);
    Tasks.waitFor(true, () -> expectedUsage.equals(usageRegistry.getDatasets(spark1)), 10, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
}
Also used : ReadOperation(io.cdap.cdap.api.lineage.field.ReadOperation) ProgramRunOperations(io.cdap.cdap.proto.metadata.lineage.ProgramRunOperations) UsageWriter(io.cdap.cdap.data2.registry.UsageWriter) MessagingUsageWriter(io.cdap.cdap.data2.registry.MessagingUsageWriter) FieldLineageReader(io.cdap.cdap.data2.metadata.lineage.field.FieldLineageReader) EndPointField(io.cdap.cdap.data2.metadata.lineage.field.EndPointField) UsageRegistry(io.cdap.cdap.data2.registry.UsageRegistry) ArrayList(java.util.ArrayList) ReadOperation(io.cdap.cdap.api.lineage.field.ReadOperation) WriteOperation(io.cdap.cdap.api.lineage.field.WriteOperation) MetadataOperation(io.cdap.cdap.data2.metadata.writer.MetadataOperation) TransformOperation(io.cdap.cdap.api.lineage.field.TransformOperation) Operation(io.cdap.cdap.api.lineage.field.Operation) TransformOperation(io.cdap.cdap.api.lineage.field.TransformOperation) NamespacedEntityId(io.cdap.cdap.proto.id.NamespacedEntityId) EntityId(io.cdap.cdap.proto.id.EntityId) NamespacedEntityId(io.cdap.cdap.proto.id.NamespacedEntityId) WriteOperation(io.cdap.cdap.api.lineage.field.WriteOperation) MessagingLineageWriter(io.cdap.cdap.data2.metadata.writer.MessagingLineageWriter) FieldLineageWriter(io.cdap.cdap.data2.metadata.writer.FieldLineageWriter) LineageWriter(io.cdap.cdap.data2.metadata.writer.LineageWriter) LineageStoreReader(io.cdap.cdap.data2.metadata.lineage.LineageStoreReader) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) FieldLineageInfo(io.cdap.cdap.data2.metadata.lineage.field.FieldLineageInfo) FieldLineageWriter(io.cdap.cdap.data2.metadata.writer.FieldLineageWriter) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 2 with UsageWriter

use of io.cdap.cdap.data2.registry.UsageWriter in project cdap by caskdata.

the class DistributedProgramContainerModule method configure.

@Override
protected void configure() {
    List<Module> modules = getCoreModules();
    RuntimeMonitorType runtimeMonitorType = SystemArguments.getRuntimeMonitorType(cConf, programOpts);
    modules.add(RuntimeMonitors.getRemoteAuthenticatorModule(runtimeMonitorType, programOpts));
    install(Modules.override(modules).with(new AbstractModule() {

        @Override
        protected void configure() {
            // Overrides the LineageWriter, UsageWriter to write to TMS instead
            bind(LineageWriter.class).to(MessagingLineageWriter.class);
            bind(FieldLineageWriter.class).to(MessagingLineageWriter.class);
            bind(UsageWriter.class).to(MessagingUsageWriter.class);
            // Overrides the metadata store to be no-op (programs never access it directly)
            bind(MetadataStorage.class).to(NoopMetadataStorage.class);
        }
    }));
    bind(RuntimeMonitorType.class).toInstance(runtimeMonitorType);
}
Also used : RuntimeMonitorType(io.cdap.cdap.runtime.spi.RuntimeMonitorType) UsageWriter(io.cdap.cdap.data2.registry.UsageWriter) MessagingUsageWriter(io.cdap.cdap.data2.registry.MessagingUsageWriter) FieldLineageWriter(io.cdap.cdap.data2.metadata.writer.FieldLineageWriter) LineageWriter(io.cdap.cdap.data2.metadata.writer.LineageWriter) MessagingLineageWriter(io.cdap.cdap.data2.metadata.writer.MessagingLineageWriter) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) NoopMetadataStorage(io.cdap.cdap.spi.metadata.noop.NoopMetadataStorage) Module(com.google.inject.Module) SecureStoreClientModule(io.cdap.cdap.security.guice.SecureStoreClientModule) ConfigModule(io.cdap.cdap.common.guice.ConfigModule) NamespaceQueryAdminModule(io.cdap.cdap.common.namespace.guice.NamespaceQueryAdminModule) SystemDatasetRuntimeModule(io.cdap.cdap.data.runtime.SystemDatasetRuntimeModule) CoreSecurityModule(io.cdap.cdap.security.guice.CoreSecurityModule) DFSLocationModule(io.cdap.cdap.common.guice.DFSLocationModule) KafkaLogAppenderModule(io.cdap.cdap.logging.guice.KafkaLogAppenderModule) MessagingClientModule(io.cdap.cdap.messaging.guice.MessagingClientModule) RemoteLogAppenderModule(io.cdap.cdap.logging.guice.RemoteLogAppenderModule) CoreSecurityRuntimeModule(io.cdap.cdap.security.guice.CoreSecurityRuntimeModule) AuthorizationEnforcementModule(io.cdap.cdap.security.authorization.AuthorizationEnforcementModule) KafkaClientModule(io.cdap.cdap.common.guice.KafkaClientModule) ZKClientModule(io.cdap.cdap.common.guice.ZKClientModule) TMSLogAppenderModule(io.cdap.cdap.logging.guice.TMSLogAppenderModule) AuditModule(io.cdap.cdap.data2.audit.AuditModule) IOModule(io.cdap.cdap.common.guice.IOModule) ZKDiscoveryModule(io.cdap.cdap.common.guice.ZKDiscoveryModule) MetricsClientRuntimeModule(io.cdap.cdap.metrics.guice.MetricsClientRuntimeModule) AbstractModule(com.google.inject.AbstractModule) AbstractModule(com.google.inject.AbstractModule) FieldLineageWriter(io.cdap.cdap.data2.metadata.writer.FieldLineageWriter)

Aggregations

FieldLineageWriter (io.cdap.cdap.data2.metadata.writer.FieldLineageWriter)2 LineageWriter (io.cdap.cdap.data2.metadata.writer.LineageWriter)2 MessagingLineageWriter (io.cdap.cdap.data2.metadata.writer.MessagingLineageWriter)2 MessagingUsageWriter (io.cdap.cdap.data2.registry.MessagingUsageWriter)2 UsageWriter (io.cdap.cdap.data2.registry.UsageWriter)2 AbstractModule (com.google.inject.AbstractModule)1 Module (com.google.inject.Module)1 Operation (io.cdap.cdap.api.lineage.field.Operation)1 ReadOperation (io.cdap.cdap.api.lineage.field.ReadOperation)1 TransformOperation (io.cdap.cdap.api.lineage.field.TransformOperation)1 WriteOperation (io.cdap.cdap.api.lineage.field.WriteOperation)1 ConfigModule (io.cdap.cdap.common.guice.ConfigModule)1 DFSLocationModule (io.cdap.cdap.common.guice.DFSLocationModule)1 IOModule (io.cdap.cdap.common.guice.IOModule)1 KafkaClientModule (io.cdap.cdap.common.guice.KafkaClientModule)1 ZKClientModule (io.cdap.cdap.common.guice.ZKClientModule)1 ZKDiscoveryModule (io.cdap.cdap.common.guice.ZKDiscoveryModule)1 NamespaceQueryAdminModule (io.cdap.cdap.common.namespace.guice.NamespaceQueryAdminModule)1 SystemDatasetRuntimeModule (io.cdap.cdap.data.runtime.SystemDatasetRuntimeModule)1 AuditModule (io.cdap.cdap.data2.audit.AuditModule)1