Examples with MetadataStore co.cask.cdap.data2.metadata.store.MetadataStore used on opensource projects

Example 11 with MetadataStore

use of co.cask.cdap.data2.metadata.store.MetadataStore in project cdap by caskdata.

the class FileStreamAdmin method updateProperties.

private StreamProperties updateProperties(StreamId streamId, StreamProperties properties) throws Exception {
    StreamConfig config = getConfig(streamId);
    StreamConfig.Builder builder = StreamConfig.builder(config);
    if (properties.getTTL() != null) {
        builder.setTTL(properties.getTTL());
    }
    if (properties.getFormat() != null) {
        builder.setFormatSpec(properties.getFormat());
    }
    if (properties.getNotificationThresholdMB() != null) {
        builder.setNotificationThreshold(properties.getNotificationThresholdMB());
    }
    // update stream description
    String description = properties.getDescription();
    if (description != null) {
        streamMetaStore.addStream(streamId, description);
    }
    final StreamConfig newConfig = builder.build();
    impersonator.doAs(streamId, new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            writeConfig(newConfig);
            return null;
        }
    });
    // Update system metadata for stream
    SystemMetadataWriter systemMetadataWriter = new StreamSystemMetadataWriter(metadataStore, streamId, newConfig, description);
    systemMetadataWriter.write();
    return new StreamProperties(config.getTTL(), config.getFormat(), config.getNotificationThresholdMB());
}

Example 12 with MetadataStore

use of co.cask.cdap.data2.metadata.store.MetadataStore in project cdap by caskdata.

the class ViewSystemMetadataWriterTest method getViewMetadataSchema.

private String getViewMetadataSchema(String format, @Nullable Schema schema) {
    StreamViewId viewId = NamespaceId.DEFAULT.stream("mystream").view("myview");
    FormatSpecification formatSpec = new FormatSpecification(format, schema);
    ViewSpecification viewSpec = new ViewSpecification(formatSpec);
    NoOpMetadataStore metadataStore = new NoOpMetadataStore();
    ViewSystemMetadataWriter writer = new ViewSystemMetadataWriter(metadataStore, viewId, viewSpec, false);
    return writer.getSchemaToAdd();
}

Example 13 with MetadataStore

use of co.cask.cdap.data2.metadata.store.MetadataStore in project cdap by caskdata.

the class BasicLineageWriterTest method testWrites.

@Test
public void testWrites() throws Exception {
    Injector injector = getInjector();
    MetadataStore metadataStore = injector.getInstance(MetadataStore.class);
    LineageStore lineageStore = injector.getInstance(LineageStore.class);
    LineageWriter lineageWriter = new BasicLineageWriter(lineageStore);
    // Define entities
    ProgramId program = new ProgramId(NamespaceId.DEFAULT.getNamespace(), "app", ProgramType.FLOW, "flow");
    StreamId stream = new StreamId(NamespaceId.DEFAULT.getNamespace(), "stream");
    ProgramRunId run1 = new ProgramRunId(program.getNamespace(), program.getApplication(), program.getType(), program.getEntityName(), RunIds.generate(10000).getId());
    ProgramRunId run2 = new ProgramRunId(program.getNamespace(), program.getApplication(), program.getType(), program.getEntityName(), RunIds.generate(20000).getId());
    // Tag stream
    metadataStore.addTags(MetadataScope.USER, stream, "stag1", "stag2");
    // Write access for run1
    lineageWriter.addAccess(run1, stream, AccessType.READ);
    Assert.assertEquals(ImmutableSet.of(program, stream), lineageStore.getEntitiesForRun(run1));
    // Record time to verify duplicate writes.
    long beforeSecondTag = System.currentTimeMillis();
    // Wait for next millisecond, since access time is stored in milliseconds.
    TimeUnit.MILLISECONDS.sleep(1);
    // Add another tag to stream
    metadataStore.addTags(MetadataScope.USER, stream, "stag3");
    // Write access for run1 again
    lineageWriter.addAccess(run1, stream, AccessType.READ);
    // The write should be no-op, and access time for run1 should not be updated
    Assert.assertTrue(lineageStore.getAccessTimesForRun(run1).get(0) < beforeSecondTag);
    // However, you can write access for another run
    lineageWriter.addAccess(run2, stream, AccessType.READ);
    // Assert new access time is written
    Assert.assertTrue(lineageStore.getAccessTimesForRun(run2).get(0) >= beforeSecondTag);
}

Example 14 with MetadataStore

use of co.cask.cdap.data2.metadata.store.MetadataStore in project cdap by caskdata.

the class SystemMetadataWriterStage method process.

@Override
public void process(ApplicationWithPrograms input) throws Exception {
    // add system metadata for apps
    ApplicationId appId = input.getApplicationId();
    ApplicationSpecification appSpec = input.getSpecification();
    // only update creation time if this is a new app
    Map<String, String> properties = metadataStore.getProperties(MetadataScope.SYSTEM, appId);
    SystemMetadataWriter appSystemMetadataWriter = new AppSystemMetadataWriter(metadataStore, appId, appSpec, !properties.isEmpty());
    appSystemMetadataWriter.write();
    // add system metadata for programs
    writeProgramSystemMetadata(appId, ProgramType.FLOW, appSpec.getFlows().values());
    writeProgramSystemMetadata(appId, ProgramType.MAPREDUCE, appSpec.getMapReduce().values());
    writeProgramSystemMetadata(appId, ProgramType.SERVICE, appSpec.getServices().values());
    writeProgramSystemMetadata(appId, ProgramType.SPARK, appSpec.getSpark().values());
    writeProgramSystemMetadata(appId, ProgramType.WORKER, appSpec.getWorkers().values());
    writeProgramSystemMetadata(appId, ProgramType.WORKFLOW, appSpec.getWorkflows().values());
    // Emit input to the next stage
    emit(input);
}

Example 15 with MetadataStore

use of co.cask.cdap.data2.metadata.store.MetadataStore in project cdap by caskdata.

the class SystemMetadataWriterStage method writeProgramSystemMetadata.

private void writeProgramSystemMetadata(ApplicationId appId, ProgramType programType, Iterable<? extends ProgramSpecification> specs) {
    for (ProgramSpecification spec : specs) {
        ProgramId programId = appId.program(programType, spec.getName());
        Map<String, String> properties = metadataStore.getProperties(MetadataScope.SYSTEM, programId);
        ProgramSystemMetadataWriter writer = new ProgramSystemMetadataWriter(metadataStore, programId, spec, !properties.isEmpty());
        writer.write();
    }
}