Examples with MetadataServiceClient io.cdap.cdap.data2.metadata.writer.MetadataServiceClient used on opensource projects

Example 6 with MetadataServiceClient

use of io.cdap.cdap.data2.metadata.writer.MetadataServiceClient in project cdap by caskdata.

the class DatasetInstanceService method publishMetadata.

private void publishMetadata(DatasetId dataset, SystemMetadata metadata) {
    if (metadata != null && !metadata.isEmpty()) {
        SystemMetadataWriter metadataWriter = new DelegateSystemMetadataWriter(metadataServiceClient, dataset, metadata);
        metadataWriter.write();
    }
}

Example 7 with MetadataServiceClient

use of io.cdap.cdap.data2.metadata.writer.MetadataServiceClient in project cdap by caskdata.

the class AppFabricTestBase method initializeAndStartServices.

protected static void initializeAndStartServices(CConfiguration cConf, Module overrides) throws Exception {
    injector = Guice.createInjector(Modules.override(new AppFabricTestModule(cConf, null)).with(overrides));
    int connectionTimeout = cConf.getInt(Constants.HTTP_CLIENT_CONNECTION_TIMEOUT_MS);
    int readTimeout = cConf.getInt(Constants.HTTP_CLIENT_READ_TIMEOUT_MS);
    httpRequestConfig = new HttpRequestConfig(connectionTimeout, readTimeout, false);
    messagingService = injector.getInstance(MessagingService.class);
    if (messagingService instanceof Service) {
        ((Service) messagingService).startAndWait();
    }
    txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();
    // Define all StructuredTable before starting any services that need StructuredTable
    StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
    metadataStorage = injector.getInstance(MetadataStorage.class);
    metadataStorage.createIndex();
    dsOpService = injector.getInstance(DatasetOpExecutorService.class);
    dsOpService.startAndWait();
    datasetService = injector.getInstance(DatasetService.class);
    datasetService.startAndWait();
    appFabricServer = injector.getInstance(AppFabricServer.class);
    appFabricServer.startAndWait();
    DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class);
    appFabricEndpointStrategy = new RandomEndpointStrategy(() -> discoveryClient.discover(Constants.Service.APP_FABRIC_HTTP));
    txClient = injector.getInstance(TransactionSystemClient.class);
    metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
    metricsCollectionService.startAndWait();
    serviceStore = injector.getInstance(ServiceStore.class);
    serviceStore.startAndWait();
    metadataService = injector.getInstance(MetadataService.class);
    metadataService.startAndWait();
    metadataSubscriberService = injector.getInstance(MetadataSubscriberService.class);
    metadataSubscriberService.startAndWait();
    logQueryService = injector.getInstance(LogQueryService.class);
    logQueryService.startAndWait();
    locationFactory = getInjector().getInstance(LocationFactory.class);
    datasetClient = new DatasetClient(getClientConfig(discoveryClient, Constants.Service.DATASET_MANAGER));
    remoteClientFactory = new RemoteClientFactory(discoveryClient, new DefaultInternalAuthenticator(new AuthenticationTestContext()));
    metadataClient = new MetadataClient(getClientConfig(discoveryClient, Constants.Service.METADATA_SERVICE));
    metadataServiceClient = new DefaultMetadataServiceClient(remoteClientFactory);
    metricStore = injector.getInstance(MetricStore.class);
    Scheduler programScheduler = injector.getInstance(Scheduler.class);
    // Wait for the scheduler to be functional.
    if (programScheduler instanceof CoreSchedulerService) {
        try {
            ((CoreSchedulerService) programScheduler).waitUntilFunctional(10, TimeUnit.SECONDS);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
    createNamespaces();
}

Example 8 with MetadataServiceClient

use of io.cdap.cdap.data2.metadata.writer.MetadataServiceClient in project cdap by caskdata.

the class MetadataWriterStage method collectProgramSystemMetadata.

private void collectProgramSystemMetadata(ApplicationId appId, ProgramType programType, Iterable<? extends ProgramSpecification> specs, List<MetadataMutation> mutations) {
    for (ProgramSpecification spec : specs) {
        ProgramId programId = appId.program(programType, spec.getName());
        mutations.add(new ProgramSystemMetadataWriter(metadataServiceClient, programId, spec, creationTime).getMetadataMutation());
    }
}

Example 9 with MetadataServiceClient

use of io.cdap.cdap.data2.metadata.writer.MetadataServiceClient in project cdap by caskdata.

the class MetadataWriterStage method process.

@Override
public void process(ApplicationWithPrograms input) {
    // use current time as creation time for app and all programs
    creationTime = String.valueOf(System.currentTimeMillis());
    // add system metadata for apps
    ApplicationId appId = input.getApplicationId();
    ApplicationSpecification appSpec = input.getSpecification();
    List<MetadataMutation> mutations = new ArrayList<>();
    // get the system metadata and combine it with our own system metadata, metadata creation mutation will
    // completely override the previous metadata in the scope, so we have to combine these together
    Metadata systemAppMetadata = input.getMetadata().get(MetadataScope.SYSTEM);
    mutations.add(new AppSystemMetadataWriter(metadataServiceClient, appId, appSpec, input.getApplicationClass(), creationTime, systemAppMetadata).getMetadataMutation());
    // collect system metadata for programs
    collectProgramSystemMetadata(appId, ProgramType.MAPREDUCE, appSpec.getMapReduce().values(), mutations);
    collectProgramSystemMetadata(appId, ProgramType.SERVICE, appSpec.getServices().values(), mutations);
    collectProgramSystemMetadata(appId, ProgramType.SPARK, appSpec.getSpark().values(), mutations);
    collectProgramSystemMetadata(appId, ProgramType.WORKER, appSpec.getWorkers().values(), mutations);
    collectProgramSystemMetadata(appId, ProgramType.WORKFLOW, appSpec.getWorkflows().values(), mutations);
    // add the rest user defined metadata
    Metadata userAppMetadata = input.getMetadata().get(MetadataScope.USER);
    if (userAppMetadata != null && (!userAppMetadata.getProperties().isEmpty() || !userAppMetadata.getTags().isEmpty())) {
        mutations.add(new MetadataMutation.Create(appId.toMetadataEntity(), new io.cdap.cdap.spi.metadata.Metadata(MetadataScope.USER, userAppMetadata.getTags(), userAppMetadata.getProperties()), Collections.emptyMap()));
    }
    // write all metadata
    metadataServiceClient.batch(mutations);
    // Emit input to the next stage
    emit(input);
}

Example 10 with MetadataServiceClient

use of io.cdap.cdap.data2.metadata.writer.MetadataServiceClient in project cdap by cdapio.

the class DefaultArtifactRepository method writeSystemMetadata.

private void writeSystemMetadata(io.cdap.cdap.proto.id.ArtifactId artifactId, ArtifactInfo artifactInfo) {
    // add system metadata for artifacts
    ArtifactSystemMetadataWriter writer = new ArtifactSystemMetadataWriter(metadataServiceClient, artifactId, artifactInfo);
    writer.write();
}