Search in sources :

Example 1 with DefaultNamespaceStore

use of io.cdap.cdap.store.DefaultNamespaceStore in project cdap by caskdata.

the class MetadataSubscriberService method preProcess.

@Override
protected void preProcess() {
    if (didBackfill) {
        return;
    }
    if (backfillAttempts > 10) {
        LOG.info("Skipping attempt to back-fill plugin metadata after 10 failures.");
        return;
    }
    // Backfill plugin metadata
    backfillAttempts++;
    LOG.info("Starting back-fill process(attempt {}) for plugin metadata", backfillAttempts);
    boolean updateFailed = false;
    NamespaceStore namespaceStore = new DefaultNamespaceStore(this.transactionRunner);
    List<String> namespaces = namespaceStore.list().stream().map(NamespaceMeta::getName).collect(Collectors.toList());
    LOG.debug("Back-filling plugin metadata for {} namespaces", namespaces.size());
    for (String namespace : namespaces) {
        List<ApplicationMeta> apps = TransactionRunners.run(this.transactionRunner, context -> {
            AppMetadataStore appMetadataStore = AppMetadataStore.create(context);
            return appMetadataStore.getAllApplications(namespace);
        });
        LOG.debug("Back-filling plugin metadata for namespace '{}' with {} applications", namespace, apps.size());
        try {
            this.getPluginCounts(namespace, apps);
        } catch (IOException e) {
            updateFailed = true;
            LOG.warn("Failed to write plugin metadata updates for namespace '{}': {}", namespace, e);
        }
    }
    if (!updateFailed) {
        LOG.info("Successfully back-filled plugin metadata for {} namespaces.", namespaces.size());
        didBackfill = true;
        TransactionRunners.run(transactionRunner, (TxRunnable) context -> AppMetadataStore.create(context).persistSubscriberState(getTopicId().getTopic(), BACKFILL_SUBSCRIBER_NAME, "true"));
    }
}
Also used : ImmutablePair(io.cdap.cdap.common.utils.ImmutablePair) TransactionRunners(io.cdap.cdap.spi.data.transaction.TransactionRunners) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Inject(com.google.inject.Inject) LoggerFactory(org.slf4j.LoggerFactory) RetryStrategies(io.cdap.cdap.common.service.RetryStrategies) UsageTable(io.cdap.cdap.data2.registry.UsageTable) GsonBuilder(com.google.gson.GsonBuilder) PluginId(io.cdap.cdap.proto.id.PluginId) DataAccessLineage(io.cdap.cdap.data2.metadata.writer.DataAccessLineage) EndPointField(io.cdap.cdap.data2.metadata.lineage.field.EndPointField) MetadataOperationTypeAdapter(io.cdap.cdap.data2.metadata.writer.MetadataOperationTypeAdapter) Gson(com.google.gson.Gson) InvalidMetadataException(io.cdap.cdap.common.InvalidMetadataException) Metadata(io.cdap.cdap.spi.metadata.Metadata) Map(java.util.Map) EntityType(io.cdap.cdap.proto.element.EntityType) MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) MetadataMutation(io.cdap.cdap.spi.metadata.MetadataMutation) MetadataMessage(io.cdap.cdap.data2.metadata.writer.MetadataMessage) BasicWorkflowToken(io.cdap.cdap.internal.app.runtime.workflow.BasicWorkflowToken) DefaultNamespaceStore(io.cdap.cdap.store.DefaultNamespaceStore) ImmutableMap(com.google.common.collect.ImmutableMap) ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) MessagingService(io.cdap.cdap.messaging.MessagingService) Set(java.util.Set) StructuredTableContext(io.cdap.cdap.spi.data.StructuredTableContext) Collectors(java.util.stream.Collectors) NamespaceMeta(io.cdap.cdap.proto.NamespaceMeta) List(java.util.List) MultiThreadMessagingContext(io.cdap.cdap.messaging.context.MultiThreadMessagingContext) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) MetadataOperation(io.cdap.cdap.data2.metadata.writer.MetadataOperation) TransactionRunner(io.cdap.cdap.spi.data.transaction.TransactionRunner) Constants(io.cdap.cdap.common.conf.Constants) MessagingContext(io.cdap.cdap.api.messaging.MessagingContext) TxCallable(io.cdap.cdap.spi.data.transaction.TxCallable) HashMap(java.util.HashMap) ApplicationMeta(io.cdap.cdap.internal.app.store.ApplicationMeta) EntityId(io.cdap.cdap.proto.id.EntityId) Function(java.util.function.Function) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) NamespaceStore(io.cdap.cdap.store.NamespaceStore) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) DatasetUsage(io.cdap.cdap.data2.registry.DatasetUsage) EntityIdTypeAdapter(io.cdap.cdap.proto.codec.EntityIdTypeAdapter) LineageTable(io.cdap.cdap.data2.metadata.lineage.LineageTable) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) Operation(io.cdap.cdap.api.lineage.field.Operation) Nullable(javax.annotation.Nullable) WorkflowNodeStateDetail(io.cdap.cdap.proto.WorkflowNodeStateDetail) AppMetadataStore(io.cdap.cdap.internal.app.store.AppMetadataStore) ProfileMetadataMessageProcessor(io.cdap.cdap.metadata.profile.ProfileMetadataMessageProcessor) FieldLineageTable(io.cdap.cdap.data2.metadata.lineage.field.FieldLineageTable) Logger(org.slf4j.Logger) Message(io.cdap.cdap.api.messaging.Message) Iterator(java.util.Iterator) ProgramId(io.cdap.cdap.proto.id.ProgramId) TxRunnable(io.cdap.cdap.spi.data.transaction.TxRunnable) TxConstants(org.apache.tephra.TxConstants) IOException(java.io.IOException) ConflictException(io.cdap.cdap.common.ConflictException) EndpointFieldDeserializer(io.cdap.cdap.data2.metadata.lineage.field.EndpointFieldDeserializer) AbstractMessagingSubscriberService(io.cdap.cdap.messaging.subscriber.AbstractMessagingSubscriberService) MetadataKind(io.cdap.cdap.spi.metadata.MetadataKind) OperationTypeAdapter(io.cdap.cdap.proto.codec.OperationTypeAdapter) TableNotFoundException(io.cdap.cdap.spi.data.TableNotFoundException) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) FieldLineageInfo(io.cdap.cdap.data2.metadata.lineage.field.FieldLineageInfo) MutationOptions(io.cdap.cdap.spi.metadata.MutationOptions) Collections(java.util.Collections) AppMetadataStore(io.cdap.cdap.internal.app.store.AppMetadataStore) DefaultNamespaceStore(io.cdap.cdap.store.DefaultNamespaceStore) NamespaceStore(io.cdap.cdap.store.NamespaceStore) DefaultNamespaceStore(io.cdap.cdap.store.DefaultNamespaceStore) IOException(java.io.IOException) ApplicationMeta(io.cdap.cdap.internal.app.store.ApplicationMeta)

Example 2 with DefaultNamespaceStore

use of io.cdap.cdap.store.DefaultNamespaceStore in project cdap by caskdata.

the class AppFabricServer method startUp.

/**
 * Configures the AppFabricService pre-start.
 */
@Override
protected void startUp() throws Exception {
    LoggingContextAccessor.setLoggingContext(new ServiceLoggingContext(NamespaceId.SYSTEM.getNamespace(), Constants.Logging.COMPONENT_NAME, Constants.Service.APP_FABRIC_HTTP));
    Futures.allAsList(ImmutableList.of(provisioningService.start(), applicationLifecycleService.start(), bootstrapService.start(), programRuntimeService.start(), programNotificationSubscriberService.start(), runRecordCorrectorService.start(), coreSchedulerService.start(), eventPublishManager.start(), runRecordCounterService.start())).get();
    // Create handler hooks
    List<MetricsReporterHook> handlerHooks = handlerHookNames.stream().map(name -> new MetricsReporterHook(metricsCollectionService, name)).collect(Collectors.toList());
    // Run http service on random port
    NettyHttpService.Builder httpServiceBuilder = new CommonNettyHttpServiceBuilder(cConf, Constants.Service.APP_FABRIC_HTTP).setHost(hostname.getCanonicalHostName()).setHandlerHooks(handlerHooks).setHttpHandlers(handlers).setConnectionBacklog(cConf.getInt(Constants.AppFabric.BACKLOG_CONNECTIONS, Constants.AppFabric.DEFAULT_BACKLOG)).setExecThreadPoolSize(cConf.getInt(Constants.AppFabric.EXEC_THREADS, Constants.AppFabric.DEFAULT_EXEC_THREADS)).setBossThreadPoolSize(cConf.getInt(Constants.AppFabric.BOSS_THREADS, Constants.AppFabric.DEFAULT_BOSS_THREADS)).setWorkerThreadPoolSize(cConf.getInt(Constants.AppFabric.WORKER_THREADS, Constants.AppFabric.DEFAULT_WORKER_THREADS)).setPort(cConf.getInt(Constants.AppFabric.SERVER_PORT));
    if (sslEnabled) {
        new HttpsEnabler().configureKeyStore(cConf, sConf).enable(httpServiceBuilder);
    }
    cancelHttpService = startHttpService(httpServiceBuilder.build());
    long applicationCount = TransactionRunners.run(transactionRunner, (TxCallable<Long>) context -> AppMetadataStore.create(context).getApplicationCount());
    long namespaceCount = new DefaultNamespaceStore(transactionRunner).getNamespaceCount();
    metricsCollectionService.getContext(Collections.emptyMap()).gauge(Constants.Metrics.Program.APPLICATION_COUNT, applicationCount);
    metricsCollectionService.getContext(Collections.emptyMap()).gauge(Constants.Metrics.Program.NAMESPACE_COUNT, namespaceCount);
}
Also used : HttpsEnabler(io.cdap.cdap.common.security.HttpsEnabler) ResolvingDiscoverable(io.cdap.cdap.common.discovery.ResolvingDiscoverable) TransactionRunners(io.cdap.cdap.spi.data.transaction.TransactionRunners) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Inject(com.google.inject.Inject) MetricsReporterHook(io.cdap.cdap.common.metrics.MetricsReporterHook) LoggerFactory(org.slf4j.LoggerFactory) TxCallable(io.cdap.cdap.spi.data.transaction.TxCallable) ArrayList(java.util.ArrayList) InetAddress(java.net.InetAddress) ProvisioningService(io.cdap.cdap.internal.provision.ProvisioningService) ImmutableList(com.google.common.collect.ImmutableList) AbstractIdleService(com.google.common.util.concurrent.AbstractIdleService) NettyHttpService(io.cdap.http.NettyHttpService) Cancellable(org.apache.twill.common.Cancellable) Nullable(javax.annotation.Nullable) DiscoveryService(org.apache.twill.discovery.DiscoveryService) AppMetadataStore(io.cdap.cdap.internal.app.store.AppMetadataStore) DefaultNamespaceStore(io.cdap.cdap.store.DefaultNamespaceStore) Logger(org.slf4j.Logger) URIScheme(io.cdap.cdap.common.discovery.URIScheme) Set(java.util.Set) LoggingContextAccessor(io.cdap.cdap.common.logging.LoggingContextAccessor) InetSocketAddress(java.net.InetSocketAddress) Collectors(java.util.stream.Collectors) ProgramRuntimeService(io.cdap.cdap.app.runtime.ProgramRuntimeService) CoreSchedulerService(io.cdap.cdap.scheduler.CoreSchedulerService) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService) SystemAppManagementService(io.cdap.cdap.internal.sysapp.SystemAppManagementService) HttpHandler(io.cdap.http.HttpHandler) Futures(com.google.common.util.concurrent.Futures) CommonNettyHttpServiceBuilder(io.cdap.cdap.common.http.CommonNettyHttpServiceBuilder) List(java.util.List) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) TransactionRunner(io.cdap.cdap.spi.data.transaction.TransactionRunner) Named(com.google.inject.name.Named) Constants(io.cdap.cdap.common.conf.Constants) BootstrapService(io.cdap.cdap.internal.bootstrap.BootstrapService) EventPublishManager(io.cdap.cdap.internal.events.EventPublishManager) Collections(java.util.Collections) ServiceLoggingContext(io.cdap.cdap.common.logging.ServiceLoggingContext) SConfiguration(io.cdap.cdap.common.conf.SConfiguration) MetricsReporterHook(io.cdap.cdap.common.metrics.MetricsReporterHook) CommonNettyHttpServiceBuilder(io.cdap.cdap.common.http.CommonNettyHttpServiceBuilder) NettyHttpService(io.cdap.http.NettyHttpService) DefaultNamespaceStore(io.cdap.cdap.store.DefaultNamespaceStore) HttpsEnabler(io.cdap.cdap.common.security.HttpsEnabler) ServiceLoggingContext(io.cdap.cdap.common.logging.ServiceLoggingContext)

Example 3 with DefaultNamespaceStore

use of io.cdap.cdap.store.DefaultNamespaceStore in project cdap by caskdata.

the class SqlDefaultStoreTest method beforeClass.

@BeforeClass
public static void beforeClass() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    pg = PostgresInstantiator.createAndStart(cConf, TEMP_FOLDER.newFolder());
    Injector injector = AppFabricTestHelper.getInjector(cConf);
    StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
    TransactionRunner transactionRunner = injector.getInstance(TransactionRunner.class);
    store = new DefaultStore(transactionRunner);
    nsStore = new DefaultNamespaceStore(transactionRunner);
    nsAdmin = new DefaultNamespaceAdmin(nsStore, store, injector.getInstance(DatasetFramework.class), injector.getInstance(MetricsCollectionService.class), injector.getProvider(NamespaceResourceDeleter.class), injector.getProvider(StorageProviderNamespaceAdmin.class), injector.getInstance(CConfiguration.class), injector.getInstance(Impersonator.class), injector.getInstance(AccessEnforcer.class), injector.getInstance(AuthenticationContext.class));
}
Also used : Injector(com.google.inject.Injector) StructuredTableAdmin(io.cdap.cdap.spi.data.StructuredTableAdmin) TransactionRunner(io.cdap.cdap.spi.data.transaction.TransactionRunner) DefaultNamespaceStore(io.cdap.cdap.store.DefaultNamespaceStore) DefaultNamespaceAdmin(io.cdap.cdap.internal.app.namespace.DefaultNamespaceAdmin) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) BeforeClass(org.junit.BeforeClass)

Aggregations

CConfiguration (io.cdap.cdap.common.conf.CConfiguration)3 Inject (com.google.inject.Inject)2 MetricsCollectionService (io.cdap.cdap.api.metrics.MetricsCollectionService)2 Constants (io.cdap.cdap.common.conf.Constants)2 AppMetadataStore (io.cdap.cdap.internal.app.store.AppMetadataStore)2 TransactionRunner (io.cdap.cdap.spi.data.transaction.TransactionRunner)2 DefaultNamespaceStore (io.cdap.cdap.store.DefaultNamespaceStore)2 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 AbstractIdleService (com.google.common.util.concurrent.AbstractIdleService)1 Futures (com.google.common.util.concurrent.Futures)1 Gson (com.google.gson.Gson)1 GsonBuilder (com.google.gson.GsonBuilder)1 Injector (com.google.inject.Injector)1 Named (com.google.inject.name.Named)1 ApplicationSpecification (io.cdap.cdap.api.app.ApplicationSpecification)1 Operation (io.cdap.cdap.api.lineage.field.Operation)1 Message (io.cdap.cdap.api.messaging.Message)1 MessagingContext (io.cdap.cdap.api.messaging.MessagingContext)1 MetadataEntity (io.cdap.cdap.api.metadata.MetadataEntity)1