Search in sources :

Example 1 with AppMetadataStore

use of io.cdap.cdap.internal.app.store.AppMetadataStore in project cdap by caskdata.

the class PreviewTMSLogSubscriber method storeMessageId.

@Override
protected void storeMessageId(StructuredTableContext context, String messageId) throws Exception {
    AppMetadataStore appMetadataStore = AppMetadataStore.create(context);
    appMetadataStore.persistSubscriberState(getTopicId().getTopic(), CONSUMER_NAME, messageId);
}
Also used : AppMetadataStore(io.cdap.cdap.internal.app.store.AppMetadataStore)

Example 2 with AppMetadataStore

use of io.cdap.cdap.internal.app.store.AppMetadataStore in project cdap by caskdata.

the class MetadataSubscriberService method preProcess.

@Override
protected void preProcess() {
    if (didBackfill) {
        return;
    }
    if (backfillAttempts > 10) {
        LOG.info("Skipping attempt to back-fill plugin metadata after 10 failures.");
        return;
    }
    // Backfill plugin metadata
    backfillAttempts++;
    LOG.info("Starting back-fill process(attempt {}) for plugin metadata", backfillAttempts);
    boolean updateFailed = false;
    NamespaceStore namespaceStore = new DefaultNamespaceStore(this.transactionRunner);
    List<String> namespaces = namespaceStore.list().stream().map(NamespaceMeta::getName).collect(Collectors.toList());
    LOG.debug("Back-filling plugin metadata for {} namespaces", namespaces.size());
    for (String namespace : namespaces) {
        List<ApplicationMeta> apps = TransactionRunners.run(this.transactionRunner, context -> {
            AppMetadataStore appMetadataStore = AppMetadataStore.create(context);
            return appMetadataStore.getAllApplications(namespace);
        });
        LOG.debug("Back-filling plugin metadata for namespace '{}' with {} applications", namespace, apps.size());
        try {
            this.getPluginCounts(namespace, apps);
        } catch (IOException e) {
            updateFailed = true;
            LOG.warn("Failed to write plugin metadata updates for namespace '{}': {}", namespace, e);
        }
    }
    if (!updateFailed) {
        LOG.info("Successfully back-filled plugin metadata for {} namespaces.", namespaces.size());
        didBackfill = true;
        TransactionRunners.run(transactionRunner, (TxRunnable) context -> AppMetadataStore.create(context).persistSubscriberState(getTopicId().getTopic(), BACKFILL_SUBSCRIBER_NAME, "true"));
    }
}
Also used : ImmutablePair(io.cdap.cdap.common.utils.ImmutablePair) TransactionRunners(io.cdap.cdap.spi.data.transaction.TransactionRunners) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Inject(com.google.inject.Inject) LoggerFactory(org.slf4j.LoggerFactory) RetryStrategies(io.cdap.cdap.common.service.RetryStrategies) UsageTable(io.cdap.cdap.data2.registry.UsageTable) GsonBuilder(com.google.gson.GsonBuilder) PluginId(io.cdap.cdap.proto.id.PluginId) DataAccessLineage(io.cdap.cdap.data2.metadata.writer.DataAccessLineage) EndPointField(io.cdap.cdap.data2.metadata.lineage.field.EndPointField) MetadataOperationTypeAdapter(io.cdap.cdap.data2.metadata.writer.MetadataOperationTypeAdapter) Gson(com.google.gson.Gson) InvalidMetadataException(io.cdap.cdap.common.InvalidMetadataException) Metadata(io.cdap.cdap.spi.metadata.Metadata) Map(java.util.Map) EntityType(io.cdap.cdap.proto.element.EntityType) MetadataEntity(io.cdap.cdap.api.metadata.MetadataEntity) MetadataMutation(io.cdap.cdap.spi.metadata.MetadataMutation) MetadataMessage(io.cdap.cdap.data2.metadata.writer.MetadataMessage) BasicWorkflowToken(io.cdap.cdap.internal.app.runtime.workflow.BasicWorkflowToken) DefaultNamespaceStore(io.cdap.cdap.store.DefaultNamespaceStore) ImmutableMap(com.google.common.collect.ImmutableMap) ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) MessagingService(io.cdap.cdap.messaging.MessagingService) Set(java.util.Set) StructuredTableContext(io.cdap.cdap.spi.data.StructuredTableContext) Collectors(java.util.stream.Collectors) NamespaceMeta(io.cdap.cdap.proto.NamespaceMeta) List(java.util.List) MultiThreadMessagingContext(io.cdap.cdap.messaging.context.MultiThreadMessagingContext) MetadataScope(io.cdap.cdap.api.metadata.MetadataScope) MetadataOperation(io.cdap.cdap.data2.metadata.writer.MetadataOperation) TransactionRunner(io.cdap.cdap.spi.data.transaction.TransactionRunner) Constants(io.cdap.cdap.common.conf.Constants) MessagingContext(io.cdap.cdap.api.messaging.MessagingContext) TxCallable(io.cdap.cdap.spi.data.transaction.TxCallable) HashMap(java.util.HashMap) ApplicationMeta(io.cdap.cdap.internal.app.store.ApplicationMeta) EntityId(io.cdap.cdap.proto.id.EntityId) Function(java.util.function.Function) MetadataStorage(io.cdap.cdap.spi.metadata.MetadataStorage) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) NamespaceStore(io.cdap.cdap.store.NamespaceStore) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) DatasetUsage(io.cdap.cdap.data2.registry.DatasetUsage) EntityIdTypeAdapter(io.cdap.cdap.proto.codec.EntityIdTypeAdapter) LineageTable(io.cdap.cdap.data2.metadata.lineage.LineageTable) ScopedNameOfKind(io.cdap.cdap.spi.metadata.ScopedNameOfKind) Operation(io.cdap.cdap.api.lineage.field.Operation) Nullable(javax.annotation.Nullable) WorkflowNodeStateDetail(io.cdap.cdap.proto.WorkflowNodeStateDetail) AppMetadataStore(io.cdap.cdap.internal.app.store.AppMetadataStore) ProfileMetadataMessageProcessor(io.cdap.cdap.metadata.profile.ProfileMetadataMessageProcessor) FieldLineageTable(io.cdap.cdap.data2.metadata.lineage.field.FieldLineageTable) Logger(org.slf4j.Logger) Message(io.cdap.cdap.api.messaging.Message) Iterator(java.util.Iterator) ProgramId(io.cdap.cdap.proto.id.ProgramId) TxRunnable(io.cdap.cdap.spi.data.transaction.TxRunnable) TxConstants(org.apache.tephra.TxConstants) IOException(java.io.IOException) ConflictException(io.cdap.cdap.common.ConflictException) EndpointFieldDeserializer(io.cdap.cdap.data2.metadata.lineage.field.EndpointFieldDeserializer) AbstractMessagingSubscriberService(io.cdap.cdap.messaging.subscriber.AbstractMessagingSubscriberService) MetadataKind(io.cdap.cdap.spi.metadata.MetadataKind) OperationTypeAdapter(io.cdap.cdap.proto.codec.OperationTypeAdapter) TableNotFoundException(io.cdap.cdap.spi.data.TableNotFoundException) MetricsCollectionService(io.cdap.cdap.api.metrics.MetricsCollectionService) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) FieldLineageInfo(io.cdap.cdap.data2.metadata.lineage.field.FieldLineageInfo) MutationOptions(io.cdap.cdap.spi.metadata.MutationOptions) Collections(java.util.Collections) AppMetadataStore(io.cdap.cdap.internal.app.store.AppMetadataStore) DefaultNamespaceStore(io.cdap.cdap.store.DefaultNamespaceStore) NamespaceStore(io.cdap.cdap.store.NamespaceStore) DefaultNamespaceStore(io.cdap.cdap.store.DefaultNamespaceStore) IOException(java.io.IOException) ApplicationMeta(io.cdap.cdap.internal.app.store.ApplicationMeta)

Example 3 with AppMetadataStore

use of io.cdap.cdap.internal.app.store.AppMetadataStore in project cdap by caskdata.

the class TetheringAgentService method initializeMessageIds.

private void initializeMessageIds() {
    List<String> peers;
    try {
        peers = store.getPeers().stream().map(PeerInfo::getName).collect(Collectors.toList());
    } catch (IOException e) {
        LOG.warn("Failed to get peer information", e);
        return;
    }
    TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore appMetadataStore = AppMetadataStore.create(context);
        for (String peer : peers) {
            String messageId = appMetadataStore.retrieveSubscriberState(peer, SUBSCRIBER);
            lastMessageIds.put(peer, messageId);
        }
    });
}
Also used : AppMetadataStore(io.cdap.cdap.internal.app.store.AppMetadataStore) IOException(java.io.IOException)

Example 4 with AppMetadataStore

use of io.cdap.cdap.internal.app.store.AppMetadataStore in project cdap by caskdata.

the class TetheringAgentService method runTask.

@Override
protected long runTask() {
    List<PeerInfo> peers;
    try {
        peers = store.getPeers().stream().filter(p -> p.getTetheringStatus() != TetheringStatus.REJECTED).collect(Collectors.toList());
    } catch (IOException e) {
        LOG.warn("Failed to get peer information", e);
        return connectionInterval;
    }
    for (PeerInfo peer : peers) {
        try {
            Preconditions.checkArgument(peer.getEndpoint() != null, "Peer %s doesn't have an endpoint", peer.getName());
            String uri = CONNECT_CONTROL_CHANNEL + instanceName;
            String lastMessageId = lastMessageIds.get(peer);
            if (lastMessageId != null) {
                uri = uri + "?messageId=" + URLEncoder.encode(lastMessageId, "UTF-8");
            }
            HttpResponse resp = TetheringUtils.sendHttpRequest(remoteAuthenticator, HttpMethod.GET, new URI(peer.getEndpoint()).resolve(uri));
            switch(resp.getResponseCode()) {
                case HttpURLConnection.HTTP_OK:
                    handleResponse(resp, peer);
                    break;
                case HttpURLConnection.HTTP_NOT_FOUND:
                    handleNotFound(peer);
                    break;
                case HttpURLConnection.HTTP_FORBIDDEN:
                    handleForbidden(peer);
                    break;
                default:
                    LOG.error("Peer {} returned unexpected error code {} body {}", peer.getName(), resp.getResponseCode(), resp.getResponseBodyAsString(StandardCharsets.UTF_8));
            }
        } catch (Exception e) {
            LOG.debug("Failed to create control channel to {}", peer, e);
        }
    }
    // Update last message ids in the store for all peers
    TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore appMetadataStore = AppMetadataStore.create(context);
        for (Map.Entry<String, String> entry : lastMessageIds.entrySet()) {
            appMetadataStore.persistSubscriberState(entry.getKey(), SUBSCRIBER, entry.getValue());
        }
    });
    return connectionInterval;
}
Also used : AppMetadataStore(io.cdap.cdap.internal.app.store.AppMetadataStore) HttpResponse(io.cdap.common.http.HttpResponse) IOException(java.io.IOException) URI(java.net.URI) HashMap(java.util.HashMap) Map(java.util.Map) URISyntaxException(java.net.URISyntaxException) TopicAlreadyExistsException(io.cdap.cdap.api.messaging.TopicAlreadyExistsException) IOException(java.io.IOException) TopicNotFoundException(io.cdap.cdap.api.messaging.TopicNotFoundException)

Example 5 with AppMetadataStore

use of io.cdap.cdap.internal.app.store.AppMetadataStore in project cdap by caskdata.

the class DirectRuntimeRequestValidatorTest method testValidProgramInStoppingState.

@Test
public void testValidProgramInStoppingState() throws BadRequestException {
    ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").spark("spark").run(RunIds.generate());
    // Insert the run
    TransactionRunners.run(txRunner, context -> {
        AppMetadataStore store = AppMetadataStore.create(context);
        store.recordProgramProvisioning(programRunId, Collections.emptyMap(), Collections.singletonMap(SystemArguments.PROFILE_NAME, "system:default"), createSourceId(1), ARTIFACT_ID);
        store.recordProgramProvisioned(programRunId, 1, createSourceId(2));
        store.recordProgramStart(programRunId, null, Collections.emptyMap(), createSourceId(3));
        store.recordProgramRunning(programRunId, System.currentTimeMillis(), null, createSourceId(3));
        store.recordProgramStopping(programRunId, createSourceId(3), System.currentTimeMillis(), System.currentTimeMillis() + 1000);
    });
    // Validation should pass
    RuntimeRequestValidator validator = new DirectRuntimeRequestValidator(cConf, txRunner, new MockProgramRunRecordFetcher(), accessEnforcer, authenticationContext);
    ProgramRunInfo programRunInfo = validator.getProgramRunStatus(programRunId, new DefaultHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/"));
    // After recording the program start in AppMetaDataStore the expected state is Stopping
    Assert.assertEquals(ProgramRunStatus.STOPPING, programRunInfo.getProgramRunStatus());
    Assert.assertNotNull("Payload isn't null when program status is Stopping", programRunInfo.getPayload());
}
Also used : AppMetadataStore(io.cdap.cdap.internal.app.store.AppMetadataStore) DefaultHttpRequest(io.netty.handler.codec.http.DefaultHttpRequest) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) Test(org.junit.Test)

Aggregations

AppMetadataStore (io.cdap.cdap.internal.app.store.AppMetadataStore)19 ProgramRunId (io.cdap.cdap.proto.id.ProgramRunId)10 ProgramId (io.cdap.cdap.proto.id.ProgramId)6 IOException (java.io.IOException)6 ArrayList (java.util.ArrayList)6 HashMap (java.util.HashMap)6 Map (java.util.Map)6 Test (org.junit.Test)6 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)5 Constants (io.cdap.cdap.common.conf.Constants)5 ProfileId (io.cdap.cdap.proto.id.ProfileId)5 TransactionRunner (io.cdap.cdap.spi.data.transaction.TransactionRunner)5 Collections (java.util.Collections)5 ImmutableMap (com.google.common.collect.ImmutableMap)4 RunRecordDetail (io.cdap.cdap.internal.app.store.RunRecordDetail)4 ProgramRunStatus (io.cdap.cdap.proto.ProgramRunStatus)4 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)4 Profile (io.cdap.cdap.proto.profile.Profile)4 Gson (com.google.gson.Gson)3 Inject (com.google.inject.Inject)3