Examples with AckReplyConsumer com.google.cloud.pubsub.v1.AckReplyConsumer used on opensource projects

Example 11 with AckReplyConsumer

use of com.google.cloud.pubsub.v1.AckReplyConsumer in project spring-cloud-gcp by spring-cloud.

the class PubSubChannelAdaptersIntegrationTests method sendAndReceiveMessageManualAckThroughAcknowledgementHeader.

@Test
@SuppressWarnings("deprecation")
public void sendAndReceiveMessageManualAckThroughAcknowledgementHeader() {
    this.contextRunner.withUserConfiguration(PollableConfiguration.class, CommonConfiguration.class).run((context) -> {
        context.getBean(PubSubInboundChannelAdapter.class).setAckMode(AckMode.MANUAL);
        context.getBean("inputChannel", MessageChannel.class).send(MessageBuilder.withPayload("I am a message (sendAndReceiveMessageManualAckThroughAcknowledgementHeader).".getBytes()).build());
        PollableChannel channel = context.getBean("outputChannel", PollableChannel.class);
        Message<?> message = channel.receive(RECEIVE_TIMEOUT_MS);
        assertThat(message).isNotNull();
        AckReplyConsumer acker = (AckReplyConsumer) message.getHeaders().get(GcpPubSubHeaders.ACKNOWLEDGEMENT);
        assertThat(acker).isNotNull();
        acker.ack();
        message = channel.receive(RECEIVE_TIMEOUT_MS);
        assertThat(message).isNull();
        assertThat(this.outputCaptureRule.getOut()).contains("ACKNOWLEDGEMENT header is deprecated");
    });
}

Example 12 with AckReplyConsumer

use of com.google.cloud.pubsub.v1.AckReplyConsumer in project curiostack by curioswitch.

the class Subscriber method onNext.

@Override
public void onNext(StreamingPullResponse value) {
    if (ctx == null) {
        ctx = RequestContext.current();
    }
    streamReconnectBackoff = INITIAL_CHANNEL_RECONNECT_BACKOFF;
    receivedMessages.increment(value.getReceivedMessagesCount());
    AtomicInteger pendingAcks = new AtomicInteger(value.getReceivedMessagesCount());
    for (ReceivedMessage message : value.getReceivedMessagesList()) {
        TraceContextOrSamplingFlags contextOrFlags = traceExtractor.extract(message.getMessage());
        // Add an artificial span modeling the time spent within Pub/Sub until getting here.
        Span span = contextOrFlags.context() != null ? tracer.joinSpan(contextOrFlags.context()) : // entire stream.
        tracer.newTrace();
        span.kind(Kind.SERVER).name("google.pubsub.v1.Publisher.Publish").tag("subscription", options.getSubscription()).start(Timestamps.toMicros(message.getMessage().getPublishTime())).finish();
        StreamObserver<StreamingPullRequest> requestObserver = this.requestObserver;
        long startTimeNanos = System.nanoTime();
        options.getMessageReceiver().receiveMessage(message.getMessage(), new AckReplyConsumer() {

            @Override
            public void ack() {
                releaseAndRecord();
                ackedMessages.increment();
                checkNotNull(requestObserver, "onNext called before start()");
                requestObserver.onNext(StreamingPullRequest.newBuilder().addAckIds(message.getAckId()).build());
            }

            @Override
            public void nack() {
                releaseAndRecord();
                nackedMessages.increment();
                checkNotNull(requestObserver, "onNext called before start()");
                requestObserver.onNext(StreamingPullRequest.newBuilder().addModifyDeadlineAckIds(message.getAckId()).addModifyDeadlineSeconds(0).build());
            }

            private void releaseAndRecord() {
                if (options.getUnsafeWrapBuffers() && pendingAcks.decrementAndGet() == 0) {
                    GrpcUnsafeBufferUtil.releaseBuffer(value, ctx);
                }
                messageProcessingTime.record(Duration.ofNanos(System.nanoTime() - startTimeNanos));
            }
        });
    }
}

Example 13 with AckReplyConsumer

use of com.google.cloud.pubsub.v1.AckReplyConsumer in project google-cloud-java by GoogleCloudPlatform.

the class SubscriberSnippets method createSubscriber.

private void createSubscriber() throws Exception {
    // [START pubsub_pull]
    String projectId = "my-project-id";
    String subscriptionId = "my-subscription-id";
    SubscriptionName subscriptionName = SubscriptionName.create(projectId, subscriptionId);
    // Instantiate an asynchronous message receiver
    MessageReceiver receiver = new MessageReceiver() {

        @Override
        public void receiveMessage(PubsubMessage message, AckReplyConsumer consumer) {
            // handle incoming message, then ack/nack the received message
            System.out.println("Id : " + message.getMessageId());
            System.out.println("Data : " + message.getData().toStringUtf8());
            consumer.ack();
        }
    };
    Subscriber subscriber = null;
    try {
        // Create a subscriber for "my-subscription-id" bound to the message receiver
        subscriber = Subscriber.defaultBuilder(subscriptionName, receiver).build();
        subscriber.startAsync();
    // ...
    } finally {
        // stop receiving messages
        if (subscriber != null) {
            subscriber.stopAsync();
        }
    }
// [END pubsub_pull]
}

Example 14 with AckReplyConsumer

use of com.google.cloud.pubsub.v1.AckReplyConsumer in project java-docs-samples by GoogleCloudPlatform.

the class Inspect method inspectGcsFile.

// [END dlp_inspect_file]
// [START dlp_inspect_gcs]
/**
 * Inspect GCS file for Info types and wait on job completion using Google Cloud Pub/Sub
 * notification
 *
 * @param bucketName The name of the bucket where the file resides.
 * @param fileName The path to the file within the bucket to inspect (can include wildcards, eg.
 *     my-image.*)
 * @param minLikelihood The minimum likelihood required before returning a match
 * @param infoTypes The infoTypes of information to match
 * @param maxFindings The maximum number of findings to report (0 = server maximum)
 * @param topicId Google Cloud Pub/Sub topic Id to notify of job status
 * @param subscriptionId Google Cloud Subscription to above topic to listen for job status updates
 * @param projectId Google Cloud project ID
 */
private static void inspectGcsFile(String bucketName, String fileName, Likelihood minLikelihood, List<InfoType> infoTypes, int maxFindings, String topicId, String subscriptionId, String projectId) throws Exception {
    // Instantiates a client
    try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) {
        CloudStorageOptions cloudStorageOptions = CloudStorageOptions.newBuilder().setFileSet(CloudStorageOptions.FileSet.newBuilder().setUrl("gs://" + bucketName + "/" + fileName)).build();
        StorageConfig storageConfig = StorageConfig.newBuilder().setCloudStorageOptions(cloudStorageOptions).build();
        FindingLimits findingLimits = FindingLimits.newBuilder().setMaxFindingsPerRequest(maxFindings).build();
        InspectConfig inspectConfig = InspectConfig.newBuilder().addAllInfoTypes(infoTypes).setMinLikelihood(minLikelihood).setLimits(findingLimits).build();
        String pubSubTopic = String.format("projects/%s/topics/%s", projectId, topicId);
        Action.PublishToPubSub publishToPubSub = Action.PublishToPubSub.newBuilder().setTopic(pubSubTopic).build();
        Action action = Action.newBuilder().setPubSub(publishToPubSub).build();
        InspectJobConfig inspectJobConfig = InspectJobConfig.newBuilder().setStorageConfig(storageConfig).setInspectConfig(inspectConfig).addActions(action).build();
        // Semi-synchronously submit an inspect job, and wait on results
        CreateDlpJobRequest createDlpJobRequest = CreateDlpJobRequest.newBuilder().setParent(ProjectName.of(projectId).toString()).setInspectJob(inspectJobConfig).build();
        DlpJob dlpJob = dlpServiceClient.createDlpJob(createDlpJobRequest);
        System.out.println("Job created with ID:" + dlpJob.getName());
        final SettableApiFuture<Boolean> done = SettableApiFuture.create();
        // Set up a Pub/Sub subscriber to listen on the job completion status
        Subscriber subscriber = Subscriber.newBuilder(ProjectSubscriptionName.of(projectId, subscriptionId), (pubsubMessage, ackReplyConsumer) -> {
            if (pubsubMessage.getAttributesCount() > 0 && pubsubMessage.getAttributesMap().get("DlpJobName").equals(dlpJob.getName())) {
                // notify job completion
                done.set(true);
                ackReplyConsumer.ack();
            }
        }).build();
        subscriber.startAsync();
        // For long jobs, consider using a truly asynchronous execution model such as Cloud Functions
        try {
            done.get(1, TimeUnit.MINUTES);
            // Wait for the job to become available
            Thread.sleep(500);
        } catch (Exception e) {
            System.out.println("Unable to verify job completion.");
        }
        DlpJob completedJob = dlpServiceClient.getDlpJob(GetDlpJobRequest.newBuilder().setName(dlpJob.getName()).build());
        System.out.println("Job status: " + completedJob.getState());
        InspectDataSourceDetails inspectDataSourceDetails = completedJob.getInspectDetails();
        InspectDataSourceDetails.Result result = inspectDataSourceDetails.getResult();
        if (result.getInfoTypeStatsCount() > 0) {
            System.out.println("Findings: ");
            for (InfoTypeStats infoTypeStat : result.getInfoTypeStatsList()) {
                System.out.print("\tInfo type: " + infoTypeStat.getInfoType().getName());
                System.out.println("\tCount: " + infoTypeStat.getCount());
            }
        } else {
            System.out.println("No findings.");
        }
    }
}

Example 15 with AckReplyConsumer

use of com.google.cloud.pubsub.v1.AckReplyConsumer in project java-docs-samples by GoogleCloudPlatform.

the class RiskAnalysis method calculateKAnonymity.

// [END dlp_categorical_stats]
// [START dlp_k_anonymity]
/**
 * Calculate k-anonymity for quasi-identifiers in a BigQuery table using the DLP API.
 *
 * @param projectId The Google Cloud Platform project ID to run the API call under.
 * @param datasetId The BigQuery dataset to analyze.
 * @param tableId The BigQuery table to analyze.
 * @param quasiIds The names of columns that form a composite key ('quasi-identifiers').
 * @param topicId The name of the Pub/Sub topic to notify once the job completes
 * @param subscriptionId The name of the Pub/Sub subscription to use when listening for job
 *     completion status.
 */
private static void calculateKAnonymity(String projectId, String datasetId, String tableId, List<String> quasiIds, String topicId, String subscriptionId) throws Exception {
    // Instantiates a client
    try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) {
        List<FieldId> quasiIdFields = quasiIds.stream().map(columnName -> FieldId.newBuilder().setName(columnName).build()).collect(Collectors.toList());
        KAnonymityConfig kanonymityConfig = KAnonymityConfig.newBuilder().addAllQuasiIds(quasiIdFields).build();
        BigQueryTable bigQueryTable = BigQueryTable.newBuilder().setProjectId(projectId).setDatasetId(datasetId).setTableId(tableId).build();
        PrivacyMetric privacyMetric = PrivacyMetric.newBuilder().setKAnonymityConfig(kanonymityConfig).build();
        String topicName = String.format("projects/%s/topics/%s", projectId, topicId);
        PublishToPubSub publishToPubSub = PublishToPubSub.newBuilder().setTopic(topicName).build();
        // Create action to publish job status notifications over Google Cloud Pub/Sub
        Action action = Action.newBuilder().setPubSub(publishToPubSub).build();
        RiskAnalysisJobConfig riskAnalysisJobConfig = RiskAnalysisJobConfig.newBuilder().setSourceTable(bigQueryTable).setPrivacyMetric(privacyMetric).addActions(action).build();
        CreateDlpJobRequest createDlpJobRequest = CreateDlpJobRequest.newBuilder().setParent(ProjectName.of(projectId).toString()).setRiskJob(riskAnalysisJobConfig).build();
        DlpJob dlpJob = dlpServiceClient.createDlpJob(createDlpJobRequest);
        String dlpJobName = dlpJob.getName();
        final SettableApiFuture<Boolean> done = SettableApiFuture.create();
        // Set up a Pub/Sub subscriber to listen on the job completion status
        Subscriber subscriber = Subscriber.newBuilder(ProjectSubscriptionName.newBuilder().setProject(projectId).setSubscription(subscriptionId).build(), (pubsubMessage, ackReplyConsumer) -> {
            if (pubsubMessage.getAttributesCount() > 0 && pubsubMessage.getAttributesMap().get("DlpJobName").equals(dlpJobName)) {
                // notify job completion
                done.set(true);
                ackReplyConsumer.ack();
            }
        }).build();
        subscriber.startAsync();
        // For long jobs, consider using a truly asynchronous execution model such as Cloud Functions
        try {
            done.get(1, TimeUnit.MINUTES);
            // Wait for the job to become available
            Thread.sleep(500);
        } catch (TimeoutException e) {
            System.out.println("Unable to verify job completion.");
        }
        // Retrieve completed job status
        DlpJob completedJob = dlpServiceClient.getDlpJob(GetDlpJobRequest.newBuilder().setName(dlpJobName).build());
        System.out.println("Job status: " + completedJob.getState());
        AnalyzeDataSourceRiskDetails riskDetails = completedJob.getRiskDetails();
        KAnonymityResult kanonymityResult = riskDetails.getKAnonymityResult();
        for (KAnonymityHistogramBucket result : kanonymityResult.getEquivalenceClassHistogramBucketsList()) {
            System.out.printf("Bucket size range: [%d, %d]\n", result.getEquivalenceClassSizeLowerBound(), result.getEquivalenceClassSizeUpperBound());
            for (KAnonymityEquivalenceClass bucket : result.getBucketValuesList()) {
                List<String> quasiIdValues = bucket.getQuasiIdsValuesList().stream().map(v -> v.toString()).collect(Collectors.toList());
                System.out.println("\tQuasi-ID values: " + String.join(", ", quasiIdValues));
                System.out.println("\tClass size: " + bucket.getEquivalenceClassSize());
            }
        }
    } catch (Exception e) {
        System.out.println("Error in calculateKAnonymity: " + e.getMessage());
    }
}