Examples with AckReplyConsumer com.google.cloud.pubsub.v1.AckReplyConsumer used on opensource projects

Example 16 with AckReplyConsumer

use of com.google.cloud.pubsub.v1.AckReplyConsumer in project java-docs-samples by GoogleCloudPlatform.

the class RiskAnalysis method numericalStatsAnalysis.

// [START dlp_numerical_stats]
/**
 * Calculate numerical statistics for a column in a BigQuery table using the DLP API.
 *
 * @param projectId The Google Cloud Platform project ID to run the API call under.
 * @param datasetId The BigQuery dataset to analyze.
 * @param tableId The BigQuery table to analyze.
 * @param columnName The name of the column to analyze, which must contain only numerical data.
 * @param topicId The name of the Pub/Sub topic to notify once the job completes
 * @param subscriptionId The name of the Pub/Sub subscription to use when listening for job
 *     completion status.
 */
private static void numericalStatsAnalysis(String projectId, String datasetId, String tableId, String columnName, String topicId, String subscriptionId) throws Exception {
    // Instantiates a client
    try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) {
        BigQueryTable bigQueryTable = BigQueryTable.newBuilder().setTableId(tableId).setDatasetId(datasetId).setProjectId(projectId).build();
        FieldId fieldId = FieldId.newBuilder().setName(columnName).build();
        NumericalStatsConfig numericalStatsConfig = NumericalStatsConfig.newBuilder().setField(fieldId).build();
        PrivacyMetric privacyMetric = PrivacyMetric.newBuilder().setNumericalStatsConfig(numericalStatsConfig).build();
        String topicName = String.format("projects/%s/topics/%s", projectId, topicId);
        PublishToPubSub publishToPubSub = PublishToPubSub.newBuilder().setTopic(topicName).build();
        // Create action to publish job status notifications over Google Cloud Pub/Sub
        Action action = Action.newBuilder().setPubSub(publishToPubSub).build();
        RiskAnalysisJobConfig riskAnalysisJobConfig = RiskAnalysisJobConfig.newBuilder().setSourceTable(bigQueryTable).setPrivacyMetric(privacyMetric).addActions(action).build();
        CreateDlpJobRequest createDlpJobRequest = CreateDlpJobRequest.newBuilder().setParent(ProjectName.of(projectId).toString()).setRiskJob(riskAnalysisJobConfig).build();
        DlpJob dlpJob = dlpServiceClient.createDlpJob(createDlpJobRequest);
        String dlpJobName = dlpJob.getName();
        final SettableApiFuture<Boolean> done = SettableApiFuture.create();
        // Set up a Pub/Sub subscriber to listen on the job completion status
        Subscriber subscriber = Subscriber.newBuilder(ProjectSubscriptionName.newBuilder().setProject(projectId).setSubscription(subscriptionId).build(), (pubsubMessage, ackReplyConsumer) -> {
            if (pubsubMessage.getAttributesCount() > 0 && pubsubMessage.getAttributesMap().get("DlpJobName").equals(dlpJobName)) {
                // notify job completion
                done.set(true);
                ackReplyConsumer.ack();
            }
        }).build();
        subscriber.startAsync();
        // For long jobs, consider using a truly asynchronous execution model such as Cloud Functions
        try {
            done.get(1, TimeUnit.MINUTES);
            // Wait for the job to become available
            Thread.sleep(500);
        } catch (TimeoutException e) {
            System.out.println("Unable to verify job completion.");
        }
        // Retrieve completed job status
        DlpJob completedJob = dlpServiceClient.getDlpJob(GetDlpJobRequest.newBuilder().setName(dlpJobName).build());
        System.out.println("Job status: " + completedJob.getState());
        AnalyzeDataSourceRiskDetails riskDetails = completedJob.getRiskDetails();
        AnalyzeDataSourceRiskDetails.NumericalStatsResult result = riskDetails.getNumericalStatsResult();
        System.out.printf("Value range : [%.3f, %.3f]\n", result.getMinValue().getFloatValue(), result.getMaxValue().getFloatValue());
        int percent = 1;
        Double lastValue = null;
        for (Value quantileValue : result.getQuantileValuesList()) {
            Double currentValue = quantileValue.getFloatValue();
            if (lastValue == null || !lastValue.equals(currentValue)) {
                System.out.printf("Value at %s %% quantile : %.3f", percent, currentValue);
            }
            lastValue = currentValue;
        }
    } catch (Exception e) {
        System.out.println("Error in categoricalStatsAnalysis: " + e.getMessage());
    }
}

Example 17 with AckReplyConsumer

use of com.google.cloud.pubsub.v1.AckReplyConsumer in project beam by apache.

the class TestPubsub method waitForNMessages.

/**
 * Repeatedly pull messages from {@link #subscriptionPath()}, returns after receiving {@code n}
 * messages or after waiting for {@code timeoutDuration}.
 */
public List<PubsubMessage> waitForNMessages(int n, Duration timeoutDuration) throws IOException, InterruptedException {
    Preconditions.checkNotNull(subscriptionPath);
    BlockingQueue<com.google.pubsub.v1.PubsubMessage> receivedMessages = new LinkedBlockingDeque<>(n);
    MessageReceiver receiver = (com.google.pubsub.v1.PubsubMessage message, AckReplyConsumer replyConsumer) -> {
        if (receivedMessages.offer(message)) {
            replyConsumer.ack();
        } else {
            replyConsumer.nack();
        }
    };
    Subscriber subscriber = Subscriber.newBuilder(subscriptionPath.getPath(), receiver).setCredentialsProvider(pipelineOptions::getGcpCredential).setChannelProvider(channelProvider).setEndpoint(pubsubEndpoint).build();
    subscriber.startAsync();
    DateTime startTime = new DateTime();
    int timeoutSeconds = timeoutDuration.toStandardSeconds().getSeconds();
    while (receivedMessages.size() < n && Seconds.secondsBetween(startTime, new DateTime()).getSeconds() < timeoutSeconds) {
        try {
            Thread.sleep(1000);
        } catch (InterruptedException ignored) {
        }
    }
    subscriber.stopAsync();
    subscriber.awaitTerminated();
    return receivedMessages.stream().map((message) -> new PubsubMessage(message.getData().toByteArray(), message.getAttributesMap(), message.getMessageId())).collect(Collectors.toList());
}

Example 18 with AckReplyConsumer

use of com.google.cloud.pubsub.v1.AckReplyConsumer in project spring-cloud-gcp by spring-cloud.

the class PubSubInboundChannelAdapter method consumeMessage.

@SuppressWarnings("deprecation")
private void consumeMessage(ConvertedBasicAcknowledgeablePubsubMessage<?> message) {
    Map<String, Object> messageHeaders = this.headerMapper.toHeaders(message.getPubsubMessage().getAttributesMap());
    // Send the original message downstream so that the user can decide on when to
    // ack/nack, or just have access to the original message for any other reason.
    messageHeaders.put(GcpPubSubHeaders.ORIGINAL_MESSAGE, message);
    if (this.ackMode == AckMode.MANUAL) {
        // Deprecated mechanism for manual (n)acking.
        messageHeaders.put(GcpPubSubHeaders.ACKNOWLEDGEMENT, new AckReplyConsumer() {

            @Override
            public void ack() {
                LOGGER.warn("ACKNOWLEDGEMENT header is deprecated. Please use ORIGINAL_MESSAGE header to ack.");
                message.ack();
            }

            @Override
            public void nack() {
                LOGGER.warn("ACKNOWLEDGEMENT header is deprecated. Please use ORIGINAL_MESSAGE header to nack.");
                message.nack();
            }
        });
    }
    try {
        sendMessage(getMessageBuilderFactory().withPayload(message.getPayload()).copyHeaders(messageHeaders).build());
        if (this.ackMode == AckMode.AUTO_ACK || this.ackMode == AckMode.AUTO) {
            message.ack();
        }
    } catch (RuntimeException re) {
        if (this.ackMode == AckMode.AUTO) {
            message.nack();
            LOGGER.warn("Sending Spring message [" + message.getPubsubMessage().getMessageId() + "] failed; message nacked automatically.", re);
        } else {
            LOGGER.warn("Sending Spring message [" + message.getPubsubMessage().getMessageId() + "] failed; message neither acked nor nacked.", re);
        }
    }
}