use of com.google.cloud.pubsub.spi.v1.Subscriber in project java-docs-samples by GoogleCloudPlatform.
the class RiskAnalysis method numericalStatsAnalysis.
// [START dlp_numerical_stats]
/**
* Calculate numerical statistics for a column in a BigQuery table using the DLP API.
*
* @param projectId The Google Cloud Platform project ID to run the API call under.
* @param datasetId The BigQuery dataset to analyze.
* @param tableId The BigQuery table to analyze.
* @param columnName The name of the column to analyze, which must contain only numerical data.
* @param topicId The name of the Pub/Sub topic to notify once the job completes
* @param subscriptionId The name of the Pub/Sub subscription to use when listening for job
* completion status.
*/
private static void numericalStatsAnalysis(String projectId, String datasetId, String tableId, String columnName, String topicId, String subscriptionId) throws Exception {
// Instantiates a client
try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) {
BigQueryTable bigQueryTable = BigQueryTable.newBuilder().setTableId(tableId).setDatasetId(datasetId).setProjectId(projectId).build();
FieldId fieldId = FieldId.newBuilder().setName(columnName).build();
NumericalStatsConfig numericalStatsConfig = NumericalStatsConfig.newBuilder().setField(fieldId).build();
PrivacyMetric privacyMetric = PrivacyMetric.newBuilder().setNumericalStatsConfig(numericalStatsConfig).build();
String topicName = String.format("projects/%s/topics/%s", projectId, topicId);
PublishToPubSub publishToPubSub = PublishToPubSub.newBuilder().setTopic(topicName).build();
// Create action to publish job status notifications over Google Cloud Pub/Sub
Action action = Action.newBuilder().setPubSub(publishToPubSub).build();
RiskAnalysisJobConfig riskAnalysisJobConfig = RiskAnalysisJobConfig.newBuilder().setSourceTable(bigQueryTable).setPrivacyMetric(privacyMetric).addActions(action).build();
CreateDlpJobRequest createDlpJobRequest = CreateDlpJobRequest.newBuilder().setParent(ProjectName.of(projectId).toString()).setRiskJob(riskAnalysisJobConfig).build();
DlpJob dlpJob = dlpServiceClient.createDlpJob(createDlpJobRequest);
String dlpJobName = dlpJob.getName();
final SettableApiFuture<Boolean> done = SettableApiFuture.create();
// Set up a Pub/Sub subscriber to listen on the job completion status
Subscriber subscriber = Subscriber.newBuilder(ProjectSubscriptionName.newBuilder().setProject(projectId).setSubscription(subscriptionId).build(), (pubsubMessage, ackReplyConsumer) -> {
if (pubsubMessage.getAttributesCount() > 0 && pubsubMessage.getAttributesMap().get("DlpJobName").equals(dlpJobName)) {
// notify job completion
done.set(true);
ackReplyConsumer.ack();
}
}).build();
subscriber.startAsync();
// For long jobs, consider using a truly asynchronous execution model such as Cloud Functions
try {
done.get(1, TimeUnit.MINUTES);
// Wait for the job to become available
Thread.sleep(500);
} catch (TimeoutException e) {
System.out.println("Unable to verify job completion.");
}
// Retrieve completed job status
DlpJob completedJob = dlpServiceClient.getDlpJob(GetDlpJobRequest.newBuilder().setName(dlpJobName).build());
System.out.println("Job status: " + completedJob.getState());
AnalyzeDataSourceRiskDetails riskDetails = completedJob.getRiskDetails();
AnalyzeDataSourceRiskDetails.NumericalStatsResult result = riskDetails.getNumericalStatsResult();
System.out.printf("Value range : [%.3f, %.3f]\n", result.getMinValue().getFloatValue(), result.getMaxValue().getFloatValue());
int percent = 1;
Double lastValue = null;
for (Value quantileValue : result.getQuantileValuesList()) {
Double currentValue = quantileValue.getFloatValue();
if (lastValue == null || !lastValue.equals(currentValue)) {
System.out.printf("Value at %s %% quantile : %.3f", percent, currentValue);
}
lastValue = currentValue;
}
} catch (Exception e) {
System.out.println("Error in categoricalStatsAnalysis: " + e.getMessage());
}
}
use of com.google.cloud.pubsub.spi.v1.Subscriber in project spring-cloud-gcp by spring-cloud.
the class PubSubTemplate method subscribe.
@Override
public Subscriber subscribe(String subscription, MessageReceiver messageHandler) {
Subscriber subscriber = this.subscriberFactory.createSubscriber(subscription, messageHandler);
subscriber.startAsync();
return subscriber;
}
use of com.google.cloud.pubsub.spi.v1.Subscriber in project beam by apache.
the class TestPubsub method waitForNMessages.
/**
* Repeatedly pull messages from {@link #subscriptionPath()}, returns after receiving {@code n}
* messages or after waiting for {@code timeoutDuration}.
*/
public List<PubsubMessage> waitForNMessages(int n, Duration timeoutDuration) throws IOException, InterruptedException {
Preconditions.checkNotNull(subscriptionPath);
BlockingQueue<com.google.pubsub.v1.PubsubMessage> receivedMessages = new LinkedBlockingDeque<>(n);
MessageReceiver receiver = (com.google.pubsub.v1.PubsubMessage message, AckReplyConsumer replyConsumer) -> {
if (receivedMessages.offer(message)) {
replyConsumer.ack();
} else {
replyConsumer.nack();
}
};
Subscriber subscriber = Subscriber.newBuilder(subscriptionPath.getPath(), receiver).setCredentialsProvider(pipelineOptions::getGcpCredential).setChannelProvider(channelProvider).setEndpoint(pubsubEndpoint).build();
subscriber.startAsync();
DateTime startTime = new DateTime();
int timeoutSeconds = timeoutDuration.toStandardSeconds().getSeconds();
while (receivedMessages.size() < n && Seconds.secondsBetween(startTime, new DateTime()).getSeconds() < timeoutSeconds) {
try {
Thread.sleep(1000);
} catch (InterruptedException ignored) {
}
}
subscriber.stopAsync();
subscriber.awaitTerminated();
return receivedMessages.stream().map((message) -> new PubsubMessage(message.getData().toByteArray(), message.getAttributesMap(), message.getMessageId())).collect(Collectors.toList());
}
use of com.google.cloud.pubsub.spi.v1.Subscriber in project flink by apache.
the class PubsubHelper method subscribeToSubscription.
public Subscriber subscribeToSubscription(String project, String subscription, MessageReceiver messageReceiver) {
ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.of(project, subscription);
Subscriber subscriber = Subscriber.newBuilder(subscriptionName, messageReceiver).setChannelProvider(channelProvider).setCredentialsProvider(EmulatorCredentialsProvider.create()).build();
subscriber.startAsync();
return subscriber;
}
use of com.google.cloud.pubsub.spi.v1.Subscriber in project spring-cloud-gcp by spring-cloud.
the class WebController method subscribe.
@GetMapping("/subscribe")
public RedirectView subscribe(@RequestParam("subscription") String subscriptionName) {
Subscriber subscriber = this.pubSubTemplate.subscribe(subscriptionName, (message) -> {
LOGGER.info("Message received from " + subscriptionName + " subscription: " + message.getPubsubMessage().getData().toStringUtf8());
message.ack();
});
this.allSubscribers.add(subscriber);
return buildStatusView("Subscribed.");
}
Aggregations