use of com.google.cloud.pubsub.spi.v1.Subscriber in project google-cloud-java by GoogleCloudPlatform.
the class SubscriberTest method testModifyAckDeadline.
@Test
public void testModifyAckDeadline() throws Exception {
Subscriber subscriber = startSubscriber(getTestSubscriberBuilder(testReceiver).setAckExpirationPadding(Duration.ofSeconds(1)).setMaxAckExtensionPeriod(Duration.ofSeconds(13)));
// Send messages to be acked
List<String> testAckIdsBatch = ImmutableList.of("A", "B", "C");
testReceiver.setExplicitAck(true);
// A modify ack deadline should be scheduled for the next 9s
fakeExecutor.setupScheduleExpectation(Duration.ofSeconds(9));
sendMessages(testAckIdsBatch);
// To ensure first modify ack deadline got scheduled
fakeExecutor.waitForExpectedWork();
fakeExecutor.advanceTime(Duration.ofSeconds(9));
assertEquivalentWithTransformation(testAckIdsBatch, fakeSubscriberServiceImpl.waitAndConsumeModifyAckDeadlines(3), new Function<String, ModifyAckDeadline>() {
@Override
public ModifyAckDeadline apply(String ack) {
return new ModifyAckDeadline(ack, INITIAL_ACK_DEADLINE_EXTENSION_SECS);
}
});
fakeExecutor.advanceTime(Duration.ofSeconds(1));
assertEquivalentWithTransformation(testAckIdsBatch, fakeSubscriberServiceImpl.waitAndConsumeModifyAckDeadlines(3), new Function<String, ModifyAckDeadline>() {
@Override
public ModifyAckDeadline apply(String ack) {
// It is expected that the deadline is renewed
return new ModifyAckDeadline(ack, 3);
// only three more seconds to not pass the max
// ack deadline ext.
}
});
// No more modify ack deadline extension should be triggered at this point
fakeExecutor.advanceTime(Duration.ofSeconds(20));
assertTrue(fakeSubscriberServiceImpl.getModifyAckDeadlines().isEmpty());
testReceiver.replyAllOutstandingMessage();
subscriber.stopAsync().awaitTerminated();
}
use of com.google.cloud.pubsub.spi.v1.Subscriber in project google-cloud-java by GoogleCloudPlatform.
the class SubscriberTest method testReceiverError_NacksMessage.
@Test
public void testReceiverError_NacksMessage() throws Exception {
testReceiver.setErrorReply(new RuntimeException("Can't process message"));
Subscriber subscriber = startSubscriber(getTestSubscriberBuilder(testReceiver));
sendMessages(ImmutableList.of("A"));
// Trigger nack sending
subscriber.stopAsync().awaitTerminated();
assertEquivalent(ImmutableList.of(new ModifyAckDeadline("A", 0)), fakeSubscriberServiceImpl.waitAndConsumeModifyAckDeadlines(1));
}
use of com.google.cloud.pubsub.spi.v1.Subscriber in project java-docs-samples by GoogleCloudPlatform.
the class SubscriberExample method main.
/**
* Receive messages over a subscription.
*/
public static void main(String... args) throws Exception {
// set subscriber id, eg. my-sub
String subscriptionId = args[0];
ProjectSubscriptionName subscriptionName = ProjectSubscriptionName.of(PROJECT_ID, subscriptionId);
Subscriber subscriber = null;
try {
// create a subscriber bound to the asynchronous message receiver
subscriber = Subscriber.newBuilder(subscriptionName, new MessageReceiverExample()).build();
subscriber.startAsync().awaitRunning();
// Continue to listen to messages
while (true) {
PubsubMessage message = messages.take();
System.out.println("Message Id: " + message.getMessageId());
System.out.println("Data: " + message.getData().toStringUtf8());
}
} finally {
if (subscriber != null) {
subscriber.stopAsync();
}
}
}
use of com.google.cloud.pubsub.spi.v1.Subscriber in project java-docs-samples by GoogleCloudPlatform.
the class Inspect method inspectGcsFile.
// [END dlp_inspect_file]
// [START dlp_inspect_gcs]
/**
* Inspect GCS file for Info types and wait on job completion using Google Cloud Pub/Sub
* notification
*
* @param bucketName The name of the bucket where the file resides.
* @param fileName The path to the file within the bucket to inspect (can include wildcards, eg.
* my-image.*)
* @param minLikelihood The minimum likelihood required before returning a match
* @param infoTypes The infoTypes of information to match
* @param maxFindings The maximum number of findings to report (0 = server maximum)
* @param topicId Google Cloud Pub/Sub topic Id to notify of job status
* @param subscriptionId Google Cloud Subscription to above topic to listen for job status updates
* @param projectId Google Cloud project ID
*/
private static void inspectGcsFile(String bucketName, String fileName, Likelihood minLikelihood, List<InfoType> infoTypes, int maxFindings, String topicId, String subscriptionId, String projectId) throws Exception {
// Instantiates a client
try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) {
CloudStorageOptions cloudStorageOptions = CloudStorageOptions.newBuilder().setFileSet(CloudStorageOptions.FileSet.newBuilder().setUrl("gs://" + bucketName + "/" + fileName)).build();
StorageConfig storageConfig = StorageConfig.newBuilder().setCloudStorageOptions(cloudStorageOptions).build();
FindingLimits findingLimits = FindingLimits.newBuilder().setMaxFindingsPerRequest(maxFindings).build();
InspectConfig inspectConfig = InspectConfig.newBuilder().addAllInfoTypes(infoTypes).setMinLikelihood(minLikelihood).setLimits(findingLimits).build();
String pubSubTopic = String.format("projects/%s/topics/%s", projectId, topicId);
Action.PublishToPubSub publishToPubSub = Action.PublishToPubSub.newBuilder().setTopic(pubSubTopic).build();
Action action = Action.newBuilder().setPubSub(publishToPubSub).build();
InspectJobConfig inspectJobConfig = InspectJobConfig.newBuilder().setStorageConfig(storageConfig).setInspectConfig(inspectConfig).addActions(action).build();
// Semi-synchronously submit an inspect job, and wait on results
CreateDlpJobRequest createDlpJobRequest = CreateDlpJobRequest.newBuilder().setParent(ProjectName.of(projectId).toString()).setInspectJob(inspectJobConfig).build();
DlpJob dlpJob = dlpServiceClient.createDlpJob(createDlpJobRequest);
System.out.println("Job created with ID:" + dlpJob.getName());
final SettableApiFuture<Boolean> done = SettableApiFuture.create();
// Set up a Pub/Sub subscriber to listen on the job completion status
Subscriber subscriber = Subscriber.newBuilder(ProjectSubscriptionName.of(projectId, subscriptionId), (pubsubMessage, ackReplyConsumer) -> {
if (pubsubMessage.getAttributesCount() > 0 && pubsubMessage.getAttributesMap().get("DlpJobName").equals(dlpJob.getName())) {
// notify job completion
done.set(true);
ackReplyConsumer.ack();
}
}).build();
subscriber.startAsync();
// For long jobs, consider using a truly asynchronous execution model such as Cloud Functions
try {
done.get(1, TimeUnit.MINUTES);
// Wait for the job to become available
Thread.sleep(500);
} catch (Exception e) {
System.out.println("Unable to verify job completion.");
}
DlpJob completedJob = dlpServiceClient.getDlpJob(GetDlpJobRequest.newBuilder().setName(dlpJob.getName()).build());
System.out.println("Job status: " + completedJob.getState());
InspectDataSourceDetails inspectDataSourceDetails = completedJob.getInspectDetails();
InspectDataSourceDetails.Result result = inspectDataSourceDetails.getResult();
if (result.getInfoTypeStatsCount() > 0) {
System.out.println("Findings: ");
for (InfoTypeStats infoTypeStat : result.getInfoTypeStatsList()) {
System.out.print("\tInfo type: " + infoTypeStat.getInfoType().getName());
System.out.println("\tCount: " + infoTypeStat.getCount());
}
} else {
System.out.println("No findings.");
}
}
}
use of com.google.cloud.pubsub.spi.v1.Subscriber in project java-docs-samples by GoogleCloudPlatform.
the class RiskAnalysis method calculateKAnonymity.
// [END dlp_categorical_stats]
// [START dlp_k_anonymity]
/**
* Calculate k-anonymity for quasi-identifiers in a BigQuery table using the DLP API.
*
* @param projectId The Google Cloud Platform project ID to run the API call under.
* @param datasetId The BigQuery dataset to analyze.
* @param tableId The BigQuery table to analyze.
* @param quasiIds The names of columns that form a composite key ('quasi-identifiers').
* @param topicId The name of the Pub/Sub topic to notify once the job completes
* @param subscriptionId The name of the Pub/Sub subscription to use when listening for job
* completion status.
*/
private static void calculateKAnonymity(String projectId, String datasetId, String tableId, List<String> quasiIds, String topicId, String subscriptionId) throws Exception {
// Instantiates a client
try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) {
List<FieldId> quasiIdFields = quasiIds.stream().map(columnName -> FieldId.newBuilder().setName(columnName).build()).collect(Collectors.toList());
KAnonymityConfig kanonymityConfig = KAnonymityConfig.newBuilder().addAllQuasiIds(quasiIdFields).build();
BigQueryTable bigQueryTable = BigQueryTable.newBuilder().setProjectId(projectId).setDatasetId(datasetId).setTableId(tableId).build();
PrivacyMetric privacyMetric = PrivacyMetric.newBuilder().setKAnonymityConfig(kanonymityConfig).build();
String topicName = String.format("projects/%s/topics/%s", projectId, topicId);
PublishToPubSub publishToPubSub = PublishToPubSub.newBuilder().setTopic(topicName).build();
// Create action to publish job status notifications over Google Cloud Pub/Sub
Action action = Action.newBuilder().setPubSub(publishToPubSub).build();
RiskAnalysisJobConfig riskAnalysisJobConfig = RiskAnalysisJobConfig.newBuilder().setSourceTable(bigQueryTable).setPrivacyMetric(privacyMetric).addActions(action).build();
CreateDlpJobRequest createDlpJobRequest = CreateDlpJobRequest.newBuilder().setParent(ProjectName.of(projectId).toString()).setRiskJob(riskAnalysisJobConfig).build();
DlpJob dlpJob = dlpServiceClient.createDlpJob(createDlpJobRequest);
String dlpJobName = dlpJob.getName();
final SettableApiFuture<Boolean> done = SettableApiFuture.create();
// Set up a Pub/Sub subscriber to listen on the job completion status
Subscriber subscriber = Subscriber.newBuilder(ProjectSubscriptionName.newBuilder().setProject(projectId).setSubscription(subscriptionId).build(), (pubsubMessage, ackReplyConsumer) -> {
if (pubsubMessage.getAttributesCount() > 0 && pubsubMessage.getAttributesMap().get("DlpJobName").equals(dlpJobName)) {
// notify job completion
done.set(true);
ackReplyConsumer.ack();
}
}).build();
subscriber.startAsync();
// For long jobs, consider using a truly asynchronous execution model such as Cloud Functions
try {
done.get(1, TimeUnit.MINUTES);
// Wait for the job to become available
Thread.sleep(500);
} catch (TimeoutException e) {
System.out.println("Unable to verify job completion.");
}
// Retrieve completed job status
DlpJob completedJob = dlpServiceClient.getDlpJob(GetDlpJobRequest.newBuilder().setName(dlpJobName).build());
System.out.println("Job status: " + completedJob.getState());
AnalyzeDataSourceRiskDetails riskDetails = completedJob.getRiskDetails();
KAnonymityResult kanonymityResult = riskDetails.getKAnonymityResult();
for (KAnonymityHistogramBucket result : kanonymityResult.getEquivalenceClassHistogramBucketsList()) {
System.out.printf("Bucket size range: [%d, %d]\n", result.getEquivalenceClassSizeLowerBound(), result.getEquivalenceClassSizeUpperBound());
for (KAnonymityEquivalenceClass bucket : result.getBucketValuesList()) {
List<String> quasiIdValues = bucket.getQuasiIdsValuesList().stream().map(v -> v.toString()).collect(Collectors.toList());
System.out.println("\tQuasi-ID values: " + String.join(", ", quasiIdValues));
System.out.println("\tClass size: " + bucket.getEquivalenceClassSize());
}
}
} catch (Exception e) {
System.out.println("Error in calculateKAnonymity: " + e.getMessage());
}
}
Aggregations