Search in sources :

Example 1 with CloudStorageOptions

use of com.google.privacy.dlp.v2.CloudStorageOptions in project java-docs-samples by GoogleCloudPlatform.

the class Inspect method inspectGcsFile.

// [END dlp_inspect_file]
// [START dlp_inspect_gcs]
/**
 * Inspect GCS file for Info types and wait on job completion using Google Cloud Pub/Sub
 * notification
 *
 * @param bucketName The name of the bucket where the file resides.
 * @param fileName The path to the file within the bucket to inspect (can include wildcards, eg.
 *     my-image.*)
 * @param minLikelihood The minimum likelihood required before returning a match
 * @param infoTypes The infoTypes of information to match
 * @param maxFindings The maximum number of findings to report (0 = server maximum)
 * @param topicId Google Cloud Pub/Sub topic Id to notify of job status
 * @param subscriptionId Google Cloud Subscription to above topic to listen for job status updates
 * @param projectId Google Cloud project ID
 */
private static void inspectGcsFile(String bucketName, String fileName, Likelihood minLikelihood, List<InfoType> infoTypes, int maxFindings, String topicId, String subscriptionId, String projectId) throws Exception {
    // Instantiates a client
    try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) {
        CloudStorageOptions cloudStorageOptions = CloudStorageOptions.newBuilder().setFileSet(CloudStorageOptions.FileSet.newBuilder().setUrl("gs://" + bucketName + "/" + fileName)).build();
        StorageConfig storageConfig = StorageConfig.newBuilder().setCloudStorageOptions(cloudStorageOptions).build();
        FindingLimits findingLimits = FindingLimits.newBuilder().setMaxFindingsPerRequest(maxFindings).build();
        InspectConfig inspectConfig = InspectConfig.newBuilder().addAllInfoTypes(infoTypes).setMinLikelihood(minLikelihood).setLimits(findingLimits).build();
        String pubSubTopic = String.format("projects/%s/topics/%s", projectId, topicId);
        Action.PublishToPubSub publishToPubSub = Action.PublishToPubSub.newBuilder().setTopic(pubSubTopic).build();
        Action action = Action.newBuilder().setPubSub(publishToPubSub).build();
        InspectJobConfig inspectJobConfig = InspectJobConfig.newBuilder().setStorageConfig(storageConfig).setInspectConfig(inspectConfig).addActions(action).build();
        // Semi-synchronously submit an inspect job, and wait on results
        CreateDlpJobRequest createDlpJobRequest = CreateDlpJobRequest.newBuilder().setParent(ProjectName.of(projectId).toString()).setInspectJob(inspectJobConfig).build();
        DlpJob dlpJob = dlpServiceClient.createDlpJob(createDlpJobRequest);
        System.out.println("Job created with ID:" + dlpJob.getName());
        final SettableApiFuture<Boolean> done = SettableApiFuture.create();
        // Set up a Pub/Sub subscriber to listen on the job completion status
        Subscriber subscriber = Subscriber.newBuilder(ProjectSubscriptionName.of(projectId, subscriptionId), (pubsubMessage, ackReplyConsumer) -> {
            if (pubsubMessage.getAttributesCount() > 0 && pubsubMessage.getAttributesMap().get("DlpJobName").equals(dlpJob.getName())) {
                // notify job completion
                done.set(true);
                ackReplyConsumer.ack();
            }
        }).build();
        subscriber.startAsync();
        // For long jobs, consider using a truly asynchronous execution model such as Cloud Functions
        try {
            done.get(1, TimeUnit.MINUTES);
            // Wait for the job to become available
            Thread.sleep(500);
        } catch (Exception e) {
            System.out.println("Unable to verify job completion.");
        }
        DlpJob completedJob = dlpServiceClient.getDlpJob(GetDlpJobRequest.newBuilder().setName(dlpJob.getName()).build());
        System.out.println("Job status: " + completedJob.getState());
        InspectDataSourceDetails inspectDataSourceDetails = completedJob.getInspectDetails();
        InspectDataSourceDetails.Result result = inspectDataSourceDetails.getResult();
        if (result.getInfoTypeStatsCount() > 0) {
            System.out.println("Findings: ");
            for (InfoTypeStats infoTypeStat : result.getInfoTypeStatsList()) {
                System.out.print("\tInfo type: " + infoTypeStat.getInfoType().getName());
                System.out.println("\tCount: " + infoTypeStat.getCount());
            }
        } else {
            System.out.println("No findings.");
        }
    }
}
Also used : ByteContentItem(com.google.privacy.dlp.v2.ByteContentItem) InspectResult(com.google.privacy.dlp.v2.InspectResult) KindExpression(com.google.privacy.dlp.v2.KindExpression) Options(org.apache.commons.cli.Options) Likelihood(com.google.privacy.dlp.v2.Likelihood) Subscriber(com.google.cloud.pubsub.v1.Subscriber) BigQueryOptions(com.google.privacy.dlp.v2.BigQueryOptions) HelpFormatter(org.apache.commons.cli.HelpFormatter) MimetypesFileTypeMap(javax.activation.MimetypesFileTypeMap) CloudStorageOptions(com.google.privacy.dlp.v2.CloudStorageOptions) ArrayList(java.util.ArrayList) DefaultParser(org.apache.commons.cli.DefaultParser) InspectDataSourceDetails(com.google.privacy.dlp.v2.InspectDataSourceDetails) FindingLimits(com.google.privacy.dlp.v2.InspectConfig.FindingLimits) InspectContentResponse(com.google.privacy.dlp.v2.InspectContentResponse) ServiceOptions(com.google.cloud.ServiceOptions) URLConnection(java.net.URLConnection) StorageConfig(com.google.privacy.dlp.v2.StorageConfig) CommandLine(org.apache.commons.cli.CommandLine) PartitionId(com.google.privacy.dlp.v2.PartitionId) Action(com.google.privacy.dlp.v2.Action) DatastoreOptions(com.google.privacy.dlp.v2.DatastoreOptions) ProjectTopicName(com.google.pubsub.v1.ProjectTopicName) Option(org.apache.commons.cli.Option) InspectJobConfig(com.google.privacy.dlp.v2.InspectJobConfig) DlpServiceClient(com.google.cloud.dlp.v2.DlpServiceClient) Finding(com.google.privacy.dlp.v2.Finding) CreateDlpJobRequest(com.google.privacy.dlp.v2.CreateDlpJobRequest) Files(java.nio.file.Files) CommandLineParser(org.apache.commons.cli.CommandLineParser) ContentItem(com.google.privacy.dlp.v2.ContentItem) InfoType(com.google.privacy.dlp.v2.InfoType) InfoTypeStats(com.google.privacy.dlp.v2.InfoTypeStats) SettableApiFuture(com.google.api.core.SettableApiFuture) ByteString(com.google.protobuf.ByteString) TimeUnit(java.util.concurrent.TimeUnit) InspectConfig(com.google.privacy.dlp.v2.InspectConfig) List(java.util.List) ProjectName(com.google.privacy.dlp.v2.ProjectName) GetDlpJobRequest(com.google.privacy.dlp.v2.GetDlpJobRequest) Paths(java.nio.file.Paths) ParseException(org.apache.commons.cli.ParseException) BigQueryTable(com.google.privacy.dlp.v2.BigQueryTable) ProjectSubscriptionName(com.google.pubsub.v1.ProjectSubscriptionName) OptionGroup(org.apache.commons.cli.OptionGroup) DlpJob(com.google.privacy.dlp.v2.DlpJob) InspectContentRequest(com.google.privacy.dlp.v2.InspectContentRequest) Collections(java.util.Collections) Action(com.google.privacy.dlp.v2.Action) FindingLimits(com.google.privacy.dlp.v2.InspectConfig.FindingLimits) StorageConfig(com.google.privacy.dlp.v2.StorageConfig) InspectDataSourceDetails(com.google.privacy.dlp.v2.InspectDataSourceDetails) ByteString(com.google.protobuf.ByteString) InspectConfig(com.google.privacy.dlp.v2.InspectConfig) CreateDlpJobRequest(com.google.privacy.dlp.v2.CreateDlpJobRequest) ParseException(org.apache.commons.cli.ParseException) InfoTypeStats(com.google.privacy.dlp.v2.InfoTypeStats) Subscriber(com.google.cloud.pubsub.v1.Subscriber) DlpServiceClient(com.google.cloud.dlp.v2.DlpServiceClient) CloudStorageOptions(com.google.privacy.dlp.v2.CloudStorageOptions) DlpJob(com.google.privacy.dlp.v2.DlpJob) InspectJobConfig(com.google.privacy.dlp.v2.InspectJobConfig)

Example 2 with CloudStorageOptions

use of com.google.privacy.dlp.v2.CloudStorageOptions in project java-docs-samples by GoogleCloudPlatform.

the class Triggers method createTrigger.

// [START dlp_create_trigger]
/**
 * Schedule a DLP inspection trigger for a GCS location.
 *
 * @param triggerId (Optional) name of the trigger to be created
 * @param displayName (Optional) display name for the trigger to be created
 * @param description (Optional) description for the trigger to be created
 * @param scanPeriod How often to wait between scans, in days (minimum = 1 day)
 * @param infoTypes infoTypes of information to match eg. InfoType.PHONE_NUMBER,
 *     InfoType.EMAIL_ADDRESS
 * @param minLikelihood minimum likelihood required before returning a match
 * @param maxFindings maximum number of findings to report per request (0 = server maximum)
 * @param projectId The project ID to run the API call under
 */
private static void createTrigger(String triggerId, String displayName, String description, String bucketName, String fileName, int scanPeriod, List<InfoType> infoTypes, Likelihood minLikelihood, int maxFindings, String projectId) throws Exception {
    // instantiate a client
    DlpServiceClient dlpServiceClient = DlpServiceClient.create();
    try {
        CloudStorageOptions cloudStorageOptions = CloudStorageOptions.newBuilder().setFileSet(CloudStorageOptions.FileSet.newBuilder().setUrl("gs://" + bucketName + "/" + fileName)).build();
        StorageConfig storageConfig = StorageConfig.newBuilder().setCloudStorageOptions(cloudStorageOptions).build();
        InspectConfig.FindingLimits findingLimits = InspectConfig.FindingLimits.newBuilder().setMaxFindingsPerRequest(maxFindings).build();
        InspectConfig inspectConfig = InspectConfig.newBuilder().addAllInfoTypes(infoTypes).setMinLikelihood(minLikelihood).setLimits(findingLimits).build();
        InspectJobConfig inspectJobConfig = InspectJobConfig.newBuilder().setInspectConfig(inspectConfig).setStorageConfig(storageConfig).build();
        // Schedule scan of GCS bucket every scanPeriod number of days (minimum = 1 day)
        Duration duration = Duration.newBuilder().setSeconds(scanPeriod * 24 * 3600).build();
        Schedule schedule = Schedule.newBuilder().setRecurrencePeriodDuration(duration).build();
        JobTrigger.Trigger trigger = JobTrigger.Trigger.newBuilder().setSchedule(schedule).build();
        JobTrigger jobTrigger = JobTrigger.newBuilder().setInspectJob(inspectJobConfig).setName(triggerId).setDisplayName(displayName).setDescription(description).setStatus(JobTrigger.Status.HEALTHY).addTriggers(trigger).build();
        // Create scan request
        CreateJobTriggerRequest createJobTriggerRequest = CreateJobTriggerRequest.newBuilder().setParent(ProjectName.of(projectId).toString()).setJobTrigger(jobTrigger).build();
        JobTrigger createdJobTrigger = dlpServiceClient.createJobTrigger(createJobTriggerRequest);
        System.out.println("Created Trigger: " + createdJobTrigger.getName());
    } catch (Exception e) {
        System.out.println("Error creating trigger: " + e.getMessage());
    }
}
Also used : StorageConfig(com.google.privacy.dlp.v2.StorageConfig) DlpServiceClient(com.google.cloud.dlp.v2.DlpServiceClient) Schedule(com.google.privacy.dlp.v2.Schedule) CloudStorageOptions(com.google.privacy.dlp.v2.CloudStorageOptions) Duration(com.google.protobuf.Duration) InspectConfig(com.google.privacy.dlp.v2.InspectConfig) JobTrigger(com.google.privacy.dlp.v2.JobTrigger) ParseException(org.apache.commons.cli.ParseException) InspectJobConfig(com.google.privacy.dlp.v2.InspectJobConfig) CreateJobTriggerRequest(com.google.privacy.dlp.v2.CreateJobTriggerRequest)

Aggregations

DlpServiceClient (com.google.cloud.dlp.v2.DlpServiceClient)2 CloudStorageOptions (com.google.privacy.dlp.v2.CloudStorageOptions)2 InspectConfig (com.google.privacy.dlp.v2.InspectConfig)2 InspectJobConfig (com.google.privacy.dlp.v2.InspectJobConfig)2 StorageConfig (com.google.privacy.dlp.v2.StorageConfig)2 ParseException (org.apache.commons.cli.ParseException)2 SettableApiFuture (com.google.api.core.SettableApiFuture)1 ServiceOptions (com.google.cloud.ServiceOptions)1 Subscriber (com.google.cloud.pubsub.v1.Subscriber)1 Action (com.google.privacy.dlp.v2.Action)1 BigQueryOptions (com.google.privacy.dlp.v2.BigQueryOptions)1 BigQueryTable (com.google.privacy.dlp.v2.BigQueryTable)1 ByteContentItem (com.google.privacy.dlp.v2.ByteContentItem)1 ContentItem (com.google.privacy.dlp.v2.ContentItem)1 CreateDlpJobRequest (com.google.privacy.dlp.v2.CreateDlpJobRequest)1 CreateJobTriggerRequest (com.google.privacy.dlp.v2.CreateJobTriggerRequest)1 DatastoreOptions (com.google.privacy.dlp.v2.DatastoreOptions)1 DlpJob (com.google.privacy.dlp.v2.DlpJob)1 Finding (com.google.privacy.dlp.v2.Finding)1 GetDlpJobRequest (com.google.privacy.dlp.v2.GetDlpJobRequest)1