Search in sources :

Example 1 with DlpServiceClient

use of com.google.cloud.dlp.v2.DlpServiceClient in project java-docs-samples by GoogleCloudPlatform.

the class DeIdentification method deIdentifyWithMask.

// [START dlp_deidentify_masking]
/**
 * Deidentify a string by masking sensitive information with a character using the DLP API.
 *
 * @param string The string to deidentify.
 * @param maskingCharacter (Optional) The character to mask sensitive data with.
 * @param numberToMask (Optional) The number of characters' worth of sensitive data to mask.
 *     Omitting this value or setting it to 0 masks all sensitive chars.
 * @param projectId ID of Google Cloud project to run the API under.
 */
private static void deIdentifyWithMask(String string, Character maskingCharacter, int numberToMask, String projectId) {
    // instantiate a client
    try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) {
        ContentItem contentItem = ContentItem.newBuilder().setValue(string).build();
        CharacterMaskConfig characterMaskConfig = CharacterMaskConfig.newBuilder().setMaskingCharacter(maskingCharacter.toString()).setNumberToMask(numberToMask).build();
        // Create the deidentification transformation configuration
        PrimitiveTransformation primitiveTransformation = PrimitiveTransformation.newBuilder().setCharacterMaskConfig(characterMaskConfig).build();
        InfoTypeTransformation infoTypeTransformationObject = InfoTypeTransformation.newBuilder().setPrimitiveTransformation(primitiveTransformation).build();
        InfoTypeTransformations infoTypeTransformationArray = InfoTypeTransformations.newBuilder().addTransformations(infoTypeTransformationObject).build();
        DeidentifyConfig deidentifyConfig = DeidentifyConfig.newBuilder().setInfoTypeTransformations(infoTypeTransformationArray).build();
        // Create the deidentification request object
        DeidentifyContentRequest request = DeidentifyContentRequest.newBuilder().setParent(ProjectName.of(projectId).toString()).setDeidentifyConfig(deidentifyConfig).setItem(contentItem).build();
        // Execute the deidentification request
        DeidentifyContentResponse response = dlpServiceClient.deidentifyContent(request);
        // Print the character-masked input value
        // e.g. "My SSN is 123456789" --> "My SSN is *********"
        String result = response.getItem().getValue();
        System.out.println(result);
    } catch (Exception e) {
        System.out.println("Error in deidentifyWithMask: " + e.getMessage());
    }
}
Also used : InfoTypeTransformations(com.google.privacy.dlp.v2.InfoTypeTransformations) DeidentifyContentRequest(com.google.privacy.dlp.v2.DeidentifyContentRequest) PrimitiveTransformation(com.google.privacy.dlp.v2.PrimitiveTransformation) DlpServiceClient(com.google.cloud.dlp.v2.DlpServiceClient) DeidentifyConfig(com.google.privacy.dlp.v2.DeidentifyConfig) CharacterMaskConfig(com.google.privacy.dlp.v2.CharacterMaskConfig) ByteString(com.google.protobuf.ByteString) InfoTypeTransformation(com.google.privacy.dlp.v2.InfoTypeTransformations.InfoTypeTransformation) ContentItem(com.google.privacy.dlp.v2.ContentItem) DeidentifyContentResponse(com.google.privacy.dlp.v2.DeidentifyContentResponse) DateTimeParseException(java.time.format.DateTimeParseException) ParseException(org.apache.commons.cli.ParseException)

Example 2 with DlpServiceClient

use of com.google.cloud.dlp.v2.DlpServiceClient in project java-docs-samples by GoogleCloudPlatform.

the class DeIdentification method deidentifyWithDateShift.

// [END dlp_reidentify_fpe]
// [START dlp_deidentify_date_shift]
/**
 * @param inputCsvPath The path to the CSV file to deidentify
 * @param outputCsvPath (Optional) path to the output CSV file
 * @param dateFields The list of (date) fields in the CSV file to date shift
 * @param lowerBoundDays The maximum number of days to shift a date backward
 * @param upperBoundDays The maximum number of days to shift a date forward
 * @param contextFieldId (Optional) The column to determine date shift, default : a random shift
 *     amount
 * @param wrappedKey (Optional) The encrypted ('wrapped') AES-256 key to use when shifting dates
 * @param keyName (Optional) The name of the Cloud KMS key used to encrypt ('wrap') the AES-256
 *     key
 * @param projectId ID of Google Cloud project to run the API under.
 */
private static void deidentifyWithDateShift(Path inputCsvPath, Path outputCsvPath, String[] dateFields, int lowerBoundDays, int upperBoundDays, String contextFieldId, String wrappedKey, String keyName, String projectId) throws Exception {
    // instantiate a client
    try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) {
        // Set the maximum days to shift a day backward (lowerbound), forward (upperbound)
        DateShiftConfig.Builder dateShiftConfigBuilder = DateShiftConfig.newBuilder().setLowerBoundDays(lowerBoundDays).setUpperBoundDays(upperBoundDays);
        // If contextFieldId, keyName or wrappedKey is set: all three arguments must be valid
        if (contextFieldId != null && keyName != null && wrappedKey != null) {
            dateShiftConfigBuilder.setContext(FieldId.newBuilder().setName(contextFieldId).build());
            KmsWrappedCryptoKey kmsWrappedCryptoKey = KmsWrappedCryptoKey.newBuilder().setCryptoKeyName(keyName).setWrappedKey(ByteString.copyFrom(BaseEncoding.base64().decode(wrappedKey))).build();
            dateShiftConfigBuilder.setCryptoKey(CryptoKey.newBuilder().setKmsWrapped(kmsWrappedCryptoKey).build());
        } else if (contextFieldId != null || keyName != null || wrappedKey != null) {
            throw new IllegalArgumentException("You must set either ALL or NONE of {contextFieldId, keyName, wrappedKey}!");
        }
        // Read and parse the CSV file
        BufferedReader br = null;
        String line;
        List<Table.Row> rows = new ArrayList<>();
        List<FieldId> headers;
        br = new BufferedReader(new FileReader(inputCsvPath.toFile()));
        // convert csv header to FieldId
        headers = Arrays.stream(br.readLine().split(",")).map(header -> FieldId.newBuilder().setName(header).build()).collect(Collectors.toList());
        while ((line = br.readLine()) != null) {
            // convert csv rows to Table.Row
            rows.add(convertCsvRowToTableRow(line));
        }
        br.close();
        Table table = Table.newBuilder().addAllHeaders(headers).addAllRows(rows).build();
        List<FieldId> dateFieldIds = Arrays.stream(dateFields).map(field -> FieldId.newBuilder().setName(field).build()).collect(Collectors.toList());
        DateShiftConfig dateShiftConfig = dateShiftConfigBuilder.build();
        FieldTransformation fieldTransformation = FieldTransformation.newBuilder().addAllFields(dateFieldIds).setPrimitiveTransformation(PrimitiveTransformation.newBuilder().setDateShiftConfig(dateShiftConfig).build()).build();
        DeidentifyConfig deidentifyConfig = DeidentifyConfig.newBuilder().setRecordTransformations(RecordTransformations.newBuilder().addFieldTransformations(fieldTransformation).build()).build();
        ContentItem tableItem = ContentItem.newBuilder().setTable(table).build();
        DeidentifyContentRequest request = DeidentifyContentRequest.newBuilder().setParent(ProjectName.of(projectId).toString()).setDeidentifyConfig(deidentifyConfig).setItem(tableItem).build();
        // Execute the deidentification request
        DeidentifyContentResponse response = dlpServiceClient.deidentifyContent(request);
        // Write out the response as a CSV file
        List<FieldId> outputHeaderFields = response.getItem().getTable().getHeadersList();
        List<Table.Row> outputRows = response.getItem().getTable().getRowsList();
        List<String> outputHeaders = outputHeaderFields.stream().map(FieldId::getName).collect(Collectors.toList());
        File outputFile = outputCsvPath.toFile();
        if (!outputFile.exists()) {
            outputFile.createNewFile();
        }
        BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(outputFile));
        // write out headers
        bufferedWriter.append(String.join(",", outputHeaders) + "\n");
        // write out each row
        for (Table.Row outputRow : outputRows) {
            String row = outputRow.getValuesList().stream().map(value -> value.getStringValue()).collect(Collectors.joining(","));
            bufferedWriter.append(row + "\n");
        }
        bufferedWriter.flush();
        bufferedWriter.close();
        System.out.println("Successfully saved date-shift output to: " + outputCsvPath.getFileName());
    } catch (Exception e) {
        System.out.println("Error in deidentifyWithDateShift: " + e.getMessage());
    }
}
Also used : Arrays(java.util.Arrays) Date(com.google.type.Date) CryptoKey(com.google.privacy.dlp.v2.CryptoKey) DefaultParser(org.apache.commons.cli.DefaultParser) Path(java.nio.file.Path) Value(com.google.privacy.dlp.v2.Value) DateShiftConfig(com.google.privacy.dlp.v2.DateShiftConfig) InfoTypeTransformation(com.google.privacy.dlp.v2.InfoTypeTransformations.InfoTypeTransformation) FieldTransformation(com.google.privacy.dlp.v2.FieldTransformation) ContentItem(com.google.privacy.dlp.v2.ContentItem) Collectors(java.util.stream.Collectors) ByteString(com.google.protobuf.ByteString) ReidentifyContentRequest(com.google.privacy.dlp.v2.ReidentifyContentRequest) DateTimeParseException(java.time.format.DateTimeParseException) List(java.util.List) ParseException(org.apache.commons.cli.ParseException) LocalDate(java.time.LocalDate) RecordTransformations(com.google.privacy.dlp.v2.RecordTransformations) FfxCommonNativeAlphabet(com.google.privacy.dlp.v2.CryptoReplaceFfxFpeConfig.FfxCommonNativeAlphabet) CharacterMaskConfig(com.google.privacy.dlp.v2.CharacterMaskConfig) Options(org.apache.commons.cli.Options) KmsWrappedCryptoKey(com.google.privacy.dlp.v2.KmsWrappedCryptoKey) PrimitiveTransformation(com.google.privacy.dlp.v2.PrimitiveTransformation) HelpFormatter(org.apache.commons.cli.HelpFormatter) ArrayList(java.util.ArrayList) CryptoReplaceFfxFpeConfig(com.google.privacy.dlp.v2.CryptoReplaceFfxFpeConfig) ServiceOptions(com.google.cloud.ServiceOptions) DeidentifyConfig(com.google.privacy.dlp.v2.DeidentifyConfig) CommandLine(org.apache.commons.cli.CommandLine) FieldId(com.google.privacy.dlp.v2.FieldId) Option(org.apache.commons.cli.Option) DeidentifyContentResponse(com.google.privacy.dlp.v2.DeidentifyContentResponse) DlpServiceClient(com.google.cloud.dlp.v2.DlpServiceClient) InfoTypeTransformations(com.google.privacy.dlp.v2.InfoTypeTransformations) BaseEncoding(com.google.common.io.BaseEncoding) CommandLineParser(org.apache.commons.cli.CommandLineParser) BufferedWriter(java.io.BufferedWriter) Table(com.google.privacy.dlp.v2.Table) FileWriter(java.io.FileWriter) ReidentifyContentResponse(com.google.privacy.dlp.v2.ReidentifyContentResponse) SurrogateType(com.google.privacy.dlp.v2.CustomInfoType.SurrogateType) InfoType(com.google.privacy.dlp.v2.InfoType) DeidentifyContentRequest(com.google.privacy.dlp.v2.DeidentifyContentRequest) File(java.io.File) InspectConfig(com.google.privacy.dlp.v2.InspectConfig) ProjectName(com.google.privacy.dlp.v2.ProjectName) Paths(java.nio.file.Paths) OptionGroup(org.apache.commons.cli.OptionGroup) BufferedReader(java.io.BufferedReader) FileReader(java.io.FileReader) CustomInfoType(com.google.privacy.dlp.v2.CustomInfoType) FileWriter(java.io.FileWriter) ArrayList(java.util.ArrayList) ByteString(com.google.protobuf.ByteString) BufferedWriter(java.io.BufferedWriter) DeidentifyConfig(com.google.privacy.dlp.v2.DeidentifyConfig) FileReader(java.io.FileReader) FieldTransformation(com.google.privacy.dlp.v2.FieldTransformation) DeidentifyContentRequest(com.google.privacy.dlp.v2.DeidentifyContentRequest) Table(com.google.privacy.dlp.v2.Table) DateTimeParseException(java.time.format.DateTimeParseException) ParseException(org.apache.commons.cli.ParseException) DateShiftConfig(com.google.privacy.dlp.v2.DateShiftConfig) DlpServiceClient(com.google.cloud.dlp.v2.DlpServiceClient) FieldId(com.google.privacy.dlp.v2.FieldId) BufferedReader(java.io.BufferedReader) KmsWrappedCryptoKey(com.google.privacy.dlp.v2.KmsWrappedCryptoKey) File(java.io.File) ContentItem(com.google.privacy.dlp.v2.ContentItem) DeidentifyContentResponse(com.google.privacy.dlp.v2.DeidentifyContentResponse)

Example 3 with DlpServiceClient

use of com.google.cloud.dlp.v2.DlpServiceClient in project java-docs-samples by GoogleCloudPlatform.

the class Inspect method inspectDatastore.

// [END dlp_inspect_gcs]
// [START dlp_inspect_datastore]
/**
 * Inspect a Datastore kind
 *
 * @param projectId The project ID containing the target Datastore
 * @param namespaceId The ID namespace of the Datastore document to inspect
 * @param kind The kind of the Datastore entity to inspect
 * @param minLikelihood The minimum likelihood required before returning a match
 * @param infoTypes The infoTypes of information to match
 * @param maxFindings max number of findings
 * @param topicId Google Cloud Pub/Sub topic to notify job status updates
 * @param subscriptionId Google Cloud Pub/Sub subscription to above topic to receive status
 *     updates
 */
private static void inspectDatastore(String projectId, String namespaceId, String kind, Likelihood minLikelihood, List<InfoType> infoTypes, int maxFindings, String topicId, String subscriptionId) {
    // Instantiates a client
    try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) {
        // Reference to the Datastore namespace
        PartitionId partitionId = PartitionId.newBuilder().setProjectId(projectId).setNamespaceId(namespaceId).build();
        // Reference to the Datastore kind
        KindExpression kindExpression = KindExpression.newBuilder().setName(kind).build();
        DatastoreOptions datastoreOptions = DatastoreOptions.newBuilder().setKind(kindExpression).setPartitionId(partitionId).build();
        // Construct Datastore configuration to be inspected
        StorageConfig storageConfig = StorageConfig.newBuilder().setDatastoreOptions(datastoreOptions).build();
        FindingLimits findingLimits = FindingLimits.newBuilder().setMaxFindingsPerRequest(maxFindings).build();
        InspectConfig inspectConfig = InspectConfig.newBuilder().addAllInfoTypes(infoTypes).setMinLikelihood(minLikelihood).setLimits(findingLimits).build();
        String pubSubTopic = String.format("projects/%s/topics/%s", projectId, topicId);
        Action.PublishToPubSub publishToPubSub = Action.PublishToPubSub.newBuilder().setTopic(pubSubTopic).build();
        Action action = Action.newBuilder().setPubSub(publishToPubSub).build();
        InspectJobConfig inspectJobConfig = InspectJobConfig.newBuilder().setStorageConfig(storageConfig).setInspectConfig(inspectConfig).addActions(action).build();
        // Asynchronously submit an inspect job, and wait on results
        CreateDlpJobRequest createDlpJobRequest = CreateDlpJobRequest.newBuilder().setParent(ProjectName.of(projectId).toString()).setInspectJob(inspectJobConfig).build();
        DlpJob dlpJob = dlpServiceClient.createDlpJob(createDlpJobRequest);
        System.out.println("Job created with ID:" + dlpJob.getName());
        final SettableApiFuture<Boolean> done = SettableApiFuture.create();
        // Set up a Pub/Sub subscriber to listen on the job completion status
        Subscriber subscriber = Subscriber.newBuilder(ProjectSubscriptionName.of(projectId, subscriptionId), (pubsubMessage, ackReplyConsumer) -> {
            if (pubsubMessage.getAttributesCount() > 0 && pubsubMessage.getAttributesMap().get("DlpJobName").equals(dlpJob.getName())) {
                // notify job completion
                done.set(true);
                ackReplyConsumer.ack();
            }
        }).build();
        subscriber.startAsync();
        // For long jobs, consider using a truly asynchronous execution model such as Cloud Functions
        try {
            done.get(1, TimeUnit.MINUTES);
            // Wait for the job to become available
            Thread.sleep(500);
        } catch (Exception e) {
            System.out.println("Unable to verify job completion.");
        }
        DlpJob completedJob = dlpServiceClient.getDlpJob(GetDlpJobRequest.newBuilder().setName(dlpJob.getName()).build());
        System.out.println("Job status: " + completedJob.getState());
        InspectDataSourceDetails inspectDataSourceDetails = completedJob.getInspectDetails();
        InspectDataSourceDetails.Result result = inspectDataSourceDetails.getResult();
        if (result.getInfoTypeStatsCount() > 0) {
            System.out.println("Findings: ");
            for (InfoTypeStats infoTypeStat : result.getInfoTypeStatsList()) {
                System.out.print("\tInfo type: " + infoTypeStat.getInfoType().getName());
                System.out.println("\tCount: " + infoTypeStat.getCount());
            }
        } else {
            System.out.println("No findings.");
        }
    } catch (Exception e) {
        System.out.println("inspectDatastore Problems: " + e.getMessage());
    }
}
Also used : ByteContentItem(com.google.privacy.dlp.v2.ByteContentItem) InspectResult(com.google.privacy.dlp.v2.InspectResult) KindExpression(com.google.privacy.dlp.v2.KindExpression) Options(org.apache.commons.cli.Options) Likelihood(com.google.privacy.dlp.v2.Likelihood) Subscriber(com.google.cloud.pubsub.v1.Subscriber) BigQueryOptions(com.google.privacy.dlp.v2.BigQueryOptions) HelpFormatter(org.apache.commons.cli.HelpFormatter) MimetypesFileTypeMap(javax.activation.MimetypesFileTypeMap) CloudStorageOptions(com.google.privacy.dlp.v2.CloudStorageOptions) ArrayList(java.util.ArrayList) DefaultParser(org.apache.commons.cli.DefaultParser) InspectDataSourceDetails(com.google.privacy.dlp.v2.InspectDataSourceDetails) FindingLimits(com.google.privacy.dlp.v2.InspectConfig.FindingLimits) InspectContentResponse(com.google.privacy.dlp.v2.InspectContentResponse) ServiceOptions(com.google.cloud.ServiceOptions) URLConnection(java.net.URLConnection) StorageConfig(com.google.privacy.dlp.v2.StorageConfig) CommandLine(org.apache.commons.cli.CommandLine) PartitionId(com.google.privacy.dlp.v2.PartitionId) Action(com.google.privacy.dlp.v2.Action) DatastoreOptions(com.google.privacy.dlp.v2.DatastoreOptions) ProjectTopicName(com.google.pubsub.v1.ProjectTopicName) Option(org.apache.commons.cli.Option) InspectJobConfig(com.google.privacy.dlp.v2.InspectJobConfig) DlpServiceClient(com.google.cloud.dlp.v2.DlpServiceClient) Finding(com.google.privacy.dlp.v2.Finding) CreateDlpJobRequest(com.google.privacy.dlp.v2.CreateDlpJobRequest) Files(java.nio.file.Files) CommandLineParser(org.apache.commons.cli.CommandLineParser) ContentItem(com.google.privacy.dlp.v2.ContentItem) InfoType(com.google.privacy.dlp.v2.InfoType) InfoTypeStats(com.google.privacy.dlp.v2.InfoTypeStats) SettableApiFuture(com.google.api.core.SettableApiFuture) ByteString(com.google.protobuf.ByteString) TimeUnit(java.util.concurrent.TimeUnit) InspectConfig(com.google.privacy.dlp.v2.InspectConfig) List(java.util.List) ProjectName(com.google.privacy.dlp.v2.ProjectName) GetDlpJobRequest(com.google.privacy.dlp.v2.GetDlpJobRequest) Paths(java.nio.file.Paths) ParseException(org.apache.commons.cli.ParseException) BigQueryTable(com.google.privacy.dlp.v2.BigQueryTable) ProjectSubscriptionName(com.google.pubsub.v1.ProjectSubscriptionName) OptionGroup(org.apache.commons.cli.OptionGroup) DlpJob(com.google.privacy.dlp.v2.DlpJob) InspectContentRequest(com.google.privacy.dlp.v2.InspectContentRequest) Collections(java.util.Collections) Action(com.google.privacy.dlp.v2.Action) FindingLimits(com.google.privacy.dlp.v2.InspectConfig.FindingLimits) StorageConfig(com.google.privacy.dlp.v2.StorageConfig) InspectDataSourceDetails(com.google.privacy.dlp.v2.InspectDataSourceDetails) ByteString(com.google.protobuf.ByteString) PartitionId(com.google.privacy.dlp.v2.PartitionId) InspectConfig(com.google.privacy.dlp.v2.InspectConfig) CreateDlpJobRequest(com.google.privacy.dlp.v2.CreateDlpJobRequest) ParseException(org.apache.commons.cli.ParseException) InfoTypeStats(com.google.privacy.dlp.v2.InfoTypeStats) Subscriber(com.google.cloud.pubsub.v1.Subscriber) DlpServiceClient(com.google.cloud.dlp.v2.DlpServiceClient) DlpJob(com.google.privacy.dlp.v2.DlpJob) DatastoreOptions(com.google.privacy.dlp.v2.DatastoreOptions) KindExpression(com.google.privacy.dlp.v2.KindExpression) InspectJobConfig(com.google.privacy.dlp.v2.InspectJobConfig)

Example 4 with DlpServiceClient

use of com.google.cloud.dlp.v2.DlpServiceClient in project java-docs-samples by GoogleCloudPlatform.

the class Inspect method inspectBigquery.

// [END dlp_inspect_datastore]
// [START dlp_inspect_bigquery]
/**
 * Inspect a BigQuery table
 *
 * @param projectId The project ID to run the API call under
 * @param datasetId The ID of the dataset to inspect, e.g. 'my_dataset'
 * @param tableId The ID of the table to inspect, e.g. 'my_table'
 * @param minLikelihood The minimum likelihood required before returning a match
 * @param infoTypes The infoTypes of information to match
 * @param maxFindings The maximum number of findings to report (0 = server maximum)
 * @param topicId Topic ID for pubsub.
 * @param subscriptionId Subscription ID for pubsub.
 */
private static void inspectBigquery(String projectId, String datasetId, String tableId, Likelihood minLikelihood, List<InfoType> infoTypes, int maxFindings, String topicId, String subscriptionId) {
    // Instantiates a client
    try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) {
        // Reference to the BigQuery table
        BigQueryTable tableReference = BigQueryTable.newBuilder().setProjectId(projectId).setDatasetId(datasetId).setTableId(tableId).build();
        BigQueryOptions bigQueryOptions = BigQueryOptions.newBuilder().setTableReference(tableReference).build();
        // Construct BigQuery configuration to be inspected
        StorageConfig storageConfig = StorageConfig.newBuilder().setBigQueryOptions(bigQueryOptions).build();
        FindingLimits findingLimits = FindingLimits.newBuilder().setMaxFindingsPerRequest(maxFindings).build();
        InspectConfig inspectConfig = InspectConfig.newBuilder().addAllInfoTypes(infoTypes).setMinLikelihood(minLikelihood).setLimits(findingLimits).build();
        ProjectTopicName topic = ProjectTopicName.of(projectId, topicId);
        Action.PublishToPubSub publishToPubSub = Action.PublishToPubSub.newBuilder().setTopic(topic.toString()).build();
        Action action = Action.newBuilder().setPubSub(publishToPubSub).build();
        InspectJobConfig inspectJobConfig = InspectJobConfig.newBuilder().setStorageConfig(storageConfig).setInspectConfig(inspectConfig).addActions(action).build();
        // Asynchronously submit an inspect job, and wait on results
        CreateDlpJobRequest createDlpJobRequest = CreateDlpJobRequest.newBuilder().setParent(ProjectName.of(projectId).toString()).setInspectJob(inspectJobConfig).build();
        DlpJob dlpJob = dlpServiceClient.createDlpJob(createDlpJobRequest);
        System.out.println("Job created with ID:" + dlpJob.getName());
        // Wait for job completion semi-synchronously
        // For long jobs, consider using a truly asynchronous execution model such as Cloud Functions
        final SettableApiFuture<Boolean> done = SettableApiFuture.create();
        // Set up a Pub/Sub subscriber to listen on the job completion status
        Subscriber subscriber = Subscriber.newBuilder(ProjectSubscriptionName.of(projectId, subscriptionId), (pubsubMessage, ackReplyConsumer) -> {
            if (pubsubMessage.getAttributesCount() > 0 && pubsubMessage.getAttributesMap().get("DlpJobName").equals(dlpJob.getName())) {
                // notify job completion
                done.set(true);
                ackReplyConsumer.ack();
            }
        }).build();
        subscriber.startAsync();
        try {
            done.get(1, TimeUnit.MINUTES);
            // Wait for the job to become available
            Thread.sleep(500);
        } catch (Exception e) {
            System.out.println("Unable to verify job completion.");
        }
        DlpJob completedJob = dlpServiceClient.getDlpJob(GetDlpJobRequest.newBuilder().setName(dlpJob.getName()).build());
        System.out.println("Job status: " + completedJob.getState());
        InspectDataSourceDetails inspectDataSourceDetails = completedJob.getInspectDetails();
        InspectDataSourceDetails.Result result = inspectDataSourceDetails.getResult();
        if (result.getInfoTypeStatsCount() > 0) {
            System.out.println("Findings: ");
            for (InfoTypeStats infoTypeStat : result.getInfoTypeStatsList()) {
                System.out.print("\tInfo type: " + infoTypeStat.getInfoType().getName());
                System.out.println("\tCount: " + infoTypeStat.getCount());
            }
        } else {
            System.out.println("No findings.");
        }
    } catch (Exception e) {
        System.out.println("inspectBigquery Problems: " + e.getMessage());
    }
}
Also used : ByteContentItem(com.google.privacy.dlp.v2.ByteContentItem) InspectResult(com.google.privacy.dlp.v2.InspectResult) KindExpression(com.google.privacy.dlp.v2.KindExpression) Options(org.apache.commons.cli.Options) Likelihood(com.google.privacy.dlp.v2.Likelihood) Subscriber(com.google.cloud.pubsub.v1.Subscriber) BigQueryOptions(com.google.privacy.dlp.v2.BigQueryOptions) HelpFormatter(org.apache.commons.cli.HelpFormatter) MimetypesFileTypeMap(javax.activation.MimetypesFileTypeMap) CloudStorageOptions(com.google.privacy.dlp.v2.CloudStorageOptions) ArrayList(java.util.ArrayList) DefaultParser(org.apache.commons.cli.DefaultParser) InspectDataSourceDetails(com.google.privacy.dlp.v2.InspectDataSourceDetails) FindingLimits(com.google.privacy.dlp.v2.InspectConfig.FindingLimits) InspectContentResponse(com.google.privacy.dlp.v2.InspectContentResponse) ServiceOptions(com.google.cloud.ServiceOptions) URLConnection(java.net.URLConnection) StorageConfig(com.google.privacy.dlp.v2.StorageConfig) CommandLine(org.apache.commons.cli.CommandLine) PartitionId(com.google.privacy.dlp.v2.PartitionId) Action(com.google.privacy.dlp.v2.Action) DatastoreOptions(com.google.privacy.dlp.v2.DatastoreOptions) ProjectTopicName(com.google.pubsub.v1.ProjectTopicName) Option(org.apache.commons.cli.Option) InspectJobConfig(com.google.privacy.dlp.v2.InspectJobConfig) DlpServiceClient(com.google.cloud.dlp.v2.DlpServiceClient) Finding(com.google.privacy.dlp.v2.Finding) CreateDlpJobRequest(com.google.privacy.dlp.v2.CreateDlpJobRequest) Files(java.nio.file.Files) CommandLineParser(org.apache.commons.cli.CommandLineParser) ContentItem(com.google.privacy.dlp.v2.ContentItem) InfoType(com.google.privacy.dlp.v2.InfoType) InfoTypeStats(com.google.privacy.dlp.v2.InfoTypeStats) SettableApiFuture(com.google.api.core.SettableApiFuture) ByteString(com.google.protobuf.ByteString) TimeUnit(java.util.concurrent.TimeUnit) InspectConfig(com.google.privacy.dlp.v2.InspectConfig) List(java.util.List) ProjectName(com.google.privacy.dlp.v2.ProjectName) GetDlpJobRequest(com.google.privacy.dlp.v2.GetDlpJobRequest) Paths(java.nio.file.Paths) ParseException(org.apache.commons.cli.ParseException) BigQueryTable(com.google.privacy.dlp.v2.BigQueryTable) ProjectSubscriptionName(com.google.pubsub.v1.ProjectSubscriptionName) OptionGroup(org.apache.commons.cli.OptionGroup) DlpJob(com.google.privacy.dlp.v2.DlpJob) InspectContentRequest(com.google.privacy.dlp.v2.InspectContentRequest) Collections(java.util.Collections) Action(com.google.privacy.dlp.v2.Action) FindingLimits(com.google.privacy.dlp.v2.InspectConfig.FindingLimits) StorageConfig(com.google.privacy.dlp.v2.StorageConfig) InspectDataSourceDetails(com.google.privacy.dlp.v2.InspectDataSourceDetails) BigQueryOptions(com.google.privacy.dlp.v2.BigQueryOptions) InspectConfig(com.google.privacy.dlp.v2.InspectConfig) CreateDlpJobRequest(com.google.privacy.dlp.v2.CreateDlpJobRequest) ParseException(org.apache.commons.cli.ParseException) InfoTypeStats(com.google.privacy.dlp.v2.InfoTypeStats) Subscriber(com.google.cloud.pubsub.v1.Subscriber) DlpServiceClient(com.google.cloud.dlp.v2.DlpServiceClient) BigQueryTable(com.google.privacy.dlp.v2.BigQueryTable) DlpJob(com.google.privacy.dlp.v2.DlpJob) ProjectTopicName(com.google.pubsub.v1.ProjectTopicName) InspectJobConfig(com.google.privacy.dlp.v2.InspectJobConfig)

Example 5 with DlpServiceClient

use of com.google.cloud.dlp.v2.DlpServiceClient in project java-docs-samples by GoogleCloudPlatform.

the class Inspect method inspectString.

/**
 * [START dlp_inspect_string] Inspect a text for given InfoTypes
 *
 * @param string String to instpect
 * @param minLikelihood The minimum likelihood required before returning a match
 * @param maxFindings The maximum number of findings to report (0 = server maximum)
 * @param infoTypes The infoTypes of information to match
 * @param includeQuote Whether to include the matching string
 * @param projectId Google Cloud project ID
 */
private static void inspectString(String string, Likelihood minLikelihood, int maxFindings, List<InfoType> infoTypes, boolean includeQuote, String projectId) {
    // instantiate a client
    try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) {
        FindingLimits findingLimits = FindingLimits.newBuilder().setMaxFindingsPerRequest(maxFindings).build();
        InspectConfig inspectConfig = InspectConfig.newBuilder().addAllInfoTypes(infoTypes).setMinLikelihood(minLikelihood).setLimits(findingLimits).setIncludeQuote(includeQuote).build();
        ByteContentItem byteContentItem = ByteContentItem.newBuilder().setType(ByteContentItem.BytesType.TEXT_UTF8).setData(ByteString.copyFromUtf8(string)).build();
        ContentItem contentItem = ContentItem.newBuilder().setByteItem(byteContentItem).build();
        InspectContentRequest request = InspectContentRequest.newBuilder().setParent(ProjectName.of(projectId).toString()).setInspectConfig(inspectConfig).setItem(contentItem).build();
        InspectContentResponse response = dlpServiceClient.inspectContent(request);
        if (response.getResult().getFindingsCount() > 0) {
            System.out.println("Findings: ");
            for (Finding finding : response.getResult().getFindingsList()) {
                if (includeQuote) {
                    System.out.print("\tQuote: " + finding.getQuote());
                }
                System.out.print("\tInfo type: " + finding.getInfoType().getName());
                System.out.println("\tLikelihood: " + finding.getLikelihood());
            }
        } else {
            System.out.println("No findings.");
        }
    } catch (Exception e) {
        System.out.println("Error in inspectString: " + e.getMessage());
    }
}
Also used : FindingLimits(com.google.privacy.dlp.v2.InspectConfig.FindingLimits) DlpServiceClient(com.google.cloud.dlp.v2.DlpServiceClient) InspectContentResponse(com.google.privacy.dlp.v2.InspectContentResponse) Finding(com.google.privacy.dlp.v2.Finding) ByteContentItem(com.google.privacy.dlp.v2.ByteContentItem) InspectContentRequest(com.google.privacy.dlp.v2.InspectContentRequest) InspectConfig(com.google.privacy.dlp.v2.InspectConfig) ByteContentItem(com.google.privacy.dlp.v2.ByteContentItem) ContentItem(com.google.privacy.dlp.v2.ContentItem) ParseException(org.apache.commons.cli.ParseException)

Aggregations

DlpServiceClient (com.google.cloud.dlp.v2.DlpServiceClient)25 ParseException (org.apache.commons.cli.ParseException)22 InfoType (com.google.privacy.dlp.v2.InfoType)13 InspectConfig (com.google.privacy.dlp.v2.InspectConfig)12 ServiceOptions (com.google.cloud.ServiceOptions)10 ContentItem (com.google.privacy.dlp.v2.ContentItem)10 ProjectName (com.google.privacy.dlp.v2.ProjectName)10 ByteString (com.google.protobuf.ByteString)10 ArrayList (java.util.ArrayList)10 List (java.util.List)10 CommandLine (org.apache.commons.cli.CommandLine)10 CommandLineParser (org.apache.commons.cli.CommandLineParser)10 DefaultParser (org.apache.commons.cli.DefaultParser)10 HelpFormatter (org.apache.commons.cli.HelpFormatter)10 Option (org.apache.commons.cli.Option)10 Options (org.apache.commons.cli.Options)10 DlpJob (com.google.privacy.dlp.v2.DlpJob)9 OptionGroup (org.apache.commons.cli.OptionGroup)9 SettableApiFuture (com.google.api.core.SettableApiFuture)8 Subscriber (com.google.cloud.pubsub.v1.Subscriber)8