Search in sources :

Example 51 with Value

use of com.google.firestore.v1beta1.Value in project java-docs-samples by GoogleCloudPlatform.

the class RiskAnalysis method calculateLDiversity.

// [END dlp_k_anonymity]
// [START dlp_l_diversity]
/**
 * Calculate l-diversity for an attribute relative to quasi-identifiers in a BigQuery table.
 *
 * @param projectId The Google Cloud Platform project ID to run the API call under.
 * @param datasetId The BigQuery dataset to analyze.
 * @param tableId The BigQuery table to analyze.
 * @param sensitiveAttribute The name of the attribute to compare the quasi-ID against
 * @param quasiIds A set of column names that form a composite key ('quasi-identifiers').
 * @param topicId The name of the Pub/Sub topic to notify once the job completes
 * @param subscriptionId The name of the Pub/Sub subscription to use when listening for job
 *     completion status.
 */
private static void calculateLDiversity(String projectId, String datasetId, String tableId, String sensitiveAttribute, List<String> quasiIds, String topicId, String subscriptionId) throws Exception {
    // Instantiates a client
    try (DlpServiceClient dlpServiceClient = DlpServiceClient.create()) {
        FieldId sensitiveAttributeField = FieldId.newBuilder().setName(sensitiveAttribute).build();
        List<FieldId> quasiIdFields = quasiIds.stream().map(columnName -> FieldId.newBuilder().setName(columnName).build()).collect(Collectors.toList());
        LDiversityConfig ldiversityConfig = LDiversityConfig.newBuilder().addAllQuasiIds(quasiIdFields).setSensitiveAttribute(sensitiveAttributeField).build();
        BigQueryTable bigQueryTable = BigQueryTable.newBuilder().setProjectId(projectId).setDatasetId(datasetId).setTableId(tableId).build();
        PrivacyMetric privacyMetric = PrivacyMetric.newBuilder().setLDiversityConfig(ldiversityConfig).build();
        String topicName = String.format("projects/%s/topics/%s", projectId, topicId);
        PublishToPubSub publishToPubSub = PublishToPubSub.newBuilder().setTopic(topicName).build();
        // Create action to publish job status notifications over Google Cloud Pub/Sub
        Action action = Action.newBuilder().setPubSub(publishToPubSub).build();
        RiskAnalysisJobConfig riskAnalysisJobConfig = RiskAnalysisJobConfig.newBuilder().setSourceTable(bigQueryTable).setPrivacyMetric(privacyMetric).addActions(action).build();
        CreateDlpJobRequest createDlpJobRequest = CreateDlpJobRequest.newBuilder().setParent(ProjectName.of(projectId).toString()).setRiskJob(riskAnalysisJobConfig).build();
        DlpJob dlpJob = dlpServiceClient.createDlpJob(createDlpJobRequest);
        String dlpJobName = dlpJob.getName();
        final SettableApiFuture<Boolean> done = SettableApiFuture.create();
        // Set up a Pub/Sub subscriber to listen on the job completion status
        Subscriber subscriber = Subscriber.newBuilder(ProjectSubscriptionName.newBuilder().setProject(projectId).setSubscription(subscriptionId).build(), (pubsubMessage, ackReplyConsumer) -> {
            if (pubsubMessage.getAttributesCount() > 0 && pubsubMessage.getAttributesMap().get("DlpJobName").equals(dlpJobName)) {
                // notify job completion
                done.set(true);
                ackReplyConsumer.ack();
            }
        }).build();
        subscriber.startAsync();
        // For long jobs, consider using a truly asynchronous execution model such as Cloud Functions
        try {
            done.get(1, TimeUnit.MINUTES);
            // Wait for the job to become available
            Thread.sleep(500);
        } catch (TimeoutException e) {
            System.out.println("Unable to verify job completion.");
        }
        // retrieve completed job status
        DlpJob completedJob = dlpServiceClient.getDlpJob(GetDlpJobRequest.newBuilder().setName(dlpJobName).build());
        System.out.println("Job status: " + completedJob.getState());
        AnalyzeDataSourceRiskDetails riskDetails = completedJob.getRiskDetails();
        LDiversityResult ldiversityResult = riskDetails.getLDiversityResult();
        for (LDiversityHistogramBucket result : ldiversityResult.getSensitiveValueFrequencyHistogramBucketsList()) {
            for (LDiversityEquivalenceClass bucket : result.getBucketValuesList()) {
                List<String> quasiIdValues = bucket.getQuasiIdsValuesList().stream().map(Value::toString).collect(Collectors.toList());
                System.out.println("\tQuasi-ID values: " + String.join(", ", quasiIdValues));
                System.out.println("\tClass size: " + bucket.getEquivalenceClassSize());
                for (ValueFrequency valueFrequency : bucket.getTopSensitiveValuesList()) {
                    System.out.printf("\t\tSensitive value %s occurs %d time(s).\n", valueFrequency.getValue().toString(), valueFrequency.getCount());
                }
            }
        }
    } catch (Exception e) {
        System.out.println("Error in calculateLDiversity: " + e.getMessage());
    }
}
Also used : Arrays(java.util.Arrays) TimeoutException(java.util.concurrent.TimeoutException) Subscriber(com.google.cloud.pubsub.v1.Subscriber) DefaultParser(org.apache.commons.cli.DefaultParser) KMapEstimationHistogramBucket(com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.KMapEstimationResult.KMapEstimationHistogramBucket) LDiversityEquivalenceClass(com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.LDiversityResult.LDiversityEquivalenceClass) ValueFrequency(com.google.privacy.dlp.v2.ValueFrequency) LDiversityConfig(com.google.privacy.dlp.v2.PrivacyMetric.LDiversityConfig) NumericalStatsConfig(com.google.privacy.dlp.v2.PrivacyMetric.NumericalStatsConfig) Action(com.google.privacy.dlp.v2.Action) KMapEstimationConfig(com.google.privacy.dlp.v2.PrivacyMetric.KMapEstimationConfig) CategoricalStatsHistogramBucket(com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.CategoricalStatsResult.CategoricalStatsHistogramBucket) KAnonymityEquivalenceClass(com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.KAnonymityResult.KAnonymityEquivalenceClass) Value(com.google.privacy.dlp.v2.Value) TaggedField(com.google.privacy.dlp.v2.PrivacyMetric.KMapEstimationConfig.TaggedField) RiskAnalysisJobConfig(com.google.privacy.dlp.v2.RiskAnalysisJobConfig) Collectors(java.util.stream.Collectors) SettableApiFuture(com.google.api.core.SettableApiFuture) List(java.util.List) KAnonymityResult(com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.KAnonymityResult) ParseException(org.apache.commons.cli.ParseException) BigQueryTable(com.google.privacy.dlp.v2.BigQueryTable) ProjectSubscriptionName(com.google.pubsub.v1.ProjectSubscriptionName) AnalyzeDataSourceRiskDetails(com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails) Options(org.apache.commons.cli.Options) HelpFormatter(org.apache.commons.cli.HelpFormatter) CategoricalStatsConfig(com.google.privacy.dlp.v2.PrivacyMetric.CategoricalStatsConfig) ArrayList(java.util.ArrayList) ServiceOptions(com.google.cloud.ServiceOptions) CommandLine(org.apache.commons.cli.CommandLine) FieldId(com.google.privacy.dlp.v2.FieldId) ProjectTopicName(com.google.pubsub.v1.ProjectTopicName) Option(org.apache.commons.cli.Option) DlpServiceClient(com.google.cloud.dlp.v2.DlpServiceClient) Iterator(java.util.Iterator) CreateDlpJobRequest(com.google.privacy.dlp.v2.CreateDlpJobRequest) CommandLineParser(org.apache.commons.cli.CommandLineParser) KMapEstimationResult(com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.KMapEstimationResult) KMapEstimationQuasiIdValues(com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.KMapEstimationResult.KMapEstimationQuasiIdValues) InfoType(com.google.privacy.dlp.v2.InfoType) LDiversityResult(com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.LDiversityResult) KAnonymityConfig(com.google.privacy.dlp.v2.PrivacyMetric.KAnonymityConfig) TimeUnit(java.util.concurrent.TimeUnit) PublishToPubSub(com.google.privacy.dlp.v2.Action.PublishToPubSub) ProjectName(com.google.privacy.dlp.v2.ProjectName) GetDlpJobRequest(com.google.privacy.dlp.v2.GetDlpJobRequest) LDiversityHistogramBucket(com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.LDiversityResult.LDiversityHistogramBucket) KAnonymityHistogramBucket(com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.KAnonymityResult.KAnonymityHistogramBucket) PrivacyMetric(com.google.privacy.dlp.v2.PrivacyMetric) OptionGroup(org.apache.commons.cli.OptionGroup) DlpJob(com.google.privacy.dlp.v2.DlpJob) Collections(java.util.Collections) LDiversityHistogramBucket(com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.LDiversityResult.LDiversityHistogramBucket) LDiversityConfig(com.google.privacy.dlp.v2.PrivacyMetric.LDiversityConfig) Action(com.google.privacy.dlp.v2.Action) RiskAnalysisJobConfig(com.google.privacy.dlp.v2.RiskAnalysisJobConfig) PrivacyMetric(com.google.privacy.dlp.v2.PrivacyMetric) CreateDlpJobRequest(com.google.privacy.dlp.v2.CreateDlpJobRequest) TimeoutException(java.util.concurrent.TimeoutException) ParseException(org.apache.commons.cli.ParseException) PublishToPubSub(com.google.privacy.dlp.v2.Action.PublishToPubSub) Subscriber(com.google.cloud.pubsub.v1.Subscriber) DlpServiceClient(com.google.cloud.dlp.v2.DlpServiceClient) BigQueryTable(com.google.privacy.dlp.v2.BigQueryTable) ValueFrequency(com.google.privacy.dlp.v2.ValueFrequency) FieldId(com.google.privacy.dlp.v2.FieldId) LDiversityResult(com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.LDiversityResult) DlpJob(com.google.privacy.dlp.v2.DlpJob) AnalyzeDataSourceRiskDetails(com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails) LDiversityEquivalenceClass(com.google.privacy.dlp.v2.AnalyzeDataSourceRiskDetails.LDiversityResult.LDiversityEquivalenceClass) TimeoutException(java.util.concurrent.TimeoutException)

Example 52 with Value

use of com.google.firestore.v1beta1.Value in project structr by structr.

the class SessionTransaction method getStrings.

public QueryResult<String> getStrings(final String statement, final Map<String, Object> map) {
    final long t0 = System.currentTimeMillis();
    try {
        final StatementResult result = tx.run(statement, map);
        final Record record = result.next();
        final Value value = record.get(0);
        return new QueryResult<String>() {

            @Override
            public void close() {
                result.consume();
            }

            @Override
            public Iterator<String> iterator() {
                return value.asList(Values.ofString()).iterator();
            }
        };
    } catch (TransientException tex) {
        closed = true;
        throw new RetryException(tex);
    } catch (NoSuchRecordException nex) {
        throw new NotFoundException(nex);
    } catch (ServiceUnavailableException ex) {
        throw new NetworkException(ex.getMessage(), ex);
    } finally {
        logQuery(statement, map, t0);
    }
}
Also used : StatementResult(org.neo4j.driver.v1.StatementResult) QueryResult(org.structr.api.QueryResult) TransientException(org.neo4j.driver.v1.exceptions.TransientException) Value(org.neo4j.driver.v1.Value) NotFoundException(org.structr.api.NotFoundException) Record(org.neo4j.driver.v1.Record) ServiceUnavailableException(org.neo4j.driver.v1.exceptions.ServiceUnavailableException) RetryException(org.structr.api.RetryException) NetworkException(org.structr.api.NetworkException) NoSuchRecordException(org.neo4j.driver.v1.exceptions.NoSuchRecordException)

Example 53 with Value

use of com.google.firestore.v1beta1.Value in project googleads-java-lib by googleads.

the class Pql method getApiValue.

/**
 * Gets the underlying value of the {@code Value} object that's comparable to what would be
 * returned in any other API object (i.e. DateTimeValue will return an API DateTime, not a Joda
 * DateTime).
 *
 * @param value the value to convert
 * @return the native value of {@code Value} or {@code null} if the underlying value is null
 * @throws IllegalArgumentException if value cannot be converted
 */
public static Object getApiValue(Value value) {
    if (value instanceof BooleanValue) {
        return ((BooleanValue) value).isValue();
    } else if (value instanceof NumberValue) {
        if (((NumberValue) value).getValue() == null) {
            return null;
        } else {
            try {
                return NumberFormat.getInstance().parse(((NumberValue) value).getValue());
            } catch (ParseException e) {
                throw new IllegalStateException("Received invalid number format from API: " + ((NumberValue) value).getValue());
            }
        }
    } else if (value instanceof TextValue) {
        return ((TextValue) value).getValue();
    } else if (value instanceof DateTimeValue) {
        return ((DateTimeValue) value).getValue();
    } else if (value instanceof DateValue) {
        return ((DateValue) value).getValue();
    } else if (value instanceof TargetingValue) {
        return ((TargetingValue) value).getValue();
    } else if (value instanceof SetValue) {
        List<Value> setValues = ((SetValue) value).getValues();
        Set<Object> apiValue = new LinkedHashSet<Object>();
        if (setValues != null) {
            for (Value setValue : setValues) {
                validateSetValueEntryForSet(getApiValue(setValue), apiValue);
                apiValue.add(getApiValue(setValue));
            }
        }
        return apiValue;
    } else {
        throw new IllegalArgumentException("Unsupported Value type [" + value.getClass() + "]");
    }
}
Also used : LinkedHashSet(java.util.LinkedHashSet) DateTimeValue(com.google.api.ads.admanager.jaxws.v202108.DateTimeValue) NumberValue(com.google.api.ads.admanager.jaxws.v202108.NumberValue) TextValue(com.google.api.ads.admanager.jaxws.v202108.TextValue) DateValue(com.google.api.ads.admanager.jaxws.v202108.DateValue) BooleanValue(com.google.api.ads.admanager.jaxws.v202108.BooleanValue) DateTimeValue(com.google.api.ads.admanager.jaxws.v202108.DateTimeValue) NumberValue(com.google.api.ads.admanager.jaxws.v202108.NumberValue) TextValue(com.google.api.ads.admanager.jaxws.v202108.TextValue) SetValue(com.google.api.ads.admanager.jaxws.v202108.SetValue) BooleanValue(com.google.api.ads.admanager.jaxws.v202108.BooleanValue) DateValue(com.google.api.ads.admanager.jaxws.v202108.DateValue) TargetingValue(com.google.api.ads.admanager.jaxws.v202108.TargetingValue) Value(com.google.api.ads.admanager.jaxws.v202108.Value) ParseException(java.text.ParseException) TargetingValue(com.google.api.ads.admanager.jaxws.v202108.TargetingValue) SetValue(com.google.api.ads.admanager.jaxws.v202108.SetValue)

Example 54 with Value

use of com.google.firestore.v1beta1.Value in project googleads-java-lib by googleads.

the class Pql method createValue.

/**
 * Creates a {@link Value} from the value i.e. a {@link TextValue} for a value of type {@code
 * String}, {@link BooleanValue} for type {@code Boolean}, {@link NumberValue} for type {@code
 * Double}, {@code Long}, or {@code Integer}, and {@link DateTimeValue} for type {@link DateTime}.
 * If the value is a {@code Value}, the value is returned. If the value is {@code null}, an empty
 * {@link TextValue} is returned.
 *
 * @param value the value to convert
 * @return the constructed value of the appropriate type
 * @throws IllegalArgumentException if value cannot be converted
 */
public static Value createValue(Object value) {
    if (value instanceof Value) {
        return (Value) value;
    } else if (value == null) {
        return new TextValue();
    } else {
        if (value instanceof Boolean) {
            BooleanValue booleanValue = new BooleanValue();
            booleanValue.setValue((Boolean) value);
            return booleanValue;
        } else if (value instanceof Double || value instanceof Long || value instanceof Integer) {
            NumberValue numberValue = new NumberValue();
            numberValue.setValue(value.toString());
            return numberValue;
        } else if (value instanceof String) {
            TextValue textValue = new TextValue();
            textValue.setValue((String) value);
            return textValue;
        } else if (value instanceof DateTime) {
            DateTimeValue dateTimeValue = new DateTimeValue();
            dateTimeValue.setValue((DateTime) value);
            return dateTimeValue;
        } else if (value instanceof Date) {
            DateValue dateValue = new DateValue();
            dateValue.setValue((Date) value);
            return dateValue;
        } else if (value instanceof Targeting) {
            TargetingValue targetingValue = new TargetingValue();
            targetingValue.setValue((Targeting) value);
            return targetingValue;
        } else if (value instanceof Set<?>) {
            SetValue setValue = new SetValue();
            Set<Value> values = new LinkedHashSet<Value>();
            for (Object entry : (Set<?>) value) {
                validateSetValueEntryForSet(createValue(entry), values);
                values.add(createValue(entry));
            }
            setValue.getValues().addAll(values);
            return setValue;
        } else {
            throw new IllegalArgumentException("Unsupported Value type [" + value.getClass() + "]");
        }
    }
}
Also used : Set(java.util.Set) ResultSet(com.google.api.ads.admanager.jaxws.v202108.ResultSet) LinkedHashSet(java.util.LinkedHashSet) DateTimeValue(com.google.api.ads.admanager.jaxws.v202108.DateTimeValue) Targeting(com.google.api.ads.admanager.jaxws.v202108.Targeting) DateTime(com.google.api.ads.admanager.jaxws.v202108.DateTime) Date(com.google.api.ads.admanager.jaxws.v202108.Date) NumberValue(com.google.api.ads.admanager.jaxws.v202108.NumberValue) TextValue(com.google.api.ads.admanager.jaxws.v202108.TextValue) DateValue(com.google.api.ads.admanager.jaxws.v202108.DateValue) BooleanValue(com.google.api.ads.admanager.jaxws.v202108.BooleanValue) DateTimeValue(com.google.api.ads.admanager.jaxws.v202108.DateTimeValue) NumberValue(com.google.api.ads.admanager.jaxws.v202108.NumberValue) TextValue(com.google.api.ads.admanager.jaxws.v202108.TextValue) SetValue(com.google.api.ads.admanager.jaxws.v202108.SetValue) BooleanValue(com.google.api.ads.admanager.jaxws.v202108.BooleanValue) DateValue(com.google.api.ads.admanager.jaxws.v202108.DateValue) TargetingValue(com.google.api.ads.admanager.jaxws.v202108.TargetingValue) Value(com.google.api.ads.admanager.jaxws.v202108.Value) TargetingValue(com.google.api.ads.admanager.jaxws.v202108.TargetingValue) SetValue(com.google.api.ads.admanager.jaxws.v202108.SetValue)

Example 55 with Value

use of com.google.firestore.v1beta1.Value in project googleads-java-lib by googleads.

the class Pql method getApiValue.

/**
 * Gets the underlying value of the {@code Value} object that's comparable to what would be
 * returned in any other API object (i.e. DateTimeValue will return an API DateTime, not a Joda
 * DateTime).
 *
 * @param value the value to convert
 * @return the native value of {@code Value} or {@code null} if the underlying value is null
 * @throws IllegalArgumentException if value cannot be converted
 */
public static Object getApiValue(Value value) {
    if (value instanceof BooleanValue) {
        return ((BooleanValue) value).isValue();
    } else if (value instanceof NumberValue) {
        if (((NumberValue) value).getValue() == null) {
            return null;
        } else {
            try {
                return NumberFormat.getInstance().parse(((NumberValue) value).getValue());
            } catch (ParseException e) {
                throw new IllegalStateException("Received invalid number format from API: " + ((NumberValue) value).getValue());
            }
        }
    } else if (value instanceof TextValue) {
        return ((TextValue) value).getValue();
    } else if (value instanceof DateTimeValue) {
        return ((DateTimeValue) value).getValue();
    } else if (value instanceof DateValue) {
        return ((DateValue) value).getValue();
    } else if (value instanceof TargetingValue) {
        return ((TargetingValue) value).getValue();
    } else if (value instanceof SetValue) {
        List<Value> setValues = ((SetValue) value).getValues();
        Set<Object> apiValue = new LinkedHashSet<Object>();
        if (setValues != null) {
            for (Value setValue : setValues) {
                validateSetValueEntryForSet(getApiValue(setValue), apiValue);
                apiValue.add(getApiValue(setValue));
            }
        }
        return apiValue;
    } else {
        throw new IllegalArgumentException("Unsupported Value type [" + value.getClass() + "]");
    }
}
Also used : LinkedHashSet(java.util.LinkedHashSet) DateTimeValue(com.google.api.ads.admanager.jaxws.v202111.DateTimeValue) NumberValue(com.google.api.ads.admanager.jaxws.v202111.NumberValue) TextValue(com.google.api.ads.admanager.jaxws.v202111.TextValue) DateValue(com.google.api.ads.admanager.jaxws.v202111.DateValue) BooleanValue(com.google.api.ads.admanager.jaxws.v202111.BooleanValue) DateTimeValue(com.google.api.ads.admanager.jaxws.v202111.DateTimeValue) TextValue(com.google.api.ads.admanager.jaxws.v202111.TextValue) SetValue(com.google.api.ads.admanager.jaxws.v202111.SetValue) NumberValue(com.google.api.ads.admanager.jaxws.v202111.NumberValue) TargetingValue(com.google.api.ads.admanager.jaxws.v202111.TargetingValue) Value(com.google.api.ads.admanager.jaxws.v202111.Value) BooleanValue(com.google.api.ads.admanager.jaxws.v202111.BooleanValue) DateValue(com.google.api.ads.admanager.jaxws.v202111.DateValue) ParseException(java.text.ParseException) TargetingValue(com.google.api.ads.admanager.jaxws.v202111.TargetingValue) SetValue(com.google.api.ads.admanager.jaxws.v202111.SetValue)

Aggregations

Test (org.junit.Test)126 Value (com.google.firestore.v1.Value)108 ArrayValue (com.google.firestore.v1.ArrayValue)73 LinkedHashSet (java.util.LinkedHashSet)71 ObjectValue (com.google.firebase.firestore.model.ObjectValue)53 NullValue (com.google.protobuf.NullValue)50 MapValue (com.google.firestore.v1.MapValue)47 ArrayList (java.util.ArrayList)30 HashMap (java.util.HashMap)25 Value (com.google.datastore.v1.Value)20 Map (java.util.Map)20 TableFieldSchema (com.google.api.services.bigquery.model.TableFieldSchema)17 List (java.util.List)17 Record (org.apache.avro.generic.GenericData.Record)16 SchemaAndRecord (org.apache.beam.sdk.io.gcp.bigquery.SchemaAndRecord)16 CoreMatchers.notNullValue (org.hamcrest.CoreMatchers.notNullValue)16 Set (java.util.Set)14 TestUtil.wrapObject (com.google.firebase.firestore.testutil.TestUtil.wrapObject)13 Nullable (androidx.annotation.Nullable)10 Value (com.google.privacy.dlp.v2.Value)9