Examples with BatchPartialColumnDlpTable com.google.cloud.solutions.autotokenize.dlp.PartialBatchAccumulator.BatchPartialColumnDlpTable used on opensource projects

Example 1 with BatchPartialColumnDlpTable

use of com.google.cloud.solutions.autotokenize.dlp.PartialBatchAccumulator.BatchPartialColumnDlpTable in project auto-data-tokenize by GoogleCloudPlatform.

the class PartialBatchAccumulatorTest method batch_arrayFields_deidConfigContainsOnlyFieldReference.

@Test
public void batch_arrayFields_deidConfigContainsOnlyFieldReference() {
    PartialBatchAccumulator accumulator = PartialBatchAccumulator.withConfig(DlpEncryptConfig.newBuilder().addTransforms(ColumnTransform.newBuilder().setColumnId("$.multi_level_arrays.simple_field1").setTransform(CRYPTO_UNWRAPPED_TRANSFORM)).addTransforms(ColumnTransform.newBuilder().setColumnId("$.multi_level_arrays.level1_array.level1_array_record.level2_simple_field").setTransform(CRYPTO_UNWRAPPED_TRANSFORM)).addTransforms(ColumnTransform.newBuilder().setColumnId("$.multi_level_arrays.level1_array.level1_array_record.level2_array").setTransform(CRYPTO_UNWRAPPED_TRANSFORM)).build());
    FlatRecord record = RecordFlattener.forGenericRecord().flatten(TestResourceLoader.classPath().forAvro().withSchemaFile("avro_records/records_with_two_levels_of_arrays/two_level_arrays_schema.avsc").loadRecord("avro_records/records_with_two_levels_of_arrays/simple_two_level_array_record.json"));
    accumulator.addElement(record.toBuilder().setRecordId(UUID.randomUUID().toString()).build());
    BatchPartialColumnDlpTable batch = accumulator.makeBatch();
    ImmutableList<FieldId> deidConfigTokenizeFields = batch.get().getDeidentifyConfig().getRecordTransformations().getFieldTransformationsList().stream().map(FieldTransformation::getFieldsList).flatMap(List::stream).collect(toImmutableList());
    assertThat(deidConfigTokenizeFields).containsExactlyElementsIn(DeidentifyColumns.fieldIdsFor(ImmutableList.of("$.simple_field1", "$.level1_array.[\"level1_array_record\"].level2_simple_field.string", "$.level1_array.[\"level1_array_record\"].level2_array.string")));
}

Example 2 with BatchPartialColumnDlpTable

use of com.google.cloud.solutions.autotokenize.dlp.PartialBatchAccumulator.BatchPartialColumnDlpTable in project auto-data-tokenize by GoogleCloudPlatform.

the class PartialBatchAccumulatorTest method batch_nullableUnionField_valid.

@Test
public void batch_nullableUnionField_valid() {
    PartialBatchAccumulator accumulator = PartialBatchAccumulator.withConfig(TestResourceLoader.classPath().forProto(DlpEncryptConfig.class).loadJson("email_cc_dlp_encrypt_config.json"));
    var flatRecords = TestResourceLoader.classPath().forAvro().withSchemaFile("avro_records/userdata_records/schema.json").loadAllRecords("avro_records/userdata_records/record-2.json", "avro_records/userdata_records/record-3-cc-null.json").stream().map(RecordFlattener.forGenericRecord()::flatten).map(record -> record.toBuilder().setRecordId(UUID.randomUUID().toString()).build()).collect(toImmutableList());
    accumulator.addAllElements(flatRecords);
    BatchPartialColumnDlpTable batch = accumulator.makeBatch();
    FieldIdMatchesTokenizeColumns.withRecordIdColumn("__AUTOTOKENIZE__RECORD_ID__").assertExpectedHeadersOnly(batch.get().getTable().getHeadersList()).contains(TokenizingColPatternChecker.of("$.email", "$.cc"));
}

Example 3 with BatchPartialColumnDlpTable

use of com.google.cloud.solutions.autotokenize.dlp.PartialBatchAccumulator.BatchPartialColumnDlpTable in project auto-data-tokenize by GoogleCloudPlatform.

the class PartialBatchAccumulatorTest method batch_arrayFields_itemTableContainsFlattenedEntries.

@Test
public void batch_arrayFields_itemTableContainsFlattenedEntries() {
    PartialBatchAccumulator accumulator = PartialBatchAccumulator.withConfig(DlpEncryptConfig.newBuilder().addTransforms(ColumnTransform.newBuilder().setColumnId("$.multi_level_arrays.simple_field1").setTransform(CRYPTO_UNWRAPPED_TRANSFORM)).addTransforms(ColumnTransform.newBuilder().setColumnId("$.multi_level_arrays.level1_array.level1_array_record.level2_simple_field").setTransform(CRYPTO_UNWRAPPED_TRANSFORM)).addTransforms(ColumnTransform.newBuilder().setColumnId("$.multi_level_arrays.level1_array.level1_array_record.level2_array").setTransform(CRYPTO_UNWRAPPED_TRANSFORM)).build());
    FlatRecord record = RecordFlattener.forGenericRecord().flatten(TestResourceLoader.classPath().forAvro().withSchemaFile("avro_records/records_with_two_levels_of_arrays/two_level_arrays_schema.avsc").loadRecord("avro_records/records_with_two_levels_of_arrays/simple_two_level_array_record.json"));
    accumulator.addElement(record.toBuilder().setRecordId(UUID.randomUUID().toString()).build());
    BatchPartialColumnDlpTable batch = accumulator.makeBatch();
    assertThat(batch.get().getTable().getHeadersList()).containsExactlyElementsIn(DeidentifyColumns.fieldIdsFor(ImmutableList.of("__AUTOTOKENIZE__RECORD_ID__", "$.simple_field1", "$.level1_array[0].[\"level1_array_record\"].level2_simple_field.string", "$.level1_array[1].[\"level1_array_record\"].level2_array[1].string", "$.level1_array[0].[\"level1_array_record\"].level2_array[0].string", "$.level1_array[0].[\"level1_array_record\"].level2_array[1].string", "$.level1_array[1].[\"level1_array_record\"].level2_simple_field.string", "$.level1_array[1].[\"level1_array_record\"].level2_array[0].string")));
}