Examples with GCSPersistentSetImpl com.google.cloud.pso.bq_pii_classifier.services.set.GCSPersistentSetImpl used on opensource projects

Example 1 with GCSPersistentSetImpl

use of com.google.cloud.pso.bq_pii_classifier.services.set.GCSPersistentSetImpl in project bq-pii-classifier by GoogleCloudPlatform.

the class InspectorController method receiveMessage.

@RequestMapping(value = "/", method = RequestMethod.POST)
public ResponseEntity receiveMessage(@RequestBody PubSubEvent requestBody) {
    String trackingId = "0000000000000-z";
    DlpService dlpService = null;
    BigQueryService bqService = null;
    try {
        if (requestBody == null || requestBody.getMessage() == null) {
            String msg = "Bad Request: invalid message format";
            logger.logSevereWithTracker(trackingId, msg);
            throw new NonRetryableApplicationException("Request body or message is Null.");
        }
        String requestJsonString = requestBody.getMessage().dataToUtf8String();
        // remove any escape characters (e.g. from Terraform
        requestJsonString = requestJsonString.replace("\\", "");
        logger.logInfoWithTracker(trackingId, String.format("Received payload: %s", requestJsonString));
        Operation operation = gson.fromJson(requestJsonString, Operation.class);
        trackingId = operation.getTrackingId();
        logger.logInfoWithTracker(trackingId, String.format("Parsed Request: %s", operation.toString()));
        dlpService = new DlpServiceImpl();
        bqService = new BigQueryServiceImpl();
        Inspector inspector = new Inspector(environment.toConfig(), dlpService, bqService, new GCSPersistentSetImpl(environment.getGcsFlagsBucket()), "inspector-flags");
        inspector.execute(operation, trackingId, requestBody.getMessage().getMessageId());
        return new ResponseEntity("Process completed successfully.", HttpStatus.OK);
    } catch (Exception e) {
        return ControllerExceptionHelper.handleException(e, logger, trackingId);
    } finally {
        if (dlpService != null) {
            dlpService.shutDown();
        }
    }
}

Example 2 with GCSPersistentSetImpl

use of com.google.cloud.pso.bq_pii_classifier.services.set.GCSPersistentSetImpl in project bq-pii-classifier by GoogleCloudPlatform.

the class TaggingDispatcherController method receiveMessage.

@RequestMapping(value = "/", method = RequestMethod.POST)
public ResponseEntity receiveMessage(@RequestBody PubSubEvent requestBody) {
    String runId = TrackingHelper.generateTaggingRunId();
    String state = "";
    try {
        if (requestBody == null || requestBody.getMessage() == null) {
            String msg = "Bad Request: invalid message format";
            logger.logSevereWithTracker(runId, msg);
            throw new NonRetryableApplicationException("Request body or message is Null.");
        }
        String requestJsonString = requestBody.getMessage().dataToUtf8String();
        // remove any escape characters (e.g. from Terraform
        requestJsonString = requestJsonString.replace("\\", "");
        logger.logInfoWithTracker(runId, String.format("Received payload: %s", requestJsonString));
        BigQueryScope bqScope = gson.fromJson(requestJsonString, BigQueryScope.class);
        logger.logInfoWithTracker(runId, String.format("Parsed JSON input %s ", bqScope.toString()));
        Scanner dlpResultsScanner;
        if (environment.getIsAutoDlpMode()) {
            dlpResultsScanner = new AutoDlpResultsScannerImpl(environment.getProjectId(), environment.getSolutionDataset(), environment.getDlpTableAuto(), new BigQueryServiceImpl());
        } else {
            dlpResultsScanner = new StandardDlpResultsScannerImpl(environment.getProjectId(), environment.getSolutionDataset(), environment.getDlpTableStandard(), environment.getLoggingTable(), new BigQueryServiceImpl());
        }
        Dispatcher dispatcher = new Dispatcher(environment.toConfig(), new BigQueryServiceImpl(), new PubSubServiceImpl(), dlpResultsScanner, new GCSPersistentSetImpl(environment.getGcsFlagsBucket()), "tagging-dispatcher-flags", runId);
        PubSubPublishResults results = dispatcher.execute(bqScope, requestBody.getMessage().getMessageId());
        state = String.format("Publishing results: %s SUCCESS MESSAGES and %s FAILED MESSAGES", results.getSuccessMessages().size(), results.getFailedMessages().size());
        logger.logInfoWithTracker(runId, state);
    } catch (Exception e) {
        logger.logNonRetryableExceptions(runId, e);
        state = String.format("ERROR '%s'", e.getMessage());
    }
    return new ResponseEntity(String.format("Process completed with state = %s", state), HttpStatus.OK);
}

Example 3 with GCSPersistentSetImpl

use of com.google.cloud.pso.bq_pii_classifier.services.set.GCSPersistentSetImpl in project bq-pii-classifier by GoogleCloudPlatform.

the class TaggerController method receiveMessage.

@RequestMapping(value = "/", method = RequestMethod.POST)
public ResponseEntity receiveMessage(@RequestBody PubSubEvent requestBody) {
    String defaultTrackingId = "0000000000000-z";
    Operation taggerRequest = null;
    try {
        if (requestBody == null || requestBody.getMessage() == null) {
            String msg = "Bad Request: invalid message format";
            logger.logSevereWithTracker(defaultTrackingId, msg);
            throw new NonRetryableApplicationException("Request body or message is Null.");
        }
        // The pubsub message could come from different sources with different formats
        // 1. From the Dispatcher as JSON encoded "Operation" object (in standard and auto-dlp modes)
        // 2. From Standard DLP inspection job notifications (PubSub Actions) as a string message with the DLP job name
        // 3. From Auto DLP notifications (PubSub Actions) as Protobuf encoded "DataProfilePubSubMessage" (in AutoDLP mode only)
        // Here we try and parse it directly as an "Operation" object or indirectly via the "DataProfilePubSubMessage" proto
        taggerRequest = parseEvent(requestBody);
        BigQueryService bigQueryService = new BigQueryServiceImpl();
        // determine the Type of DLP FindingsReader based on env
        FindingsReader findingsReader = FindingsReaderFactory.getNewReader(FindingsReaderFactory.findReader(environment.getIsAutoDlpMode(), environment.getPromoteMixedTypes()), bigQueryService, environment.getProjectId(), environment.getDlpDataset(), environment.getIsAutoDlpMode() ? environment.getDlpTableAuto() : environment.getDlpTableStandard(), environment.getConfigViewDatasetDomainMap(), environment.getConfigViewProjectDomainMap(), environment.getConfigViewInfoTypePolicyTagsMap());
        Tagger tagger = new Tagger(environment.toConfig(), bigQueryService, findingsReader, new GCSPersistentSetImpl(environment.getGcsFlagsBucket()), "tagger-flags");
        tagger.execute(taggerRequest, requestBody.getMessage().getMessageId());
        return new ResponseEntity("Process completed successfully.", HttpStatus.OK);
    } catch (Exception e) {
        String trackingId = taggerRequest == null ? defaultTrackingId : taggerRequest.getTrackingId();
        return ControllerExceptionHelper.handleException(e, logger, trackingId);
    }
}

Example 4 with GCSPersistentSetImpl

use of com.google.cloud.pso.bq_pii_classifier.services.set.GCSPersistentSetImpl in project bq-pii-classifier by GoogleCloudPlatform.

the class InspectionDispatcherController method receiveMessage.

@RequestMapping(value = "/", method = RequestMethod.POST)
public ResponseEntity receiveMessage(@RequestBody PubSubEvent requestBody) {
    String runId = TrackingHelper.generateInspectionRunId();
    String state = "";
    try {
        if (requestBody == null || requestBody.getMessage() == null) {
            String msg = "Bad Request: invalid message format";
            logger.logSevereWithTracker(runId, msg);
            throw new NonRetryableApplicationException("Request body or message is Null.");
        }
        String requestJsonString = requestBody.getMessage().dataToUtf8String();
        // remove any escape characters (e.g. from Terraform
        requestJsonString = requestJsonString.replace("\\", "");
        logger.logInfoWithTracker(runId, String.format("Received payload: %s", requestJsonString));
        BigQueryScope bqScope = gson.fromJson(requestJsonString, BigQueryScope.class);
        logger.logInfoWithTracker(runId, String.format("Parsed JSON input %s ", bqScope.toString()));
        Dispatcher dispatcher = new Dispatcher(environment.toConfig(), new BigQueryServiceImpl(), new PubSubServiceImpl(), new BigQueryScannerImpl(), new GCSPersistentSetImpl(environment.getGcsFlagsBucket()), "inspection-dispatcher-flags", runId);
        PubSubPublishResults results = dispatcher.execute(bqScope, requestBody.getMessage().getMessageId());
        state = String.format("Publishing results: %s SUCCESS MESSAGES and %s FAILED MESSAGES", results.getSuccessMessages().size(), results.getFailedMessages().size());
        logger.logInfoWithTracker(runId, state);
    } catch (Exception e) {
        logger.logNonRetryableExceptions(runId, e);
        state = String.format("ERROR '%s'", e.getMessage());
    }
    return new ResponseEntity(String.format("Process completed with state = %s", state), HttpStatus.OK);
}