Examples with JobStorage com.mercedesbenz.sechub.storage.core.JobStorage used on opensource projects

Example 11 with JobStorage

use of com.mercedesbenz.sechub.storage.core.JobStorage in project sechub by mercedes-benz.

the class PDSFileUploadJobService method startUpload.

private void startUpload(UUID jobUUID, HttpServletRequest request, String fileName) throws FileUploadException, IOException, UnsupportedEncodingException {
    /* prepare */
    LOG.debug("Start upload file: {} for PDS job: {}", fileName, jobUUID);
    String checksumFromUser = null;
    String checksumCalculated = null;
    boolean fileDefinedByUser = false;
    boolean checkSumDefinedByUser = false;
    JobStorage jobStorage = storageService.getJobStorage(jobUUID);
    ServletFileUpload upload = new ServletFileUpload();
    long maxUploadSize = configuration.getMaxUploadSizeInBytes();
    // we accept 600 bytes more for header, checksum etc.
    upload.setSizeMax(maxUploadSize + 600);
    upload.setFileSizeMax(maxUploadSize);
    /*
         * Important: this next call of "upload.getItemIterator(..)" looks very simple,
         * but it creates a new <code>FileItemIteratorImpl</code> instances which
         * internally does some heavy things on creation: It does create a new input
         * stream, checks for max size handling and much more. We want to avoid creating
         * the iterator multiple times!
         *
         * Also any access to the origin request to access the parameter/field names
         * does always trigger a multipart resolving which uses again the underlying
         * standard Servlet mechanism and the configured max sizes there!
         *
         * So we could only check parameters with another item iterator when we want to
         * handle this specialized, but the item iterator should be created only one
         * time (see explained reason before).
         *
         * This is the reason, why we do not check the user input at the beginning but
         * only at the end. This is maybe inconvenient for the user when forgetting to
         * define a field, but this normally happens only one time and the benefit of
         * avoiding side effects. In addition, the performance (speed) does matter here.
         *
         * ------------------------- So please do NOT change! -------------------------
         */
    FileItemIterator iterStream = upload.getItemIterator(request);
    while (iterStream.hasNext()) {
        FileItemStream item = iterStream.next();
        String fieldName = item.getFieldName();
        switch(fieldName) {
            case MULTIPART_CHECKSUM:
                try (InputStream checkSumInputStream = item.openStream()) {
                    checksumFromUser = Streams.asString(checkSumInputStream);
                    checksumSHA256Service.assertValidSha256Checksum(checksumFromUser);
                    jobStorage.store(fileName + DOT_CHECKSUM, new StringInputStream(checksumFromUser));
                    LOG.info("uploaded user defined checksum as file for file: {} in PDS job: {}", fileName, jobUUID);
                }
                checkSumDefinedByUser = true;
                break;
            case MULTIPART_FILE:
                try (InputStream fileInputstream = item.openStream()) {
                    MessageDigest digest = checksumSHA256Service.createSHA256MessageDigest();
                    MessageDigestCalculatingInputStream messageDigestInputStream = new MessageDigestCalculatingInputStream(fileInputstream, digest);
                    jobStorage.store(fileName, messageDigestInputStream);
                    LOG.info("uploaded file:{} for job:{}", fileName, jobUUID);
                    checksumCalculated = checksumSHA256Service.convertMessageDigestToHex(digest);
                }
                fileDefinedByUser = true;
                break;
            default:
                LOG.warn("Given field '{}' is not supported while uploading job data to project {}, {}", logSanitizer.sanitize(fieldName, 30), jobUUID);
        }
    }
    if (!fileDefinedByUser) {
        throw new PDSBadRequestException("No file defined by user for job data upload!");
    }
    if (!checkSumDefinedByUser) {
        throw new PDSBadRequestException("No checksum defined by user for job data upload!");
    }
    if (checksumFromUser == null) {
        throw new PDSBadRequestException("No user checksum available for job data upload!");
    }
    if (checksumCalculated == null) {
        throw new PDSBadRequestException("Upload of binaries was not possible!");
    }
    assertCheckSumCorrect(checksumFromUser, checksumCalculated);
}

Example 12 with JobStorage

use of com.mercedesbenz.sechub.storage.core.JobStorage in project sechub by mercedes-benz.

the class PDSMultiStorageService method getJobStorage.

@Override
public JobStorage getJobStorage(String storagePath, UUID jobUUID) {
    if (storagePath == null) {
        storagePath = serverConfigurationService.getStorageId();
        LOG.debug("storage path parameter was null - fallback to default:{}", storagePath);
    }
    JobStorage jobStorage = jobStorageFactory.createJobStorage(storagePath, jobUUID);
    return jobStorage;
}

Example 13 with JobStorage

use of com.mercedesbenz.sechub.storage.core.JobStorage in project sechub by mercedes-benz.

the class IntegrationTestServerRestController method getUploadedFile.

@RequestMapping(path = APIConstants.API_ANONYMOUS + "integrationtest/{projectId}/{jobUUID}/uploaded/{fileName}", method = RequestMethod.GET)
public ResponseEntity<Resource> getUploadedFile(@PathVariable("projectId") String projectId, @PathVariable("jobUUID") UUID jobUUID, @PathVariable("fileName") String fileName) throws IOException {
    ValidationResult projectIdValidationResult = projectIdValidation.validate(projectId);
    if (!projectIdValidationResult.isValid()) {
        LOG.warn("Called with illegal projectId '{}'", logSanitizer.sanitize(projectId, 30));
        return ResponseEntity.notFound().build();
    }
    LOG.info("Integration test server: getJobStorage for {} {}", logSanitizer.sanitize(projectId, 30), jobUUID);
    JobStorage storage = storageService.getJobStorage(projectId, jobUUID);
    if (!storage.isExisting(fileName)) {
        throw new NotFoundException("file not uploaded:" + fileName);
    }
    InputStream inputStream = storage.fetch(fileName);
    HttpHeaders headers = new HttpHeaders();
    headers.add("Cache-Control", "no-cache, no-store, must-revalidate");
    headers.add("Pragma", "no-cache");
    headers.add("Expires", "0");
    /* @formatter:off */
    byte[] bytes = new byte[inputStream.available()];
    new DataInputStream(inputStream).readFully(bytes);
    ByteArrayResource resource = new ByteArrayResource(bytes);
    return ResponseEntity.ok().headers(headers).contentLength(resource.contentLength()).contentType(MediaType.parseMediaType("application/octet-stream")).body(resource);
/* @formatter:on */
}

Example 14 with JobStorage

use of com.mercedesbenz.sechub.storage.core.JobStorage in project sechub by mercedes-benz.

the class SchedulerSourcecodeUploadService method storeUploadFileAndSha256Checksum.

private void storeUploadFileAndSha256Checksum(String projectId, UUID jobUUID, MultipartFile file, String checkSum, String traceLogID) {
    JobStorage jobStorage = storageService.getJobStorage(projectId, jobUUID);
    try (InputStream inputStream = file.getInputStream()) {
        jobStorage.store(FILENAME_SOURCECODE_ZIP, inputStream);
        // we also store given checksum - so can be reused by security product
        jobStorage.store(FILENAME_SOURCECODE_ZIP_CHECKSUM, new StringInputStream(checkSum));
    } catch (IOException e) {
        LOG.error("Was not able to store zipped sources! {}", traceLogID, e);
        throw new SecHubRuntimeException("Was not able to upload sources");
    }
}

Example 15 with JobStorage

use of com.mercedesbenz.sechub.storage.core.JobStorage in project sechub by mercedes-benz.

the class S3RealLiveStorageManualTest method manualTestByDeveloper.

@Test
@EnabledIfSystemProperty(named = TestConstants.MANUAL_TEST_BY_DEVELOPER, matches = "true", disabledReason = TestConstants.DESCRIPTION_DISABLED_BECAUSE_A_MANUAL_TEST_FOR_GENERATION)
void manualTestByDeveloper() throws Exception {
    /* setup */
    SharedVolumeSetup setup = createFakeSharedVolumeNotValid();
    S3Setup s3Setup = createS3SetupByEnvironmentVariables();
    MultiStorageService service = new MultiStorageService(setup, s3Setup);
    UUID jobUUID = UUID.randomUUID();
    JobStorage jobStorage = service.getJobStorage("test-only", jobUUID);
    /* check preconditions */
    boolean existsBefore = jobStorage.isExisting(S3_OBJECT_NAME);
    String testDataAsString = "This is some test data as a simple string\nJust another line...";
    /* store */
    jobStorage.store(S3_OBJECT_NAME, new StringInputStream(testDataAsString));
    boolean existsAfterStore = jobStorage.isExisting(S3_OBJECT_NAME);
    /* fetch */
    InputStream inputStream = jobStorage.fetch(S3_OBJECT_NAME);
    InputStreamReader reader = new InputStreamReader(inputStream);
    BufferedReader br = new BufferedReader(reader);
    String result = br.readLine();
    br.close();
    /* delete all */
    jobStorage.deleteAll();
    /* check delete done */
    boolean existsAfterDelete = jobStorage.isExisting(S3_OBJECT_NAME);
    System.out.println("exists before storage:" + existsBefore);
    System.out.println("exists after storage:" + existsAfterStore);
    System.out.println("fetched string from object store:" + result);
    System.out.println("exists after delete:" + existsAfterDelete);
    if (existsBefore) {
        System.err.println("existed before!");
        System.exit(1);
    }
    if (!existsAfterStore) {
        System.err.println("was not stored!");
        System.exit(1);
    }
    if (!testDataAsString.equals(result)) {
        System.err.println("result was not as expected:" + result);
        System.exit(1);
    }
    if (existsAfterDelete) {
        System.err.println("data was not as expected:" + result);
        System.exit(1);
    }
}