Examples with FireStoreFile bio.terra.service.filedata.google.firestore.FireStoreFile used on opensource projects

Example 1 with FireStoreFile

use of bio.terra.service.filedata.google.firestore.FireStoreFile in project jade-data-repo by DataBiosphere.

the class IngestFileDirectoryStep method doStep.

@Override
public StepResult doStep(FlightContext context) {
    FlightMap inputParameters = context.getInputParameters();
    FileLoadModel loadModel = inputParameters.get(JobMapKeys.REQUEST.getKeyName(), FileLoadModel.class);
    FlightMap workingMap = context.getWorkingMap();
    String fileId = workingMap.get(FileMapKeys.FILE_ID, String.class);
    workingMap.put(FileMapKeys.LOAD_COMPLETED, false);
    String datasetId = dataset.getId().toString();
    String targetPath = loadModel.getTargetPath();
    try {
        // The state logic goes like this:
        // 1. the directory entry doesn't exist. We need to create the directory entry for it.
        // 2. the directory entry exists. There are three cases:
        // a. If loadTags do not match, then we throw FileAlreadyExistsException.
        // b. directory entry loadTag matches our loadTag AND entry fileId matches our fileId:
        // means we are recovering and need to complete the file creation work.
        // c. directory entry loadTag matches our loadTag AND entry fileId does NOT match our fileId
        // means this is a re-run of a load job. We update the fileId in the working map. We don't
        // know if we are recovering or already finished. We try to retrieve the file object for
        // the entry fileId:
        // i. If that is successful, then we already loaded this file. We store "completed=true"
        // in the working map, so other steps do nothing.
        // ii. If that fails, then we are recovering: we leave completed unset (=false) in the working map.
        // 
        // Lookup the file - on a recovery, we may have already created it, but not
        // finished. Or it might already exist, created by someone else.
        FireStoreDirectoryEntry existingEntry = fileDao.lookupDirectoryEntryByPath(dataset, targetPath);
        if (existingEntry == null) {
            // Not there - create it
            FireStoreDirectoryEntry newEntry = new FireStoreDirectoryEntry().fileId(fileId).isFileRef(true).path(fireStoreUtils.getDirectoryPath(loadModel.getTargetPath())).name(fireStoreUtils.getName(loadModel.getTargetPath())).datasetId(datasetId).loadTag(loadModel.getLoadTag());
            fileDao.createDirectoryEntry(dataset, newEntry);
        } else {
            if (!StringUtils.equals(existingEntry.getLoadTag(), loadModel.getLoadTag())) {
                // (a) Exists and is not our file
                throw new FileAlreadyExistsException("Path already exists: " + targetPath);
            }
            // (b) or (c) Load tags match - check file ids
            if (!StringUtils.equals(existingEntry.getFileId(), fileId)) {
                // (c) We are in a re-run of a load job. Try to get the file entry.
                fileId = existingEntry.getFileId();
                workingMap.put(FileMapKeys.FILE_ID, fileId);
                FireStoreFile fileEntry = fileDao.lookupFile(dataset, fileId);
                if (fileEntry != null) {
                    // (c)(i) We successfully loaded this file already
                    workingMap.put(FileMapKeys.LOAD_COMPLETED, true);
                }
            // (c)(ii) We are recovering and should continue this load; leave load completed false/unset
            }
        }
    } catch (FileSystemAbortTransactionException rex) {
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, rex);
    }
    return StepResult.getStepResultSuccess();
}

Example 2 with FireStoreFile

use of bio.terra.service.filedata.google.firestore.FireStoreFile in project jade-data-repo by DataBiosphere.

the class DeleteFileLookupStep method doStep.

@Override
public StepResult doStep(FlightContext context) throws InterruptedException {
    if (configService.testInsertFault(ConfigEnum.FILE_DELETE_LOCK_CONFLICT_STOP_FAULT)) {
        logger.info("FILE_DELETE_LOCK_CONFLICT_STOP_FAULT");
        while (!configService.testInsertFault(ConfigEnum.FILE_DELETE_LOCK_CONFLICT_CONTINUE_FAULT)) {
            logger.info("Sleeping for CONTINUE FAULT");
            TimeUnit.SECONDS.sleep(5);
        }
        logger.info("FILE_DELETE_LOCK_CONFLICT_CONTINUE_FAULT");
    }
    try {
        // If we are restarting, we may have already retrieved and saved the file,
        // so we check the working map before doing the lookup.
        FlightMap workingMap = context.getWorkingMap();
        FireStoreFile fireStoreFile = workingMap.get(FileMapKeys.FIRESTORE_FILE, FireStoreFile.class);
        if (fireStoreFile == null) {
            fireStoreFile = fileDao.lookupFile(dataset, fileId);
            if (fireStoreFile != null) {
                workingMap.put(FileMapKeys.FIRESTORE_FILE, fireStoreFile);
            }
        }
        // steps know there is no file. If there is a file, check dependencies here.
        if (fireStoreFile != null) {
            if (dependencyDao.fileHasSnapshotReference(dataset, fireStoreFile.getFileId())) {
                throw new FileDependencyException("File is used by at least one snapshot and cannot be deleted");
            }
        }
    } catch (FileSystemAbortTransactionException rex) {
        return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, rex);
    }
    return StepResult.getStepResultSuccess();
}

Example 3 with FireStoreFile

use of bio.terra.service.filedata.google.firestore.FireStoreFile in project jade-data-repo by DataBiosphere.

the class DeleteFilePrimaryDataStep method doStep.

@Override
public StepResult doStep(FlightContext context) {
    FlightMap workingMap = context.getWorkingMap();
    FireStoreFile fireStoreFile = workingMap.get(FileMapKeys.FIRESTORE_FILE, FireStoreFile.class);
    if (fireStoreFile != null) {
        GoogleBucketResource bucketResource = locationService.lookupBucket(fireStoreFile.getBucketResourceId());
        gcsPdao.deleteFileByGspath(fireStoreFile.getGspath(), bucketResource);
    }
    return StepResult.getStepResultSuccess();
}

Example 4 with FireStoreFile

use of bio.terra.service.filedata.google.firestore.FireStoreFile in project jade-data-repo by DataBiosphere.

the class IngestFileFileStep method doStep.

@Override
public StepResult doStep(FlightContext context) {
    FlightMap workingMap = context.getWorkingMap();
    Boolean loadComplete = workingMap.get(FileMapKeys.LOAD_COMPLETED, Boolean.class);
    if (loadComplete == null || !loadComplete) {
        FlightMap inputParameters = context.getInputParameters();
        FileLoadModel fileLoadModel = inputParameters.get(JobMapKeys.REQUEST.getKeyName(), FileLoadModel.class);
        FSFileInfo fsFileInfo = workingMap.get(FileMapKeys.FILE_INFO, FSFileInfo.class);
        String fileId = workingMap.get(FileMapKeys.FILE_ID, String.class);
        FireStoreFile newFile = new FireStoreFile().fileId(fileId).mimeType(fileLoadModel.getMimeType()).description(fileLoadModel.getDescription()).bucketResourceId(fsFileInfo.getBucketResourceId()).fileCreatedDate(fsFileInfo.getCreatedDate()).gspath(fsFileInfo.getGspath()).checksumCrc32c(fsFileInfo.getChecksumCrc32c()).checksumMd5(fsFileInfo.getChecksumMd5()).size(fsFileInfo.getSize()).loadTag(fileLoadModel.getLoadTag());
        try {
            fileDao.createFileMetadata(dataset, newFile);
            // Retrieve to build the complete FSItem
            FSItem fsItem = fileDao.retrieveById(dataset, fileId, 1, true);
            workingMap.put(JobMapKeys.RESPONSE.getKeyName(), fileService.fileModelFromFSItem(fsItem));
        } catch (FileSystemAbortTransactionException rex) {
            return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, rex);
        }
    }
    return StepResult.getStepResultSuccess();
}