Examples with FileSystemCorruptException bio.terra.service.filedata.exception.FileSystemCorruptException used on opensource projects

Example 1 with FileSystemCorruptException

use of bio.terra.service.filedata.exception.FileSystemCorruptException in project jade-data-repo by DataBiosphere.

the class IngestDriverStep method getLoadCandidates.

private LoadCandidates getLoadCandidates(FlightContext context, UUID loadId, int concurrentLoads) throws DatabaseOperationException, InterruptedException {
    // We start by getting the database view of the state of loads.
    // For the running loads, we ask Stairway what the actual state is.
    // If they have completed, we mark them as such.
    // We then update the failure count and runnings loads list in the
    // LoadCandidates so it correctly reflects the running state
    // right now (more or less).
    LoadCandidates candidates = loadService.findCandidates(loadId, concurrentLoads);
    logger.debug("Candidates from db: failedLoads={}  runningLoads={}  candidateFiles={}", candidates.getFailedLoads(), candidates.getRunningLoads().size(), candidates.getCandidateFiles().size());
    int failureCount = candidates.getFailedLoads();
    List<LoadFile> realRunningLoads = new LinkedList<>();
    for (LoadFile loadFile : candidates.getRunningLoads()) {
        FlightState flightState = context.getStairway().getFlightState(loadFile.getFlightId());
        switch(flightState.getFlightStatus()) {
            case RUNNING:
            case WAITING:
            case READY:
            case QUEUED:
                realRunningLoads.add(loadFile);
                break;
            case ERROR:
            case FATAL:
                {
                    String error = "unknown error";
                    if (flightState.getException().isPresent()) {
                        error = flightState.getException().get().toString();
                    }
                    loadService.setLoadFileFailed(loadId, loadFile.getTargetPath(), error);
                    failureCount++;
                    break;
                }
            case SUCCESS:
                {
                    FlightMap resultMap = flightState.getResultMap().orElse(null);
                    if (resultMap == null) {
                        throw new FileSystemCorruptException("no result map in flight state");
                    }
                    String fileId = resultMap.get(FileMapKeys.FILE_ID, String.class);
                    FSFileInfo fileInfo = resultMap.get(FileMapKeys.FILE_INFO, FSFileInfo.class);
                    loadService.setLoadFileSucceeded(loadId, loadFile.getTargetPath(), fileId, fileInfo);
                    break;
                }
        }
    }
    candidates.failedLoads(failureCount).runningLoads(realRunningLoads);
    logger.debug("Candidates resolved: failedLoads={}  runningLoads={}  candidateFiles={}", candidates.getFailedLoads(), candidates.getRunningLoads().size(), candidates.getCandidateFiles().size());
    return candidates;
}

Example 2 with FileSystemCorruptException

use of bio.terra.service.filedata.exception.FileSystemCorruptException in project jade-data-repo by DataBiosphere.

the class FireStoreDependencyDao method storeSnapshotFileDependency.

public void storeSnapshotFileDependency(Dataset dataset, String snapshotId, String fileId) {
    DatasetDataProject dataProject = dataLocationService.getProjectOrThrow(dataset);
    FireStoreProject fireStoreProject = FireStoreProject.get(dataProject.getGoogleProjectId());
    String dependencyCollectionName = getDatasetDependencyId(dataset.getId().toString());
    CollectionReference depColl = fireStoreProject.getFirestore().collection(dependencyCollectionName);
    ApiFuture<Void> transaction = fireStoreProject.getFirestore().runTransaction(xn -> {
        Query query = depColl.whereEqualTo("fileId", fileId).whereEqualTo("snapshotId", snapshotId);
        ApiFuture<QuerySnapshot> querySnapshot = query.get();
        // There should be zero or one documents. Since we keep track of greater than one reference using the
        // reference count.
        List<QueryDocumentSnapshot> documents = querySnapshot.get().getDocuments();
        switch(documents.size()) {
            case 0:
                {
                    // no dependency yet. Let's make one
                    FireStoreDependency fireStoreDependency = new FireStoreDependency().snapshotId(snapshotId).fileId(fileId).refCount(1L);
                    DocumentReference docRef = depColl.document();
                    xn.set(docRef, fireStoreDependency);
                    break;
                }
            case 1:
                {
                    // existing dependency; increment the reference count
                    QueryDocumentSnapshot docSnap = documents.get(0);
                    FireStoreDependency fireStoreDependency = docSnap.toObject(FireStoreDependency.class);
                    fireStoreDependency.refCount(fireStoreDependency.getRefCount() + 1);
                    xn.set(docSnap.getReference(), fireStoreDependency);
                    break;
                }
            default:
                throw new FileSystemCorruptException("Found more than one document for a file dependency");
        }
        return null;
    });
    fireStoreUtils.transactionGet("store dependency", transaction);
}

Example 3 with FileSystemCorruptException

use of bio.terra.service.filedata.exception.FileSystemCorruptException in project jade-data-repo by DataBiosphere.

the class FireStoreDependencyDao method removeSnapshotFileDependency.

public void removeSnapshotFileDependency(Dataset dataset, String snapshotId, String fileId) {
    DatasetDataProject dataProject = dataLocationService.getProjectOrThrow(dataset);
    FireStoreProject fireStoreProject = FireStoreProject.get(dataProject.getGoogleProjectId());
    String dependencyCollectionName = getDatasetDependencyId(dataset.getId().toString());
    CollectionReference depColl = fireStoreProject.getFirestore().collection(dependencyCollectionName);
    ApiFuture<Void> transaction = fireStoreProject.getFirestore().runTransaction(xn -> {
        Query query = depColl.whereEqualTo("fileId", fileId).whereEqualTo("snapshotId", snapshotId);
        ApiFuture<QuerySnapshot> querySnapshot = query.get();
        List<QueryDocumentSnapshot> documents = querySnapshot.get().getDocuments();
        if (documents.size() == 0) {
            // No file - nothing to delete
            return null;
        }
        if (documents.size() > 1) {
            throw new FileSystemCorruptException("Found more than one document for a file dependency");
        }
        QueryDocumentSnapshot docSnap = documents.get(0);
        FireStoreDependency fireStoreDependency = docSnap.toObject(FireStoreDependency.class);
        if (fireStoreDependency.getRefCount() <= 1) {
            xn.delete(docSnap.getReference());
        } else {
            fireStoreDependency.refCount(fireStoreDependency.getRefCount() - 1);
            xn.set(docSnap.getReference(), fireStoreDependency);
        }
        return null;
    });
    fireStoreUtils.transactionGet("delete dependency", transaction);
}

Example 4 with FileSystemCorruptException

use of bio.terra.service.filedata.exception.FileSystemCorruptException in project jade-data-repo by DataBiosphere.

the class DeleteDatasetValidateStep method doStep.

@Override
public StepResult doStep(FlightContext context) {
    List<SnapshotSummary> snapshots = snapshotDao.retrieveSnapshotsForDataset(datasetId);
    Dataset dataset = datasetService.retrieve(datasetId);
    if (snapshots.size() != 0) {
        throw new ValidationException("Can not delete a dataset being used by snapshots");
    }
    // if there are no snapshots returned from retrieveSnapshotsForDataset.
    if (dependencyDao.datasetHasSnapshotReference(dataset)) {
        throw new FileSystemCorruptException("File system has snapshot dependencies; metadata does not");
    }
    return StepResult.getStepResultSuccess();
}

Example 5 with FileSystemCorruptException

use of bio.terra.service.filedata.exception.FileSystemCorruptException in project jade-data-repo by DataBiosphere.

the class FileService method fileModelFromFSItem.

public FileModel fileModelFromFSItem(FSItem fsItem) {
    FileModel fileModel = new FileModel().fileId(fsItem.getFileId().toString()).collectionId(fsItem.getCollectionId().toString()).path(fsItem.getPath()).size(fsItem.getSize()).created(fsItem.getCreatedDate().toString()).description(fsItem.getDescription()).checksums(makeChecksums(fsItem));
    if (fsItem instanceof FSFile) {
        fileModel.fileType(FileModelType.FILE);
        FSFile fsFile = (FSFile) fsItem;
        fileModel.fileDetail(new FileDetailModel().datasetId(fsFile.getDatasetId().toString()).accessUrl(fsFile.getGspath()).mimeType(fsFile.getMimeType()).loadTag(fsFile.getLoadTag()));
    } else if (fsItem instanceof FSDir) {
        fileModel.fileType(FileModelType.DIRECTORY);
        FSDir fsDir = (FSDir) fsItem;
        DirectoryDetailModel directoryDetail = new DirectoryDetailModel().enumerated(fsDir.isEnumerated());
        if (fsDir.isEnumerated()) {
            directoryDetail.contents(new ArrayList<>());
            for (FSItem fsContentsItem : fsDir.getContents()) {
                FileModel itemModel = fileModelFromFSItem(fsContentsItem);
                directoryDetail.addContentsItem(itemModel);
            }
        }
        fileModel.directoryDetail(directoryDetail);
    } else {
        throw new FileSystemCorruptException("Entry type is totally wrong; we shouldn't be here");
    }
    return fileModel;
}