Examples with SnapshotDataProject bio.terra.service.snapshot.SnapshotDataProject used on opensource projects

Example 1 with SnapshotDataProject

use of bio.terra.service.snapshot.SnapshotDataProject in project jade-data-repo by DataBiosphere.

the class FireStoreDao method snapshotCompute.

public void snapshotCompute(Snapshot snapshot) {
    SnapshotDataProject dataProject = dataLocationService.getProjectOrThrow(snapshot);
    Firestore firestore = FireStoreProject.get(dataProject.getGoogleProjectId()).getFirestore();
    String snapshotId = snapshot.getId().toString();
    FireStoreDirectoryEntry topDir = directoryDao.retrieveByPath(firestore, snapshotId, "/");
    // step. So there is nothing to compute
    if (topDir != null) {
        computeDirectory(firestore, snapshotId, topDir);
    }
}

Example 2 with SnapshotDataProject

use of bio.terra.service.snapshot.SnapshotDataProject in project jade-data-repo by DataBiosphere.

the class DataLocationService method getProject.

/**
 * Fetch existing SnapshotDataProject for the Snapshot.
 * Delete it if it's invalid, that is, the referenced cloud resource doesn't exist.
 * @param snapshot
 * @return a populated SnapshotDataProject if one exists, empty if not
 */
public Optional<SnapshotDataProject> getProject(Snapshot snapshot) {
    SnapshotDataProjectSummary snapshotDataProjectSummary = null;
    try {
        // first, check if SnapshotDataProjectSummary (= mapping btw Snapshot ID and cloud Project ID) exists
        snapshotDataProjectSummary = dataProjectDao.retrieveSnapshotDataProject(snapshot.getId());
        // second, check if the referenced cloud resource exists
        GoogleProjectResource googleProjectResource = resourceService.getProjectResourceById(snapshotDataProjectSummary.getProjectResourceId());
        // if both exist, then create the DatasetDataProject object from the summary and return here
        return Optional.of(new SnapshotDataProject(snapshotDataProjectSummary).googleProjectResource(googleProjectResource));
    } catch (DataProjectNotFoundException projNfEx) {
    // suppress exception here, will create later
    } catch (GoogleResourceNotFoundException rsrcNfEx) {
        // I don't think this null check will ever be false, but just in case
        if (snapshotDataProjectSummary != null) {
            logger.warn("metadata has a project resource id it can't resolve for snapshot: " + snapshot.getName());
            dataProjectDao.deleteSnapshotDataProject(snapshotDataProjectSummary.getId());
        }
    }
    // did not find a valid SnapshotDataProject for the given Snapshot
    return Optional.empty();
}

Example 3 with SnapshotDataProject

use of bio.terra.service.snapshot.SnapshotDataProject in project jade-data-repo by DataBiosphere.

the class SnapshotAuthzBqJobUserStep method doStep.

@Override
public StepResult doStep(FlightContext context) throws InterruptedException {
    FlightMap workingMap = context.getWorkingMap();
    Snapshot snapshot = snapshotService.retrieveByName(snapshotName);
    SnapshotDataProject projectForSnapshot = dataLocationService.getOrCreateProject(snapshot);
    Map<IamRole, String> policyMap = workingMap.get(SnapshotWorkingMapKeys.POLICY_MAP, Map.class);
    // Allow the custodian to make queries in this project.
    // The underlying service provides retries so we do not need to retry this operation
    resourceService.grantPoliciesBqJobUser(projectForSnapshot.getGoogleProjectId(), Collections.singletonList(policyMap.get(IamRole.CUSTODIAN)));
    return StepResult.getStepResultSuccess();
}

Example 4 with SnapshotDataProject

use of bio.terra.service.snapshot.SnapshotDataProject in project jade-data-repo by DataBiosphere.

the class EncodeFileTest method getFileRefIdFromSnapshot.

private String getFileRefIdFromSnapshot(SnapshotSummaryModel snapshotSummary) throws InterruptedException {
    Snapshot snapshot = snapshotDao.retrieveSnapshotByName(snapshotSummary.getName());
    SnapshotDataProject dataProject = dataLocationService.getOrCreateProject(snapshot);
    BigQueryProject bigQueryProject = BigQueryProject.get(dataProject.getGoogleProjectId());
    StringBuilder builder = new StringBuilder().append("SELECT file_ref FROM `").append(dataProject.getGoogleProjectId()).append('.').append(snapshot.getName()).append(".file` AS T").append(" WHERE T.file_ref IS NOT NULL LIMIT 1");
    String sql = builder.toString();
    try {
        QueryJobConfiguration queryConfig = QueryJobConfiguration.newBuilder(sql).build();
        TableResult result = bigQueryProject.getBigQuery().query(queryConfig);
        FieldValueList row = result.iterateAll().iterator().next();
        FieldValue idValue = row.get(0);
        String drsUri = idValue.getStringValue();
        return drsUri;
    } catch (InterruptedException ie) {
        throw new PdaoException("get file ref id from snapshot unexpectedly interrupted", ie);
    }
}

Example 5 with SnapshotDataProject

use of bio.terra.service.snapshot.SnapshotDataProject in project jade-data-repo by DataBiosphere.

the class FireStoreDao method addFilesToSnapshot.

public void addFilesToSnapshot(Dataset dataset, Snapshot snapshot, List<String> refIds) {
    DatasetDataProject datasetDataProject = dataLocationService.getProjectOrThrow(dataset);
    Firestore datasetFirestore = FireStoreProject.get(datasetDataProject.getGoogleProjectId()).getFirestore();
    SnapshotDataProject snapshotDataProject = dataLocationService.getProjectOrThrow(snapshot);
    Firestore snapshotFirestore = FireStoreProject.get(snapshotDataProject.getGoogleProjectId()).getFirestore();
    String datasetId = dataset.getId().toString();
    // TODO: Do we need to make sure the dataset name does not contain characters that are invalid for paths?
    // Added the work to figure that out to DR-325
    String datasetName = dataset.getName();
    String snapshotId = snapshot.getId().toString();
    for (String fileId : refIds) {
        directoryDao.addEntryToSnapshot(datasetFirestore, datasetId, datasetName, snapshotFirestore, snapshotId, fileId);
    }
}