Examples with Snapshot bio.terra.service.snapshot.Snapshot used on opensource projects

Example 1 with Snapshot

use of bio.terra.service.snapshot.Snapshot in project jade-data-repo by DataBiosphere.

the class BigQueryPdao method matchRowIds.

// for each table in a dataset (source), collect row id matches ON the row id
public RowIdMatch matchRowIds(Snapshot snapshot, SnapshotSource source, String tableName, List<String> rowIds) throws InterruptedException {
    // One source: grab it and navigate to the relevant parts
    BigQueryProject bigQueryProject = bigQueryProjectForSnapshot(snapshot);
    Optional<SnapshotMapTable> optTable = source.getSnapshotMapTables().stream().filter(table -> table.getFromTable().getName().equals(tableName)).findFirst();
    // create a column to point to the row id column in the source table to check that passed row ids exist in it
    Column rowIdColumn = new Column().table(optTable.get().getFromTable()).name(PDAO_ROW_ID_COLUMN);
    ST sqlTemplate = new ST(mapValuesToRowsTemplate);
    sqlTemplate.add("project", bigQueryProject.getProjectId());
    sqlTemplate.add("dataset", prefixName(source.getDataset().getName()));
    sqlTemplate.add("table", tableName);
    sqlTemplate.add("column", rowIdColumn.getName());
    sqlTemplate.add("inputVals", rowIds);
    // Execute the query building the row id match structure that tracks the matching
    // ids and the mismatched ids
    RowIdMatch rowIdMatch = new RowIdMatch();
    String sql = sqlTemplate.render();
    logger.debug("mapValuesToRows sql: " + sql);
    TableResult result = bigQueryProject.query(sql);
    for (FieldValueList row : result.iterateAll()) {
        // Test getting these by name
        FieldValue rowId = row.get(0);
        FieldValue inputValue = row.get(1);
        if (rowId.isNull()) {
            rowIdMatch.addMismatch(inputValue.getStringValue());
            logger.debug("rowId=<NULL>" + "  inVal=" + inputValue.getStringValue());
        } else {
            rowIdMatch.addMatch(inputValue.getStringValue(), rowId.getStringValue());
            logger.debug("rowId=" + rowId.getStringValue() + "  inVal=" + inputValue.getStringValue());
        }
    }
    return rowIdMatch;
}

Example 2 with Snapshot

use of bio.terra.service.snapshot.Snapshot in project jade-data-repo by DataBiosphere.

the class DeleteSnapshotPrimaryDataStep method doStep.

@Override
public StepResult doStep(FlightContext context) throws InterruptedException {
    try {
        // this fault is used by the SnapshotConnectedTest > testOverlappingDeletes
        if (configService.testInsertFault(ConfigEnum.SNAPSHOT_DELETE_LOCK_CONFLICT_STOP_FAULT)) {
            logger.info("SNAPSHOT_DELETE_LOCK_CONFLICT_STOP_FAULT");
            while (!configService.testInsertFault(ConfigEnum.SNAPSHOT_DELETE_LOCK_CONFLICT_CONTINUE_FAULT)) {
                logger.info("Sleeping for CONTINUE FAULT");
                TimeUnit.SECONDS.sleep(5);
            }
            logger.info("SNAPSHOT_DELETE_LOCK_CONFLICT_CONTINUE_FAULT");
        }
        Snapshot snapshot = snapshotService.retrieve(snapshotId);
        bigQueryPdao.deleteSnapshot(snapshot);
        // Remove snapshot file references from the underlying datasets
        for (SnapshotSource snapshotSource : snapshot.getSnapshotSources()) {
            Dataset dataset = datasetService.retrieve(snapshotSource.getDataset().getId());
            dependencyDao.deleteSnapshotFileDependencies(dataset, snapshotId.toString());
        }
        fileDao.deleteFilesFromSnapshot(snapshot);
    } catch (SnapshotNotFoundException | DatasetNotFoundException nfe) {
    // If we do not find the snapshot or dataset, we assume things are already clean
    }
    return StepResult.getStepResultSuccess();
}

Example 3 with Snapshot

use of bio.terra.service.snapshot.Snapshot in project jade-data-repo by DataBiosphere.

the class CreateSnapshotPrimaryDataFullViewStep method doStep.

@Override
public StepResult doStep(FlightContext context) throws InterruptedException {
    /*
         * from the dataset tables, we will need to get the table's live views
         */
    SnapshotRequestContentsModel contentsModel = snapshotReq.getContents().get(0);
    Snapshot snapshot = snapshotDao.retrieveSnapshotByName(snapshotReq.getName());
    Dataset dataset = datasetservice.retrieveByName(contentsModel.getDatasetName());
    bigQueryPdao.createSnapshotWithLiveViews(snapshot, dataset);
    return StepResult.getStepResultSuccess();
}

Example 4 with Snapshot

use of bio.terra.service.snapshot.Snapshot in project jade-data-repo by DataBiosphere.

the class CreateSnapshotPrimaryDataRowIdsStep method doStep.

@Override
public StepResult doStep(FlightContext context) throws InterruptedException {
    // TODO: this assumes single-dataset snapshots, will need to add a loop for multiple
    SnapshotRequestContentsModel contentsModel = snapshotReq.getContents().get(0);
    Snapshot snapshot = snapshotDao.retrieveSnapshotByName(snapshotReq.getName());
    SnapshotSource source = snapshot.getSnapshotSources().get(0);
    SnapshotRequestRowIdModel rowIdModel = contentsModel.getRowIdSpec();
    // for each table, make sure all of the row ids match
    for (SnapshotRequestRowIdTableModel table : rowIdModel.getTables()) {
        List<String> rowIds = table.getRowIds();
        if (!rowIds.isEmpty()) {
            RowIdMatch rowIdMatch = bigQueryPdao.matchRowIds(snapshot, source, table.getTableName(), rowIds);
            if (!rowIdMatch.getUnmatchedInputValues().isEmpty()) {
                String unmatchedValues = String.join("', '", rowIdMatch.getUnmatchedInputValues());
                String message = String.format("Mismatched row ids: '%s'", unmatchedValues);
                FlightUtils.setErrorResponse(context, message, HttpStatus.BAD_REQUEST);
                return new StepResult(StepStatus.STEP_RESULT_FAILURE_FATAL, new MismatchedValueException(message));
            }
        }
    }
    bigQueryPdao.createSnapshotWithProvidedIds(snapshot, contentsModel);
    return StepResult.getStepResultSuccess();
}

Example 5 with Snapshot

use of bio.terra.service.snapshot.Snapshot in project jade-data-repo by DataBiosphere.

the class SnapshotAuthzBqJobUserStep method doStep.

@Override
public StepResult doStep(FlightContext context) throws InterruptedException {
    FlightMap workingMap = context.getWorkingMap();
    Snapshot snapshot = snapshotService.retrieveByName(snapshotName);
    SnapshotDataProject projectForSnapshot = dataLocationService.getOrCreateProject(snapshot);
    Map<IamRole, String> policyMap = workingMap.get(SnapshotWorkingMapKeys.POLICY_MAP, Map.class);
    // Allow the custodian to make queries in this project.
    // The underlying service provides retries so we do not need to retry this operation
    resourceService.grantPoliciesBqJobUser(projectForSnapshot.getGoogleProjectId(), Collections.singletonList(policyMap.get(IamRole.CUSTODIAN)));
    return StepResult.getStepResultSuccess();
}