use of bio.terra.model.SnapshotRequestRowIdModel in project jade-data-repo by DataBiosphere.
the class CreateSnapshotPrimaryDataRowIdsStep method doStep.
@Override
public StepResult doStep(FlightContext context) throws InterruptedException {
// TODO: this assumes single-dataset snapshots, will need to add a loop for multiple
SnapshotRequestContentsModel contentsModel = snapshotReq.getContents().get(0);
Snapshot snapshot = snapshotDao.retrieveSnapshotByName(snapshotReq.getName());
SnapshotSource source = snapshot.getSnapshotSources().get(0);
SnapshotRequestRowIdModel rowIdModel = contentsModel.getRowIdSpec();
// for each table, make sure all of the row ids match
for (SnapshotRequestRowIdTableModel table : rowIdModel.getTables()) {
List<String> rowIds = table.getRowIds();
if (!rowIds.isEmpty()) {
RowIdMatch rowIdMatch = bigQueryPdao.matchRowIds(snapshot, source, table.getTableName(), rowIds);
if (!rowIdMatch.getUnmatchedInputValues().isEmpty()) {
String unmatchedValues = String.join("', '", rowIdMatch.getUnmatchedInputValues());
String message = String.format("Mismatched row ids: '%s'", unmatchedValues);
FlightUtils.setErrorResponse(context, message, HttpStatus.BAD_REQUEST);
return new StepResult(StepStatus.STEP_RESULT_FAILURE_FATAL, new MismatchedValueException(message));
}
}
}
bigQueryPdao.createSnapshotWithProvidedIds(snapshot, contentsModel);
return StepResult.getStepResultSuccess();
}
use of bio.terra.model.SnapshotRequestRowIdModel in project jade-data-repo by DataBiosphere.
the class SnapshotValidationTest method testSnapshotRowIdsEmptyColumns.
@Test
public void testSnapshotRowIdsEmptyColumns() throws Exception {
SnapshotRequestRowIdModel rowIdSpec = snapshotByRowIdsRequestModel.getContents().get(0).getRowIdSpec();
rowIdSpec.getTables().get(0).setColumns(Collections.emptyList());
expectBadSnapshotCreateRequest(snapshotByRowIdsRequestModel);
}
use of bio.terra.model.SnapshotRequestRowIdModel in project jade-data-repo by DataBiosphere.
the class SnapshotValidationTest method makeSnapshotRowIdsRequest.
// Generate a valid snapshot-by-rowId request, we will tweak individual pieces to test validation below
public SnapshotRequestModel makeSnapshotRowIdsRequest() {
SnapshotRequestRowIdTableModel snapshotRequestTableModel = new SnapshotRequestRowIdTableModel().tableName("snapshot").columns(Arrays.asList("col1", "col2", "col3")).rowIds(Arrays.asList("row1", "row2", "row3"));
SnapshotRequestRowIdModel rowIdSpec = new SnapshotRequestRowIdModel().tables(Collections.singletonList(snapshotRequestTableModel));
SnapshotRequestContentsModel snapshotRequestContentsModel = new SnapshotRequestContentsModel().datasetName("dataset").mode(SnapshotRequestContentsModel.ModeEnum.BYROWID).rowIdSpec(rowIdSpec);
return new SnapshotRequestModel().contents(Collections.singletonList(snapshotRequestContentsModel));
}
use of bio.terra.model.SnapshotRequestRowIdModel in project jade-data-repo by DataBiosphere.
the class SnapshotService method conjureSnapshotTablesFromRowIds.
private void conjureSnapshotTablesFromRowIds(SnapshotRequestRowIdModel requestRowIdModel, Snapshot snapshot, SnapshotSource snapshotSource) {
// TODO this will need to be changed when we have more than one dataset per snapshot (>1 contentsModel)
List<SnapshotTable> tableList = new ArrayList<>();
snapshot.snapshotTables(tableList);
List<SnapshotMapTable> mapTableList = new ArrayList<>();
snapshotSource.snapshotMapTables(mapTableList);
Dataset dataset = snapshotSource.getDataset();
// create a lookup from tableName -> table spec from the request
Map<String, SnapshotRequestRowIdTableModel> requestTableLookup = requestRowIdModel.getTables().stream().collect(Collectors.toMap(SnapshotRequestRowIdTableModel::getTableName, Function.identity()));
// for each dataset table specified in the request, create a table in the snapshot with the same name
for (DatasetTable datasetTable : dataset.getTables()) {
if (!requestTableLookup.containsKey(datasetTable.getName())) {
// only capture the dataset tables in the request model
continue;
}
List<Column> columnList = new ArrayList<>();
SnapshotTable snapshotTable = new SnapshotTable().name(datasetTable.getName()).columns(columnList);
tableList.add(snapshotTable);
List<SnapshotMapColumn> mapColumnList = new ArrayList<>();
mapTableList.add(new SnapshotMapTable().fromTable(datasetTable).toTable(snapshotTable).snapshotMapColumns(mapColumnList));
// for each dataset column specified in the request, create a column in the snapshot with the same name
Set<String> requestColumns = new HashSet<>(requestTableLookup.get(datasetTable.getName()).getColumns());
datasetTable.getColumns().stream().filter(c -> requestColumns.contains(c.getName())).forEach(datasetColumn -> {
Column snapshotColumn = new Column().name(datasetColumn.getName());
SnapshotMapColumn snapshotMapColumn = new SnapshotMapColumn().fromColumn(datasetColumn).toColumn(snapshotColumn);
columnList.add(snapshotColumn);
mapColumnList.add(snapshotMapColumn);
});
}
}
use of bio.terra.model.SnapshotRequestRowIdModel in project jade-data-repo by DataBiosphere.
the class SnapshotService method makeSnapshotFromSnapshotRequest.
/**
* Make a Snapshot structure with all of its parts from an incoming snapshot request.
* Note that the structure does not have UUIDs or created dates filled in. Those are
* updated by the DAO when it stores the snapshot in the repository metadata.
*
* @param snapshotRequestModel
* @return Snapshot
*/
public Snapshot makeSnapshotFromSnapshotRequest(SnapshotRequestModel snapshotRequestModel) {
// Make this early so we can hook up back links to it
Snapshot snapshot = new Snapshot();
List<SnapshotRequestContentsModel> requestContentsList = snapshotRequestModel.getContents();
// TODO: for MVM we only allow one source list
if (requestContentsList.size() > 1) {
throw new ValidationException("Only a single snapshot contents entry is currently allowed.");
}
SnapshotRequestContentsModel requestContents = requestContentsList.get(0);
Dataset dataset = datasetService.retrieveByName(requestContents.getDatasetName());
SnapshotSource snapshotSource = new SnapshotSource().snapshot(snapshot).dataset(dataset);
switch(snapshotRequestModel.getContents().get(0).getMode()) {
case BYASSET:
// TODO: When we implement explicit definition of snapshot tables, we will handle that here.
// For now, we generate the snapshot tables directly from the asset tables of the one source
// allowed in a snapshot.
AssetSpecification assetSpecification = getAssetSpecificationFromRequest(requestContents);
snapshotSource.assetSpecification(assetSpecification);
conjureSnapshotTablesFromAsset(snapshotSource.getAssetSpecification(), snapshot, snapshotSource);
break;
case BYFULLVIEW:
conjureSnapshotTablesFromDatasetTables(snapshot, snapshotSource);
break;
case BYQUERY:
SnapshotRequestQueryModel queryModel = requestContents.getQuerySpec();
String assetName = queryModel.getAssetName();
String snapshotQuery = queryModel.getQuery();
Query query = Query.parse(snapshotQuery);
List<String> datasetNames = query.getDatasetNames();
// TODO this makes the assumption that there is only one dataset
// (based on the validation flight step that already occurred.)
// This will change when more than 1 dataset is allowed
String datasetName = datasetNames.get(0);
Dataset queryDataset = datasetService.retrieveByName(datasetName);
AssetSpecification queryAssetSpecification = queryDataset.getAssetSpecificationByName(assetName).orElseThrow(() -> new AssetNotFoundException("This dataset does not have an asset specification with name: " + assetName));
snapshotSource.assetSpecification(queryAssetSpecification);
// TODO this is wrong? why dont we just pass the assetSpecification?
conjureSnapshotTablesFromAsset(snapshotSource.getAssetSpecification(), snapshot, snapshotSource);
break;
case BYROWID:
SnapshotRequestRowIdModel requestRowIdModel = requestContents.getRowIdSpec();
conjureSnapshotTablesFromRowIds(requestRowIdModel, snapshot, snapshotSource);
break;
default:
throw new InvalidSnapshotException("Snapshot does not have required mode information");
}
return snapshot.name(snapshotRequestModel.getName()).description(snapshotRequestModel.getDescription()).snapshotSources(Collections.singletonList(snapshotSource)).profileId(UUID.fromString(snapshotRequestModel.getProfileId())).relationships(createSnapshotRelationships(dataset.getRelationships(), snapshotSource));
}
Aggregations