Examples with DataDeletionRequest bio.terra.model.DataDeletionRequest used on opensource projects

Example 6 with DataDeletionRequest

use of bio.terra.model.DataDeletionRequest in project jade-data-repo by DataBiosphere.

the class DatasetIntegrationTest method testSoftDeleteHappyPath.

@Test
public void testSoftDeleteHappyPath() throws Exception {
    datasetId = ingestedDataset();
    // get row ids
    DatasetModel dataset = dataRepoFixtures.getDataset(steward(), datasetId);
    BigQuery bigQuery = BigQueryFixtures.getBigQuery(dataset.getDataProject(), stewardToken);
    List<String> participantRowIds = getRowIds(bigQuery, dataset, "participant", 3L);
    List<String> sampleRowIds = getRowIds(bigQuery, dataset, "sample", 2L);
    // write them to GCS
    String participantPath = writeListToScratch("softDel", participantRowIds);
    String samplePath = writeListToScratch("softDel", sampleRowIds);
    // build the deletion request with pointers to the two files with row ids to soft delete
    List<DataDeletionTableModel> dataDeletionTableModels = Arrays.asList(deletionTableFile("participant", participantPath), deletionTableFile("sample", samplePath));
    DataDeletionRequest request = dataDeletionRequest().tables(dataDeletionTableModels);
    // send off the soft delete request
    dataRepoFixtures.deleteData(steward(), datasetId, request);
    // make sure the new counts make sense
    assertTableCount(bigQuery, dataset, "participant", 2L);
    assertTableCount(bigQuery, dataset, "sample", 5L);
}

Example 7 with DataDeletionRequest

use of bio.terra.model.DataDeletionRequest in project jade-data-repo by DataBiosphere.

the class DataDeletionStep method doStep.

@Override
public StepResult doStep(FlightContext context) throws InterruptedException {
    Dataset dataset = getDataset(context, datasetService);
    String suffix = getSuffix(context);
    DataDeletionRequest dataDeletionRequest = getRequest(context);
    List<String> tableNames = dataDeletionRequest.getTables().stream().map(DataDeletionTableModel::getTableName).collect(Collectors.toList());
    bigQueryPdao.validateDeleteRequest(dataset, dataDeletionRequest.getTables(), suffix);
    if (configService.testInsertFault(ConfigEnum.SOFT_DELETE_LOCK_CONFLICT_STOP_FAULT)) {
        logger.info("SOFT_DELETE_LOCK_CONFLICT_STOP_FAULT");
        while (!configService.testInsertFault(ConfigEnum.SOFT_DELETE_LOCK_CONFLICT_CONTINUE_FAULT)) {
            logger.info("Sleeping for CONTINUE FAULT");
            TimeUnit.SECONDS.sleep(5);
        }
        logger.info("SOFT_DELETE_LOCK_CONFLICT_CONTINUE_FAULT");
    }
    bigQueryPdao.applySoftDeletes(dataset, tableNames, suffix);
    // TODO: this can be more informative, something like # rows deleted per table, or mismatched row ids
    DeleteResponseModel deleteResponseModel = new DeleteResponseModel().objectState(DeleteResponseModel.ObjectStateEnum.DELETED);
    FlightUtils.setResponse(context, deleteResponseModel, HttpStatus.OK);
    return StepResult.getStepResultSuccess();
}

Example 8 with DataDeletionRequest

use of bio.terra.model.DataDeletionRequest in project jade-data-repo by DataBiosphere.

the class DatasetConnectedTest method uploadInputFileAndBuildSoftDeleteRequest.

private DataDeletionRequest uploadInputFileAndBuildSoftDeleteRequest(String dirInCloud, String filenameInCloud, String tableName, List<String> softDeleteRowIds) throws Exception {
    Storage storage = StorageOptions.getDefaultInstance().getService();
    // load a CSV file that contains the table rows to soft delete into the test bucket
    StringBuilder csvLines = new StringBuilder();
    for (String softDeleteRowId : softDeleteRowIds) {
        csvLines.append(softDeleteRowId + "\n");
    }
    BlobInfo softDeleteBlob = BlobInfo.newBuilder(testConfig.getIngestbucket(), dirInCloud + "/" + filenameInCloud).build();
    storage.create(softDeleteBlob, csvLines.toString().getBytes(Charset.forName("UTF-8")));
    String softDeleteInputFilePath = "gs://" + testConfig.getIngestbucket() + "/" + dirInCloud + "/" + filenameInCloud;
    // make sure the JSON file gets cleaned up on test teardown
    connectedOperations.addScratchFile(dirInCloud + "/" + filenameInCloud);
    // build the soft delete request with a pointer to a file that contains the row ids to soft delete
    DataDeletionGcsFileModel softDeleteGcsFileModel = new DataDeletionGcsFileModel().fileType(DataDeletionGcsFileModel.FileTypeEnum.CSV).path(softDeleteInputFilePath);
    DataDeletionTableModel softDeleteTableModel = new DataDeletionTableModel().tableName(tableName).gcsFileSpec(softDeleteGcsFileModel);
    DataDeletionRequest softDeleteRequest = new DataDeletionRequest().deleteType(DataDeletionRequest.DeleteTypeEnum.SOFT).specType(DataDeletionRequest.SpecTypeEnum.GCSFILE).tables(Arrays.asList(softDeleteTableModel));
    return softDeleteRequest;
}

Example 9 with DataDeletionRequest

use of bio.terra.model.DataDeletionRequest in project jade-data-repo by DataBiosphere.

the class DatasetConnectedTest method testRepeatedSoftDelete.

@Test
public void testRepeatedSoftDelete() throws Exception {
    // load a CSV file that contains the table rows to load into the test bucket
    String resourceFileName = "snapshot-test-dataset-data.csv";
    String dirInCloud = "scratch/testRepeatedSoftDelete/" + UUID.randomUUID().toString();
    BlobInfo ingestTableBlob = BlobInfo.newBuilder(testConfig.getIngestbucket(), dirInCloud + "/" + resourceFileName).build();
    Storage storage = StorageOptions.getDefaultInstance().getService();
    storage.create(ingestTableBlob, IOUtils.toByteArray(getClass().getClassLoader().getResource(resourceFileName)));
    String tableIngestInputFilePath = "gs://" + testConfig.getIngestbucket() + "/" + dirInCloud + "/" + resourceFileName;
    // ingest the table
    String tableName = "thetable";
    IngestRequestModel ingestRequest = new IngestRequestModel().table(tableName).format(IngestRequestModel.FormatEnum.CSV).csvSkipLeadingRows(1).path(tableIngestInputFilePath);
    connectedOperations.ingestTableSuccess(summaryModel.getId(), ingestRequest);
    // make sure the JSON file gets cleaned up on test teardown
    connectedOperations.addScratchFile(dirInCloud + "/" + resourceFileName);
    // load a CSV file that contains the table rows to soft delete into the test bucket
    String softDeleteRowId = "8c52c63e-8d9f-4cfc-82d0-0f916b2404c1";
    List<String> softDeleteRowIds = new ArrayList<>();
    // add the same rowid twice
    softDeleteRowIds.add(softDeleteRowId);
    softDeleteRowIds.add(softDeleteRowId);
    DataDeletionRequest softDeleteRequest = uploadInputFileAndBuildSoftDeleteRequest(dirInCloud, "testRepeatedSoftDelete.csv", tableName, softDeleteRowIds);
    // make the soft delete request and wait for it to return
    connectedOperations.softDeleteSuccess(summaryModel.getId(), softDeleteRequest);
    // check that the size of the live table matches what we expect
    List<String> liveTableRowIds1 = getRowIdsFromBQTable(summaryModel.getName(), tableName);
    assertEquals("Size of live table is 3", 3, liveTableRowIds1.size());
    assertFalse("Soft deleted row id is not in live table", liveTableRowIds1.contains(softDeleteRowId));
    // note: the soft delete table name is not exposed to end users, so to check that the state of the
    // soft delete table is correct, I'm reaching into our internals to fetch the table name
    Dataset internalDatasetObj = datasetDao.retrieve(UUID.fromString(summaryModel.getId()));
    DatasetTable internalDatasetTableObj = internalDatasetObj.getTableByName(tableName).get();
    String internalSoftDeleteTableName = internalDatasetTableObj.getSoftDeleteTableName();
    // check that the size of the soft delete table matches what we expect
    List<String> softDeleteRowIds1 = getRowIdsFromBQTable(summaryModel.getName(), internalSoftDeleteTableName);
    assertEquals("Size of soft delete table is 1", 1, softDeleteRowIds1.size());
    assertTrue("Soft deleted row id is in soft delete table", softDeleteRowIds1.contains(softDeleteRowId));
    // repeat the same soft delete request and wait for it to return
    connectedOperations.softDeleteSuccess(summaryModel.getId(), softDeleteRequest);
    // check that the size of the live table has not changed
    List<String> liveTableRowIds2 = getRowIdsFromBQTable(summaryModel.getName(), tableName);
    assertEquals("Size of live table is still 3", 3, liveTableRowIds2.size());
    assertFalse("Soft deleted row id is still not in live table", liveTableRowIds2.contains(softDeleteRowId));
    // check that the size of the soft delete table has not changed
    List<String> softDeleteRowIds2 = getRowIdsFromBQTable(summaryModel.getName(), internalSoftDeleteTableName);
    assertEquals("Size of soft delete table is still 1", 1, softDeleteRowIds2.size());
    assertTrue("Soft deleted row id is still in soft delete table", softDeleteRowIds2.contains(softDeleteRowId));
    // delete the dataset and check that it succeeds
    connectedOperations.deleteTestDataset(summaryModel.getId());
    // try to fetch the dataset again and confirm nothing is returned
    connectedOperations.getDatasetExpectError(summaryModel.getId(), HttpStatus.NOT_FOUND);
}

Example 10 with DataDeletionRequest

use of bio.terra.model.DataDeletionRequest in project jade-data-repo by DataBiosphere.

the class DatasetConnectedTest method testBadSoftDelete.

@Test
public void testBadSoftDelete() throws Exception {
    // load a CSV file that contains the table rows to load into the test bucket
    String resourceFileName = "snapshot-test-dataset-data.csv";
    String dirInCloud = "scratch/testBadSoftDelete/" + UUID.randomUUID().toString();
    BlobInfo ingestTableBlob = BlobInfo.newBuilder(testConfig.getIngestbucket(), dirInCloud + "/" + resourceFileName).build();
    Storage storage = StorageOptions.getDefaultInstance().getService();
    storage.create(ingestTableBlob, IOUtils.toByteArray(getClass().getClassLoader().getResource(resourceFileName)));
    String tableIngestInputFilePath = "gs://" + testConfig.getIngestbucket() + "/" + dirInCloud + "/" + resourceFileName;
    // ingest the table
    String tableName = "thetable";
    IngestRequestModel ingestRequest = new IngestRequestModel().table(tableName).format(IngestRequestModel.FormatEnum.CSV).csvSkipLeadingRows(1).path(tableIngestInputFilePath);
    connectedOperations.ingestTableSuccess(summaryModel.getId(), ingestRequest);
    // make sure the JSON file gets cleaned up on test teardown
    connectedOperations.addScratchFile(dirInCloud + "/" + resourceFileName);
    // load a CSV file that contains the table rows to soft delete into the test bucket
    String softDeleteBadRowId = "badrowid";
    String softDeleteGoodRowId = "8c52c63e-8d9f-4cfc-82d0-0f916b2404c1";
    List<String> softDeleteRowIds = new ArrayList<>();
    softDeleteRowIds.add(softDeleteBadRowId);
    softDeleteRowIds.add(softDeleteGoodRowId);
    DataDeletionRequest softDeleteRequest = uploadInputFileAndBuildSoftDeleteRequest(dirInCloud, "testBadSoftDelete.csv", tableName, softDeleteRowIds);
    // make the soft delete request and wait for it to return
    MvcResult softDeleteResult = connectedOperations.softDeleteRaw(summaryModel.getId(), softDeleteRequest);
    MockHttpServletResponse softDeleteResponse = connectedOperations.validateJobModelAndWait(softDeleteResult);
    assertEquals("soft delete of bad row id failed", HttpStatus.BAD_REQUEST.value(), softDeleteResponse.getStatus());
    // check that the size of the live table matches what we expect
    List<String> liveTableRowIds = getRowIdsFromBQTable(summaryModel.getName(), tableName);
    assertEquals("Size of live table is 4", 4, liveTableRowIds.size());
    assertFalse("Bad row id is not in live table", liveTableRowIds.contains(softDeleteBadRowId));
    assertTrue("Good row id is in live table", liveTableRowIds.contains(softDeleteGoodRowId));
    // note: the soft delete table name is not exposed to end users, so to check that the state of the
    // soft delete table is correct, I'm reaching into our internals to fetch the table name
    Dataset internalDatasetObj = datasetDao.retrieve(UUID.fromString(summaryModel.getId()));
    DatasetTable internalDatasetTableObj = internalDatasetObj.getTableByName(tableName).get();
    String internalSoftDeleteTableName = internalDatasetTableObj.getSoftDeleteTableName();
    // check that the size of the soft delete table matches what we expect
    List<String> softDeleteRowIdsFromBQ = getRowIdsFromBQTable(summaryModel.getName(), internalSoftDeleteTableName);
    assertEquals("Size of soft delete table is 0", 0, softDeleteRowIdsFromBQ.size());
    assertFalse("Bad row id is not in soft delete table", softDeleteRowIdsFromBQ.contains(softDeleteBadRowId));
    assertFalse("Good row id is not in soft delete table", softDeleteRowIdsFromBQ.contains(softDeleteGoodRowId));
    // delete the dataset and check that it succeeds
    connectedOperations.deleteTestDataset(summaryModel.getId());
    // try to fetch the dataset again and confirm nothing is returned
    connectedOperations.getDatasetExpectError(summaryModel.getId(), HttpStatus.NOT_FOUND);
}