Examples with FileLoadModel bio.terra.model.FileLoadModel used on opensource projects

Example 6 with FileLoadModel

use of bio.terra.model.FileLoadModel in project jade-data-repo by DataBiosphere.

the class FileOperationTest method fileOperationsTest.

@Test
public void fileOperationsTest() throws Exception {
    FileLoadModel fileLoadModel = makeFileLoad(profileModel.getId());
    FileModel fileModel = connectedOperations.ingestFileSuccess(datasetSummary.getId(), fileLoadModel);
    assertThat("file path matches", fileModel.getPath(), equalTo(fileLoadModel.getTargetPath()));
    // Change the data location selector, verify that we can still delete the file
    String newBucketName = "bucket-" + UUID.randomUUID().toString();
    doReturn(newBucketName).when(dataLocationSelector).bucketForFile(any());
    connectedOperations.deleteTestFile(datasetSummary.getId(), fileModel.getFileId());
    fileModel = connectedOperations.ingestFileSuccess(datasetSummary.getId(), fileLoadModel);
    assertThat("file path reflects new bucket location", fileModel.getFileDetail().getAccessUrl(), containsString(newBucketName));
    // Track the bucket so connected ops can remove it on teardown
    connectedOperations.addBucket(newBucketName);
    // lookup the file we just created
    String url = "/api/repository/v1/datasets/" + datasetSummary.getId() + "/files/" + fileModel.getFileId();
    MvcResult result = mvc.perform(get(url)).andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8_VALUE)).andReturn();
    MockHttpServletResponse response = result.getResponse();
    assertThat("Lookup file succeeds", HttpStatus.valueOf(response.getStatus()), equalTo(HttpStatus.OK));
    FileModel lookupModel = TestUtils.mapFromJson(response.getContentAsString(), FileModel.class);
    assertTrue("Ingest file equals lookup file", lookupModel.equals(fileModel));
    // Error: Duplicate target file
    ErrorModel errorModel = connectedOperations.ingestFileFailure(datasetSummary.getId(), fileLoadModel);
    assertThat("duplicate file error", errorModel.getMessage(), containsString("already exists"));
    // Lookup the file by path
    url = "/api/repository/v1/datasets/" + datasetSummary.getId() + "/filesystem/objects";
    result = mvc.perform(get(url).param("path", fileModel.getPath())).andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8_VALUE)).andReturn();
    response = result.getResponse();
    assertThat("Lookup file by path succeeds", HttpStatus.valueOf(response.getStatus()), equalTo(HttpStatus.OK));
    lookupModel = TestUtils.mapFromJson(response.getContentAsString(), FileModel.class);
    assertTrue("Ingest file equals lookup file", lookupModel.equals(fileModel));
    // Delete the file and we should be able to create it successfully again
    connectedOperations.deleteTestFile(datasetSummary.getId(), fileModel.getFileId());
    fileModel = connectedOperations.ingestFileSuccess(datasetSummary.getId(), fileLoadModel);
    assertThat("file path matches", fileModel.getPath(), equalTo(fileLoadModel.getTargetPath()));
    // Error: Non-existent source file
    String badfile = "/I am not a file";
    String uribadfile = "gs://" + testConfig.getIngestbucket() + "/" + badfile;
    String badPath = "/dd/files/" + Names.randomizeName("dir") + badfile;
    fileLoadModel = new FileLoadModel().profileId(profileModel.getId()).sourcePath(uribadfile).description(testDescription).mimeType(testMimeType).targetPath(badPath);
    errorModel = connectedOperations.ingestFileFailure(datasetSummary.getId(), fileLoadModel);
    assertThat("source file does not exist", errorModel.getMessage(), containsString("file not found"));
    // Error: Invalid gs path - case 1: not gs
    fileLoadModel = new FileLoadModel().profileId(profileModel.getId()).sourcePath("http://jade_notabucket/foo/bar.txt").description(testDescription).mimeType(testMimeType).targetPath(makeValidUniqueFilePath());
    errorModel = connectedOperations.ingestFileFailure(datasetSummary.getId(), fileLoadModel);
    assertThat("Not a gs schema", errorModel.getMessage(), containsString("not a gs"));
    // Error: Invalid gs path - case 2: invalid bucket name
    fileLoadModel = new FileLoadModel().profileId(profileModel.getId()).sourcePath("gs://jade_notabucket:1234/foo/bar.txt").description(testDescription).mimeType(testMimeType).targetPath(makeValidUniqueFilePath());
    errorModel = connectedOperations.ingestFileFailure(datasetSummary.getId(), fileLoadModel);
    assertThat("Invalid bucket name", errorModel.getMessage(), containsString("Invalid bucket name"));
    // Error: Invalid gs path - case 3: no bucket or path
    fileLoadModel = new FileLoadModel().profileId(profileModel.getId()).sourcePath("gs:///").description(testDescription).mimeType(testMimeType).targetPath(makeValidUniqueFilePath());
    errorModel = connectedOperations.ingestFileFailure(datasetSummary.getId(), fileLoadModel);
    assertThat("No bucket or path", errorModel.getMessage(), containsString("gs path"));
}

Example 7 with FileLoadModel

use of bio.terra.model.FileLoadModel in project jade-data-repo by DataBiosphere.

the class FileOperationTest method retryAndAcquireSharedLock.

@Test
public void retryAndAcquireSharedLock() throws Exception {
    FileLoadModel fileLoadModel = makeFileLoad(profileModel.getId());
    connectedOperations.retryAcquireLockIngestFileSuccess(true, datasetSummary.getId(), fileLoadModel, configService, datasetDao);
}

Example 8 with FileLoadModel

use of bio.terra.model.FileLoadModel in project jade-data-repo by DataBiosphere.

the class SnapshotConnectedTest method testExcludeLockedFromSnapshotFileLookups.

@Test
public void testExcludeLockedFromSnapshotFileLookups() throws Exception {
    // create a dataset
    DatasetSummaryModel datasetRefSummary = createTestDataset("simple-with-filerefs-dataset.json");
    // ingest a file
    URI sourceUri = new URI("gs", "jade-testdata", "/fileloadprofiletest/1KBfile.txt", null, null);
    String targetFilePath = "/mm/" + Names.randomizeName("testdir") + "/testExcludeLockedFromSnapshotFileLookups.txt";
    FileLoadModel fileLoadModel = new FileLoadModel().sourcePath(sourceUri.toString()).description("testExcludeLockedFromSnapshotFileLookups").mimeType("text/plain").targetPath(targetFilePath).profileId(billingProfile.getId());
    FileModel fileModel = connectedOperations.ingestFileSuccess(datasetRefSummary.getId(), fileLoadModel);
    // generate a JSON file with the fileref
    String jsonLine = "{\"name\":\"name1\", \"file_ref\":\"" + fileModel.getFileId() + "\"}\n";
    // load a JSON file that contains the table rows to load into the test bucket
    String jsonFileName = "this-better-pass.json";
    String dirInCloud = "scratch/testExcludeLockedFromSnapshotFileLookups/" + UUID.randomUUID().toString();
    BlobInfo ingestTableBlob = BlobInfo.newBuilder(testConfig.getIngestbucket(), dirInCloud + "/" + jsonFileName).build();
    Storage storage = StorageOptions.getDefaultInstance().getService();
    storage.create(ingestTableBlob, jsonLine.getBytes(StandardCharsets.UTF_8));
    // make sure the JSON file gets cleaned up on test teardown
    connectedOperations.addScratchFile(dirInCloud + "/" + jsonFileName);
    // ingest the tabular data from the JSON file we just generated
    String gsPath = "gs://" + testConfig.getIngestbucket() + "/" + dirInCloud + "/" + jsonFileName;
    IngestRequestModel ingestRequest1 = new IngestRequestModel().format(IngestRequestModel.FormatEnum.JSON).table("tableA").path(gsPath);
    connectedOperations.ingestTableSuccess(datasetRefSummary.getId(), ingestRequest1);
    // create a snapshot
    SnapshotSummaryModel snapshotSummary = connectedOperations.createSnapshot(datasetRefSummary, "simple-with-filerefs-snapshot.json", "");
    // check that the snapshot metadata row is unlocked
    String exclusiveLock = snapshotDao.getExclusiveLockState(UUID.fromString(snapshotSummary.getId()));
    assertNull("snapshot row is unlocked", exclusiveLock);
    String fileUri = getFileRefIdFromSnapshot(snapshotSummary);
    DrsId drsId = drsIdService.fromUri(fileUri);
    DRSObject drsObject = connectedOperations.drsGetObjectSuccess(drsId.toDrsObjectId(), false);
    String filePath = drsObject.getAliases().get(0);
    // lookup the snapshot file by DRS id, make sure it's returned (lookupSnapshotFileSuccess will already check)
    FileModel fsObjById = connectedOperations.lookupSnapshotFileSuccess(snapshotSummary.getId(), drsId.getFsObjectId());
    assertEquals("Retrieve snapshot file by DRS id matches desc", fsObjById.getDescription(), fileLoadModel.getDescription());
    // lookup the snapshot file by DRS path and check that it's found
    FileModel fsObjByPath = connectedOperations.lookupSnapshotFileByPathSuccess(snapshotSummary.getId(), filePath, 0);
    assertEquals("Retrieve snapshot file by path matches desc", fsObjByPath.getDescription(), fileLoadModel.getDescription());
    assertThat("Retrieve snapshot file objects match", fsObjById, CoreMatchers.equalTo(fsObjByPath));
    // now the snapshot exists....let's get it locked!
    // NO ASSERTS inside the block below where hang is enabled to reduce chance of failing before disabling the hang
    // ====================================================
    // enable hang in DeleteSnapshotPrimaryDataStep
    configService.setFault(ConfigEnum.SNAPSHOT_DELETE_LOCK_CONFLICT_STOP_FAULT.name(), true);
    // kick off a request to delete the snapshot. this should hang before unlocking the snapshot object.
    // note: asserts are below outside the hang block
    MvcResult deleteResult = mvc.perform(delete("/api/repository/v1/snapshots/" + snapshotSummary.getId())).andReturn();
    // give the flight time to launch and get to the hang
    TimeUnit.SECONDS.sleep(5);
    // check that the snapshot metadata row has an exclusive lock
    exclusiveLock = snapshotDao.getExclusiveLockState(UUID.fromString(snapshotSummary.getId()));
    // lookup the snapshot file by id and check that it's NOT found
    MockHttpServletResponse failedGetSnapshotByIdResponse = connectedOperations.lookupSnapshotFileRaw(snapshotSummary.getId(), drsId.getFsObjectId());
    // lookup the snapshot file by path and check that it's NOT found
    MockHttpServletResponse failedGetSnapshotByPathResponse = connectedOperations.lookupSnapshotFileByPathRaw(snapshotSummary.getId(), filePath, 0);
    // disable hang in DeleteSnapshotPrimaryDataStep
    configService.setFault(ConfigEnum.SNAPSHOT_DELETE_LOCK_CONFLICT_CONTINUE_FAULT.name(), true);
    // ====================================================
    // check that the snapshot metadata row has an exclusive lock after kicking off the delete
    assertNotNull("snapshot row is exclusively locked", exclusiveLock);
    assertEquals("Snapshot file NOT found by DRS id lookup", HttpStatus.NOT_FOUND, HttpStatus.valueOf(failedGetSnapshotByIdResponse.getStatus()));
    assertEquals("Snapshot file NOT found by path lookup", HttpStatus.NOT_FOUND, HttpStatus.valueOf(failedGetSnapshotByPathResponse.getStatus()));
    // check the response from the snapshot delete request
    MockHttpServletResponse deleteResponse = connectedOperations.validateJobModelAndWait(deleteResult);
    DeleteResponseModel deleteResponseModel = connectedOperations.handleSuccessCase(deleteResponse, DeleteResponseModel.class);
    assertEquals("Snapshot delete returned successfully", DeleteResponseModel.ObjectStateEnum.DELETED, deleteResponseModel.getObjectState());
    // delete the dataset and check that it succeeds
    connectedOperations.deleteTestDataset(datasetRefSummary.getId());
    // remove the file from the connectedoperation bookkeeping list
    connectedOperations.removeFile(datasetRefSummary.getId(), fileModel.getFileId());
    // try to fetch the snapshot again and confirm nothing is returned
    connectedOperations.getSnapshotExpectError(snapshotSummary.getId(), HttpStatus.NOT_FOUND);
    // try to fetch the dataset again and confirm nothing is returned
    connectedOperations.getDatasetExpectError(datasetRefSummary.getId(), HttpStatus.NOT_FOUND);
}

Example 9 with FileLoadModel

use of bio.terra.model.FileLoadModel in project jade-data-repo by DataBiosphere.

the class EncodeFileTest method makeFileLoadModel.

private FileLoadModel makeFileLoadModel(String gspath) throws Exception {
    URI uri = URI.create(gspath);
    FileLoadModel fileLoadModel = new FileLoadModel().sourcePath(gspath).profileId(profileModel.getId()).description(null).mimeType("application/octet-string").targetPath(uri.getPath()).loadTag(loadTag);
    return fileLoadModel;
}

Example 10 with FileLoadModel

use of bio.terra.model.FileLoadModel in project jade-data-repo by DataBiosphere.

the class IngestFileFileStep method doStep.

@Override
public StepResult doStep(FlightContext context) {
    FlightMap workingMap = context.getWorkingMap();
    Boolean loadComplete = workingMap.get(FileMapKeys.LOAD_COMPLETED, Boolean.class);
    if (loadComplete == null || !loadComplete) {
        FlightMap inputParameters = context.getInputParameters();
        FileLoadModel fileLoadModel = inputParameters.get(JobMapKeys.REQUEST.getKeyName(), FileLoadModel.class);
        FSFileInfo fsFileInfo = workingMap.get(FileMapKeys.FILE_INFO, FSFileInfo.class);
        String fileId = workingMap.get(FileMapKeys.FILE_ID, String.class);
        FireStoreFile newFile = new FireStoreFile().fileId(fileId).mimeType(fileLoadModel.getMimeType()).description(fileLoadModel.getDescription()).bucketResourceId(fsFileInfo.getBucketResourceId()).fileCreatedDate(fsFileInfo.getCreatedDate()).gspath(fsFileInfo.getGspath()).checksumCrc32c(fsFileInfo.getChecksumCrc32c()).checksumMd5(fsFileInfo.getChecksumMd5()).size(fsFileInfo.getSize()).loadTag(fileLoadModel.getLoadTag());
        try {
            fileDao.createFileMetadata(dataset, newFile);
            // Retrieve to build the complete FSItem
            FSItem fsItem = fileDao.retrieveById(dataset, fileId, 1, true);
            workingMap.put(JobMapKeys.RESPONSE.getKeyName(), fileService.fileModelFromFSItem(fsItem));
        } catch (FileSystemAbortTransactionException rex) {
            return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, rex);
        }
    }
    return StepResult.getStepResultSuccess();
}