use of bio.terra.stairway.FlightMap in project jade-data-repo by DataBiosphere.
the class IngestDriverStep method launchLoads.
private void launchLoads(FlightContext context, int launchCount, List<LoadFile> loadFiles, String profileId, UUID loadId, GoogleBucketResource bucketInfo) throws DatabaseOperationException, StairwayExecutionException, InterruptedException {
Stairway stairway = context.getStairway();
for (int i = 0; i < launchCount; i++) {
LoadFile loadFile = loadFiles.get(i);
String flightId = stairway.createFlightId();
FileLoadModel fileLoadModel = new FileLoadModel().sourcePath(loadFile.getSourcePath()).targetPath(loadFile.getTargetPath()).mimeType(loadFile.getMimeType()).profileId(profileId).loadTag(loadTag).description(loadFile.getDescription());
FlightMap inputParameters = new FlightMap();
inputParameters.put(FileMapKeys.DATASET_ID, datasetId);
inputParameters.put(FileMapKeys.REQUEST, fileLoadModel);
inputParameters.put(FileMapKeys.BUCKET_INFO, bucketInfo);
loadService.setLoadFileRunning(loadId, loadFile.getTargetPath(), flightId);
// NOTE: this is the window where we have recorded a flight as RUNNING in the load_file
// table, but it has not yet been launched. A failure in this window leaves "orphan"
// loads that are marked running, but not actually started. We handle this
// with the check for launch orphans at the beginning of the do() method.
// We use submitToQueue to spread the file loaders across multiple instances of datarepo.
stairway.submitToQueue(flightId, FileIngestWorkerFlight.class, inputParameters);
}
}
use of bio.terra.stairway.FlightMap in project jade-data-repo by DataBiosphere.
the class IngestFileDirectoryStep method doStep.
@Override
public StepResult doStep(FlightContext context) {
FlightMap inputParameters = context.getInputParameters();
FileLoadModel loadModel = inputParameters.get(JobMapKeys.REQUEST.getKeyName(), FileLoadModel.class);
FlightMap workingMap = context.getWorkingMap();
String fileId = workingMap.get(FileMapKeys.FILE_ID, String.class);
workingMap.put(FileMapKeys.LOAD_COMPLETED, false);
String datasetId = dataset.getId().toString();
String targetPath = loadModel.getTargetPath();
try {
// The state logic goes like this:
// 1. the directory entry doesn't exist. We need to create the directory entry for it.
// 2. the directory entry exists. There are three cases:
// a. If loadTags do not match, then we throw FileAlreadyExistsException.
// b. directory entry loadTag matches our loadTag AND entry fileId matches our fileId:
// means we are recovering and need to complete the file creation work.
// c. directory entry loadTag matches our loadTag AND entry fileId does NOT match our fileId
// means this is a re-run of a load job. We update the fileId in the working map. We don't
// know if we are recovering or already finished. We try to retrieve the file object for
// the entry fileId:
// i. If that is successful, then we already loaded this file. We store "completed=true"
// in the working map, so other steps do nothing.
// ii. If that fails, then we are recovering: we leave completed unset (=false) in the working map.
//
// Lookup the file - on a recovery, we may have already created it, but not
// finished. Or it might already exist, created by someone else.
FireStoreDirectoryEntry existingEntry = fileDao.lookupDirectoryEntryByPath(dataset, targetPath);
if (existingEntry == null) {
// Not there - create it
FireStoreDirectoryEntry newEntry = new FireStoreDirectoryEntry().fileId(fileId).isFileRef(true).path(fireStoreUtils.getDirectoryPath(loadModel.getTargetPath())).name(fireStoreUtils.getName(loadModel.getTargetPath())).datasetId(datasetId).loadTag(loadModel.getLoadTag());
fileDao.createDirectoryEntry(dataset, newEntry);
} else {
if (!StringUtils.equals(existingEntry.getLoadTag(), loadModel.getLoadTag())) {
// (a) Exists and is not our file
throw new FileAlreadyExistsException("Path already exists: " + targetPath);
}
// (b) or (c) Load tags match - check file ids
if (!StringUtils.equals(existingEntry.getFileId(), fileId)) {
// (c) We are in a re-run of a load job. Try to get the file entry.
fileId = existingEntry.getFileId();
workingMap.put(FileMapKeys.FILE_ID, fileId);
FireStoreFile fileEntry = fileDao.lookupFile(dataset, fileId);
if (fileEntry != null) {
// (c)(i) We successfully loaded this file already
workingMap.put(FileMapKeys.LOAD_COMPLETED, true);
}
// (c)(ii) We are recovering and should continue this load; leave load completed false/unset
}
}
} catch (FileSystemAbortTransactionException rex) {
return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, rex);
}
return StepResult.getStepResultSuccess();
}
use of bio.terra.stairway.FlightMap in project jade-data-repo by DataBiosphere.
the class IngestFileIdStep method doStep.
@Override
public StepResult doStep(FlightContext context) {
FlightMap workingMap = context.getWorkingMap();
String fileId = UUID.randomUUID().toString();
workingMap.put(FileMapKeys.FILE_ID, fileId);
if (configService.testInsertFault(ConfigEnum.FILE_INGEST_LOCK_CONFLICT_STOP_FAULT)) {
try {
logger.info("FILE_INGEST_LOCK_CONFLICT_STOP_FAULT");
while (!configService.testInsertFault(ConfigEnum.FILE_INGEST_LOCK_CONFLICT_CONTINUE_FAULT)) {
logger.info("Sleeping for CONTINUE FAULT");
TimeUnit.SECONDS.sleep(5);
}
logger.info("FILE_INGEST_LOCK_CONFLICT_CONTINUE_FAULT");
} catch (InterruptedException intEx) {
Thread.currentThread().interrupt();
throw new DatasetLockException("Unexpected interrupt during file ingest lock fault", intEx);
}
}
return StepResult.getStepResultSuccess();
}
use of bio.terra.stairway.FlightMap in project jade-data-repo by DataBiosphere.
the class IngestFilePrimaryDataLocationStep method doStep.
@Override
public StepResult doStep(FlightContext context) throws InterruptedException {
FlightMap workingMap = context.getWorkingMap();
Boolean loadComplete = workingMap.get(FileMapKeys.LOAD_COMPLETED, Boolean.class);
if (loadComplete == null || !loadComplete) {
try {
GoogleBucketResource bucketForFile = locationService.getOrCreateBucketForFile(profileId, context.getFlightId());
workingMap.put(FileMapKeys.BUCKET_INFO, bucketForFile);
} catch (BucketLockException blEx) {
return new StepResult(StepStatus.STEP_RESULT_FAILURE_RETRY, blEx);
}
}
return StepResult.getStepResultSuccess();
}
use of bio.terra.stairway.FlightMap in project jade-data-repo by DataBiosphere.
the class IngestFilePrimaryDataStep method undoStep.
@Override
public StepResult undoStep(FlightContext context) {
FlightMap workingMap = context.getWorkingMap();
String fileId = workingMap.get(FileMapKeys.FILE_ID, String.class);
GoogleBucketResource bucketResource = workingMap.get(FileMapKeys.BUCKET_INFO, GoogleBucketResource.class);
gcsPdao.deleteFileById(dataset, fileId, bucketResource);
return StepResult.getStepResultSuccess();
}
Aggregations