use of bio.terra.model.IngestRequestModel in project jade-data-repo by DataBiosphere.
the class IngestSetupStep method doStep.
@Override
public StepResult doStep(FlightContext context) throws InterruptedException {
if (configService.testInsertFault(ConfigEnum.TABLE_INGEST_LOCK_CONFLICT_STOP_FAULT)) {
logger.info("TABLE_INGEST_LOCK_CONFLICT_STOP_FAULT");
while (!configService.testInsertFault(ConfigEnum.TABLE_INGEST_LOCK_CONFLICT_CONTINUE_FAULT)) {
logger.info("Sleeping for CONTINUE FAULT");
TimeUnit.SECONDS.sleep(5);
}
logger.info("TABLE_INGEST_LOCK_CONFLICT_CONTINUE_FAULT");
}
IngestRequestModel ingestRequestModel = IngestUtils.getIngestRequestModel(context);
// We don't actually care about the output here since BQ takes the raw "gs://" string as input.
// As long as parsing succeeds, we're good to move forward.
IngestUtils.parseBlobUri(ingestRequestModel.getPath());
Dataset dataset = IngestUtils.getDataset(context, datasetService);
IngestUtils.putDatasetName(context, dataset.getName());
DatasetTable targetTable = IngestUtils.getDatasetTable(context, dataset);
String sgName = DatasetUtils.generateAuxTableName(targetTable, "st");
IngestUtils.putStagingTableName(context, sgName);
return StepResult.getStepResultSuccess();
}
use of bio.terra.model.IngestRequestModel in project jade-data-repo by DataBiosphere.
the class IngestUtils method getDatasetTable.
public static DatasetTable getDatasetTable(FlightContext context, Dataset dataset) {
IngestRequestModel ingestRequest = getIngestRequestModel(context);
Optional<DatasetTable> optTable = dataset.getTableByName(ingestRequest.getTable());
if (!optTable.isPresent()) {
throw new TableNotFoundException("Table not found: " + ingestRequest.getTable());
}
return optTable.get();
}
use of bio.terra.model.IngestRequestModel in project jade-data-repo by DataBiosphere.
the class IngestRequestValidator method validate.
@Override
@SuppressFBWarnings(value = "UC_USELESS_VOID_METHOD", justification = "FB mistake - this clearly validates and returns data in errors")
public void validate(@NotNull Object target, Errors errors) {
if (target instanceof IngestRequestModel) {
IngestRequestModel ingestRequest = (IngestRequestModel) target;
validateTableName(ingestRequest.getTable(), errors);
} else if (target instanceof FileLoadModel) {
FileLoadModel fileLoadModel = (FileLoadModel) target;
if (fileLoadModel.getProfileId() == null) {
errors.rejectValue("profileId", "ProfileIdMissing", "File ingest requires a profile id.");
}
}
}
use of bio.terra.model.IngestRequestModel in project jade-data-repo by DataBiosphere.
the class IngestInsertIntoDatasetTableStep method doStep.
@Override
public StepResult doStep(FlightContext context) throws InterruptedException {
Dataset dataset = IngestUtils.getDataset(context, datasetService);
DatasetTable targetTable = IngestUtils.getDatasetTable(context, dataset);
String stagingTableName = IngestUtils.getStagingTableName(context);
IngestRequestModel ingestRequest = IngestUtils.getIngestRequestModel(context);
PdaoLoadStatistics loadStatistics = IngestUtils.getIngestStatistics(context);
IngestResponseModel ingestResponse = new IngestResponseModel().dataset(dataset.getName()).datasetId(dataset.getId().toString()).table(ingestRequest.getTable()).path(ingestRequest.getPath()).loadTag(ingestRequest.getLoadTag()).badRowCount(loadStatistics.getBadRecords()).rowCount(loadStatistics.getRowCount());
context.getWorkingMap().put(JobMapKeys.RESPONSE.getKeyName(), ingestResponse);
bigQueryPdao.insertIntoDatasetTable(dataset, targetTable, stagingTableName);
return StepResult.getStepResultSuccess();
}
use of bio.terra.model.IngestRequestModel in project jade-data-repo by DataBiosphere.
the class IngestLoadTableStep method doStep.
@Override
public StepResult doStep(FlightContext context) throws InterruptedException {
Dataset dataset = IngestUtils.getDataset(context, datasetService);
DatasetTable targetTable = IngestUtils.getDatasetTable(context, dataset);
String stagingTableName = IngestUtils.getStagingTableName(context);
IngestRequestModel ingestRequest = IngestUtils.getIngestRequestModel(context);
PdaoLoadStatistics ingestStatistics = bigQueryPdao.loadToStagingTable(dataset, targetTable, stagingTableName, ingestRequest);
// Save away the stats in the working map. We will use some of them later
// when we make the annotations. Others are returned on the ingest response.
IngestUtils.putIngestStatistics(context, ingestStatistics);
return StepResult.getStepResultSuccess();
}
Aggregations