Examples with PdaoException bio.terra.common.exception.PdaoException used on opensource projects

Example 1 with PdaoException

use of bio.terra.common.exception.PdaoException in project jade-data-repo by DataBiosphere.

the class BigQueryPdao method buildFormatOptions.

private FormatOptions buildFormatOptions(IngestRequestModel ingestRequest) {
    FormatOptions options;
    switch(ingestRequest.getFormat()) {
        case CSV:
            CsvOptions csvDefaults = FormatOptions.csv();
            options = CsvOptions.newBuilder().setFieldDelimiter(ingestRequest.getCsvFieldDelimiter() == null ? csvDefaults.getFieldDelimiter() : ingestRequest.getCsvFieldDelimiter()).setQuote(ingestRequest.getCsvQuote() == null ? csvDefaults.getQuote() : ingestRequest.getCsvQuote()).setSkipLeadingRows(ingestRequest.getCsvSkipLeadingRows() == null ? csvDefaults.getSkipLeadingRows() : ingestRequest.getCsvSkipLeadingRows()).setAllowQuotedNewLines(ingestRequest.isCsvAllowQuotedNewlines() == null ? csvDefaults.allowQuotedNewLines() : ingestRequest.isCsvAllowQuotedNewlines()).build();
            break;
        case JSON:
            options = FormatOptions.json();
            break;
        default:
            throw new PdaoException("Invalid format option: " + ingestRequest.getFormat());
    }
    return options;
}

Example 2 with PdaoException

use of bio.terra.common.exception.PdaoException in project jade-data-repo by DataBiosphere.

the class BigQueryPdao method createStagingLoadHistoryTable.

public void createStagingLoadHistoryTable(Dataset dataset, String tableName_FlightId) throws InterruptedException {
    BigQueryProject bigQueryProject = bigQueryProjectForDataset(dataset);
    try {
        String datasetName = prefixName(dataset.getName());
        if (bigQueryProject.tableExists(datasetName, PDAO_LOAD_HISTORY_STAGING_TABLE_PREFIX + tableName_FlightId)) {
            bigQueryProject.deleteTable(datasetName, PDAO_LOAD_HISTORY_STAGING_TABLE_PREFIX + tableName_FlightId);
        }
        bigQueryProject.createTable(datasetName, PDAO_LOAD_HISTORY_STAGING_TABLE_PREFIX + tableName_FlightId, buildLoadDatasetSchema());
    } catch (Exception ex) {
        throw new PdaoException("create staging load history table failed for " + dataset.getName(), ex);
    }
}

Example 3 with PdaoException

use of bio.terra.common.exception.PdaoException in project jade-data-repo by DataBiosphere.

the class BigQueryPdao method createSnapshotWithLiveViews.

public void createSnapshotWithLiveViews(Snapshot snapshot, Dataset dataset) throws InterruptedException {
    BigQueryProject bigQueryProject = bigQueryProjectForSnapshot(snapshot);
    String projectId = bigQueryProject.getProjectId();
    String snapshotName = snapshot.getName();
    BigQuery bigQuery = bigQueryProject.getBigQuery();
    String datasetBqDatasetName = prefixName(dataset.getName());
    // create snapshot BQ dataset
    snapshotCreateBQDataset(bigQueryProject, snapshot);
    // create the row id table (row id col and table id col)
    bigQueryProject.createTable(snapshotName, PDAO_ROW_ID_TABLE, rowIdTableSchema());
    // get source dataset table live views
    List<DatasetTable> tables = dataset.getTables();
    // create a snapshot table based on the live view data row ids
    String liveViewTables = createSnaphotTableFromLiveViews(bigQueryProject, tables, datasetBqDatasetName);
    ST sqlTemplate = new ST(insertAllLiveViewDataTemplate);
    sqlTemplate.add("project", projectId);
    sqlTemplate.add("snapshot", snapshotName);
    sqlTemplate.add("dataRepoTable", PDAO_ROW_ID_TABLE);
    sqlTemplate.add("dataRepoTableId", PDAO_TABLE_ID_COLUMN);
    sqlTemplate.add("dataRepoRowId", PDAO_ROW_ID_COLUMN);
    sqlTemplate.add("liveViewTables", liveViewTables);
    bigQueryProject.query(sqlTemplate.render());
    ST sqlValidateSnapshotTemplate = new ST(validateSnapshotSizeTemplate);
    sqlValidateSnapshotTemplate.add("project", projectId);
    sqlValidateSnapshotTemplate.add("snapshot", snapshotName);
    sqlValidateSnapshotTemplate.add("dataRepoTable", PDAO_ROW_ID_TABLE);
    TableResult result = bigQueryProject.query(sqlValidateSnapshotTemplate.render());
    if (result.getTotalRows() <= 0) {
        throw new PdaoException("This snapshot is empty");
    }
    snapshotViewCreation(datasetBqDatasetName, snapshotName, snapshot, projectId, bigQuery, bigQueryProject);
}

Example 4 with PdaoException

use of bio.terra.common.exception.PdaoException in project jade-data-repo by DataBiosphere.

the class BigQueryProject method datasetExists.

// TODO: REVIEWERS PLEASE CHECK: Should these methods be in here? On the one hand, it is convenient. But it
// mixes the duties of this class: it is supplying both the cache and the BQ methods. Unfortunately, it
// doesn't supply all of the BQ methods, so sometimes getBigQuery is needed. Seems like it could be
// improved.
// 
// In general, BigQueryPdao is too big and needs refactoring. Perhaps when that is done, we can also
// re-think this code structure.
public boolean datasetExists(String datasetName) {
    try {
        DatasetId datasetId = DatasetId.of(projectId, datasetName);
        Dataset dataset = bigQuery.getDataset(datasetId);
        return (dataset != null);
    } catch (Exception ex) {
        throw new PdaoException("existence check failed for " + datasetName, ex);
    }
}

Example 5 with PdaoException

use of bio.terra.common.exception.PdaoException in project jade-data-repo by DataBiosphere.

the class BigQueryProject method tableExists.

public boolean tableExists(String datasetName, String tableName) {
    try {
        TableId tableId = TableId.of(projectId, datasetName, tableName);
        Table table = bigQuery.getTable(tableId);
        return (table != null);
    } catch (Exception ex) {
        throw new PdaoException("existence check failed for " + datasetName + "." + tableName, ex);
    }
}