Search in sources :

Example 6 with Command

use of org.apache.gobblin.source.extractor.extract.Command in project incubator-gobblin by apache.

the class JdbcExtractorTest method testUnsignedInt.

/**
 * Test for the metadata query to see if the check for unsigned int is present
 */
@Test
public void testUnsignedInt() throws SchemaException {
    State state = new WorkUnitState();
    state.setId("id");
    MysqlExtractor mysqlExtractor = new MysqlExtractor((WorkUnitState) state);
    List<Command> commands = mysqlExtractor.getSchemaMetadata("db", "table");
    assertTrue(commands.get(0).getCommandType() == JdbcCommand.JdbcCommandType.QUERY);
    assertTrue(commands.get(0).getParams().get(0).contains("bigint"));
    assertTrue(commands.get(1).getCommandType() == JdbcCommand.JdbcCommandType.QUERYPARAMS);
    assertTrue(!commands.get(1).getParams().get(0).contains("unsigned"));
    // set option to promote unsigned int to bigint
    state.setProp(ConfigurationKeys.SOURCE_QUERYBASED_PROMOTE_UNSIGNED_INT_TO_BIGINT, "true");
    commands = mysqlExtractor.getSchemaMetadata("db", "table");
    assertTrue(commands.get(0).getCommandType() == JdbcCommand.JdbcCommandType.QUERY);
    assertTrue(commands.get(0).getParams().get(0).contains("bigint"));
    assertTrue(commands.get(1).getCommandType() == JdbcCommand.JdbcCommandType.QUERYPARAMS);
    assertTrue(commands.get(1).getParams().get(0).contains("unsigned"));
}
Also used : Command(org.apache.gobblin.source.extractor.extract.Command) WorkUnitState(org.apache.gobblin.configuration.WorkUnitState) State(org.apache.gobblin.configuration.State) WorkUnitState(org.apache.gobblin.configuration.WorkUnitState) Test(org.testng.annotations.Test)

Example 7 with Command

use of org.apache.gobblin.source.extractor.extract.Command in project incubator-gobblin by apache.

the class RestApiExtractor method getRecordSet.

@Override
public Iterator<JsonElement> getRecordSet(String schema, String entity, WorkUnit workUnit, List<Predicate> predicateList) throws DataRecordException {
    log.debug("Get data records using Rest Api");
    Iterator<JsonElement> rs = null;
    List<Command> cmds;
    try {
        boolean success = true;
        if (this.connector.isConnectionClosed()) {
            success = this.connector.connect();
        }
        if (!success) {
            throw new DataRecordException("Failed to connect.");
        }
        log.debug("Connected successfully.");
        if (this.getPullStatus() == false) {
            return null;
        }
        if (this.getNextUrl() == null) {
            cmds = this.getDataMetadata(schema, entity, workUnit, predicateList);
        } else {
            cmds = RestApiConnector.constructGetCommand(this.getNextUrl());
        }
        CommandOutput<?, ?> response = this.connector.getResponse(cmds);
        rs = this.getData(response);
        return rs;
    } catch (Exception e) {
        throw new DataRecordException("Failed to get records using rest api; error - " + e.getMessage(), e);
    }
}
Also used : Command(org.apache.gobblin.source.extractor.extract.Command) JsonElement(com.google.gson.JsonElement) DataRecordException(org.apache.gobblin.source.extractor.DataRecordException) SchemaException(org.apache.gobblin.source.extractor.exception.SchemaException) RestApiProcessingException(org.apache.gobblin.source.extractor.exception.RestApiProcessingException) IOException(java.io.IOException) HighWatermarkException(org.apache.gobblin.source.extractor.exception.HighWatermarkException) RecordCountException(org.apache.gobblin.source.extractor.exception.RecordCountException) DataRecordException(org.apache.gobblin.source.extractor.DataRecordException) RestApiConnectionException(org.apache.gobblin.source.extractor.exception.RestApiConnectionException)

Example 8 with Command

use of org.apache.gobblin.source.extractor.extract.Command in project incubator-gobblin by apache.

the class RestApiExtractor method getMaxWatermark.

@Override
public long getMaxWatermark(String schema, String entity, String watermarkColumn, List<Predicate> predicateList, String watermarkSourceFormat) throws HighWatermarkException {
    log.info("Get high watermark using Rest Api");
    long CalculatedHighWatermark = -1;
    try {
        boolean success = this.connector.connect();
        if (!success) {
            throw new HighWatermarkException("Failed to connect.");
        }
        log.debug("Connected successfully.");
        List<Command> cmds = this.getHighWatermarkMetadata(schema, entity, watermarkColumn, predicateList);
        CommandOutput<?, ?> response = this.connector.getResponse(cmds);
        CalculatedHighWatermark = this.getHighWatermark(response, watermarkColumn, watermarkSourceFormat);
        log.info("High watermark:" + CalculatedHighWatermark);
        return CalculatedHighWatermark;
    } catch (Exception e) {
        throw new HighWatermarkException("Failed to get high watermark using rest api; error - " + e.getMessage(), e);
    }
}
Also used : Command(org.apache.gobblin.source.extractor.extract.Command) HighWatermarkException(org.apache.gobblin.source.extractor.exception.HighWatermarkException) SchemaException(org.apache.gobblin.source.extractor.exception.SchemaException) RestApiProcessingException(org.apache.gobblin.source.extractor.exception.RestApiProcessingException) IOException(java.io.IOException) HighWatermarkException(org.apache.gobblin.source.extractor.exception.HighWatermarkException) RecordCountException(org.apache.gobblin.source.extractor.exception.RecordCountException) DataRecordException(org.apache.gobblin.source.extractor.DataRecordException) RestApiConnectionException(org.apache.gobblin.source.extractor.exception.RestApiConnectionException)

Example 9 with Command

use of org.apache.gobblin.source.extractor.extract.Command in project incubator-gobblin by apache.

the class RestApiExtractor method getSourceCount.

@Override
public long getSourceCount(String schema, String entity, WorkUnit workUnit, List<Predicate> predicateList) throws RecordCountException {
    log.info("Get source record count using Rest Api");
    long count = 0;
    try {
        boolean success = this.connector.connect();
        if (!success) {
            throw new RecordCountException("Failed to connect.");
        }
        log.debug("Connected successfully.");
        List<Command> cmds = this.getCountMetadata(schema, entity, workUnit, predicateList);
        CommandOutput<?, ?> response = this.connector.getResponse(cmds);
        count = getCount(response);
        log.info("Source record count:" + count);
        return count;
    } catch (Exception e) {
        throw new RecordCountException("Failed to get record count using rest api; error - " + e.getMessage(), e);
    }
}
Also used : RecordCountException(org.apache.gobblin.source.extractor.exception.RecordCountException) Command(org.apache.gobblin.source.extractor.extract.Command) SchemaException(org.apache.gobblin.source.extractor.exception.SchemaException) RestApiProcessingException(org.apache.gobblin.source.extractor.exception.RestApiProcessingException) IOException(java.io.IOException) HighWatermarkException(org.apache.gobblin.source.extractor.exception.HighWatermarkException) RecordCountException(org.apache.gobblin.source.extractor.exception.RecordCountException) DataRecordException(org.apache.gobblin.source.extractor.DataRecordException) RestApiConnectionException(org.apache.gobblin.source.extractor.exception.RestApiConnectionException)

Example 10 with Command

use of org.apache.gobblin.source.extractor.extract.Command in project incubator-gobblin by apache.

the class JdbcExtractor method getMaxWatermark.

@Override
public long getMaxWatermark(String schema, String entity, String watermarkColumn, List<Predicate> predicateList, String watermarkSourceFormat) throws HighWatermarkException {
    this.log.info("Get high watermark using JDBC");
    long calculatedHighWatermark = ConfigurationKeys.DEFAULT_WATERMARK_VALUE;
    try {
        List<Command> cmds = this.getHighWatermarkMetadata(schema, entity, watermarkColumn, predicateList);
        CommandOutput<?, ?> response = this.executeSql(cmds);
        calculatedHighWatermark = this.getHighWatermark(response, watermarkColumn, watermarkSourceFormat);
        return calculatedHighWatermark;
    } catch (Exception e) {
        throw new HighWatermarkException("Failed to get high watermark using JDBC; error - " + e.getMessage(), e);
    }
}
Also used : Command(org.apache.gobblin.source.extractor.extract.Command) HighWatermarkException(org.apache.gobblin.source.extractor.exception.HighWatermarkException) ParseException(java.text.ParseException) RecordCountException(org.apache.gobblin.source.extractor.exception.RecordCountException) DataRecordException(org.apache.gobblin.source.extractor.DataRecordException) SqlParseException(org.apache.calcite.sql.parser.SqlParseException) SchemaException(org.apache.gobblin.source.extractor.exception.SchemaException) SQLException(java.sql.SQLException) IOException(java.io.IOException) HighWatermarkException(org.apache.gobblin.source.extractor.exception.HighWatermarkException)

Aggregations

Command (org.apache.gobblin.source.extractor.extract.Command)11 IOException (java.io.IOException)10 SchemaException (org.apache.gobblin.source.extractor.exception.SchemaException)10 DataRecordException (org.apache.gobblin.source.extractor.DataRecordException)8 HighWatermarkException (org.apache.gobblin.source.extractor.exception.HighWatermarkException)8 RecordCountException (org.apache.gobblin.source.extractor.exception.RecordCountException)8 SQLException (java.sql.SQLException)5 ParseException (java.text.ParseException)5 SqlParseException (org.apache.calcite.sql.parser.SqlParseException)5 RestApiConnectionException (org.apache.gobblin.source.extractor.exception.RestApiConnectionException)4 RestApiProcessingException (org.apache.gobblin.source.extractor.exception.RestApiProcessingException)4 JsonElement (com.google.gson.JsonElement)3 JsonArray (com.google.gson.JsonArray)2 JsonObject (com.google.gson.JsonObject)2 PreparedStatement (java.sql.PreparedStatement)2 ResultSet (java.sql.ResultSet)2 Schema (org.apache.gobblin.source.extractor.schema.Schema)2 JdbcCommandType (org.apache.gobblin.source.jdbc.JdbcCommand.JdbcCommandType)2 Splitter (com.google.common.base.Splitter)1 Statement (java.sql.Statement)1