Examples with Command org.apache.gobblin.source.extractor.extract.Command used on opensource projects

Example 6 with Command

use of org.apache.gobblin.source.extractor.extract.Command in project incubator-gobblin by apache.

the class JdbcExtractorTest method testUnsignedInt.

/**
 * Test for the metadata query to see if the check for unsigned int is present
 */
@Test
public void testUnsignedInt() throws SchemaException {
    State state = new WorkUnitState();
    state.setId("id");
    MysqlExtractor mysqlExtractor = new MysqlExtractor((WorkUnitState) state);
    List<Command> commands = mysqlExtractor.getSchemaMetadata("db", "table");
    assertTrue(commands.get(0).getCommandType() == JdbcCommand.JdbcCommandType.QUERY);
    assertTrue(commands.get(0).getParams().get(0).contains("bigint"));
    assertTrue(commands.get(1).getCommandType() == JdbcCommand.JdbcCommandType.QUERYPARAMS);
    assertTrue(!commands.get(1).getParams().get(0).contains("unsigned"));
    // set option to promote unsigned int to bigint
    state.setProp(ConfigurationKeys.SOURCE_QUERYBASED_PROMOTE_UNSIGNED_INT_TO_BIGINT, "true");
    commands = mysqlExtractor.getSchemaMetadata("db", "table");
    assertTrue(commands.get(0).getCommandType() == JdbcCommand.JdbcCommandType.QUERY);
    assertTrue(commands.get(0).getParams().get(0).contains("bigint"));
    assertTrue(commands.get(1).getCommandType() == JdbcCommand.JdbcCommandType.QUERYPARAMS);
    assertTrue(commands.get(1).getParams().get(0).contains("unsigned"));
}

Example 7 with Command

use of org.apache.gobblin.source.extractor.extract.Command in project incubator-gobblin by apache.

the class RestApiExtractor method getRecordSet.

@Override
public Iterator<JsonElement> getRecordSet(String schema, String entity, WorkUnit workUnit, List<Predicate> predicateList) throws DataRecordException {
    log.debug("Get data records using Rest Api");
    Iterator<JsonElement> rs = null;
    List<Command> cmds;
    try {
        boolean success = true;
        if (this.connector.isConnectionClosed()) {
            success = this.connector.connect();
        }
        if (!success) {
            throw new DataRecordException("Failed to connect.");
        }
        log.debug("Connected successfully.");
        if (this.getPullStatus() == false) {
            return null;
        }
        if (this.getNextUrl() == null) {
            cmds = this.getDataMetadata(schema, entity, workUnit, predicateList);
        } else {
            cmds = RestApiConnector.constructGetCommand(this.getNextUrl());
        }
        CommandOutput<?, ?> response = this.connector.getResponse(cmds);
        rs = this.getData(response);
        return rs;
    } catch (Exception e) {
        throw new DataRecordException("Failed to get records using rest api; error - " + e.getMessage(), e);
    }
}

Example 8 with Command

use of org.apache.gobblin.source.extractor.extract.Command in project incubator-gobblin by apache.

the class RestApiExtractor method getMaxWatermark.

@Override
public long getMaxWatermark(String schema, String entity, String watermarkColumn, List<Predicate> predicateList, String watermarkSourceFormat) throws HighWatermarkException {
    log.info("Get high watermark using Rest Api");
    long CalculatedHighWatermark = -1;
    try {
        boolean success = this.connector.connect();
        if (!success) {
            throw new HighWatermarkException("Failed to connect.");
        }
        log.debug("Connected successfully.");
        List<Command> cmds = this.getHighWatermarkMetadata(schema, entity, watermarkColumn, predicateList);
        CommandOutput<?, ?> response = this.connector.getResponse(cmds);
        CalculatedHighWatermark = this.getHighWatermark(response, watermarkColumn, watermarkSourceFormat);
        log.info("High watermark:" + CalculatedHighWatermark);
        return CalculatedHighWatermark;
    } catch (Exception e) {
        throw new HighWatermarkException("Failed to get high watermark using rest api; error - " + e.getMessage(), e);
    }
}

Example 9 with Command

use of org.apache.gobblin.source.extractor.extract.Command in project incubator-gobblin by apache.

the class RestApiExtractor method getSourceCount.

@Override
public long getSourceCount(String schema, String entity, WorkUnit workUnit, List<Predicate> predicateList) throws RecordCountException {
    log.info("Get source record count using Rest Api");
    long count = 0;
    try {
        boolean success = this.connector.connect();
        if (!success) {
            throw new RecordCountException("Failed to connect.");
        }
        log.debug("Connected successfully.");
        List<Command> cmds = this.getCountMetadata(schema, entity, workUnit, predicateList);
        CommandOutput<?, ?> response = this.connector.getResponse(cmds);
        count = getCount(response);
        log.info("Source record count:" + count);
        return count;
    } catch (Exception e) {
        throw new RecordCountException("Failed to get record count using rest api; error - " + e.getMessage(), e);
    }
}

Example 10 with Command

use of org.apache.gobblin.source.extractor.extract.Command in project incubator-gobblin by apache.

the class JdbcExtractor method getMaxWatermark.

@Override
public long getMaxWatermark(String schema, String entity, String watermarkColumn, List<Predicate> predicateList, String watermarkSourceFormat) throws HighWatermarkException {
    this.log.info("Get high watermark using JDBC");
    long calculatedHighWatermark = ConfigurationKeys.DEFAULT_WATERMARK_VALUE;
    try {
        List<Command> cmds = this.getHighWatermarkMetadata(schema, entity, watermarkColumn, predicateList);
        CommandOutput<?, ?> response = this.executeSql(cmds);
        calculatedHighWatermark = this.getHighWatermark(response, watermarkColumn, watermarkSourceFormat);
        return calculatedHighWatermark;
    } catch (Exception e) {
        throw new HighWatermarkException("Failed to get high watermark using JDBC; error - " + e.getMessage(), e);
    }
}