use of org.apache.gobblin.source.extractor.extract.Command in project incubator-gobblin by apache.
the class JdbcExtractorTest method testUnsignedInt.
/**
* Test for the metadata query to see if the check for unsigned int is present
*/
@Test
public void testUnsignedInt() throws SchemaException {
State state = new WorkUnitState();
state.setId("id");
MysqlExtractor mysqlExtractor = new MysqlExtractor((WorkUnitState) state);
List<Command> commands = mysqlExtractor.getSchemaMetadata("db", "table");
assertTrue(commands.get(0).getCommandType() == JdbcCommand.JdbcCommandType.QUERY);
assertTrue(commands.get(0).getParams().get(0).contains("bigint"));
assertTrue(commands.get(1).getCommandType() == JdbcCommand.JdbcCommandType.QUERYPARAMS);
assertTrue(!commands.get(1).getParams().get(0).contains("unsigned"));
// set option to promote unsigned int to bigint
state.setProp(ConfigurationKeys.SOURCE_QUERYBASED_PROMOTE_UNSIGNED_INT_TO_BIGINT, "true");
commands = mysqlExtractor.getSchemaMetadata("db", "table");
assertTrue(commands.get(0).getCommandType() == JdbcCommand.JdbcCommandType.QUERY);
assertTrue(commands.get(0).getParams().get(0).contains("bigint"));
assertTrue(commands.get(1).getCommandType() == JdbcCommand.JdbcCommandType.QUERYPARAMS);
assertTrue(commands.get(1).getParams().get(0).contains("unsigned"));
}
use of org.apache.gobblin.source.extractor.extract.Command in project incubator-gobblin by apache.
the class RestApiExtractor method getRecordSet.
@Override
public Iterator<JsonElement> getRecordSet(String schema, String entity, WorkUnit workUnit, List<Predicate> predicateList) throws DataRecordException {
log.debug("Get data records using Rest Api");
Iterator<JsonElement> rs = null;
List<Command> cmds;
try {
boolean success = true;
if (this.connector.isConnectionClosed()) {
success = this.connector.connect();
}
if (!success) {
throw new DataRecordException("Failed to connect.");
}
log.debug("Connected successfully.");
if (this.getPullStatus() == false) {
return null;
}
if (this.getNextUrl() == null) {
cmds = this.getDataMetadata(schema, entity, workUnit, predicateList);
} else {
cmds = RestApiConnector.constructGetCommand(this.getNextUrl());
}
CommandOutput<?, ?> response = this.connector.getResponse(cmds);
rs = this.getData(response);
return rs;
} catch (Exception e) {
throw new DataRecordException("Failed to get records using rest api; error - " + e.getMessage(), e);
}
}
use of org.apache.gobblin.source.extractor.extract.Command in project incubator-gobblin by apache.
the class RestApiExtractor method getMaxWatermark.
@Override
public long getMaxWatermark(String schema, String entity, String watermarkColumn, List<Predicate> predicateList, String watermarkSourceFormat) throws HighWatermarkException {
log.info("Get high watermark using Rest Api");
long CalculatedHighWatermark = -1;
try {
boolean success = this.connector.connect();
if (!success) {
throw new HighWatermarkException("Failed to connect.");
}
log.debug("Connected successfully.");
List<Command> cmds = this.getHighWatermarkMetadata(schema, entity, watermarkColumn, predicateList);
CommandOutput<?, ?> response = this.connector.getResponse(cmds);
CalculatedHighWatermark = this.getHighWatermark(response, watermarkColumn, watermarkSourceFormat);
log.info("High watermark:" + CalculatedHighWatermark);
return CalculatedHighWatermark;
} catch (Exception e) {
throw new HighWatermarkException("Failed to get high watermark using rest api; error - " + e.getMessage(), e);
}
}
use of org.apache.gobblin.source.extractor.extract.Command in project incubator-gobblin by apache.
the class RestApiExtractor method getSourceCount.
@Override
public long getSourceCount(String schema, String entity, WorkUnit workUnit, List<Predicate> predicateList) throws RecordCountException {
log.info("Get source record count using Rest Api");
long count = 0;
try {
boolean success = this.connector.connect();
if (!success) {
throw new RecordCountException("Failed to connect.");
}
log.debug("Connected successfully.");
List<Command> cmds = this.getCountMetadata(schema, entity, workUnit, predicateList);
CommandOutput<?, ?> response = this.connector.getResponse(cmds);
count = getCount(response);
log.info("Source record count:" + count);
return count;
} catch (Exception e) {
throw new RecordCountException("Failed to get record count using rest api; error - " + e.getMessage(), e);
}
}
use of org.apache.gobblin.source.extractor.extract.Command in project incubator-gobblin by apache.
the class JdbcExtractor method getMaxWatermark.
@Override
public long getMaxWatermark(String schema, String entity, String watermarkColumn, List<Predicate> predicateList, String watermarkSourceFormat) throws HighWatermarkException {
this.log.info("Get high watermark using JDBC");
long calculatedHighWatermark = ConfigurationKeys.DEFAULT_WATERMARK_VALUE;
try {
List<Command> cmds = this.getHighWatermarkMetadata(schema, entity, watermarkColumn, predicateList);
CommandOutput<?, ?> response = this.executeSql(cmds);
calculatedHighWatermark = this.getHighWatermark(response, watermarkColumn, watermarkSourceFormat);
return calculatedHighWatermark;
} catch (Exception e) {
throw new HighWatermarkException("Failed to get high watermark using JDBC; error - " + e.getMessage(), e);
}
}
Aggregations