use of org.apache.drill.exec.store.http.util.SimpleHttp in project drill by apache.
the class HttpXMLBatchReader method open.
@Override
public boolean open(SchemaNegotiator negotiator) {
HttpUrl url = buildUrl();
// Result set loader setup
String tempDirPath = negotiator.drillConfig().getString(ExecConstants.DRILL_TMP_DIR);
// Create user-friendly error context
CustomErrorContext errorContext = new ChildErrorContext(negotiator.parentErrorContext()) {
@Override
public void addContext(UserException.Builder builder) {
super.addContext(builder);
builder.addContext("URL", url.toString());
}
};
negotiator.setErrorContext(errorContext);
// Http client setup
SimpleHttp http = SimpleHttp.builder().scanDefn(subScan).url(url).tempDir(new File(tempDirPath)).paginator(paginator).proxyConfig(proxySettings(negotiator.drillConfig(), url)).errorContext(errorContext).build();
// Get the input stream
inStream = http.getInputStream();
// Initialize the XMLReader the reader
try {
xmlReader = new XMLReader(inStream, dataLevel, maxRecords);
resultLoader = negotiator.build();
if (implicitColumnsAreProjected()) {
implicitColumns = new ImplicitColumns(resultLoader.writer());
buildImplicitColumns();
populateImplicitFieldMap(http);
}
RowSetLoader rootRowWriter = resultLoader.writer();
xmlReader.open(rootRowWriter, errorContext);
xmlReader.implicitFields(implicitColumns);
} catch (XMLStreamException e) {
throw UserException.dataReadError(e).message("Error opening XML stream: " + e.getMessage()).addContext(errorContext).build(logger);
}
return true;
}
use of org.apache.drill.exec.store.http.util.SimpleHttp in project drill by apache.
the class HttpBatchReader method open.
@Override
public boolean open(SchemaNegotiator negotiator) {
// Result set loader setup
String tempDirPath = negotiator.drillConfig().getString(ExecConstants.DRILL_TMP_DIR);
HttpUrl url = buildUrl();
logger.debug("Final URL: {}", url);
CustomErrorContext errorContext = new ChildErrorContext(negotiator.parentErrorContext()) {
@Override
public void addContext(UserException.Builder builder) {
super.addContext(builder);
builder.addContext("URL", url.toString());
}
};
negotiator.setErrorContext(errorContext);
// Http client setup
SimpleHttp http = SimpleHttp.builder().scanDefn(subScan).url(url).tempDir(new File(tempDirPath)).proxyConfig(proxySettings(negotiator.drillConfig(), url)).errorContext(errorContext).build();
// JSON loader setup
resultSetLoader = negotiator.build();
if (implicitColumnsAreProjected()) {
implicitColumns = new ImplicitColumns(resultSetLoader.writer());
buildImplicitColumns();
}
InputStream inStream = http.getInputStream();
populateImplicitFieldMap(http);
try {
JsonLoaderBuilder jsonBuilder = new JsonLoaderBuilder().implicitFields(implicitColumns).resultSetLoader(resultSetLoader).standardOptions(negotiator.queryOptions()).maxRows(maxRecords).dataPath(subScan.tableSpec().connectionConfig().dataPath()).errorContext(errorContext).fromStream(inStream);
if (subScan.tableSpec().connectionConfig().jsonOptions() != null) {
JsonLoaderOptions jsonOptions = subScan.tableSpec().connectionConfig().jsonOptions().getJsonOptions(negotiator.queryOptions());
jsonBuilder.options(jsonOptions);
} else {
jsonBuilder.standardOptions(negotiator.queryOptions());
}
jsonLoader = jsonBuilder.build();
} catch (Throwable t) {
// Paranoia: ensure stream is closed if anything goes wrong.
// After this, the JSON loader will close the stream.
AutoCloseables.closeSilently(inStream);
throw t;
}
return true;
}
use of org.apache.drill.exec.store.http.util.SimpleHttp in project drill by apache.
the class HttpCSVBatchReader method open.
@Override
public boolean open(SchemaNegotiator negotiator) {
// Result set loader setup
String tempDirPath = negotiator.drillConfig().getString(ExecConstants.DRILL_TMP_DIR);
HttpUrl url = buildUrl();
CustomErrorContext errorContext = new ChildErrorContext(negotiator.parentErrorContext()) {
@Override
public void addContext(UserException.Builder builder) {
super.addContext(builder);
builder.addContext("URL", url.toString());
}
};
negotiator.setErrorContext(errorContext);
// Http client setup
SimpleHttp http = SimpleHttp.builder().scanDefn(subScan).url(url).tempDir(new File(tempDirPath)).proxyConfig(proxySettings(negotiator.drillConfig(), url)).errorContext(errorContext).build();
// CSV loader setup
inStream = http.getInputStream();
this.csvReader = new CsvParser(csvSettings);
csvReader.beginParsing(inStream);
// Build the Schema
builder = new SchemaBuilder();
TupleMetadata drillSchema = buildSchema();
negotiator.tableSchema(drillSchema, true);
resultLoader = negotiator.build();
// Add implicit columns
if (implicitColumnsAreProjected()) {
implicitColumns = new ImplicitColumns(resultLoader.writer());
buildImplicitColumns();
populateImplicitFieldMap(http);
}
// Create ScalarWriters
rowWriter = resultLoader.writer();
populateWriterArray();
return true;
}
Aggregations