use of org.locationtech.geowave.datastore.accumulo.operations.AccumuloOperations in project geowave by locationtech.
the class AccumuloUtils method setLocalityGroup.
/**
* Set locality group.
*/
public static void setLocalityGroup(final Connector connector, final String namespace, final Index index, final InternalDataAdapter<?> adapter) throws AccumuloException, AccumuloSecurityException, IOException, TableNotFoundException {
final AccumuloOperations operations = new AccumuloOperations(connector, namespace, new AccumuloOptions());
operations.addLocalityGroup(index.getName(), adapter.getTypeName(), adapter.getAdapterId());
}
use of org.locationtech.geowave.datastore.accumulo.operations.AccumuloOperations in project geowave by locationtech.
the class AccumuloUtils method getEntries.
/**
* Get number of entries per index.
*/
public static long getEntries(final BaseDataStore dataStore, final Connector connector, final String namespace, final Index index) throws AccumuloException, AccumuloSecurityException, IOException {
long counter = 0L;
final AccumuloOptions options = new AccumuloOptions();
final AccumuloOperations operations = new AccumuloOperations(connector, namespace, options);
final IndexStore indexStore = new IndexStoreImpl(operations, options);
if (indexStore.indexExists(index.getName())) {
try (final CloseableIterator<?> iterator = new AccumuloDataStore(operations, options).query(QueryBuilder.newBuilder().build())) {
while (iterator.hasNext()) {
counter++;
iterator.next();
}
}
}
return counter;
}
use of org.locationtech.geowave.datastore.accumulo.operations.AccumuloOperations in project geowave by locationtech.
the class AccumuloUtils method getIterator.
private static CloseableIterator<Entry<Key, Value>> getIterator(final Connector connector, final String namespace, final Index index) throws AccumuloException, AccumuloSecurityException, IOException, TableNotFoundException {
CloseableIterator<Entry<Key, Value>> iterator = null;
final AccumuloOptions options = new AccumuloOptions();
final AccumuloOperations operations = new AccumuloOperations(connector, namespace, new AccumuloOptions());
final IndexStore indexStore = new IndexStoreImpl(operations, options);
final PersistentAdapterStore adapterStore = new AdapterStoreImpl(operations, options);
final AdapterIndexMappingStore mappingStore = new AdapterIndexMappingStoreImpl(operations, options);
if (indexStore.indexExists(index.getName())) {
final ScannerBase scanner = operations.createBatchScanner(index.getName());
((BatchScanner) scanner).setRanges(AccumuloUtils.byteArrayRangesToAccumuloRanges(null));
final IteratorSetting iteratorSettings = new IteratorSetting(10, "GEOWAVE_WHOLE_ROW_ITERATOR", WholeRowIterator.class);
scanner.addScanIterator(iteratorSettings);
final Iterator<Entry<Key, Value>> it = new IteratorWrapper(adapterStore, mappingStore, index, scanner.iterator(), new QueryFilter[] { new DedupeFilter() });
iterator = new CloseableIteratorWrapper<>(new ScannerClosableWrapper(scanner), it);
}
return iterator;
}
use of org.locationtech.geowave.datastore.accumulo.operations.AccumuloOperations in project geowave by locationtech.
the class OSMRunner method enableLocalityGroups.
private void enableLocalityGroups(final OSMIngestCommandArgs argv) throws AccumuloSecurityException, AccumuloException, TableNotFoundException, IOException {
final AccumuloOperations bao = new AccumuloOperations(accumuloOptions.getZookeeper(), accumuloOptions.getInstance(), accumuloOptions.getUser(), accumuloOptions.getPasswordOrKeytab(), accumuloOptions.isUseSasl(), accumuloOptions.getGeoWaveNamespace(), new AccumuloOptions());
bao.createTable(argv.getOsmTableName(), true, true);
}
use of org.locationtech.geowave.datastore.accumulo.operations.AccumuloOperations in project geowave by locationtech.
the class OSMConversionRunner method run.
@Override
public int run(final String[] args) throws Exception {
final Configuration conf = getConf();
final AccumuloRequiredOptions accumuloOptions = (AccumuloRequiredOptions) inputStoreOptions.getFactoryOptions();
// job settings
final Job job = Job.getInstance(conf, ingestOptions.getJobName() + "NodeConversion");
job.setJarByClass(OSMConversionRunner.class);
job.getConfiguration().set("osm_mapping", ingestOptions.getMappingContents());
job.getConfiguration().set("arguments", ingestOptions.serializeToString());
if (ingestOptions.getVisibilityOptions().getGlobalVisibility() != null) {
job.getConfiguration().set(AbstractMapReduceIngest.GLOBAL_VISIBILITY_KEY, ingestOptions.getVisibilityOptions().getGlobalVisibility());
}
// input format
AbstractInputFormat.setConnectorInfo(job, accumuloOptions.getUser(), new PasswordToken(accumuloOptions.getPassword()));
InputFormatBase.setInputTableName(job, ingestOptions.getQualifiedTableName());
AbstractInputFormat.setZooKeeperInstance(job, ClientConfiguration.create().withInstance(accumuloOptions.getInstance()).withZkHosts(accumuloOptions.getZookeeper()));
AbstractInputFormat.setScanAuthorizations(job, new Authorizations(ingestOptions.getVisibilityOptions().getGlobalVisibility()));
final IteratorSetting is = new IteratorSetting(50, "WholeRow", WholeRowIterator.class);
InputFormatBase.addIterator(job, is);
job.setInputFormatClass(AccumuloInputFormat.class);
final Range r = new Range();
// final ArrayList<Pair<Text, Text>> columns = new ArrayList<>();
InputFormatBase.setRanges(job, Arrays.asList(r));
// output format
GeoWaveOutputFormat.setStoreOptions(job.getConfiguration(), inputStoreOptions);
final AccumuloOptions options = new AccumuloOptions();
final AdapterStore as = new AdapterStoreImpl(new AccumuloOperations(accumuloOptions.getZookeeper(), accumuloOptions.getInstance(), accumuloOptions.getUser(), accumuloOptions.getPasswordOrKeytab(), accumuloOptions.isUseSasl(), accumuloOptions.getGeoWaveNamespace(), options), options);
for (final FeatureDataAdapter fda : FeatureDefinitionSet.featureAdapters.values()) {
as.addAdapter(fda);
GeoWaveOutputFormat.addDataAdapter(job.getConfiguration(), fda);
}
final Index primaryIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions());
GeoWaveOutputFormat.addIndex(job.getConfiguration(), primaryIndex);
job.getConfiguration().set(AbstractMapReduceIngest.INDEX_NAMES_KEY, primaryIndex.getName());
job.setOutputFormatClass(GeoWaveOutputFormat.class);
job.setMapOutputKeyClass(GeoWaveOutputKey.class);
job.setMapOutputValueClass(SimpleFeature.class);
// mappper
job.setMapperClass(OSMConversionMapper.class);
// reducer
job.setNumReduceTasks(0);
return job.waitForCompletion(true) ? 0 : -1;
}
Aggregations