use of org.locationtech.geowave.datastore.filesystem.FileSystemDataFormatter.IndexFormatter in project geowave by locationtech.
the class FileSystemUtils method recurseDirectoriesToString.
private static Set<ByteArray> recurseDirectoriesToString(final Path currentPath, final String subdirectoryName, final Set<ByteArray> partitionDirectories, final IndexFormatter indexFormatter, final String indexName, final String typeName) {
try {
final AtomicBoolean atLeastOneRegularFile = new AtomicBoolean(false);
Files.list(currentPath).filter(p -> {
if (Files.isDirectory(p)) {
return true;
} else {
atLeastOneRegularFile.set(true);
return false;
}
}).forEach(path -> recurseDirectoriesToString(path, (subdirectoryName == null) || subdirectoryName.isEmpty() ? path.getFileName().toString() : subdirectoryName + "/" + path.getFileName().toString(), partitionDirectories, indexFormatter, indexName, typeName));
if (atLeastOneRegularFile.get()) {
partitionDirectories.add(new ByteArray(indexFormatter.getPartitionKey(indexName, typeName, subdirectoryName)));
}
} catch (final IOException e) {
LOGGER.warn("Cannot list files in " + subdirectoryName, e);
}
return partitionDirectories;
}
use of org.locationtech.geowave.datastore.filesystem.FileSystemDataFormatter.IndexFormatter in project geowave by locationtech.
the class FileSystemReader method createIteratorForReader.
private CloseableIterator<T> createIteratorForReader(final FileSystemClient client, final ReaderParams<T> readerParams, final GeoWaveRowIteratorTransformer<T> rowTransformer, final boolean async) {
final Collection<SinglePartitionQueryRanges> ranges = readerParams.getQueryRanges().getPartitionQueryRanges();
final Set<String> authorizations = Sets.newHashSet(readerParams.getAdditionalAuthorizations());
if ((ranges != null) && !ranges.isEmpty()) {
return createIterator(client, readerParams, readerParams.getRowTransformer(), ranges, authorizations, async);
} else {
final List<CloseableIterator<GeoWaveRow>> iterators = new ArrayList<>();
final IndexFormatter indexFormatter = DataFormatterCache.getInstance().getFormatter(client.getFormat(), client.isVisibilityEnabled()).getIndexFormatter();
final String indexName = readerParams.getIndex().getName();
for (final short adapterId : readerParams.getAdapterIds()) {
final Pair<Boolean, Boolean> groupByRowAndSortByTime = FileSystemUtils.isGroupByRowAndIsSortByTime(readerParams, adapterId);
final String typeName = readerParams.getInternalAdapterStore().getTypeName(adapterId);
final String indexDirectory = indexFormatter.getDirectoryName(indexName, typeName);
final Stream<CloseableIterator<GeoWaveRow>> streamIt = FileSystemUtils.getPartitions(FileSystemUtils.getSubdirectory(client.getSubDirectory(), indexDirectory), indexFormatter, indexName, typeName).stream().map(p -> FileSystemUtils.getIndexTable(client, adapterId, typeName, indexName, p.getBytes(), groupByRowAndSortByTime.getRight()).iterator());
iterators.addAll(streamIt.collect(Collectors.toList()));
}
return wrapResults(new Closeable() {
AtomicBoolean closed = new AtomicBoolean(false);
@Override
public void close() throws IOException {
if (!closed.getAndSet(true)) {
iterators.forEach(it -> it.close());
}
}
}, Iterators.concat(iterators.iterator()), readerParams, rowTransformer, authorizations, client.isVisibilityEnabled());
}
}
Aggregations