Search in sources :

Example 1 with IndexFormatter

use of org.locationtech.geowave.datastore.filesystem.FileSystemDataFormatter.IndexFormatter in project geowave by locationtech.

the class FileSystemUtils method recurseDirectoriesToString.

private static Set<ByteArray> recurseDirectoriesToString(final Path currentPath, final String subdirectoryName, final Set<ByteArray> partitionDirectories, final IndexFormatter indexFormatter, final String indexName, final String typeName) {
    try {
        final AtomicBoolean atLeastOneRegularFile = new AtomicBoolean(false);
        Files.list(currentPath).filter(p -> {
            if (Files.isDirectory(p)) {
                return true;
            } else {
                atLeastOneRegularFile.set(true);
                return false;
            }
        }).forEach(path -> recurseDirectoriesToString(path, (subdirectoryName == null) || subdirectoryName.isEmpty() ? path.getFileName().toString() : subdirectoryName + "/" + path.getFileName().toString(), partitionDirectories, indexFormatter, indexName, typeName));
        if (atLeastOneRegularFile.get()) {
            partitionDirectories.add(new ByteArray(indexFormatter.getPartitionKey(indexName, typeName, subdirectoryName)));
        }
    } catch (final IOException e) {
        LOGGER.warn("Cannot list files in " + subdirectoryName, e);
    }
    return partitionDirectories;
}
Also used : ByteArray(org.locationtech.geowave.core.index.ByteArray) SortedSet(java.util.SortedSet) LoggerFactory(org.slf4j.LoggerFactory) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) IndexFormatter(org.locationtech.geowave.datastore.filesystem.FileSystemDataFormatter.IndexFormatter) Function(java.util.function.Function) Supplier(java.util.function.Supplier) TreeSet(java.util.TreeSet) HashSet(java.util.HashSet) MetadataType(org.locationtech.geowave.core.store.operations.MetadataType) Pair(org.apache.commons.lang3.tuple.Pair) Path(java.nio.file.Path) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) UnsignedBytes(com.google.common.primitives.UnsignedBytes) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) Files(java.nio.file.Files) Set(java.util.Set) IOException(java.io.IOException) NavigableSet(java.util.NavigableSet) Streams(com.google.common.collect.Streams) Collectors(java.util.stream.Collectors) Serializable(java.io.Serializable) Consumer(java.util.function.Consumer) RangeReaderParams(org.locationtech.geowave.core.store.operations.RangeReaderParams) Paths(java.nio.file.Paths) RowMergingDataAdapter(org.locationtech.geowave.core.store.adapter.RowMergingDataAdapter) ByteArrayUtils(org.locationtech.geowave.core.index.ByteArrayUtils) Comparator(java.util.Comparator) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ByteArray(org.locationtech.geowave.core.index.ByteArray) IOException(java.io.IOException)

Example 2 with IndexFormatter

use of org.locationtech.geowave.datastore.filesystem.FileSystemDataFormatter.IndexFormatter in project geowave by locationtech.

the class FileSystemReader method createIteratorForReader.

private CloseableIterator<T> createIteratorForReader(final FileSystemClient client, final ReaderParams<T> readerParams, final GeoWaveRowIteratorTransformer<T> rowTransformer, final boolean async) {
    final Collection<SinglePartitionQueryRanges> ranges = readerParams.getQueryRanges().getPartitionQueryRanges();
    final Set<String> authorizations = Sets.newHashSet(readerParams.getAdditionalAuthorizations());
    if ((ranges != null) && !ranges.isEmpty()) {
        return createIterator(client, readerParams, readerParams.getRowTransformer(), ranges, authorizations, async);
    } else {
        final List<CloseableIterator<GeoWaveRow>> iterators = new ArrayList<>();
        final IndexFormatter indexFormatter = DataFormatterCache.getInstance().getFormatter(client.getFormat(), client.isVisibilityEnabled()).getIndexFormatter();
        final String indexName = readerParams.getIndex().getName();
        for (final short adapterId : readerParams.getAdapterIds()) {
            final Pair<Boolean, Boolean> groupByRowAndSortByTime = FileSystemUtils.isGroupByRowAndIsSortByTime(readerParams, adapterId);
            final String typeName = readerParams.getInternalAdapterStore().getTypeName(adapterId);
            final String indexDirectory = indexFormatter.getDirectoryName(indexName, typeName);
            final Stream<CloseableIterator<GeoWaveRow>> streamIt = FileSystemUtils.getPartitions(FileSystemUtils.getSubdirectory(client.getSubDirectory(), indexDirectory), indexFormatter, indexName, typeName).stream().map(p -> FileSystemUtils.getIndexTable(client, adapterId, typeName, indexName, p.getBytes(), groupByRowAndSortByTime.getRight()).iterator());
            iterators.addAll(streamIt.collect(Collectors.toList()));
        }
        return wrapResults(new Closeable() {

            AtomicBoolean closed = new AtomicBoolean(false);

            @Override
            public void close() throws IOException {
                if (!closed.getAndSet(true)) {
                    iterators.forEach(it -> it.close());
                }
            }
        }, Iterators.concat(iterators.iterator()), readerParams, rowTransformer, authorizations, client.isVisibilityEnabled());
    }
}
Also used : Arrays(java.util.Arrays) RecordReaderParams(org.locationtech.geowave.mapreduce.splits.RecordReaderParams) SinglePartitionQueryRanges(org.locationtech.geowave.core.index.SinglePartitionQueryRanges) GeoWaveRowIteratorTransformer(org.locationtech.geowave.core.store.entities.GeoWaveRowIteratorTransformer) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ArrayUtils(org.apache.commons.lang3.ArrayUtils) IndexFormatter(org.locationtech.geowave.datastore.filesystem.FileSystemDataFormatter.IndexFormatter) Iterators(com.google.common.collect.Iterators) ArrayList(java.util.ArrayList) Pair(org.apache.commons.lang3.tuple.Pair) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper) ReaderParams(org.locationtech.geowave.core.store.operations.ReaderParams) ByteArrayRange(org.locationtech.geowave.core.index.ByteArrayRange) ClientVisibilityFilter(org.locationtech.geowave.core.store.query.filter.ClientVisibilityFilter) FileSystemDataIndexTable(org.locationtech.geowave.datastore.filesystem.util.FileSystemDataIndexTable) DataFormatterCache(org.locationtech.geowave.datastore.filesystem.util.DataFormatterCache) DataStoreUtils(org.locationtech.geowave.core.store.util.DataStoreUtils) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) Iterator(java.util.Iterator) GeoWaveRowMergingIterator(org.locationtech.geowave.core.store.entities.GeoWaveRowMergingIterator) DataIndexReaderParams(org.locationtech.geowave.core.store.operations.DataIndexReaderParams) RowReader(org.locationtech.geowave.core.store.operations.RowReader) Collection(java.util.Collection) Set(java.util.Set) FileSystemUtils(org.locationtech.geowave.datastore.filesystem.util.FileSystemUtils) IOException(java.io.IOException) Streams(com.google.common.collect.Streams) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) RangeReaderParams(org.locationtech.geowave.core.store.operations.RangeReaderParams) List(java.util.List) Stream(java.util.stream.Stream) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) Closeable(java.io.Closeable) GeoWaveRowRange(org.locationtech.geowave.mapreduce.splits.GeoWaveRowRange) Collections(java.util.Collections) FileSystemClient(org.locationtech.geowave.datastore.filesystem.util.FileSystemClient) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) Closeable(java.io.Closeable) SinglePartitionQueryRanges(org.locationtech.geowave.core.index.SinglePartitionQueryRanges) ArrayList(java.util.ArrayList) IOException(java.io.IOException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) IndexFormatter(org.locationtech.geowave.datastore.filesystem.FileSystemDataFormatter.IndexFormatter) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean)

Aggregations

Streams (com.google.common.collect.Streams)2 IOException (java.io.IOException)2 Iterator (java.util.Iterator)2 Set (java.util.Set)2 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)2 Collectors (java.util.stream.Collectors)2 Pair (org.apache.commons.lang3.tuple.Pair)2 GeoWaveRow (org.locationtech.geowave.core.store.entities.GeoWaveRow)2 RangeReaderParams (org.locationtech.geowave.core.store.operations.RangeReaderParams)2 IndexFormatter (org.locationtech.geowave.datastore.filesystem.FileSystemDataFormatter.IndexFormatter)2 Iterators (com.google.common.collect.Iterators)1 Sets (com.google.common.collect.Sets)1 UnsignedBytes (com.google.common.primitives.UnsignedBytes)1 Closeable (java.io.Closeable)1 Serializable (java.io.Serializable)1 Files (java.nio.file.Files)1 Path (java.nio.file.Path)1 Paths (java.nio.file.Paths)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1