Examples with GeoWaveBlockingRpcCallback org.locationtech.geowave.datastore.hbase.util.GeoWaveBlockingRpcCallback used on opensource projects

Example 1 with GeoWaveBlockingRpcCallback

use of org.locationtech.geowave.datastore.hbase.util.GeoWaveBlockingRpcCallback in project geowave by locationtech.

the class HBaseOperations method bulkDelete.

public void bulkDelete(final ReaderParams readerParams) {
    final String tableName = readerParams.getIndex().getName();
    final short[] adapterIds = readerParams.getAdapterIds();
    Long total = 0L;
    try {
        // Use the row count coprocessor
        if (options.isVerifyCoprocessors()) {
            verifyCoprocessor(tableName, "org.locationtech.geowave.datastore.hbase.coprocessors.HBaseBulkDeleteEndpoint", options.getCoprocessorJar());
        }
        final HBaseBulkDeleteProtosClient.BulkDeleteRequest.Builder requestBuilder = HBaseBulkDeleteProtosClient.BulkDeleteRequest.newBuilder();
        requestBuilder.setDeleteType(DELETE_TYPE);
        requestBuilder.setRowBatchSize(DELETE_BATCH_SIZE);
        if (readerParams.getFilter() != null) {
            final List<QueryFilter> distFilters = new ArrayList();
            distFilters.add(readerParams.getFilter());
            final byte[] filterBytes = PersistenceUtils.toBinary(distFilters);
            final ByteString filterByteString = ByteString.copyFrom(filterBytes);
            requestBuilder.setFilter(filterByteString);
        } else {
            final List<MultiDimensionalCoordinateRangesArray> coords = readerParams.getCoordinateRanges();
            if ((coords != null) && !coords.isEmpty()) {
                final byte[] filterBytes = new HBaseNumericIndexStrategyFilter(readerParams.getIndex().getIndexStrategy(), coords.toArray(new MultiDimensionalCoordinateRangesArray[] {})).toByteArray();
                final ByteString filterByteString = ByteString.copyFrom(new byte[] { 0 }).concat(ByteString.copyFrom(filterBytes));
                requestBuilder.setNumericIndexStrategyFilter(filterByteString);
            }
        }
        requestBuilder.setModel(ByteString.copyFrom(PersistenceUtils.toBinary(readerParams.getIndex().getIndexModel())));
        final MultiRowRangeFilter multiFilter = getMultiRowRangeFilter(readerParams.getQueryRanges().getCompositeQueryRanges());
        if (multiFilter != null) {
            requestBuilder.setRangeFilter(ByteString.copyFrom(multiFilter.toByteArray()));
        }
        if ((adapterIds != null) && (adapterIds.length > 0)) {
            final ByteBuffer buf = ByteBuffer.allocate(2 * adapterIds.length);
            for (final Short a : adapterIds) {
                buf.putShort(a);
            }
            requestBuilder.setAdapterIds(ByteString.copyFrom(buf.array()));
        }
        final Table table = getTable(tableName);
        final HBaseBulkDeleteProtosClient.BulkDeleteRequest request = requestBuilder.build();
        byte[] startRow = null;
        byte[] endRow = null;
        final List<ByteArrayRange> ranges = readerParams.getQueryRanges().getCompositeQueryRanges();
        if ((ranges != null) && !ranges.isEmpty()) {
            final ByteArrayRange aggRange = ByteArrayUtils.getSingleRange(ranges);
            startRow = aggRange.getStart();
            endRow = aggRange.getEnd();
        }
        final Map<byte[], Long> results = table.coprocessorService(HBaseBulkDeleteProtosClient.BulkDeleteService.class, startRow, endRow, new Batch.Call<HBaseBulkDeleteProtosClient.BulkDeleteService, Long>() {

            @Override
            public Long call(final HBaseBulkDeleteProtosClient.BulkDeleteService counter) throws IOException {
                final GeoWaveBlockingRpcCallback<HBaseBulkDeleteProtosClient.BulkDeleteResponse> rpcCallback = new GeoWaveBlockingRpcCallback<>();
                counter.delete(null, request, rpcCallback);
                final BulkDeleteResponse response = rpcCallback.get();
                return response.hasRowsDeleted() ? response.getRowsDeleted() : null;
            }
        });
        int regionCount = 0;
        for (final Map.Entry<byte[], Long> entry : results.entrySet()) {
            regionCount++;
            final Long value = entry.getValue();
            if (value != null) {
                LOGGER.debug("Value from region " + regionCount + " is " + value);
                total += value;
            } else {
                LOGGER.debug("Empty response for region " + regionCount);
            }
        }
    } catch (final Exception e) {
        LOGGER.error("Error during bulk delete.", e);
    } catch (final Throwable e) {
        LOGGER.error("Error during bulkdelete.", e);
    }
}

Example 2 with GeoWaveBlockingRpcCallback

use of org.locationtech.geowave.datastore.hbase.util.GeoWaveBlockingRpcCallback in project geowave by locationtech.

the class HBaseOperations method aggregateServerSide.

public <T> Iterator<GeoWaveRow> aggregateServerSide(final ReaderParams<T> readerParams) {
    final String tableName = readerParams.getIndex().getName();
    try {
        // Use the row count coprocessor
        if (options.isVerifyCoprocessors()) {
            verifyCoprocessor(tableName, "org.locationtech.geowave.datastore.hbase.coprocessors.AggregationEndpoint", options.getCoprocessorJar());
        }
        final Aggregation aggregation = readerParams.getAggregation().getRight();
        final AggregationProtosClient.AggregationType.Builder aggregationBuilder = AggregationProtosClient.AggregationType.newBuilder();
        aggregationBuilder.setClassId(ByteString.copyFrom(URLClassloaderUtils.toClassId(aggregation)));
        if (aggregation.getParameters() != null) {
            final byte[] paramBytes = URLClassloaderUtils.toBinary(aggregation.getParameters());
            aggregationBuilder.setParams(ByteString.copyFrom(paramBytes));
        }
        final AggregationProtosClient.AggregationRequest.Builder requestBuilder = AggregationProtosClient.AggregationRequest.newBuilder();
        requestBuilder.setAggregation(aggregationBuilder.build());
        if (readerParams.getFilter() != null) {
            final List<QueryFilter> distFilters = new ArrayList<>();
            distFilters.add(readerParams.getFilter());
            final byte[] filterBytes = URLClassloaderUtils.toBinary(distFilters);
            final ByteString filterByteString = ByteString.copyFrom(filterBytes);
            requestBuilder.setFilter(filterByteString);
        } else {
            final List<MultiDimensionalCoordinateRangesArray> coords = readerParams.getCoordinateRanges();
            if (!coords.isEmpty()) {
                final byte[] filterBytes = new HBaseNumericIndexStrategyFilter(readerParams.getIndex().getIndexStrategy(), coords.toArray(new MultiDimensionalCoordinateRangesArray[] {})).toByteArray();
                final ByteString filterByteString = ByteString.copyFrom(new byte[] { 0 }).concat(ByteString.copyFrom(filterBytes));
                requestBuilder.setNumericIndexStrategyFilter(filterByteString);
            }
        }
        requestBuilder.setModel(ByteString.copyFrom(URLClassloaderUtils.toBinary(readerParams.getIndex().getIndexModel())));
        final int maxRangeDecomposition = readerParams.getMaxRangeDecomposition() == null ? options.getAggregationMaxRangeDecomposition() : readerParams.getMaxRangeDecomposition();
        final MultiRowRangeFilter multiFilter = getMultiRowRangeFilter(DataStoreUtils.constraintsToQueryRanges(readerParams.getConstraints(), readerParams.getIndex(), null, maxRangeDecomposition).getCompositeQueryRanges());
        if (multiFilter != null) {
            requestBuilder.setRangeFilter(ByteString.copyFrom(multiFilter.toByteArray()));
        }
        if (readerParams.getAggregation().getLeft() != null) {
            if (!(readerParams.getAggregation().getRight() instanceof CommonIndexAggregation)) {
                final byte[] adapterBytes = URLClassloaderUtils.toBinary(readerParams.getAggregation().getLeft());
                requestBuilder.setAdapter(ByteString.copyFrom(adapterBytes));
                final byte[] mappingBytes = URLClassloaderUtils.toBinary(readerParams.getAdapterIndexMappingStore().getMapping(readerParams.getAggregation().getLeft().getAdapterId(), readerParams.getIndex().getName()));
                requestBuilder.setIndexMapping(ByteString.copyFrom(mappingBytes));
            }
            requestBuilder.setInternalAdapterId(ByteString.copyFrom(ByteArrayUtils.shortToByteArray(readerParams.getAggregation().getLeft().getAdapterId())));
        }
        if ((readerParams.getAdditionalAuthorizations() != null) && (readerParams.getAdditionalAuthorizations().length > 0)) {
            requestBuilder.setVisLabels(ByteString.copyFrom(StringUtils.stringsToBinary(readerParams.getAdditionalAuthorizations())));
        }
        if (readerParams.isMixedVisibility()) {
            requestBuilder.setWholeRowFilter(true);
        }
        requestBuilder.setPartitionKeyLength(readerParams.getIndex().getIndexStrategy().getPartitionKeyLength());
        final AggregationProtosClient.AggregationRequest request = requestBuilder.build();
        byte[] startRow = null;
        byte[] endRow = null;
        final List<ByteArrayRange> ranges = readerParams.getQueryRanges().getCompositeQueryRanges();
        if ((ranges != null) && !ranges.isEmpty()) {
            final ByteArrayRange aggRange = ByteArrayUtils.getSingleRange(ranges);
            startRow = aggRange.getStart();
            endRow = aggRange.getEnd();
        }
        Map<byte[], ByteString> results = null;
        boolean shouldRetry;
        int retries = 0;
        do {
            shouldRetry = false;
            try (final Table table = getTable(tableName)) {
                results = table.coprocessorService(AggregationProtosClient.AggregationService.class, startRow, endRow, new Batch.Call<AggregationProtosClient.AggregationService, ByteString>() {

                    @Override
                    public ByteString call(final AggregationProtosClient.AggregationService counter) throws IOException {
                        final GeoWaveBlockingRpcCallback<AggregationProtosClient.AggregationResponse> rpcCallback = new GeoWaveBlockingRpcCallback<>();
                        counter.aggregate(null, request, rpcCallback);
                        final AggregationProtosClient.AggregationResponse response = rpcCallback.get();
                        if (response == null) {
                            // Region returned no response
                            throw new RegionException();
                        }
                        return response.hasValue() ? response.getValue() : null;
                    }
                });
                break;
            } catch (final RegionException e) {
                retries++;
                if (retries <= MAX_AGGREGATE_RETRIES) {
                    LOGGER.warn("Aggregate timed out due to unavailable region. Retrying (" + retries + " of " + MAX_AGGREGATE_RETRIES + ")");
                    shouldRetry = true;
                }
            }
        } while (shouldRetry);
        if (results == null) {
            LOGGER.error("Aggregate timed out and exceeded max retries.");
            return null;
        }
        return Iterators.transform(results.values().iterator(), b -> new GeoWaveRowImpl(null, new GeoWaveValue[] { new GeoWaveValueImpl(null, null, b.toByteArray()) }));
    } catch (final Exception e) {
        LOGGER.error("Error during aggregation.", e);
    } catch (final Throwable e) {
        LOGGER.error("Error during aggregation.", e);
    }
    return null;
}

Example 3 with GeoWaveBlockingRpcCallback

use of org.locationtech.geowave.datastore.hbase.util.GeoWaveBlockingRpcCallback in project geowave by locationtech.

the class HBaseOperations method getVersion.

@Override
public String getVersion() {
    String version = null;
    if ((options == null) || !options.isServerSideLibraryEnabled()) {
        LOGGER.warn("Serverside library not enabled, serverside version is irrelevant");
        return null;
    }
    try {
        // use Index as the type to check for version (for hbase type
        // doesn't matter anyways)
        final MetadataType type = MetadataType.INDEX;
        final String tableName = getMetadataTableName(type);
        if (!indexExists(tableName)) {
            createTable(new byte[0][], getMetadataCFAndVersioning(), StringColumnFamilyFactory.getSingletonInstance(), getTableName(getQualifiedTableName(tableName)));
        }
        // Use the row count coprocessor
        if (options.isVerifyCoprocessors()) {
            verifyCoprocessor(tableName, "org.locationtech.geowave.datastore.hbase.coprocessors.VersionEndpoint", options.getCoprocessorJar());
        }
        final Table table = getTable(tableName);
        final Map<byte[], List<String>> versionInfoResponse = table.coprocessorService(VersionProtosClient.VersionService.class, null, null, new Batch.Call<VersionProtosClient.VersionService, List<String>>() {

            @Override
            public List<String> call(final VersionProtosClient.VersionService versionService) throws IOException {
                final GeoWaveBlockingRpcCallback<VersionProtosClient.VersionResponse> rpcCallback = new GeoWaveBlockingRpcCallback<>();
                versionService.version(null, VersionRequest.getDefaultInstance(), rpcCallback);
                final VersionProtosClient.VersionResponse response = rpcCallback.get();
                return response.getVersionInfoList();
            }
        });
        table.close();
        if ((versionInfoResponse == null) || versionInfoResponse.isEmpty()) {
            LOGGER.error("No response from version coprocessor");
        } else {
            final Iterator<List<String>> values = versionInfoResponse.values().iterator();
            final List<String> value = values.next();
            while (values.hasNext()) {
                final List<String> newValue = values.next();
                if (!value.equals(newValue)) {
                    LOGGER.error("Version Info '" + Arrays.toString(value.toArray()) + "' and '" + Arrays.toString(newValue.toArray()) + "' differ.  This may mean that different regions are using different versions of GeoWave.");
                }
            }
            version = VersionUtils.asLineDelimitedString(value);
        }
    } catch (final Throwable e) {
        LOGGER.warn("Unable to check metadata table for version", e);
    }
    return version;
}