Examples with VisibilityHandler org.locationtech.geowave.core.store.api.VisibilityHandler used on opensource projects

Example 1 with VisibilityHandler

use of org.locationtech.geowave.core.store.api.VisibilityHandler in project geowave by locationtech.

the class AbstractMapReduceIngest method run.

@Override
public int run(final String[] args) throws Exception {
    final Configuration conf = getConf();
    conf.set(INGEST_PLUGIN_KEY, ByteArrayUtils.byteArrayToString(PersistenceUtils.toBinary(ingestPlugin)));
    final VisibilityHandler visibilityHandler = visibilityOptions.getConfiguredVisibilityHandler();
    if (visibilityHandler != null) {
        conf.set(GLOBAL_VISIBILITY_KEY, ByteArrayUtils.byteArrayToString(PersistenceUtils.toBinary(visibilityHandler)));
    }
    final Job job = new Job(conf, getJobName());
    final StringBuilder indexNames = new StringBuilder();
    final List<Index> indexes = new ArrayList<>();
    for (final Index primaryIndex : indices) {
        indexes.add(primaryIndex);
        if (primaryIndex != null) {
            // add index
            GeoWaveOutputFormat.addIndex(job.getConfiguration(), primaryIndex);
            if (indexNames.length() != 0) {
                indexNames.append(",");
            }
            indexNames.append(primaryIndex.getName());
        }
    }
    job.getConfiguration().set(INDEX_NAMES_KEY, indexNames.toString());
    job.setJarByClass(AbstractMapReduceIngest.class);
    job.setInputFormatClass(AvroKeyInputFormat.class);
    AvroJob.setInputKeySchema(job, parentPlugin.getAvroSchema());
    FileInputFormat.setInputPaths(job, inputFile);
    setupMapper(job);
    setupReducer(job);
    // set geowave output format
    job.setOutputFormatClass(GeoWaveOutputFormat.class);
    GeoWaveOutputFormat.setStoreOptions(job.getConfiguration(), dataStoreOptions);
    final DataStore store = dataStoreOptions.createDataStore();
    final PersistentAdapterStore adapterStore = dataStoreOptions.createAdapterStore();
    final InternalAdapterStore internalAdapterStore = dataStoreOptions.createInternalAdapterStore();
    final DataTypeAdapter<?>[] dataAdapters = ingestPlugin.getDataAdapters();
    final Index[] indices = indexes.toArray(new Index[indexes.size()]);
    if ((dataAdapters != null) && (dataAdapters.length > 0)) {
        for (final DataTypeAdapter<?> dataAdapter : dataAdapters) {
            // metadata for which there is no data
            try {
                store.addType(dataAdapter, visibilityOptions.getConfiguredVisibilityHandler(), Lists.newArrayList(), indices);
                final short adapterId = internalAdapterStore.getAdapterId(dataAdapter.getTypeName());
                final InternalDataAdapter<?> internalAdapter = adapterStore.getAdapter(adapterId);
                GeoWaveOutputFormat.addDataAdapter(job.getConfiguration(), internalAdapter);
            } catch (IllegalArgumentException e) {
            // Skip any adapters that can't be mapped to the input indices
            }
        }
    } else {
        // indices from the client
        for (final Index index : indices) {
            store.addIndex(index);
        }
        if (indices.length > 0) {
            for (final MetadataType type : MetadataType.values()) {
                // stats and index metadata writers are created elsewhere
                if (!MetadataType.INDEX.equals(type) && !MetadataType.STATISTIC_VALUES.equals(type)) {
                    dataStoreOptions.createDataStoreOperations().createMetadataWriter(type).close();
                }
            }
        }
    }
    // distributed ingest
    if (dataStoreOptions.getFactoryOptions().getStoreOptions().isPersistDataStatistics()) {
        dataStoreOptions.createDataStoreOperations().createMetadataWriter(MetadataType.STATISTIC_VALUES).close();
    }
    job.setSpeculativeExecution(false);
    // add required indices
    final Index[] requiredIndices = parentPlugin.getRequiredIndices();
    if (requiredIndices != null) {
        for (final Index requiredIndex : requiredIndices) {
            GeoWaveOutputFormat.addIndex(job.getConfiguration(), requiredIndex);
        }
    }
    final int retVal = job.waitForCompletion(true) ? 0 : -1;
    // ingests
    if ((dataAdapters != null) && (dataAdapters.length > 0)) {
        AdapterIndexMappingStore adapterIndexMappingStore = null;
        for (final DataTypeAdapter<?> dataAdapter : dataAdapters) {
            final String typeName = dataAdapter.getTypeName();
            try (CloseableIterator<?> it = store.query(QueryBuilder.newBuilder().addTypeName(typeName).limit(1).build())) {
                if (!it.hasNext()) {
                    if (adapterIndexMappingStore == null) {
                        adapterIndexMappingStore = dataStoreOptions.createAdapterIndexMappingStore();
                    }
                    final Short adapterId = internalAdapterStore.getAdapterId(typeName);
                    if (adapterId != null) {
                        internalAdapterStore.remove(adapterId);
                        adapterStore.removeAdapter(adapterId);
                        adapterIndexMappingStore.remove(adapterId);
                    }
                }
            }
        }
    }
    return retVal;
}

Example 2 with VisibilityHandler

use of org.locationtech.geowave.core.store.api.VisibilityHandler in project geowave by locationtech.

the class GeoWaveDataStoreComponents method writeCommit.

public void writeCommit(final SimpleFeature feature, final GeoWaveTransaction transaction) throws IOException {
    final VisibilityHandler visibilityHandler = new GlobalVisibilityHandler(transaction.composeVisibility());
    dataStore.addType(adapter, adapterIndices);
    try (Writer<SimpleFeature> indexWriter = dataStore.createWriter(adapter.getTypeName())) {
        indexWriter.write(feature, visibilityHandler);
    }
}

Example 3 with VisibilityHandler

use of org.locationtech.geowave.core.store.api.VisibilityHandler in project geowave by locationtech.

the class LegacyInternalDataAdapterWrapper method fromBinary.

@SuppressWarnings("unchecked")
@Override
public void fromBinary(final byte[] bytes) {
    ByteBuffer buffer = ByteBuffer.wrap(bytes);
    adapterId = buffer.getShort();
    byte[] adapterBytes = new byte[buffer.remaining()];
    buffer.get(adapterBytes);
    adapter = (DataTypeAdapter<T>) PersistenceUtils.fromBinary(adapterBytes);
    VisibilityHandler visibilityHandler = new UnconstrainedVisibilityHandler();
    if (adapter instanceof LegacyFeatureDataAdapter) {
        visibilityHandler = ((LegacyFeatureDataAdapter) adapter).getVisibilityHandler();
        adapter = (DataTypeAdapter<T>) ((LegacyFeatureDataAdapter) adapter).getUpdatedAdapter();
    }
    this.updatedAdapter = adapter.asInternalAdapter(adapterId, visibilityHandler);
}

Example 4 with VisibilityHandler

use of org.locationtech.geowave.core.store.api.VisibilityHandler in project geowave by locationtech.

the class MigrationTest method testLegacyFeatureDataAdapterMigration.

@Test
public void testLegacyFeatureDataAdapterMigration() {
    final SimpleFeatureTypeBuilder builder = new SimpleFeatureTypeBuilder();
    final AttributeTypeBuilder attributeTypeBuilder = new AttributeTypeBuilder();
    builder.setName("testType");
    builder.setNamespaceURI("geowave.namespace");
    builder.add(attributeTypeBuilder.binding(String.class).nillable(true).buildDescriptor("strAttr"));
    builder.add(attributeTypeBuilder.binding(Integer.class).nillable(true).buildDescriptor("intAttr"));
    builder.add(attributeTypeBuilder.binding(Date.class).nillable(false).buildDescriptor("dateAttr"));
    builder.add(attributeTypeBuilder.binding(Point.class).nillable(false).buildDescriptor("geom"));
    builder.crs(GeometryUtils.getDefaultCRS());
    final SimpleFeatureType featureType = builder.buildFeatureType();
    final AttributeDescriptor stringAttr = featureType.getDescriptor("strAttr");
    // Configure legacy visiblity
    stringAttr.getUserData().put("visibility", Boolean.TRUE);
    featureType.getUserData().put("visibilityManagerClass", "org.locationtech.geowave.adapter.vector.plugin.visibility.JsonDefinitionColumnVisibilityManagement");
    LegacyFeatureDataAdapter adapter = new LegacyFeatureDataAdapter(featureType, "EPSG:3257");
    final byte[] adapterBinary = PersistenceUtils.toBinary(adapter);
    final Persistable persistableAdapter = PersistenceUtils.fromBinary(adapterBinary);
    assertTrue(persistableAdapter instanceof LegacyFeatureDataAdapter);
    adapter = (LegacyFeatureDataAdapter) persistableAdapter;
    assertNotNull(adapter.getUpdatedAdapter());
    final FeatureDataAdapter updatedAdapter = adapter.getUpdatedAdapter();
    assertEquals(4, updatedAdapter.getFieldDescriptors().length);
    assertEquals(String.class, updatedAdapter.getFieldDescriptor("strAttr").bindingClass());
    assertEquals(Integer.class, updatedAdapter.getFieldDescriptor("intAttr").bindingClass());
    assertTrue(TemporalFieldDescriptor.class.isAssignableFrom(updatedAdapter.getFieldDescriptor("dateAttr").getClass()));
    final TemporalFieldDescriptor<?> temporalField = (TemporalFieldDescriptor<?>) updatedAdapter.getFieldDescriptor("dateAttr");
    assertEquals(Date.class, temporalField.bindingClass());
    assertTrue(temporalField.indexHints().contains(TimeField.TIME_DIMENSION_HINT));
    assertTrue(SpatialFieldDescriptor.class.isAssignableFrom(updatedAdapter.getFieldDescriptor("geom").getClass()));
    final SpatialFieldDescriptor<?> spatialField = (SpatialFieldDescriptor<?>) updatedAdapter.getFieldDescriptor("geom");
    assertEquals(Point.class, spatialField.bindingClass());
    assertEquals(GeometryUtils.getDefaultCRS(), spatialField.crs());
    assertTrue(spatialField.indexHints().contains(SpatialField.LATITUDE_DIMENSION_HINT));
    assertTrue(spatialField.indexHints().contains(SpatialField.LONGITUDE_DIMENSION_HINT));
    assertEquals("testType", updatedAdapter.getTypeName());
    assertEquals(SimpleFeature.class, updatedAdapter.getDataClass());
    assertTrue(updatedAdapter.hasTemporalConstraints());
    assertNotNull(adapter.getVisibilityHandler());
    final VisibilityHandler visibilityHandler = adapter.getVisibilityHandler();
    assertTrue(visibilityHandler instanceof JsonFieldLevelVisibilityHandler);
    assertEquals("strAttr", ((JsonFieldLevelVisibilityHandler) visibilityHandler).getVisibilityAttribute());
}

Example 5 with VisibilityHandler

use of org.locationtech.geowave.core.store.api.VisibilityHandler in project geowave by locationtech.

the class BaseDataStore method createWriter.

@SuppressWarnings("unchecked")
private <T> Writer<T> createWriter(final InternalDataAdapter<T> adapter, final VisibilityHandler visibilityHandler, final boolean writingOriginalData, final Index... indices) {
    final boolean secondaryIndex = writingOriginalData && baseOptions.isSecondaryIndexing() && DataIndexUtils.adapterSupportsDataIndex(adapter);
    final Writer<T>[] writers = new Writer[secondaryIndex ? indices.length + 1 : indices.length];
    final VisibilityHandler resolvedVisibilityHandler = resolveVisibilityHandler(adapter, visibilityHandler);
    int i = 0;
    if (secondaryIndex) {
        final DataStoreCallbackManager callbackManager = new DataStoreCallbackManager(statisticsStore, true);
        final AdapterToIndexMapping indexMapping = indexMappingStore.getMapping(adapter.getAdapterId(), DataIndexUtils.DATA_ID_INDEX.getName());
        final List<IngestCallback<T>> callbacks = Collections.singletonList(callbackManager.getIngestCallback(adapter, indexMapping, DataIndexUtils.DATA_ID_INDEX));
        final IngestCallbackList<T> callbacksList = new IngestCallbackList<>(callbacks);
        writers[i++] = createDataIndexWriter(adapter, indexMapping, resolvedVisibilityHandler, baseOperations, baseOptions, callbacksList, callbacksList);
    }
    for (final Index index : indices) {
        final DataStoreCallbackManager callbackManager = new DataStoreCallbackManager(statisticsStore, i == 0);
        callbackManager.setPersistStats(baseOptions.isPersistDataStatistics());
        final AdapterToIndexMapping indexMapping = indexMappingStore.getMapping(adapter.getAdapterId(), index.getName());
        final List<IngestCallback<T>> callbacks = writingOriginalData ? Collections.singletonList(callbackManager.getIngestCallback(adapter, indexMapping, index)) : Collections.emptyList();
        final IngestCallbackList<T> callbacksList = new IngestCallbackList<>(callbacks);
        writers[i] = createIndexWriter(adapter, indexMapping, index, resolvedVisibilityHandler, baseOperations, baseOptions, callbacksList, callbacksList);
        if (adapter.getAdapter() instanceof IndexDependentDataAdapter) {
            writers[i] = new IndependentAdapterIndexWriter<>((IndexDependentDataAdapter<T>) adapter.getAdapter(), index, resolvedVisibilityHandler, writers[i]);
        }
        i++;
    }
    return new IndexCompositeWriter<>(writers);
}