Examples with Mutation org.apache.accumulo.core.data.Mutation used on opensource projects

Example 26 with Mutation

use of org.apache.accumulo.core.data.Mutation in project Gaffer by gchq.

the class CoreKeyGroupByAggregatorIteratorTest method testAggregatingSinglePropertySet.

public void testAggregatingSinglePropertySet(final AccumuloStore store, final AccumuloElementConverter elementConverter) throws StoreException, AccumuloElementConversionException {
    String visibilityString = "public";
    try {
        // Create edge
        final Edge edge = new Edge(TestGroups.EDGE);
        edge.setSource("1");
        edge.setDestination("2");
        edge.setDirected(true);
        edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
        edge.putProperty(AccumuloPropertyNames.COUNT, 1);
        final Properties properties1 = new Properties();
        properties1.put(AccumuloPropertyNames.COUNT, 1);
        // Accumulo key
        final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
        // Accumulo values
        final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties1);
        // Create mutation
        final Mutation m1 = new Mutation(key.getRow());
        m1.put(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp(), value1);
        // Write mutation
        final BatchWriterConfig writerConfig = new BatchWriterConfig();
        writerConfig.setMaxMemory(1000000L);
        writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
        writerConfig.setMaxWriteThreads(1);
        final BatchWriter writer = store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
        writer.addMutation(m1);
        writer.close();
        final Edge expectedEdge = new Edge(TestGroups.EDGE);
        expectedEdge.setSource("1");
        expectedEdge.setDestination("2");
        expectedEdge.setDirected(true);
        expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 8);
        expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 1);
        // Read data back and check we get one merged element
        final Authorizations authorizations = new Authorizations(visibilityString);
        final Scanner scanner = store.getConnection().createScanner(store.getProperties().getTable(), authorizations);
        final IteratorSetting iteratorSetting = new IteratorSettingBuilder(AccumuloStoreConstants.COLUMN_QUALIFIER_AGGREGATOR_ITERATOR_PRIORITY, "KeyCombiner", CoreKeyGroupByAggregatorIterator.class).all().view(new View.Builder().edge(TestGroups.EDGE, new ViewElementDefinition.Builder().groupBy().build()).build()).schema(store.getSchema()).keyConverter(store.getKeyPackage().getKeyConverter()).build();
        scanner.addScanIterator(iteratorSetting);
        final Iterator<Entry<Key, Value>> it = scanner.iterator();
        final Entry<Key, Value> entry = it.next();
        final Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
        assertEquals(expectedEdge, readEdge);
        assertEquals(8, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
        assertEquals(1, readEdge.getProperty(AccumuloPropertyNames.COUNT));
        // Check no more entries
        if (it.hasNext()) {
            fail("Additional row found.");
        }
    } catch (AccumuloException | TableNotFoundException e) {
        fail(this.getClass().getSimpleName() + " failed with exception: " + e);
    }
}

Example 27 with Mutation

use of org.apache.accumulo.core.data.Mutation in project presto by prestodb.

the class Indexer method getMetricsMutations.

private Collection<Mutation> getMetricsMutations() {
    ImmutableList.Builder<Mutation> mutationBuilder = ImmutableList.builder();
    // Mapping of column value to column to number of row IDs that contain that value
    for (Entry<MetricsKey, AtomicLong> entry : metrics.entrySet()) {
        // Row ID: Column value
        // Family: columnfamily_columnqualifier
        // Qualifier: CARDINALITY_CQ
        // Visibility: Inherited from indexed Mutation
        // Value: Cardinality
        Mutation mut = new Mutation(entry.getKey().row.array());
        mut.put(entry.getKey().family.array(), CARDINALITY_CQ, entry.getKey().visibility, ENCODER.encode(entry.getValue().get()));
        // Add to our list of mutations
        mutationBuilder.add(mut);
    }
    // Talk about your edge cases!
    if (firstRow != null && lastRow != null) {
        // Add a some columns to the special metrics table row ID for the first/last row.
        // Note that if the values on the server side are greater/lesser,
        // the configured iterator will take care of this at scan/compaction time
        Mutation firstLastMutation = new Mutation(METRICS_TABLE_ROW_ID.array());
        firstLastMutation.put(METRICS_TABLE_ROWS_CF.array(), METRICS_TABLE_FIRST_ROW_CQ.array(), firstRow);
        firstLastMutation.put(METRICS_TABLE_ROWS_CF.array(), METRICS_TABLE_LAST_ROW_CQ.array(), lastRow);
        mutationBuilder.add(firstLastMutation);
    }
    return mutationBuilder.build();
}

Example 28 with Mutation

use of org.apache.accumulo.core.data.Mutation in project presto by prestodb.

the class AccumuloPageSink method toMutation.

/**
     * Converts a {@link Row} to an Accumulo mutation.
     *
     * @param row Row object
     * @param rowIdOrdinal Ordinal in the list of columns that is the row ID. This isn't checked at all, so I hope you're right. Also, it is expected that the list of column handles is sorted in ordinal order. This is a very demanding function.
     * @param columns All column handles for the Row, sorted by ordinal.
     * @param serializer Instance of {@link AccumuloRowSerializer} used to encode the values of the row to the Mutation
     * @return Mutation
     */
public static Mutation toMutation(Row row, int rowIdOrdinal, List<AccumuloColumnHandle> columns, AccumuloRowSerializer serializer) {
    // Set our value to the row ID
    Text value = new Text();
    Field rowField = row.getField(rowIdOrdinal);
    if (rowField.isNull()) {
        throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "Column mapped as the Accumulo row ID cannot be null");
    }
    setText(rowField, value, serializer);
    // Iterate through all the column handles, setting the Mutation's columns
    Mutation mutation = new Mutation(value);
    // Store row ID in a special column
    mutation.put(ROW_ID_COLUMN, ROW_ID_COLUMN, new Value(value.copyBytes()));
    for (AccumuloColumnHandle columnHandle : columns) {
        // Skip the row ID ordinal
        if (columnHandle.getOrdinal() == rowIdOrdinal) {
            continue;
        }
        // If the value of the field is not null
        if (!row.getField(columnHandle.getOrdinal()).isNull()) {
            // Serialize the value to the text
            setText(row.getField(columnHandle.getOrdinal()), value, serializer);
            // And add the bytes to the Mutation
            mutation.put(columnHandle.getFamily().get(), columnHandle.getQualifier().get(), new Value(value.copyBytes()));
        }
    }
    return mutation;
}

Example 29 with Mutation

use of org.apache.accumulo.core.data.Mutation in project presto by prestodb.

the class AccumuloPageSink method appendPage.

@Override
public CompletableFuture<?> appendPage(Page page) {
    // For each position within the page, i.e. row
    for (int position = 0; position < page.getPositionCount(); ++position) {
        Row row = new Row();
        // For each channel within the page, i.e. column
        for (int channel = 0; channel < page.getChannelCount(); ++channel) {
            // Get the type for this channel
            Type type = columns.get(channel).getType();
            // Read the value from the page and append the field to the row
            row.addField(TypeUtils.readNativeValue(type, page.getBlock(channel), position), type);
        }
        try {
            // Convert row to a Mutation, writing and indexing it
            Mutation mutation = toMutation(row, rowIdOrdinal, columns, serializer);
            writer.addMutation(mutation);
            if (indexer.isPresent()) {
                indexer.get().index(mutation);
            }
            ++numRows;
        } catch (MutationsRejectedException e) {
            throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Mutation rejected by server", e);
        }
        // TODO Fix arbitrary flush every 100k rows
        if (numRows % 100_000 == 0) {
            flush();
        }
    }
    return NOT_BLOCKED;
}

Example 30 with Mutation

use of org.apache.accumulo.core.data.Mutation in project gora by apache.

the class AccumuloStore method deleteByQuery.

@Override
public long deleteByQuery(Query<K, T> query) {
    try {
        Scanner scanner = createScanner(query);
        // add iterator that drops values on the server side
        scanner.addScanIterator(new IteratorSetting(Integer.MAX_VALUE, SortedKeyIterator.class));
        RowIterator iterator = new RowIterator(scanner.iterator());
        long count = 0;
        while (iterator.hasNext()) {
            Iterator<Entry<Key, Value>> row = iterator.next();
            Mutation m = null;
            while (row.hasNext()) {
                Entry<Key, Value> entry = row.next();
                Key key = entry.getKey();
                if (m == null)
                    m = new Mutation(key.getRow());
                // TODO optimize to avoid continually creating column vis? prob does not matter for empty
                m.putDelete(key.getColumnFamily(), key.getColumnQualifier(), new ColumnVisibility(key.getColumnVisibility()), key.getTimestamp());
            }
            getBatchWriter().addMutation(m);
            count++;
        }
        return count;
    } catch (TableNotFoundException e) {
        // TODO return 0?
        LOG.error(e.getMessage(), e);
        return 0;
    } catch (MutationsRejectedException e) {
        LOG.error(e.getMessage(), e);
        return 0;
    } catch (IOException e) {
        LOG.error(e.getMessage(), e);
        return 0;
    }
}