Examples with ElementSerialisation uk.gov.gchq.gaffer.hbasestore.serialisation.ElementSerialisation used on opensource projects

Example 1 with ElementSerialisation

use of uk.gov.gchq.gaffer.hbasestore.serialisation.ElementSerialisation in project Gaffer by gchq.

the class AddElementsHandler method addElements.

private void addElements(final AddElements addElementsOperation, final HBaseStore store) throws OperationException {
    if (null == addElementsOperation.getInput()) {
        return;
    }
    try {
        final Table table = store.getTable();
        final Iterator<? extends Element> elements = addElementsOperation.getInput().iterator();
        final ElementSerialisation serialisation = new ElementSerialisation(store.getSchema());
        final int batchSize = store.getProperties().getWriteBufferSize();
        List<Put> puts = new ArrayList<>(batchSize);
        while (elements.hasNext()) {
            for (int i = 0; i < batchSize && elements.hasNext(); i++) {
                final Element element = elements.next();
                if (null == element) {
                    i--;
                    continue;
                }
                try {
                    final Pair<Put, Put> putPair = serialisation.getPuts(element);
                    puts.add(putPair.getFirst());
                    if (null != putPair.getSecond()) {
                        i++;
                        if (i >= batchSize) {
                            executePuts(table, puts);
                            puts = new ArrayList<>(batchSize);
                            i = 0;
                        }
                        puts.add(putPair.getSecond());
                    }
                } catch (final Exception e) {
                    if (addElementsOperation.isValidate() && !addElementsOperation.isSkipInvalidElements()) {
                        throw e;
                    }
                // otherwise just ignore the error
                }
            }
            executePuts(table, puts);
            puts = new ArrayList<>(batchSize);
        }
    } catch (final IOException | StoreException e) {
        throw new OperationException("Failed to add elements", e);
    }
}

Example 2 with ElementSerialisation

use of uk.gov.gchq.gaffer.hbasestore.serialisation.ElementSerialisation in project Gaffer by gchq.

the class AddElementsFromHdfsReducer method setup.

@Override
protected void setup(final Context context) {
    super.setup(context);
    serialisation = new ElementSerialisation(schema);
}

Example 3 with ElementSerialisation

use of uk.gov.gchq.gaffer.hbasestore.serialisation.ElementSerialisation in project Gaffer by gchq.

the class AddElementsHandlerTest method shouldAddElements.

@Test
public void shouldAddElements() throws OperationException, StoreException, IOException {
    // Given
    final AddElementsHandler handler = new AddElementsHandler();
    final List<Element> elements = createElements();
    final List<Element> elementsWithNull = new ArrayList<>(elements);
    // null should be skipped
    elementsWithNull.add(null);
    final AddElements addElements = new AddElements.Builder().input(elementsWithNull).build();
    final Context context = mock(Context.class);
    final HBaseStore store = mock(HBaseStore.class);
    final HTable table = mock(HTable.class);
    given(store.getTable()).willReturn(table);
    final HBaseProperties properties = HBaseProperties.loadStoreProperties(StreamUtil.storeProps(getClass()));
    final int writeBufferSize = 5;
    properties.setWriteBufferSize(writeBufferSize);
    given(store.getProperties()).willReturn(properties);
    given(store.getSchema()).willReturn(SCHEMA);
    // When
    handler.doOperation(addElements, context, store);
    // Then
    final ArgumentCaptor<List<Put>> putsCaptor = (ArgumentCaptor) ArgumentCaptor.forClass(List.class);
    verify(table, times(2)).put(putsCaptor.capture());
    verify(table, times(2)).flushCommits();
    final List<List<Put>> allPuts = putsCaptor.getAllValues();
    assertThat(allPuts).hasSize(2);
    final List<Put> combinedPuts = new ArrayList<>();
    combinedPuts.addAll(allPuts.get(0));
    combinedPuts.addAll(allPuts.get(1));
    final List<Element> expectedElements = new ArrayList<>();
    for (final Element element : elements) {
        expectedElements.add(element);
        if (element instanceof Edge && !((Edge) element).getSource().equals(((Edge) element).getDestination())) {
            expectedElements.add(element);
        }
    }
    final Element[] expectedElementsArr = expectedElements.toArray(new Element[expectedElements.size()]);
    final List<Element> elementsAdded = CellUtil.getElements(combinedPuts, new ElementSerialisation(SCHEMA), false);
    assertEquals(expectedElements.size(), elementsAdded.size());
    assertThat(elementsAdded).contains(expectedElementsArr);
}

Example 4 with ElementSerialisation

use of uk.gov.gchq.gaffer.hbasestore.serialisation.ElementSerialisation in project Gaffer by gchq.

the class AddElementsFromHdfsMapper method setup.

@Override
protected void setup(final Context context) {
    super.setup(context);
    serialisation = new ElementSerialisation(schema);
    Configuration conf = context.getConfiguration();
    this.kvCreator = new CellCreator(conf);
}

Example 5 with ElementSerialisation

use of uk.gov.gchq.gaffer.hbasestore.serialisation.ElementSerialisation in project Gaffer by gchq.

the class GafferCoprocessor method start.

@Override
public void start(final CoprocessorEnvironment e) throws IOException {
    final String schemaJson = StringUtil.unescapeComma(e.getConfiguration().get(HBaseStoreConstants.SCHEMA));
    schema = Schema.fromJson(Bytes.toBytes(schemaJson));
    serialisation = new ElementSerialisation(schema);
}