Examples with StoreProperties uk.gov.gchq.gaffer.store.StoreProperties used on opensource projects

Example 11 with StoreProperties

use of uk.gov.gchq.gaffer.store.StoreProperties in project Gaffer by gchq.

the class JobFactory method createJobs.

/**
 * Creates a job with the store specific job initialisation and then applies the operation specific
 * {@link uk.gov.gchq.gaffer.hdfs.operation.handler.job.initialiser.JobInitialiser}.
 *
 * @param operation The operation.
 * @param store     The store executing the operation.
 * @return The created job.
 * @throws IOException for IO issues.
 */
default List<Job> createJobs(final O operation, final Store store) throws IOException {
    final List<Job> jobs = new ArrayList<>();
    Map<String, List<String>> mapperGeneratorsToInputPathsList = new HashMap<>();
    for (final Map.Entry<String, String> entry : operation.getInputMapperPairs().entrySet()) {
        if (mapperGeneratorsToInputPathsList.containsKey(entry.getValue())) {
            mapperGeneratorsToInputPathsList.get(entry.getValue()).add(entry.getKey());
        } else {
            mapperGeneratorsToInputPathsList.put(entry.getValue(), Lists.newArrayList(entry.getKey()));
        }
    }
    for (final String mapperGeneratorClassName : mapperGeneratorsToInputPathsList.keySet()) {
        final JobConf jobConf = createJobConf(operation, mapperGeneratorClassName, store);
        final Job job = Job.getInstance(jobConf);
        final Configuration configuration = job.getConfiguration();
        final StoreProperties storeProperties = store.getProperties();
        if (storeProperties.getJsonSerialiserClass() != null) {
            configuration.set(StoreProperties.JSON_SERIALISER_CLASS, storeProperties.getJsonSerialiserClass());
        }
        if (storeProperties.getJsonSerialiserModules() != null) {
            configuration.set(StoreProperties.JSON_SERIALISER_MODULES, storeProperties.getJsonSerialiserModules());
        }
        if (storeProperties.getStrictJson() != null) {
            configuration.setBoolean(StoreProperties.STRICT_JSON, storeProperties.getStrictJson());
        }
        setupJob(job, operation, mapperGeneratorClassName, store);
        if (null != operation.getJobInitialiser()) {
            operation.getJobInitialiser().initialiseJob(job, operation, store);
        }
        jobs.add(job);
    }
    return jobs;
}

Example 12 with StoreProperties

use of uk.gov.gchq.gaffer.store.StoreProperties in project Gaffer by gchq.

the class GafferAdderTest method shouldRestartAddElementsIfPauseInIngest.

@Test
public void shouldRestartAddElementsIfPauseInIngest() throws Exception {
    // Given
    final AddElementsFromSocket op = mock(AddElementsFromSocket.class);
    final Store store = mock(Store.class);
    given(store.getProperties()).willReturn(new StoreProperties());
    given(store.getSchema()).willReturn(new Schema());
    given(op.isValidate()).willReturn(true);
    given(op.isSkipInvalidElements()).willReturn(false);
    given(op.getOption(FlinkConstants.MAX_QUEUE_SIZE)).willReturn(MAX_QUEUE_SIZE_OPTION);
    final Element element = mock(Element.class);
    final Element element2 = mock(Element.class);
    final GafferAdder adder = new GafferAdder(op, store);
    // When
    adder.add(element);
    // Then
    final ArgumentCaptor<Runnable> runnableCaptor1 = ArgumentCaptor.forClass(Runnable.class);
    verify(store).runAsync(runnableCaptor1.capture());
    runnableCaptor1.getValue().run();
    final ConsumableBlockingQueue<Element> expectedQueue = new ConsumableBlockingQueue<>(MAX_QUEUE_SIZE_VALUE);
    expectedQueue.put(element);
    verify(store).execute(Mockito.eq(new AddElements.Builder().input(expectedQueue).validate(true).skipInvalidElements(false).build()), Mockito.any());
    Mockito.reset(store);
    // When
    adder.add(element2);
    // Then
    final ArgumentCaptor<Runnable> runnableCaptor2 = ArgumentCaptor.forClass(Runnable.class);
    verify(store).runAsync(runnableCaptor2.capture());
    runnableCaptor2.getValue().run();
    // As the queue has not been consumed the original elements will still be on the queue.
    expectedQueue.put(element2);
    verify(store).execute(Mockito.eq(new AddElements.Builder().input(expectedQueue).validate(true).skipInvalidElements(false).build()), Mockito.any());
}

Example 13 with StoreProperties

use of uk.gov.gchq.gaffer.store.StoreProperties in project Gaffer by gchq.

the class OperationControllerIT method shouldPropagateStatusInformationContainedInOperationExceptionsThrownByOperationHandlers.

@Test
public void shouldPropagateStatusInformationContainedInOperationExceptionsThrownByOperationHandlers() throws IOException {
    // Given
    final StoreProperties storeProperties = new MapStoreProperties();
    storeProperties.set(StoreProperties.JOB_TRACKER_ENABLED, Boolean.FALSE.toString());
    final Graph graph = new Graph.Builder().config(StreamUtil.graphConfig(this.getClass())).storeProperties(storeProperties).addSchema(new Schema()).build();
    when(getGraphFactory().getGraph()).thenReturn(graph);
    // When
    final ResponseEntity<Error> response = post("/graph/operations/execute", new GetAllJobDetails(), Error.class);
    // Then
    assertEquals(SERVICE_UNAVAILABLE.getStatusCode(), response.getStatusCode().value());
}

Example 14 with StoreProperties

use of uk.gov.gchq.gaffer.store.StoreProperties in project Gaffer by gchq.

the class GraphConfigurationControllerTest method shouldSerialiseAndDeserialiseGetStoreTraits.

@Test
public void shouldSerialiseAndDeserialiseGetStoreTraits() throws SerialisationException {
    // Given
    Store store = mock(Store.class);
    Schema schema = new Schema();
    StoreProperties props = new StoreProperties();
    Set<StoreTrait> storeTraits = Sets.newHashSet(INGEST_AGGREGATION, PRE_AGGREGATION_FILTERING, POST_AGGREGATION_FILTERING);
    when(store.getSchema()).thenReturn(schema);
    when(store.getProperties()).thenReturn(props);
    when(store.getTraits()).thenReturn(storeTraits);
    Graph graph = new Graph.Builder().config(new GraphConfig("id")).addSchema(new Schema()).store(store).build();
    when(graphFactory.getGraph()).thenReturn(graph);
    GraphConfigurationController controller = new GraphConfigurationController(graphFactory);
    // When
    byte[] bytes = JSONSerialiser.serialise(controller.getStoreTraits());
    final Set<String> traits = JSONSerialiser.deserialise(bytes, Set.class);
    // Then
    assertEquals(Sets.newHashSet(INGEST_AGGREGATION.name(), PRE_AGGREGATION_FILTERING.name(), POST_AGGREGATION_FILTERING.name()), traits);
}

Example 15 with StoreProperties

use of uk.gov.gchq.gaffer.store.StoreProperties in project Gaffer by gchq.

the class FederatedOperationOutputHandlerTest method shouldThrowException.

@Test
public void shouldThrowException() throws Exception {
    // Given
    final String message = "Test Exception";
    final OP op = getExampleOperation();
    op.addOption(KEY_OPERATION_OPTIONS_GRAPH_IDS, TEST_GRAPH_ID);
    Schema unusedSchema = new Schema.Builder().build();
    StoreProperties storeProperties = new StoreProperties();
    Store mockStoreInner = Mockito.mock(Store.class);
    given(mockStoreInner.getSchema()).willReturn(unusedSchema);
    given(mockStoreInner.getProperties()).willReturn(storeProperties);
    given(mockStoreInner.execute(any(OperationChain.class), any(Context.class))).willThrow(new RuntimeException(message));
    FederatedStore mockStore = Mockito.mock(FederatedStore.class);
    HashSet<Graph> filteredGraphs = Sets.newHashSet(getGraphWithMockStore(mockStoreInner));
    Mockito.when(mockStore.getGraphs(user, TEST_GRAPH_ID, op)).thenReturn(filteredGraphs);
    // When
    try {
        getFederatedHandler().doOperation(op, context, mockStore);
        fail("Exception not thrown");
    } catch (OperationException e) {
        assertEquals(message, e.getCause().getMessage());
    }
}