Search in sources :

Example 6 with MockConfigurationContext

use of org.apache.nifi.util.MockConfigurationContext in project nifi by apache.

the class TestHortonworksSchemaRegistry method testCacheUsed.

@Test
public void testCacheUsed() throws Exception {
    final String text = new String(Files.readAllBytes(Paths.get("src/test/resources/empty-schema.avsc")));
    final SchemaVersionInfo info = new SchemaVersionInfo(1L, "unit-test", 2, text, System.currentTimeMillis(), "description");
    schemaVersionInfoMap.put("unit-test", info);
    final SchemaMetadata metadata = new SchemaMetadata.Builder("unit-test").compatibility(SchemaCompatibility.NONE).evolve(true).schemaGroup("group").type("AVRO").build();
    final Constructor<SchemaMetadataInfo> ctr = SchemaMetadataInfo.class.getDeclaredConstructor(SchemaMetadata.class, Long.class, Long.class);
    ctr.setAccessible(true);
    final SchemaMetadataInfo schemaMetadataInfo = ctr.newInstance(metadata, 1L, System.currentTimeMillis());
    schemaMetadataInfoMap.put("unit-test", schemaMetadataInfo);
    final Map<PropertyDescriptor, String> properties = new HashMap<>();
    properties.put(HortonworksSchemaRegistry.URL, "http://localhost:44444");
    properties.put(HortonworksSchemaRegistry.CACHE_EXPIRATION, "5 mins");
    properties.put(HortonworksSchemaRegistry.CACHE_SIZE, "1000");
    final ConfigurationContext configurationContext = new MockConfigurationContext(properties, null);
    registry.enable(configurationContext);
    for (int i = 0; i < 10000; i++) {
        final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().name("unit-test").build();
        final RecordSchema schema = registry.retrieveSchema(schemaIdentifier);
        assertNotNull(schema);
    }
    Mockito.verify(client, Mockito.times(1)).getLatestSchemaVersionInfo(any(String.class));
}
Also used : ConfigurationContext(org.apache.nifi.controller.ConfigurationContext) MockConfigurationContext(org.apache.nifi.util.MockConfigurationContext) SchemaMetadata(com.hortonworks.registries.schemaregistry.SchemaMetadata) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) HashMap(java.util.HashMap) MockConfigurationContext(org.apache.nifi.util.MockConfigurationContext) SchemaVersionInfo(com.hortonworks.registries.schemaregistry.SchemaVersionInfo) SchemaIdentifier(org.apache.nifi.serialization.record.SchemaIdentifier) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) SchemaMetadataInfo(com.hortonworks.registries.schemaregistry.SchemaMetadataInfo) Test(org.junit.Test)

Example 7 with MockConfigurationContext

use of org.apache.nifi.util.MockConfigurationContext in project nifi by apache.

the class TestHortonworksSchemaRegistry method testCacheExpires.

@Test
@Ignore("This can be useful for manual testing/debugging, but will keep ignored for now because we don't want automated builds to run this, since it depends on timing")
public void testCacheExpires() throws Exception {
    final String text = new String(Files.readAllBytes(Paths.get("src/test/resources/empty-schema.avsc")));
    final SchemaVersionInfo info = new SchemaVersionInfo(1L, "unit-test", 2, text, System.currentTimeMillis(), "description");
    schemaVersionInfoMap.put("unit-test", info);
    final SchemaMetadata metadata = new SchemaMetadata.Builder("unit-test").compatibility(SchemaCompatibility.NONE).evolve(true).schemaGroup("group").type("AVRO").build();
    final Constructor<SchemaMetadataInfo> ctr = SchemaMetadataInfo.class.getDeclaredConstructor(SchemaMetadata.class, Long.class, Long.class);
    ctr.setAccessible(true);
    final SchemaMetadataInfo schemaMetadataInfo = ctr.newInstance(metadata, 1L, System.currentTimeMillis());
    schemaMetadataInfoMap.put("unit-test", schemaMetadataInfo);
    final Map<PropertyDescriptor, String> properties = new HashMap<>();
    properties.put(HortonworksSchemaRegistry.URL, "http://localhost:44444");
    properties.put(HortonworksSchemaRegistry.CACHE_EXPIRATION, "1 sec");
    properties.put(HortonworksSchemaRegistry.CACHE_SIZE, "1000");
    final ConfigurationContext configurationContext = new MockConfigurationContext(properties, null);
    registry.enable(configurationContext);
    for (int i = 0; i < 2; i++) {
        final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().name("unit-test").build();
        final RecordSchema schema = registry.retrieveSchema(schemaIdentifier);
        assertNotNull(schema);
    }
    Mockito.verify(client, Mockito.times(1)).getLatestSchemaVersionInfo(any(String.class));
    Thread.sleep(2000L);
    for (int i = 0; i < 2; i++) {
        final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().name("unit-test").build();
        final RecordSchema schema = registry.retrieveSchema(schemaIdentifier);
        assertNotNull(schema);
    }
    Mockito.verify(client, Mockito.times(2)).getLatestSchemaVersionInfo(any(String.class));
}
Also used : ConfigurationContext(org.apache.nifi.controller.ConfigurationContext) MockConfigurationContext(org.apache.nifi.util.MockConfigurationContext) SchemaMetadata(com.hortonworks.registries.schemaregistry.SchemaMetadata) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) HashMap(java.util.HashMap) MockConfigurationContext(org.apache.nifi.util.MockConfigurationContext) SchemaVersionInfo(com.hortonworks.registries.schemaregistry.SchemaVersionInfo) SchemaIdentifier(org.apache.nifi.serialization.record.SchemaIdentifier) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) SchemaMetadataInfo(com.hortonworks.registries.schemaregistry.SchemaMetadataInfo) Ignore(org.junit.Ignore) Test(org.junit.Test)

Example 8 with MockConfigurationContext

use of org.apache.nifi.util.MockConfigurationContext in project nifi by apache.

the class TestCSVHeaderSchemaStrategy method testSimple.

@Test
public void testSimple() throws SchemaNotFoundException, IOException {
    final String headerLine = "a, b, c, d, e\\,z, f";
    final byte[] headerBytes = headerLine.getBytes();
    final Map<PropertyDescriptor, String> properties = new HashMap<>();
    properties.put(CSVUtils.CSV_FORMAT, CSVUtils.CUSTOM.getValue());
    properties.put(CSVUtils.COMMENT_MARKER, "#");
    properties.put(CSVUtils.VALUE_SEPARATOR, ",");
    properties.put(CSVUtils.TRIM_FIELDS, "true");
    properties.put(CSVUtils.QUOTE_CHAR, "\"");
    properties.put(CSVUtils.ESCAPE_CHAR, "\\");
    final ConfigurationContext context = new MockConfigurationContext(properties, null);
    final CSVHeaderSchemaStrategy strategy = new CSVHeaderSchemaStrategy(context);
    final RecordSchema schema;
    try (final InputStream bais = new ByteArrayInputStream(headerBytes)) {
        schema = strategy.getSchema(null, bais, null);
    }
    final List<String> expectedFieldNames = Arrays.asList("a", "b", "c", "d", "e,z", "f");
    assertEquals(expectedFieldNames, schema.getFieldNames());
    assertTrue(schema.getFields().stream().allMatch(field -> field.getDataType().equals(RecordFieldType.STRING.getDataType())));
}
Also used : MockConfigurationContext(org.apache.nifi.util.MockConfigurationContext) Arrays(java.util.Arrays) ConfigurationContext(org.apache.nifi.controller.ConfigurationContext) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) HashMap(java.util.HashMap) Test(org.junit.Test) SchemaNotFoundException(org.apache.nifi.schema.access.SchemaNotFoundException) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) List(java.util.List) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) ByteArrayInputStream(java.io.ByteArrayInputStream) Map(java.util.Map) RecordFieldType(org.apache.nifi.serialization.record.RecordFieldType) Assert.assertEquals(org.junit.Assert.assertEquals) InputStream(java.io.InputStream) MockConfigurationContext(org.apache.nifi.util.MockConfigurationContext) ConfigurationContext(org.apache.nifi.controller.ConfigurationContext) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) HashMap(java.util.HashMap) ByteArrayInputStream(java.io.ByteArrayInputStream) InputStream(java.io.InputStream) MockConfigurationContext(org.apache.nifi.util.MockConfigurationContext) ByteArrayInputStream(java.io.ByteArrayInputStream) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) Test(org.junit.Test)

Example 9 with MockConfigurationContext

use of org.apache.nifi.util.MockConfigurationContext in project nifi by apache.

the class TestServerAndClient method testNonPersistentMapServerAndClient.

@Test
public void testNonPersistentMapServerAndClient() throws InitializationException, IOException, InterruptedException {
    /**
     * This bypasses the test for build environments in OS X running Java 1.8 due to a JVM bug
     * See:  https://issues.apache.org/jira/browse/NIFI-437
     */
    Assume.assumeFalse("test is skipped due to build environment being OS X with JDK 1.8. See https://issues.apache.org/jira/browse/NIFI-437", SystemUtils.IS_OS_MAC && SystemUtils.IS_JAVA_1_8);
    LOGGER.info("Testing " + Thread.currentThread().getStackTrace()[1].getMethodName());
    // Create server
    final DistributedMapCacheServer server = new MapServer();
    final TestRunner runner = TestRunners.newTestRunner(Mockito.mock(Processor.class));
    runner.addControllerService("server", server);
    runner.enableControllerService(server);
    DistributedMapCacheClientService client = new DistributedMapCacheClientService();
    MockControllerServiceInitializationContext clientInitContext = new MockControllerServiceInitializationContext(client, "client");
    client.initialize(clientInitContext);
    final Map<PropertyDescriptor, String> clientProperties = new HashMap<>();
    clientProperties.put(DistributedMapCacheClientService.HOSTNAME, "localhost");
    clientProperties.put(DistributedMapCacheClientService.PORT, String.valueOf(server.getPort()));
    clientProperties.put(DistributedMapCacheClientService.COMMUNICATIONS_TIMEOUT, "360 secs");
    MockConfigurationContext clientContext = new MockConfigurationContext(clientProperties, clientInitContext.getControllerServiceLookup());
    client.cacheConfig(clientContext);
    final Serializer<String> valueSerializer = new StringSerializer();
    final Serializer<String> keySerializer = new StringSerializer();
    final Deserializer<String> deserializer = new StringDeserializer();
    final String original = client.getAndPutIfAbsent("testKey", "test", keySerializer, valueSerializer, deserializer);
    assertEquals(null, original);
    LOGGER.debug("end getAndPutIfAbsent");
    final boolean contains = client.containsKey("testKey", keySerializer);
    assertTrue(contains);
    LOGGER.debug("end containsKey");
    final boolean added = client.putIfAbsent("testKey", "test", keySerializer, valueSerializer);
    assertFalse(added);
    LOGGER.debug("end putIfAbsent");
    final String originalAfterPut = client.getAndPutIfAbsent("testKey", "test", keySerializer, valueSerializer, deserializer);
    assertEquals("test", originalAfterPut);
    LOGGER.debug("end getAndPutIfAbsent");
    final boolean removed = client.remove("testKey", keySerializer);
    assertTrue(removed);
    LOGGER.debug("end remove");
    client.put("testKey", "testValue", keySerializer, valueSerializer);
    assertTrue(client.containsKey("testKey", keySerializer));
    String removedValue = client.removeAndGet("testKey", keySerializer, deserializer);
    assertEquals("testValue", removedValue);
    removedValue = client.removeAndGet("testKey", keySerializer, deserializer);
    assertNull(removedValue);
    final Set<String> keys = client.keySet(deserializer);
    assertEquals(0, keys.size());
    // Test removeByPattern, the first two should be removed and the last should remain
    client.put("test.1", "1", keySerializer, keySerializer);
    client.put("test.2", "2", keySerializer, keySerializer);
    client.put("test3", "2", keySerializer, keySerializer);
    final long removedTwo = client.removeByPattern("test\\..*");
    assertEquals(2L, removedTwo);
    assertFalse(client.containsKey("test.1", keySerializer));
    assertFalse(client.containsKey("test.2", keySerializer));
    assertTrue(client.containsKey("test3", keySerializer));
    final boolean containedAfterRemove = client.containsKey("testKey", keySerializer);
    assertFalse(containedAfterRemove);
    client.putIfAbsent("testKey", "test", keySerializer, valueSerializer);
    client.close();
    try {
        client.containsKey("testKey", keySerializer);
        fail("Should be closed and not accessible");
    } catch (final Exception e) {
    }
    DistributedMapCacheClientService client2 = new DistributedMapCacheClientService();
    MockControllerServiceInitializationContext clientInitContext2 = new MockControllerServiceInitializationContext(client2, "client2");
    client2.initialize(clientInitContext2);
    MockConfigurationContext clientContext2 = new MockConfigurationContext(clientProperties, clientInitContext2.getControllerServiceLookup());
    client2.cacheConfig(clientContext2);
    assertFalse(client2.putIfAbsent("testKey", "test", keySerializer, valueSerializer));
    assertTrue(client2.containsKey("testKey", keySerializer));
    server.shutdownServer();
    Thread.sleep(1000);
    try {
        client2.containsKey("testKey", keySerializer);
        fail("Should have blown exception!");
    } catch (final ConnectException e) {
        client2 = null;
        clientContext2 = null;
        clientInitContext2 = null;
    }
    LOGGER.debug("end testNonPersistentMapServerAndClient");
}
Also used : Processor(org.apache.nifi.processor.Processor) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) HashMap(java.util.HashMap) TestRunner(org.apache.nifi.util.TestRunner) DistributedMapCacheServer(org.apache.nifi.distributed.cache.server.map.DistributedMapCacheServer) InitializationException(org.apache.nifi.reporting.InitializationException) ConnectException(java.net.ConnectException) IOException(java.io.IOException) SerializationException(org.apache.commons.lang3.SerializationException) DeserializationException(org.apache.nifi.distributed.cache.client.exception.DeserializationException) DistributedMapCacheClientService(org.apache.nifi.distributed.cache.client.DistributedMapCacheClientService) MockControllerServiceInitializationContext(org.apache.nifi.util.MockControllerServiceInitializationContext) MockConfigurationContext(org.apache.nifi.util.MockConfigurationContext) ConnectException(java.net.ConnectException) Test(org.junit.Test)

Example 10 with MockConfigurationContext

use of org.apache.nifi.util.MockConfigurationContext in project nifi by apache.

the class TestServerAndClient method createMapClient.

private DistributedMapCacheClientService createMapClient(final int port) throws InitializationException {
    final DistributedMapCacheClientService client = new DistributedMapCacheClientService();
    final MockControllerServiceInitializationContext clientInitContext = new MockControllerServiceInitializationContext(client, "client");
    client.initialize(clientInitContext);
    final Map<PropertyDescriptor, String> clientProperties = new HashMap<>();
    clientProperties.put(DistributedMapCacheClientService.HOSTNAME, "localhost");
    clientProperties.put(DistributedMapCacheClientService.PORT, String.valueOf(port));
    final MockConfigurationContext clientContext = new MockConfigurationContext(clientProperties, clientInitContext.getControllerServiceLookup());
    client.cacheConfig(clientContext);
    return client;
}
Also used : DistributedMapCacheClientService(org.apache.nifi.distributed.cache.client.DistributedMapCacheClientService) MockControllerServiceInitializationContext(org.apache.nifi.util.MockControllerServiceInitializationContext) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) MockConfigurationContext(org.apache.nifi.util.MockConfigurationContext) HashMap(java.util.HashMap)

Aggregations

MockConfigurationContext (org.apache.nifi.util.MockConfigurationContext)12 HashMap (java.util.HashMap)11 PropertyDescriptor (org.apache.nifi.components.PropertyDescriptor)10 Test (org.junit.Test)8 MockControllerServiceInitializationContext (org.apache.nifi.util.MockControllerServiceInitializationContext)6 DistributedMapCacheClientService (org.apache.nifi.distributed.cache.client.DistributedMapCacheClientService)5 ConfigurationContext (org.apache.nifi.controller.ConfigurationContext)4 DistributedMapCacheServer (org.apache.nifi.distributed.cache.server.map.DistributedMapCacheServer)4 Processor (org.apache.nifi.processor.Processor)3 RecordSchema (org.apache.nifi.serialization.record.RecordSchema)3 TestRunner (org.apache.nifi.util.TestRunner)3 SchemaMetadata (com.hortonworks.registries.schemaregistry.SchemaMetadata)2 SchemaMetadataInfo (com.hortonworks.registries.schemaregistry.SchemaMetadataInfo)2 SchemaVersionInfo (com.hortonworks.registries.schemaregistry.SchemaVersionInfo)2 IOException (java.io.IOException)2 SchemaIdentifier (org.apache.nifi.serialization.record.SchemaIdentifier)2 MockStateManager (org.apache.nifi.state.MockStateManager)2 MockComponentLog (org.apache.nifi.util.MockComponentLog)2 ByteArrayInputStream (java.io.ByteArrayInputStream)1 File (java.io.File)1