Search in sources :

Example 11 with ConfigurationContext

use of org.apache.nifi.controller.ConfigurationContext in project nifi by apache.

the class TestHortonworksSchemaRegistry method testCacheUsed.

@Test
public void testCacheUsed() throws Exception {
    final String text = new String(Files.readAllBytes(Paths.get("src/test/resources/empty-schema.avsc")));
    final SchemaVersionInfo info = new SchemaVersionInfo(1L, "unit-test", 2, text, System.currentTimeMillis(), "description");
    schemaVersionInfoMap.put("unit-test", info);
    final SchemaMetadata metadata = new SchemaMetadata.Builder("unit-test").compatibility(SchemaCompatibility.NONE).evolve(true).schemaGroup("group").type("AVRO").build();
    final Constructor<SchemaMetadataInfo> ctr = SchemaMetadataInfo.class.getDeclaredConstructor(SchemaMetadata.class, Long.class, Long.class);
    ctr.setAccessible(true);
    final SchemaMetadataInfo schemaMetadataInfo = ctr.newInstance(metadata, 1L, System.currentTimeMillis());
    schemaMetadataInfoMap.put("unit-test", schemaMetadataInfo);
    final Map<PropertyDescriptor, String> properties = new HashMap<>();
    properties.put(HortonworksSchemaRegistry.URL, "http://localhost:44444");
    properties.put(HortonworksSchemaRegistry.CACHE_EXPIRATION, "5 mins");
    properties.put(HortonworksSchemaRegistry.CACHE_SIZE, "1000");
    final ConfigurationContext configurationContext = new MockConfigurationContext(properties, null);
    registry.enable(configurationContext);
    for (int i = 0; i < 10000; i++) {
        final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().name("unit-test").build();
        final RecordSchema schema = registry.retrieveSchema(schemaIdentifier);
        assertNotNull(schema);
    }
    Mockito.verify(client, Mockito.times(1)).getLatestSchemaVersionInfo(any(String.class));
}
Also used : ConfigurationContext(org.apache.nifi.controller.ConfigurationContext) MockConfigurationContext(org.apache.nifi.util.MockConfigurationContext) SchemaMetadata(com.hortonworks.registries.schemaregistry.SchemaMetadata) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) HashMap(java.util.HashMap) MockConfigurationContext(org.apache.nifi.util.MockConfigurationContext) SchemaVersionInfo(com.hortonworks.registries.schemaregistry.SchemaVersionInfo) SchemaIdentifier(org.apache.nifi.serialization.record.SchemaIdentifier) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) SchemaMetadataInfo(com.hortonworks.registries.schemaregistry.SchemaMetadataInfo) Test(org.junit.Test)

Example 12 with ConfigurationContext

use of org.apache.nifi.controller.ConfigurationContext in project nifi by apache.

the class TestHortonworksSchemaRegistry method testCacheExpires.

@Test
@Ignore("This can be useful for manual testing/debugging, but will keep ignored for now because we don't want automated builds to run this, since it depends on timing")
public void testCacheExpires() throws Exception {
    final String text = new String(Files.readAllBytes(Paths.get("src/test/resources/empty-schema.avsc")));
    final SchemaVersionInfo info = new SchemaVersionInfo(1L, "unit-test", 2, text, System.currentTimeMillis(), "description");
    schemaVersionInfoMap.put("unit-test", info);
    final SchemaMetadata metadata = new SchemaMetadata.Builder("unit-test").compatibility(SchemaCompatibility.NONE).evolve(true).schemaGroup("group").type("AVRO").build();
    final Constructor<SchemaMetadataInfo> ctr = SchemaMetadataInfo.class.getDeclaredConstructor(SchemaMetadata.class, Long.class, Long.class);
    ctr.setAccessible(true);
    final SchemaMetadataInfo schemaMetadataInfo = ctr.newInstance(metadata, 1L, System.currentTimeMillis());
    schemaMetadataInfoMap.put("unit-test", schemaMetadataInfo);
    final Map<PropertyDescriptor, String> properties = new HashMap<>();
    properties.put(HortonworksSchemaRegistry.URL, "http://localhost:44444");
    properties.put(HortonworksSchemaRegistry.CACHE_EXPIRATION, "1 sec");
    properties.put(HortonworksSchemaRegistry.CACHE_SIZE, "1000");
    final ConfigurationContext configurationContext = new MockConfigurationContext(properties, null);
    registry.enable(configurationContext);
    for (int i = 0; i < 2; i++) {
        final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().name("unit-test").build();
        final RecordSchema schema = registry.retrieveSchema(schemaIdentifier);
        assertNotNull(schema);
    }
    Mockito.verify(client, Mockito.times(1)).getLatestSchemaVersionInfo(any(String.class));
    Thread.sleep(2000L);
    for (int i = 0; i < 2; i++) {
        final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().name("unit-test").build();
        final RecordSchema schema = registry.retrieveSchema(schemaIdentifier);
        assertNotNull(schema);
    }
    Mockito.verify(client, Mockito.times(2)).getLatestSchemaVersionInfo(any(String.class));
}
Also used : ConfigurationContext(org.apache.nifi.controller.ConfigurationContext) MockConfigurationContext(org.apache.nifi.util.MockConfigurationContext) SchemaMetadata(com.hortonworks.registries.schemaregistry.SchemaMetadata) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) HashMap(java.util.HashMap) MockConfigurationContext(org.apache.nifi.util.MockConfigurationContext) SchemaVersionInfo(com.hortonworks.registries.schemaregistry.SchemaVersionInfo) SchemaIdentifier(org.apache.nifi.serialization.record.SchemaIdentifier) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) SchemaMetadataInfo(com.hortonworks.registries.schemaregistry.SchemaMetadataInfo) Ignore(org.junit.Ignore) Test(org.junit.Test)

Example 13 with ConfigurationContext

use of org.apache.nifi.controller.ConfigurationContext in project nifi by apache.

the class TestCSVHeaderSchemaStrategy method testSimple.

@Test
public void testSimple() throws SchemaNotFoundException, IOException {
    final String headerLine = "a, b, c, d, e\\,z, f";
    final byte[] headerBytes = headerLine.getBytes();
    final Map<PropertyDescriptor, String> properties = new HashMap<>();
    properties.put(CSVUtils.CSV_FORMAT, CSVUtils.CUSTOM.getValue());
    properties.put(CSVUtils.COMMENT_MARKER, "#");
    properties.put(CSVUtils.VALUE_SEPARATOR, ",");
    properties.put(CSVUtils.TRIM_FIELDS, "true");
    properties.put(CSVUtils.QUOTE_CHAR, "\"");
    properties.put(CSVUtils.ESCAPE_CHAR, "\\");
    final ConfigurationContext context = new MockConfigurationContext(properties, null);
    final CSVHeaderSchemaStrategy strategy = new CSVHeaderSchemaStrategy(context);
    final RecordSchema schema;
    try (final InputStream bais = new ByteArrayInputStream(headerBytes)) {
        schema = strategy.getSchema(null, bais, null);
    }
    final List<String> expectedFieldNames = Arrays.asList("a", "b", "c", "d", "e,z", "f");
    assertEquals(expectedFieldNames, schema.getFieldNames());
    assertTrue(schema.getFields().stream().allMatch(field -> field.getDataType().equals(RecordFieldType.STRING.getDataType())));
}
Also used : MockConfigurationContext(org.apache.nifi.util.MockConfigurationContext) Arrays(java.util.Arrays) ConfigurationContext(org.apache.nifi.controller.ConfigurationContext) Assert.assertTrue(org.junit.Assert.assertTrue) IOException(java.io.IOException) HashMap(java.util.HashMap) Test(org.junit.Test) SchemaNotFoundException(org.apache.nifi.schema.access.SchemaNotFoundException) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) List(java.util.List) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) ByteArrayInputStream(java.io.ByteArrayInputStream) Map(java.util.Map) RecordFieldType(org.apache.nifi.serialization.record.RecordFieldType) Assert.assertEquals(org.junit.Assert.assertEquals) InputStream(java.io.InputStream) MockConfigurationContext(org.apache.nifi.util.MockConfigurationContext) ConfigurationContext(org.apache.nifi.controller.ConfigurationContext) PropertyDescriptor(org.apache.nifi.components.PropertyDescriptor) HashMap(java.util.HashMap) ByteArrayInputStream(java.io.ByteArrayInputStream) InputStream(java.io.InputStream) MockConfigurationContext(org.apache.nifi.util.MockConfigurationContext) ByteArrayInputStream(java.io.ByteArrayInputStream) RecordSchema(org.apache.nifi.serialization.record.RecordSchema) Test(org.junit.Test)

Aggregations

ConfigurationContext (org.apache.nifi.controller.ConfigurationContext)13 HashMap (java.util.HashMap)6 PropertyDescriptor (org.apache.nifi.components.PropertyDescriptor)6 Test (org.junit.Test)5 IOException (java.io.IOException)4 Map (java.util.Map)4 RecordSchema (org.apache.nifi.serialization.record.RecordSchema)4 MockConfigurationContext (org.apache.nifi.util.MockConfigurationContext)4 InvocationTargetException (java.lang.reflect.InvocationTargetException)3 List (java.util.List)3 CompletableFuture (java.util.concurrent.CompletableFuture)3 NarCloseable (org.apache.nifi.nar.NarCloseable)3 EventAccess (org.apache.nifi.reporting.EventAccess)3 ReportingContext (org.apache.nifi.reporting.ReportingContext)3 ReportingInitializationContext (org.apache.nifi.reporting.ReportingInitializationContext)3 SchemaIdentifier (org.apache.nifi.serialization.record.SchemaIdentifier)3 MockPropertyValue (org.apache.nifi.util.MockPropertyValue)3 SchemaMetadata (com.hortonworks.registries.schemaregistry.SchemaMetadata)2 SchemaMetadataInfo (com.hortonworks.registries.schemaregistry.SchemaMetadataInfo)2 SchemaVersionInfo (com.hortonworks.registries.schemaregistry.SchemaVersionInfo)2