Examples with SchemaVersionInfo com.hortonworks.registries.schemaregistry.SchemaVersionInfo used on opensource projects

Example 16 with SchemaVersionInfo

use of com.hortonworks.registries.schemaregistry.SchemaVersionInfo in project nifi by apache.

the class HortonworksSchemaRegistry method retrieveSchemaByIdAndVersion.

private RecordSchema retrieveSchemaByIdAndVersion(final SchemaIdentifier schemaIdentifier) throws org.apache.nifi.schema.access.SchemaNotFoundException, IOException {
    final SchemaRegistryClient client = getClient();
    final String schemaName;
    final SchemaVersionInfo versionInfo;
    final OptionalLong schemaId = schemaIdentifier.getIdentifier();
    if (!schemaId.isPresent()) {
        throw new org.apache.nifi.schema.access.SchemaNotFoundException("Cannot retrieve schema because Schema Id is not present");
    }
    final OptionalInt version = schemaIdentifier.getVersion();
    if (!version.isPresent()) {
        throw new org.apache.nifi.schema.access.SchemaNotFoundException("Cannot retrieve schema because Schema Version is not present");
    }
    try {
        final SchemaMetadataInfo info = client.getSchemaMetadataInfo(schemaId.getAsLong());
        if (info == null) {
            throw new org.apache.nifi.schema.access.SchemaNotFoundException("Could not find schema with ID '" + schemaId + "' and version '" + version + "'");
        }
        final SchemaMetadata metadata = info.getSchemaMetadata();
        schemaName = metadata.getName();
        final SchemaVersionKey schemaVersionKey = new SchemaVersionKey(schemaName, version.getAsInt());
        versionInfo = getSchemaVersionInfo(client, schemaVersionKey);
        if (versionInfo == null) {
            throw new org.apache.nifi.schema.access.SchemaNotFoundException("Could not find schema with ID '" + schemaId + "' and version '" + version + "'");
        }
    } catch (final Exception e) {
        handleException("Failed to retrieve schema with ID '" + schemaId + "' and version '" + version + "'", e);
        return null;
    }
    final String schemaText = versionInfo.getSchemaText();
    final SchemaIdentifier resultSchemaIdentifier = SchemaIdentifier.builder().name(schemaName).id(schemaId.getAsLong()).version(version.getAsInt()).build();
    final Tuple<SchemaIdentifier, String> tuple = new Tuple<>(resultSchemaIdentifier, schemaText);
    return schemaNameToSchemaMap.computeIfAbsent(tuple, t -> {
        final Schema schema = new Schema.Parser().parse(schemaText);
        return AvroTypeUtil.createSchema(schema, schemaText, resultSchemaIdentifier);
    });
}

Example 17 with SchemaVersionInfo

use of com.hortonworks.registries.schemaregistry.SchemaVersionInfo in project nifi by apache.

the class TestHortonworksSchemaRegistry method testCacheUsed.

@Test
public void testCacheUsed() throws Exception {
    final String text = new String(Files.readAllBytes(Paths.get("src/test/resources/empty-schema.avsc")));
    final SchemaVersionInfo info = new SchemaVersionInfo(1L, "unit-test", 2, text, System.currentTimeMillis(), "description");
    schemaVersionInfoMap.put("unit-test", info);
    final SchemaMetadata metadata = new SchemaMetadata.Builder("unit-test").compatibility(SchemaCompatibility.NONE).evolve(true).schemaGroup("group").type("AVRO").build();
    final Constructor<SchemaMetadataInfo> ctr = SchemaMetadataInfo.class.getDeclaredConstructor(SchemaMetadata.class, Long.class, Long.class);
    ctr.setAccessible(true);
    final SchemaMetadataInfo schemaMetadataInfo = ctr.newInstance(metadata, 1L, System.currentTimeMillis());
    schemaMetadataInfoMap.put("unit-test", schemaMetadataInfo);
    final Map<PropertyDescriptor, String> properties = new HashMap<>();
    properties.put(HortonworksSchemaRegistry.URL, "http://localhost:44444");
    properties.put(HortonworksSchemaRegistry.CACHE_EXPIRATION, "5 mins");
    properties.put(HortonworksSchemaRegistry.CACHE_SIZE, "1000");
    final ConfigurationContext configurationContext = new MockConfigurationContext(properties, null);
    registry.enable(configurationContext);
    for (int i = 0; i < 10000; i++) {
        final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().name("unit-test").build();
        final RecordSchema schema = registry.retrieveSchema(schemaIdentifier);
        assertNotNull(schema);
    }
    Mockito.verify(client, Mockito.times(1)).getLatestSchemaVersionInfo(any(String.class));
}

Example 18 with SchemaVersionInfo

use of com.hortonworks.registries.schemaregistry.SchemaVersionInfo in project nifi by apache.

the class TestHortonworksSchemaRegistry method testCacheExpires.

@Test
@Ignore("This can be useful for manual testing/debugging, but will keep ignored for now because we don't want automated builds to run this, since it depends on timing")
public void testCacheExpires() throws Exception {
    final String text = new String(Files.readAllBytes(Paths.get("src/test/resources/empty-schema.avsc")));
    final SchemaVersionInfo info = new SchemaVersionInfo(1L, "unit-test", 2, text, System.currentTimeMillis(), "description");
    schemaVersionInfoMap.put("unit-test", info);
    final SchemaMetadata metadata = new SchemaMetadata.Builder("unit-test").compatibility(SchemaCompatibility.NONE).evolve(true).schemaGroup("group").type("AVRO").build();
    final Constructor<SchemaMetadataInfo> ctr = SchemaMetadataInfo.class.getDeclaredConstructor(SchemaMetadata.class, Long.class, Long.class);
    ctr.setAccessible(true);
    final SchemaMetadataInfo schemaMetadataInfo = ctr.newInstance(metadata, 1L, System.currentTimeMillis());
    schemaMetadataInfoMap.put("unit-test", schemaMetadataInfo);
    final Map<PropertyDescriptor, String> properties = new HashMap<>();
    properties.put(HortonworksSchemaRegistry.URL, "http://localhost:44444");
    properties.put(HortonworksSchemaRegistry.CACHE_EXPIRATION, "1 sec");
    properties.put(HortonworksSchemaRegistry.CACHE_SIZE, "1000");
    final ConfigurationContext configurationContext = new MockConfigurationContext(properties, null);
    registry.enable(configurationContext);
    for (int i = 0; i < 2; i++) {
        final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().name("unit-test").build();
        final RecordSchema schema = registry.retrieveSchema(schemaIdentifier);
        assertNotNull(schema);
    }
    Mockito.verify(client, Mockito.times(1)).getLatestSchemaVersionInfo(any(String.class));
    Thread.sleep(2000L);
    for (int i = 0; i < 2; i++) {
        final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().name("unit-test").build();
        final RecordSchema schema = registry.retrieveSchema(schemaIdentifier);
        assertNotNull(schema);
    }
    Mockito.verify(client, Mockito.times(2)).getLatestSchemaVersionInfo(any(String.class));
}

Example 19 with SchemaVersionInfo

use of com.hortonworks.registries.schemaregistry.SchemaVersionInfo in project streamline by hortonworks.

the class AvroStreamsSnapshotDeserializerTest method testAvroPayloadConversions.

@Test
public void testAvroPayloadConversions() throws Exception {
    try (InputStream schemaStream = AvroStreamsSnapshotDeserializerTest.class.getResourceAsStream("/avro/complex.avsc")) {
        CustomAvroSerializer customAvroSerializer = new CustomAvroSerializer();
        customAvroSerializer.init(Collections.singletonMap("serdes.protocol.version", (byte) 1));
        final Schema schema = new Schema.Parser().parse(schemaStream);
        GenericRecord inputRecord = generateGenericRecord(schema);
        LOG.info("Generated record [{}]", inputRecord);
        byte[] serializedBytes = customAvroSerializer.customSerialize(inputRecord);
        SchemaMetadata schemaMetadata = new SchemaMetadata.Builder("topic-1").type("avro").schemaGroup("kafka").build();
        new Expectations() {

            {
                mockSchemaRegistryClient.getSchemaVersionInfo(withEqual(SCHEMA_ID_VERSION));
                // only versions of schema matters for this mock, as getVersion is only used during de-serialization
                result = new SchemaVersionInfo(1l, schemaMetadata.getName(), SCHEMA_ID_VERSION.getVersion(), "doesNotMatter", 1l, "doesNotMatter");
                mockSchemaRegistryClient.getSchemaMetadataInfo(withEqual(schemaMetadata.getName()));
                result = new SchemaMetadataInfo(schemaMetadata);
            }
        };
        AvroStreamsSnapshotDeserializer avroStreamsSnapshotDeserializer = new AvroStreamsSnapshotDeserializer() {

            @Override
            protected Schema getSchema(SchemaVersionKey schemaVersionKey) {
                return schema;
            }
        };
        Map<String, String> config = Collections.singletonMap(SchemaRegistryClient.Configuration.SCHEMA_REGISTRY_URL.name(), "http://localhost:8080/api/v1");
        avroStreamsSnapshotDeserializer.init(config);
        Object deserializedObject = avroStreamsSnapshotDeserializer.deserialize(new ByteArrayInputStream(serializedBytes), 1);
        Map<Object, Object> map = (Map<Object, Object>) deserializedObject;
        String deserializedJson = new ObjectMapper().writeValueAsString(map);
        String inputJson = GenericData.get().toString(inputRecord);
        LOG.info("inputJson #{}# " + inputJson);
        LOG.info("deserializedJson = #{}#" + deserializedJson);
        ObjectMapper objectMapper = new ObjectMapper();
        Assert.assertEquals(objectMapper.readTree(inputJson), objectMapper.readTree(deserializedJson));
    }
}

Example 20 with SchemaVersionInfo

use of com.hortonworks.registries.schemaregistry.SchemaVersionInfo in project registry by hortonworks.

the class AbstractSnapshotDeserializer method deserialize.

@Override
public O deserialize(I input, Integer readerSchemaVersion) throws SerDesException {
    if (!initialized) {
        throw new IllegalStateException("init should be invoked before invoking deserialize operation");
    }
    if (closed) {
        throw new IllegalStateException("This deserializer is already closed");
    }
    // it can be enhanced to have respective protocol handlers for different versions
    byte protocolId = retrieveProtocolId(input);
    SchemaIdVersion schemaIdVersion = retrieveSchemaIdVersion(protocolId, input);
    SchemaVersionInfo schemaVersionInfo = null;
    SchemaMetadata schemaMetadata = null;
    try {
        schemaVersionInfo = schemaRegistryClient.getSchemaVersionInfo(schemaIdVersion);
        schemaMetadata = schemaRegistryClient.getSchemaMetadataInfo(schemaVersionInfo.getName()).getSchemaMetadata();
    } catch (Exception e) {
        throw new RegistryException(e);
    }
    return doDeserialize(input, protocolId, schemaMetadata, schemaVersionInfo.getVersion(), readerSchemaVersion);
}