Examples with SchemaMetadata com.hortonworks.registries.schemaregistry.SchemaMetadata used on opensource projects

Example 21 with SchemaMetadata

use of com.hortonworks.registries.schemaregistry.SchemaMetadata in project registry by hortonworks.

the class SchemaBranchLifeCycleTest method createAlreadyExistingBranch.

@Test(expected = SchemaBranchAlreadyExistsException.class)
public void createAlreadyExistingBranch() throws IOException, SchemaBranchNotFoundException, InvalidSchemaException, SchemaNotFoundException, IncompatibleSchemaException, SchemaBranchAlreadyExistsException {
    SchemaMetadata schemaMetadata = addSchemaMetadata(testNameRule.getMethodName(), SchemaCompatibility.NONE);
    SchemaIdVersion masterSchemaIdVersion1 = addSchemaVersion(SchemaBranch.MASTER_BRANCH, schemaMetadata, "/device.avsc");
    SchemaBranch schemaBranch1 = addSchemaBranch("BRANCH1", schemaMetadata, masterSchemaIdVersion1.getSchemaVersionId());
    SchemaIdVersion masterSchemaIdVersion2 = addSchemaVersion(SchemaBranch.MASTER_BRANCH, schemaMetadata, "/device-incompat.avsc");
    addSchemaBranch(schemaBranch1.getName(), schemaMetadata, masterSchemaIdVersion2.getSchemaVersionId());
}

Example 22 with SchemaMetadata

use of com.hortonworks.registries.schemaregistry.SchemaMetadata in project nifi by apache.

the class HortonworksSchemaRegistry method retrieveSchemaByIdAndVersion.

private RecordSchema retrieveSchemaByIdAndVersion(final SchemaIdentifier schemaIdentifier) throws org.apache.nifi.schema.access.SchemaNotFoundException, IOException {
    final SchemaRegistryClient client = getClient();
    final String schemaName;
    final SchemaVersionInfo versionInfo;
    final OptionalLong schemaId = schemaIdentifier.getIdentifier();
    if (!schemaId.isPresent()) {
        throw new org.apache.nifi.schema.access.SchemaNotFoundException("Cannot retrieve schema because Schema Id is not present");
    }
    final OptionalInt version = schemaIdentifier.getVersion();
    if (!version.isPresent()) {
        throw new org.apache.nifi.schema.access.SchemaNotFoundException("Cannot retrieve schema because Schema Version is not present");
    }
    try {
        final SchemaMetadataInfo info = client.getSchemaMetadataInfo(schemaId.getAsLong());
        if (info == null) {
            throw new org.apache.nifi.schema.access.SchemaNotFoundException("Could not find schema with ID '" + schemaId + "' and version '" + version + "'");
        }
        final SchemaMetadata metadata = info.getSchemaMetadata();
        schemaName = metadata.getName();
        final SchemaVersionKey schemaVersionKey = new SchemaVersionKey(schemaName, version.getAsInt());
        versionInfo = getSchemaVersionInfo(client, schemaVersionKey);
        if (versionInfo == null) {
            throw new org.apache.nifi.schema.access.SchemaNotFoundException("Could not find schema with ID '" + schemaId + "' and version '" + version + "'");
        }
    } catch (final Exception e) {
        handleException("Failed to retrieve schema with ID '" + schemaId + "' and version '" + version + "'", e);
        return null;
    }
    final String schemaText = versionInfo.getSchemaText();
    final SchemaIdentifier resultSchemaIdentifier = SchemaIdentifier.builder().name(schemaName).id(schemaId.getAsLong()).version(version.getAsInt()).build();
    final Tuple<SchemaIdentifier, String> tuple = new Tuple<>(resultSchemaIdentifier, schemaText);
    return schemaNameToSchemaMap.computeIfAbsent(tuple, t -> {
        final Schema schema = new Schema.Parser().parse(schemaText);
        return AvroTypeUtil.createSchema(schema, schemaText, resultSchemaIdentifier);
    });
}

Example 23 with SchemaMetadata

use of com.hortonworks.registries.schemaregistry.SchemaMetadata in project nifi by apache.

the class TestHortonworksSchemaRegistry method testCacheUsed.

@Test
public void testCacheUsed() throws Exception {
    final String text = new String(Files.readAllBytes(Paths.get("src/test/resources/empty-schema.avsc")));
    final SchemaVersionInfo info = new SchemaVersionInfo(1L, "unit-test", 2, text, System.currentTimeMillis(), "description");
    schemaVersionInfoMap.put("unit-test", info);
    final SchemaMetadata metadata = new SchemaMetadata.Builder("unit-test").compatibility(SchemaCompatibility.NONE).evolve(true).schemaGroup("group").type("AVRO").build();
    final Constructor<SchemaMetadataInfo> ctr = SchemaMetadataInfo.class.getDeclaredConstructor(SchemaMetadata.class, Long.class, Long.class);
    ctr.setAccessible(true);
    final SchemaMetadataInfo schemaMetadataInfo = ctr.newInstance(metadata, 1L, System.currentTimeMillis());
    schemaMetadataInfoMap.put("unit-test", schemaMetadataInfo);
    final Map<PropertyDescriptor, String> properties = new HashMap<>();
    properties.put(HortonworksSchemaRegistry.URL, "http://localhost:44444");
    properties.put(HortonworksSchemaRegistry.CACHE_EXPIRATION, "5 mins");
    properties.put(HortonworksSchemaRegistry.CACHE_SIZE, "1000");
    final ConfigurationContext configurationContext = new MockConfigurationContext(properties, null);
    registry.enable(configurationContext);
    for (int i = 0; i < 10000; i++) {
        final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().name("unit-test").build();
        final RecordSchema schema = registry.retrieveSchema(schemaIdentifier);
        assertNotNull(schema);
    }
    Mockito.verify(client, Mockito.times(1)).getLatestSchemaVersionInfo(any(String.class));
}

Example 24 with SchemaMetadata

use of com.hortonworks.registries.schemaregistry.SchemaMetadata in project nifi by apache.

the class TestHortonworksSchemaRegistry method testCacheExpires.

@Test
@Ignore("This can be useful for manual testing/debugging, but will keep ignored for now because we don't want automated builds to run this, since it depends on timing")
public void testCacheExpires() throws Exception {
    final String text = new String(Files.readAllBytes(Paths.get("src/test/resources/empty-schema.avsc")));
    final SchemaVersionInfo info = new SchemaVersionInfo(1L, "unit-test", 2, text, System.currentTimeMillis(), "description");
    schemaVersionInfoMap.put("unit-test", info);
    final SchemaMetadata metadata = new SchemaMetadata.Builder("unit-test").compatibility(SchemaCompatibility.NONE).evolve(true).schemaGroup("group").type("AVRO").build();
    final Constructor<SchemaMetadataInfo> ctr = SchemaMetadataInfo.class.getDeclaredConstructor(SchemaMetadata.class, Long.class, Long.class);
    ctr.setAccessible(true);
    final SchemaMetadataInfo schemaMetadataInfo = ctr.newInstance(metadata, 1L, System.currentTimeMillis());
    schemaMetadataInfoMap.put("unit-test", schemaMetadataInfo);
    final Map<PropertyDescriptor, String> properties = new HashMap<>();
    properties.put(HortonworksSchemaRegistry.URL, "http://localhost:44444");
    properties.put(HortonworksSchemaRegistry.CACHE_EXPIRATION, "1 sec");
    properties.put(HortonworksSchemaRegistry.CACHE_SIZE, "1000");
    final ConfigurationContext configurationContext = new MockConfigurationContext(properties, null);
    registry.enable(configurationContext);
    for (int i = 0; i < 2; i++) {
        final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().name("unit-test").build();
        final RecordSchema schema = registry.retrieveSchema(schemaIdentifier);
        assertNotNull(schema);
    }
    Mockito.verify(client, Mockito.times(1)).getLatestSchemaVersionInfo(any(String.class));
    Thread.sleep(2000L);
    for (int i = 0; i < 2; i++) {
        final SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().name("unit-test").build();
        final RecordSchema schema = registry.retrieveSchema(schemaIdentifier);
        assertNotNull(schema);
    }
    Mockito.verify(client, Mockito.times(2)).getLatestSchemaVersionInfo(any(String.class));
}

Example 25 with SchemaMetadata

use of com.hortonworks.registries.schemaregistry.SchemaMetadata in project streamline by hortonworks.

the class AvroStreamsSnapshotDeserializerTest method testAvroPayloadConversions.

@Test
public void testAvroPayloadConversions() throws Exception {
    try (InputStream schemaStream = AvroStreamsSnapshotDeserializerTest.class.getResourceAsStream("/avro/complex.avsc")) {
        CustomAvroSerializer customAvroSerializer = new CustomAvroSerializer();
        customAvroSerializer.init(Collections.singletonMap("serdes.protocol.version", (byte) 1));
        final Schema schema = new Schema.Parser().parse(schemaStream);
        GenericRecord inputRecord = generateGenericRecord(schema);
        LOG.info("Generated record [{}]", inputRecord);
        byte[] serializedBytes = customAvroSerializer.customSerialize(inputRecord);
        SchemaMetadata schemaMetadata = new SchemaMetadata.Builder("topic-1").type("avro").schemaGroup("kafka").build();
        new Expectations() {

            {
                mockSchemaRegistryClient.getSchemaVersionInfo(withEqual(SCHEMA_ID_VERSION));
                // only versions of schema matters for this mock, as getVersion is only used during de-serialization
                result = new SchemaVersionInfo(1l, schemaMetadata.getName(), SCHEMA_ID_VERSION.getVersion(), "doesNotMatter", 1l, "doesNotMatter");
                mockSchemaRegistryClient.getSchemaMetadataInfo(withEqual(schemaMetadata.getName()));
                result = new SchemaMetadataInfo(schemaMetadata);
            }
        };
        AvroStreamsSnapshotDeserializer avroStreamsSnapshotDeserializer = new AvroStreamsSnapshotDeserializer() {

            @Override
            protected Schema getSchema(SchemaVersionKey schemaVersionKey) {
                return schema;
            }
        };
        Map<String, String> config = Collections.singletonMap(SchemaRegistryClient.Configuration.SCHEMA_REGISTRY_URL.name(), "http://localhost:8080/api/v1");
        avroStreamsSnapshotDeserializer.init(config);
        Object deserializedObject = avroStreamsSnapshotDeserializer.deserialize(new ByteArrayInputStream(serializedBytes), 1);
        Map<Object, Object> map = (Map<Object, Object>) deserializedObject;
        String deserializedJson = new ObjectMapper().writeValueAsString(map);
        String inputJson = GenericData.get().toString(inputRecord);
        LOG.info("inputJson #{}# " + inputJson);
        LOG.info("deserializedJson = #{}#" + deserializedJson);
        ObjectMapper objectMapper = new ObjectMapper();
        Assert.assertEquals(objectMapper.readTree(inputJson), objectMapper.readTree(deserializedJson));
    }
}