Examples with SchemaNotFoundException org.apache.nifi.schema.access.SchemaNotFoundException used on opensource projects

Example 16 with SchemaNotFoundException

use of org.apache.nifi.schema.access.SchemaNotFoundException in project nifi by apache.

the class RestSchemaRegistryClient method getSchema.

@Override
public RecordSchema getSchema(final int schemaId) throws IOException, SchemaNotFoundException {
    // The Confluent Schema Registry's REST API does not provide us with the 'subject' (name) of a Schema given the ID.
    // It will provide us only the text of the Schema itself. Therefore, in order to determine the name (which is required for
    // a SchemaIdentifier), we must obtain a list of all Schema names, and then request each and every one of the schemas to determine
    // if the ID requested matches the Schema's ID.
    // To make this more efficient, we will cache a mapping of Schema Name to identifier, so that we can look this up more efficiently.
    // Check if we have cached the Identifier to Name mapping
    final String schemaPath = getSchemaPath(schemaId);
    final JsonNode responseJson = fetchJsonResponse(schemaPath, "id " + schemaId);
    final JsonNode subjectsJson = fetchJsonResponse("/subjects", "subjects array");
    final ArrayNode subjectsList = (ArrayNode) subjectsJson;
    JsonNode completeSchema = null;
    for (JsonNode subject : subjectsList) {
        try {
            final String subjectName = subject.asText();
            completeSchema = postJsonResponse("/subjects/" + subjectName, responseJson, "schema id: " + schemaId);
            break;
        } catch (SchemaNotFoundException e) {
            continue;
        }
    }
    if (completeSchema == null) {
        throw new SchemaNotFoundException("could not get schema with id: " + schemaId);
    }
    final RecordSchema recordSchema = createRecordSchema(completeSchema);
    return recordSchema;
}

Example 17 with SchemaNotFoundException

use of org.apache.nifi.schema.access.SchemaNotFoundException in project nifi by apache.

the class RestSchemaRegistryClient method fetchJsonResponse.

private JsonNode fetchJsonResponse(final String pathSuffix, final String schemaDescription) throws SchemaNotFoundException, IOException {
    String errorMessage = null;
    for (final String baseUrl : baseUrls) {
        final String path = getPath(pathSuffix);
        final String trimmedBase = getTrimmedBase(baseUrl);
        final String url = trimmedBase + path;
        final WebTarget webTarget = client.target(url);
        final Response response = webTarget.request().accept(MediaType.APPLICATION_JSON).get();
        final int responseCode = response.getStatus();
        if (responseCode == Response.Status.OK.getStatusCode()) {
            final JsonNode responseJson = response.readEntity(JsonNode.class);
            return responseJson;
        }
        if (responseCode == Response.Status.NOT_FOUND.getStatusCode()) {
            throw new SchemaNotFoundException("Could not find Schema with " + schemaDescription + " from the Confluent Schema Registry located at " + baseUrl);
        }
        if (errorMessage == null) {
            errorMessage = response.readEntity(String.class);
        }
    }
    throw new IOException("Failed to retrieve Schema with " + schemaDescription + " from any of the Confluent Schema Registry URL's provided; failure response message: " + errorMessage);
}

Example 18 with SchemaNotFoundException

use of org.apache.nifi.schema.access.SchemaNotFoundException in project nifi by apache.

the class RestSchemaRegistryClient method postJsonResponse.

private JsonNode postJsonResponse(final String pathSuffix, final JsonNode schema, final String schemaDescription) throws SchemaNotFoundException {
    String errorMessage = null;
    for (final String baseUrl : baseUrls) {
        final String path = getPath(pathSuffix);
        final String trimmedBase = getTrimmedBase(baseUrl);
        final String url = trimmedBase + path;
        final WebTarget builder = client.target(url);
        final Response response = builder.request().accept(MediaType.APPLICATION_JSON).header(CONTENT_TYPE_HEADER, SCHEMA_REGISTRY_CONTENT_TYPE).post(Entity.json(schema.toString()));
        final int responseCode = response.getStatus();
        if (responseCode == Response.Status.NOT_FOUND.getStatusCode()) {
            continue;
        }
        if (responseCode == Response.Status.OK.getStatusCode()) {
            final JsonNode responseJson = response.readEntity(JsonNode.class);
            return responseJson;
        }
    }
    throw new SchemaNotFoundException("Failed to retrieve Schema with " + schemaDescription + " from any of the Confluent Schema Registry URL's provided; failure response message: " + errorMessage);
}

Example 19 with SchemaNotFoundException

use of org.apache.nifi.schema.access.SchemaNotFoundException in project nifi by apache.

the class PutElasticsearchHttpRecord method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final RecordReaderFactory readerFactory = context.getProperty(RECORD_READER).asControllerService(RecordReaderFactory.class);
    // Authentication
    final String username = context.getProperty(USERNAME).evaluateAttributeExpressions(flowFile).getValue();
    final String password = context.getProperty(PASSWORD).evaluateAttributeExpressions(flowFile).getValue();
    OkHttpClient okHttpClient = getClient();
    final ComponentLog logger = getLogger();
    final String baseUrl = trimToEmpty(context.getProperty(ES_URL).evaluateAttributeExpressions().getValue());
    HttpUrl.Builder urlBuilder = HttpUrl.parse(baseUrl).newBuilder().addPathSegment("_bulk");
    // Find the user-added properties and set them as query parameters on the URL
    for (Map.Entry<PropertyDescriptor, String> property : context.getProperties().entrySet()) {
        PropertyDescriptor pd = property.getKey();
        if (pd.isDynamic()) {
            if (property.getValue() != null) {
                urlBuilder = urlBuilder.addQueryParameter(pd.getName(), context.getProperty(pd).evaluateAttributeExpressions().getValue());
            }
        }
    }
    final URL url = urlBuilder.build().url();
    final String index = context.getProperty(INDEX).evaluateAttributeExpressions(flowFile).getValue();
    if (StringUtils.isEmpty(index)) {
        logger.error("No value for index in for {}, transferring to failure", new Object[] { flowFile });
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    final String docType = context.getProperty(TYPE).evaluateAttributeExpressions(flowFile).getValue();
    String indexOp = context.getProperty(INDEX_OP).evaluateAttributeExpressions(flowFile).getValue();
    if (StringUtils.isEmpty(indexOp)) {
        logger.error("No Index operation specified for {}, transferring to failure.", new Object[] { flowFile });
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    switch(indexOp.toLowerCase()) {
        case "index":
        case "update":
        case "upsert":
        case "delete":
            break;
        default:
            logger.error("Index operation {} not supported for {}, transferring to failure.", new Object[] { indexOp, flowFile });
            session.transfer(flowFile, REL_FAILURE);
            return;
    }
    this.nullSuppression = context.getProperty(SUPPRESS_NULLS).getValue();
    final String id_path = context.getProperty(ID_RECORD_PATH).evaluateAttributeExpressions(flowFile).getValue();
    final RecordPath recordPath = StringUtils.isEmpty(id_path) ? null : recordPathCache.getCompiled(id_path);
    final StringBuilder sb = new StringBuilder();
    try (final InputStream in = session.read(flowFile);
        final RecordReader reader = readerFactory.createRecordReader(flowFile, in, getLogger())) {
        Record record;
        while ((record = reader.nextRecord()) != null) {
            final String id;
            if (recordPath != null) {
                Optional<FieldValue> idPathValue = recordPath.evaluate(record).getSelectedFields().findFirst();
                if (!idPathValue.isPresent() || idPathValue.get().getValue() == null) {
                    throw new IdentifierNotFoundException("Identifier Record Path specified but no value was found, transferring {} to failure.");
                }
                id = idPathValue.get().getValue().toString();
            } else {
                id = null;
            }
            // a missing ID indicates one is to be auto-generated by Elasticsearch
            if (id == null && !indexOp.equalsIgnoreCase("index")) {
                throw new IdentifierNotFoundException("Index operation {} requires a valid identifier value from a flow file attribute, transferring to failure.");
            }
            final StringBuilder json = new StringBuilder();
            ByteArrayOutputStream out = new ByteArrayOutputStream();
            JsonGenerator generator = factory.createJsonGenerator(out);
            writeRecord(record, record.getSchema(), generator);
            generator.flush();
            generator.close();
            json.append(out.toString());
            if (indexOp.equalsIgnoreCase("index")) {
                sb.append("{\"index\": { \"_index\": \"");
                sb.append(index);
                sb.append("\", \"_type\": \"");
                sb.append(docType);
                sb.append("\"");
                if (!StringUtils.isEmpty(id)) {
                    sb.append(", \"_id\": \"");
                    sb.append(id);
                    sb.append("\"");
                }
                sb.append("}}\n");
                sb.append(json);
                sb.append("\n");
            } else if (indexOp.equalsIgnoreCase("upsert") || indexOp.equalsIgnoreCase("update")) {
                sb.append("{\"update\": { \"_index\": \"");
                sb.append(index);
                sb.append("\", \"_type\": \"");
                sb.append(docType);
                sb.append("\", \"_id\": \"");
                sb.append(id);
                sb.append("\" }\n");
                sb.append("{\"doc\": ");
                sb.append(json);
                sb.append(", \"doc_as_upsert\": ");
                sb.append(indexOp.equalsIgnoreCase("upsert"));
                sb.append(" }\n");
            } else if (indexOp.equalsIgnoreCase("delete")) {
                sb.append("{\"delete\": { \"_index\": \"");
                sb.append(index);
                sb.append("\", \"_type\": \"");
                sb.append(docType);
                sb.append("\", \"_id\": \"");
                sb.append(id);
                sb.append("\" }\n");
            }
        }
    } catch (IdentifierNotFoundException infe) {
        logger.error(infe.getMessage(), new Object[] { flowFile });
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    } catch (final IOException | SchemaNotFoundException | MalformedRecordException e) {
        logger.error("Could not parse incoming data", e);
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    RequestBody requestBody = RequestBody.create(MediaType.parse("application/json"), sb.toString());
    final Response getResponse;
    try {
        getResponse = sendRequestToElasticsearch(okHttpClient, url, username, password, "PUT", requestBody);
    } catch (final Exception e) {
        logger.error("Routing to {} due to exception: {}", new Object[] { REL_FAILURE.getName(), e }, e);
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    final int statusCode = getResponse.code();
    if (isSuccess(statusCode)) {
        ResponseBody responseBody = getResponse.body();
        try {
            final byte[] bodyBytes = responseBody.bytes();
            JsonNode responseJson = parseJsonResponse(new ByteArrayInputStream(bodyBytes));
            boolean errors = responseJson.get("errors").asBoolean(false);
            // ES has no rollback, so if errors occur, log them and route the whole flow file to failure
            if (errors) {
                ArrayNode itemNodeArray = (ArrayNode) responseJson.get("items");
                if (itemNodeArray.size() > 0) {
                    // All items are returned whether they succeeded or failed, so iterate through the item array
                    // at the same time as the flow file list, moving each to success or failure accordingly,
                    // but only keep the first error for logging
                    String errorReason = null;
                    for (int i = itemNodeArray.size() - 1; i >= 0; i--) {
                        JsonNode itemNode = itemNodeArray.get(i);
                        int status = itemNode.findPath("status").asInt();
                        if (!isSuccess(status)) {
                            if (errorReason == null) {
                                // Use "result" if it is present; this happens for status codes like 404 Not Found, which may not have an error/reason
                                String reason = itemNode.findPath("//result").asText();
                                if (StringUtils.isEmpty(reason)) {
                                    // If there was no result, we expect an error with a string description in the "reason" field
                                    reason = itemNode.findPath("//error/reason").asText();
                                }
                                errorReason = reason;
                                logger.error("Failed to process {} due to {}, transferring to failure", new Object[] { flowFile, errorReason });
                            }
                        }
                    }
                }
                session.transfer(flowFile, REL_FAILURE);
            } else {
                session.transfer(flowFile, REL_SUCCESS);
                session.getProvenanceReporter().send(flowFile, url.toString());
            }
        } catch (IOException ioe) {
            // Something went wrong when parsing the response, log the error and route to failure
            logger.error("Error parsing Bulk API response: {}", new Object[] { ioe.getMessage() }, ioe);
            session.transfer(flowFile, REL_FAILURE);
            context.yield();
        }
    } else if (statusCode / 100 == 5) {
        // 5xx -> RETRY, but a server error might last a while, so yield
        logger.warn("Elasticsearch returned code {} with message {}, transferring flow file to retry. This is likely a server problem, yielding...", new Object[] { statusCode, getResponse.message() });
        session.transfer(flowFile, REL_RETRY);
        context.yield();
    } else {
        // 1xx, 3xx, 4xx, etc. -> NO RETRY
        logger.warn("Elasticsearch returned code {} with message {}, transferring flow file to failure", new Object[] { statusCode, getResponse.message() });
        session.transfer(flowFile, REL_FAILURE);
    }
    getResponse.close();
}

Example 20 with SchemaNotFoundException

use of org.apache.nifi.schema.access.SchemaNotFoundException in project nifi by apache.

the class TestAvroSchemaRegistry method validateSchemaRegistrationFromrDynamicProperties.

@Test
public void validateSchemaRegistrationFromrDynamicProperties() throws Exception {
    String schemaName = "fooSchema";
    PropertyDescriptor fooSchema = new PropertyDescriptor.Builder().name(schemaName).dynamic(true).build();
    String fooSchemaText = "{\"namespace\": \"example.avro\", " + "\"type\": \"record\", " + "\"name\": \"User\", " + "\"fields\": [ " + "{\"name\": \"name\", \"type\": [\"string\", \"null\"]}, " + "{\"name\": \"favorite_number\",  \"type\": [\"int\", \"null\"]}, " + "{\"name\": \"foo\",  \"type\": [\"int\", \"null\"]}, " + "{\"name\": \"favorite_color\", \"type\": [\"string\", \"null\"]} " + "]" + "}";
    PropertyDescriptor barSchema = new PropertyDescriptor.Builder().name("barSchema").dynamic(false).build();
    AvroSchemaRegistry delegate = new AvroSchemaRegistry();
    delegate.onPropertyModified(fooSchema, null, fooSchemaText);
    delegate.onPropertyModified(barSchema, null, "");
    SchemaIdentifier schemaIdentifier = SchemaIdentifier.builder().name(schemaName).build();
    RecordSchema locatedSchema = delegate.retrieveSchema(schemaIdentifier);
    assertEquals(fooSchemaText, locatedSchema.getSchemaText().get());
    try {
        delegate.retrieveSchema(SchemaIdentifier.builder().name("barSchema").build());
        Assert.fail("Expected a SchemaNotFoundException to be thrown but it was not");
    } catch (final SchemaNotFoundException expected) {
    }
}