Examples with MapRecord org.apache.nifi.serialization.record.MapRecord used on opensource projects

Example 6 with MapRecord

use of org.apache.nifi.serialization.record.MapRecord in project nifi by apache.

the class HBase_1_1_2_RecordLookupService method lookup.

@Override
public Optional<Record> lookup(Map<String, Object> coordinates) throws LookupFailureException {
    if (coordinates.get(ROW_KEY_KEY) == null) {
        return Optional.empty();
    }
    final String rowKey = coordinates.get(ROW_KEY_KEY).toString();
    if (StringUtils.isBlank(rowKey)) {
        return Optional.empty();
    }
    final byte[] rowKeyBytes = rowKey.getBytes(StandardCharsets.UTF_8);
    try {
        final Map<String, Object> values = new HashMap<>();
        hBaseClientService.scan(tableName, rowKeyBytes, rowKeyBytes, columns, (byte[] row, ResultCell[] resultCells) -> {
            for (final ResultCell cell : resultCells) {
                final byte[] qualifier = Arrays.copyOfRange(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierOffset() + cell.getQualifierLength());
                final byte[] value = Arrays.copyOfRange(cell.getValueArray(), cell.getValueOffset(), cell.getValueOffset() + cell.getValueLength());
                values.put(new String(qualifier, charset), new String(value, charset));
            }
        });
        if (values.size() > 0) {
            final List<RecordField> fields = new ArrayList<>();
            for (String key : values.keySet()) {
                fields.add(new RecordField(key, RecordFieldType.STRING.getDataType()));
            }
            final RecordSchema schema = new SimpleRecordSchema(fields);
            return Optional.ofNullable(new MapRecord(schema, values));
        } else {
            return Optional.empty();
        }
    } catch (IOException e) {
        getLogger().error("Error occurred loading {}", new Object[] { coordinates.get("rowKey") }, e);
        throw new LookupFailureException(e);
    }
}

Example 7 with MapRecord

use of org.apache.nifi.serialization.record.MapRecord in project nifi by apache.

the class CSVRecordLookupService method loadCache.

private void loadCache() throws IllegalStateException, IOException {
    if (lock.tryLock()) {
        try {
            final ComponentLog logger = getLogger();
            if (logger.isDebugEnabled()) {
                logger.debug("Loading lookup table from file: " + csvFile);
            }
            final FileReader reader = new FileReader(csvFile);
            final CSVParser records = csvFormat.withFirstRecordAsHeader().parse(reader);
            ConcurrentHashMap<String, Record> cache = new ConcurrentHashMap<>();
            RecordSchema lookupRecordSchema = null;
            for (final CSVRecord record : records) {
                final String key = record.get(lookupKeyColumn);
                if (StringUtils.isBlank(key)) {
                    throw new IllegalStateException("Empty lookup key encountered in: " + csvFile);
                } else if (!ignoreDuplicates && cache.containsKey(key)) {
                    throw new IllegalStateException("Duplicate lookup key encountered: " + key + " in " + csvFile);
                } else if (ignoreDuplicates && cache.containsKey(key)) {
                    logger.warn("Duplicate lookup key encountered: {} in {}", new Object[] { key, csvFile });
                }
                // Put each key/value pair (except the lookup) into the properties
                final Map<String, Object> properties = new HashMap<>();
                record.toMap().forEach((k, v) -> {
                    if (!lookupKeyColumn.equals(k)) {
                        properties.put(k, v);
                    }
                });
                if (lookupRecordSchema == null) {
                    List<RecordField> recordFields = new ArrayList<>(properties.size());
                    properties.forEach((k, v) -> recordFields.add(new RecordField(k, RecordFieldType.STRING.getDataType())));
                    lookupRecordSchema = new SimpleRecordSchema(recordFields);
                }
                cache.put(key, new MapRecord(lookupRecordSchema, properties));
            }
            this.cache = cache;
            if (cache.isEmpty()) {
                logger.warn("Lookup table is empty after reading file: " + csvFile);
            }
        } finally {
            lock.unlock();
        }
    }
}

Example 8 with MapRecord

use of org.apache.nifi.serialization.record.MapRecord in project nifi by apache.

the class IPLookupService method createContainerRecord.

private Record createContainerRecord(final Record geoRecord, final Record ispRecord, final String domainName, final String connectionType, final Record anonymousIpRecord) {
    final Map<String, Object> values = new HashMap<>(4);
    values.put("geo", geoRecord);
    values.put("isp", ispRecord);
    values.put("domainName", domainName);
    values.put("connectionType", connectionType);
    values.put("anonymousIp", anonymousIpRecord);
    final Record containerRecord = new MapRecord(ContainerSchema.CONTAINER_SCHEMA, values);
    return containerRecord;
}

Example 9 with MapRecord

use of org.apache.nifi.serialization.record.MapRecord in project nifi by apache.

the class MongoDBLookupService method lookup.

@Override
public Optional<Object> lookup(Map<String, Object> coordinates) throws LookupFailureException {
    Map<String, Object> clean = new HashMap<>();
    clean.putAll(coordinates);
    Document query = new Document(clean);
    if (coordinates.size() == 0) {
        throw new LookupFailureException("No keys were configured. Mongo query would return random documents.");
    }
    try {
        Document result = this.findOne(query);
        if (result == null) {
            return Optional.empty();
        } else if (!StringUtils.isEmpty(lookupValueField)) {
            return Optional.ofNullable(result.get(lookupValueField));
        } else {
            final List<RecordField> fields = new ArrayList<>();
            for (String key : result.keySet()) {
                if (key.equals("_id")) {
                    continue;
                }
                fields.add(new RecordField(key, RecordFieldType.STRING.getDataType()));
            }
            final RecordSchema schema = new SimpleRecordSchema(fields);
            return Optional.ofNullable(new MapRecord(schema, result));
        }
    } catch (Exception ex) {
        getLogger().error("Error during lookup {}", new Object[] { query.toJson() }, ex);
        throw new LookupFailureException(ex);
    }
}

Example 10 with MapRecord

use of org.apache.nifi.serialization.record.MapRecord in project nifi by apache.

the class MongoDBLookupServiceIT method testLookupRecord.

@Test
public void testLookupRecord() throws Exception {
    runner.setProperty(service, MongoDBLookupService.LOOKUP_VALUE_FIELD, "");
    runner.enableControllerService(service);
    Document document = service.convertJson("{ \"uuid\": \"x-y-z\", \"message\": \"Hello, world\" }");
    service.insert(document);
    Map<String, Object> criteria = new HashMap<>();
    criteria.put("uuid", "x-y-z");
    Optional result = service.lookup(criteria);
    Assert.assertNotNull("The value was null.", result.get());
    Assert.assertTrue("The value was wrong.", result.get() instanceof MapRecord);
    MapRecord record = (MapRecord) result.get();
    Assert.assertEquals("The value was wrong.", "Hello, world", record.getAsString("message"));
    Assert.assertEquals("The value was wrong.", "x-y-z", record.getAsString("uuid"));
    Map<String, Object> clean = new HashMap<>();
    clean.putAll(criteria);
    service.delete(new Document(clean));
    try {
        result = service.lookup(criteria);
    } catch (LookupFailureException ex) {
        Assert.fail();
    }
    Assert.assertTrue(!result.isPresent());
}