Examples with JsonDecoder org.apache.avro.io.JsonDecoder used on opensource projects

Example 1 with JsonDecoder

use of org.apache.avro.io.JsonDecoder in project components by Talend.

the class FixedDatasetRuntime method getValues.

public List<IndexedRecord> getValues(int limit) {
    List<IndexedRecord> values = new ArrayList<>();
    switch(properties.format.getValue()) {
        case CSV:
            try {
                CsvRecordToIndexedRecordConverter converter = new CsvRecordToIndexedRecordConverter(getSchema());
                for (CSVRecord r : // 
                CSVFormat.RFC4180.withDelimiter(// 
                properties.getFieldDelimiter().charAt(0)).withRecordSeparator(properties.getRecordDelimiter()).parse(new StringReader(properties.values.getValue()))) values.add(converter.convertToAvro(r));
            } catch (IOException e) {
                throw LocalIOErrorCode.createCannotParseSchema(e, properties.values.getValue());
            }
            break;
        case JSON:
            ObjectMapper mapper = new ObjectMapper();
            JsonSchemaInferrer jsonSchemaInferrer = new JsonSchemaInferrer(mapper);
            JsonGenericRecordConverter converter = null;
            JsonFactory jsonFactory = new JsonFactory();
            try (StringReader r = new StringReader(properties.values.getValue())) {
                Iterator<JsonNode> value = mapper.readValues(jsonFactory.createParser(r), JsonNode.class);
                int count = 0;
                while (value.hasNext() && count++ < limit) {
                    String json = value.next().toString();
                    if (converter == null) {
                        Schema jsonSchema = jsonSchemaInferrer.inferSchema(json);
                        converter = new JsonGenericRecordConverter(jsonSchema);
                    }
                    values.add(converter.convertToAvro(json));
                }
            } catch (IOException e) {
                throw LocalIOErrorCode.createCannotParseJson(e, properties.schema.getValue(), properties.values.getValue());
            }
            break;
        case AVRO:
            Schema schema = getSchema();
            if (isRandom()) {
                GeneratorFunction<IndexedRecord> gf = (GeneratorFunction<IndexedRecord>) GeneratorFunctions.of(getSchema());
                GeneratorFunction.GeneratorContext ctx = GeneratorFunction.GeneratorContext.of(0, 0L);
                for (int i = 0; i < limit; i++) {
                    ctx.setRowId(i);
                    values.add(gf.apply(ctx));
                }
            } else {
                try (ByteArrayInputStream bais = new ByteArrayInputStream(properties.values.getValue().trim().getBytes())) {
                    JsonDecoder decoder = DecoderFactory.get().jsonDecoder(schema, bais);
                    DatumReader<IndexedRecord> reader = new GenericDatumReader<>(schema);
                    int count = 0;
                    while (count++ < limit) {
                        values.add(reader.read(null, decoder));
                    }
                } catch (EOFException e) {
                // Indicates the end of the values.
                } catch (IOException e) {
                    throw LocalIOErrorCode.createCannotParseAvroJson(e, properties.schema.getValue(), properties.values.getValue());
                }
            }
            break;
    }
    return values;
}

Example 2 with JsonDecoder

use of org.apache.avro.io.JsonDecoder in project databus by linkedin.

the class AvroConverter method convert.

public List<GenericRecord> convert(InputStream in) throws IOException {
    Decoder inputDecoder = (_inputFormat == AvroFormat.BINARY) ? DecoderFactory.defaultFactory().createBinaryDecoder(in, null) : (AvroFormat.JSON == _inputFormat) ? new JsonDecoder(_inputSchema, in) : null;
    ArrayList<GenericRecord> result = new ArrayList<GenericRecord>();
    GenericDatumReader<GenericRecord> genericReader = _inputSchema != _outputSchema ? new GenericDatumReader<GenericRecord>(_inputSchema, _outputSchema) : new GenericDatumReader<GenericRecord>(_inputSchema);
    switch(_inputFormat) {
        case BINARY:
        case JSON:
            {
                GenericRecord r = genericReader.read(null, inputDecoder);
                result.add(r);
                break;
            }
        case JSON_LINES:
            {
                InputStreamReader inReader = new InputStreamReader(in);
                try {
                    BufferedReader lineIn = new BufferedReader(inReader);
                    try {
                        String line;
                        while (null != (line = lineIn.readLine())) {
                            inputDecoder = new JsonDecoder(_inputSchema, line);
                            GenericRecord r = genericReader.read(null, inputDecoder);
                            result.add(r);
                            break;
                        }
                    } finally {
                        lineIn.close();
                    }
                } finally {
                    inReader.close();
                }
            }
        default:
            {
                throw new RuntimeException("Unimplemented input format: " + _inputFormat);
            }
    }
    return result;
}

Example 3 with JsonDecoder

use of org.apache.avro.io.JsonDecoder in project voldemort by voldemort.

the class ClientConfigUtil method readSingleClientConfigAvro.

/**
 * Parses a string that contains single fat client config string in avro
 * format
 *
 * @param configAvro Input string of avro format, that contains config for
 *        multiple stores
 * @return Properties of single fat client config
 */
@SuppressWarnings("unchecked")
public static Properties readSingleClientConfigAvro(String configAvro) {
    Properties props = new Properties();
    try {
        JsonDecoder decoder = new JsonDecoder(CLIENT_CONFIG_AVRO_SCHEMA, configAvro);
        GenericDatumReader<Object> datumReader = new GenericDatumReader<Object>(CLIENT_CONFIG_AVRO_SCHEMA);
        Map<Utf8, Utf8> flowMap = (Map<Utf8, Utf8>) datumReader.read(null, decoder);
        for (Utf8 key : flowMap.keySet()) {
            props.put(key.toString(), flowMap.get(key).toString());
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
    return props;
}

Example 4 with JsonDecoder

use of org.apache.avro.io.JsonDecoder in project voldemort by voldemort.

the class ClientConfigUtil method readMultipleClientConfigAvro.

/**
 * Parses a string that contains multiple fat client configs in avro format
 *
 * @param configAvro Input string of avro format, that contains config for
 *        multiple stores
 * @return Map of store names to store config properties
 */
@SuppressWarnings("unchecked")
public static Map<String, Properties> readMultipleClientConfigAvro(String configAvro) {
    Map<String, Properties> mapStoreToProps = Maps.newHashMap();
    try {
        JsonDecoder decoder = new JsonDecoder(CLIENT_CONFIGS_AVRO_SCHEMA, configAvro);
        GenericDatumReader<Object> datumReader = new GenericDatumReader<Object>(CLIENT_CONFIGS_AVRO_SCHEMA);
        Map<Utf8, Map<Utf8, Utf8>> storeConfigs = (Map<Utf8, Map<Utf8, Utf8>>) datumReader.read(null, decoder);
        // Store config props to return back
        for (Utf8 storeName : storeConfigs.keySet()) {
            Properties props = new Properties();
            Map<Utf8, Utf8> singleConfig = storeConfigs.get(storeName);
            for (Utf8 key : singleConfig.keySet()) {
                props.put(key.toString(), singleConfig.get(key).toString());
            }
            if (storeName == null || storeName.length() == 0) {
                throw new Exception("Invalid store name found!");
            }
            mapStoreToProps.put(storeName.toString(), props);
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
    return mapStoreToProps;
}

Example 5 with JsonDecoder

use of org.apache.avro.io.JsonDecoder in project voldemort by voldemort.

the class VoldemortAdminTool method executeQueryKey.

private static void executeQueryKey(final Integer nodeId, AdminClient adminClient, List<String> storeNames, String keyString, String keyFormat) throws IOException {
    // decide queryingNode(s) for Key
    List<Integer> queryingNodes = new ArrayList<Integer>();
    if (nodeId < 0) {
        // means all nodes
        for (Node node : adminClient.getAdminClientCluster().getNodes()) {
            queryingNodes.add(node.getId());
        }
    } else {
        queryingNodes.add(nodeId);
    }
    // get basic info
    List<StoreDefinition> storeDefinitionList = getStoreDefinitions(adminClient, nodeId);
    Map<String, StoreDefinition> storeDefinitions = new HashMap<String, StoreDefinition>();
    for (StoreDefinition storeDef : storeDefinitionList) {
        storeDefinitions.put(storeDef.getName(), storeDef);
    }
    BufferedWriter out = new BufferedWriter(new OutputStreamWriter(System.out));
    // iterate through stores
    for (final String storeName : storeNames) {
        // store definition
        StoreDefinition storeDefinition = storeDefinitions.get(storeName);
        if (storeDefinition == null) {
            throw new StoreNotFoundException("Store " + storeName + " not found");
        }
        out.write("STORE_NAME: " + storeDefinition.getName() + "\n");
        // k-v serializer
        final SerializerDefinition keySerializerDef = storeDefinition.getKeySerializer();
        final SerializerDefinition valueSerializerDef = storeDefinition.getValueSerializer();
        SerializerFactory serializerFactory = new DefaultSerializerFactory();
        @SuppressWarnings("unchecked") final Serializer<Object> keySerializer = (Serializer<Object>) serializerFactory.getSerializer(keySerializerDef);
        @SuppressWarnings("unchecked") final Serializer<Object> valueSerializer = (Serializer<Object>) serializerFactory.getSerializer(valueSerializerDef);
        // compression strategy
        final CompressionStrategy keyCompressionStrategy;
        final CompressionStrategy valueCompressionStrategy;
        if (keySerializerDef != null && keySerializerDef.hasCompression()) {
            keyCompressionStrategy = new CompressionStrategyFactory().get(keySerializerDef.getCompression());
        } else {
            keyCompressionStrategy = null;
        }
        if (valueSerializerDef != null && valueSerializerDef.hasCompression()) {
            valueCompressionStrategy = new CompressionStrategyFactory().get(valueSerializerDef.getCompression());
        } else {
            valueCompressionStrategy = null;
        }
        if (keyCompressionStrategy == null) {
            out.write("KEY_COMPRESSION_STRATEGY: None\n");
        } else {
            out.write("KEY_COMPRESSION_STRATEGY: " + keyCompressionStrategy.getType() + "\n");
        }
        out.write("KEY_SERIALIZER_NAME: " + keySerializerDef.getName() + "\n");
        for (Map.Entry<Integer, String> entry : keySerializerDef.getAllSchemaInfoVersions().entrySet()) {
            out.write(String.format("KEY_SCHEMA VERSION=%d\n", entry.getKey()));
            out.write("====================================\n");
            out.write(entry.getValue());
            out.write("\n====================================\n");
        }
        out.write("\n");
        if (valueCompressionStrategy == null) {
            out.write("VALUE_COMPRESSION_STRATEGY: None\n");
        } else {
            out.write("VALUE_COMPRESSION_STRATEGY: " + valueCompressionStrategy.getType() + "\n");
        }
        out.write("VALUE_SERIALIZER_NAME: " + valueSerializerDef.getName() + "\n");
        for (Map.Entry<Integer, String> entry : valueSerializerDef.getAllSchemaInfoVersions().entrySet()) {
            out.write(String.format("VALUE_SCHEMA %d\n", entry.getKey()));
            out.write("====================================\n");
            out.write(entry.getValue());
            out.write("\n====================================\n");
        }
        out.write("\n");
        // although the streamingOps support multiple keys, we only query
        // one key here
        ByteArray key;
        try {
            if (keyFormat.equals("readable")) {
                Object keyObject;
                String keySerializerName = keySerializerDef.getName();
                if (isAvroSchema(keySerializerName)) {
                    Schema keySchema = Schema.parse(keySerializerDef.getCurrentSchemaInfo());
                    JsonDecoder decoder = new JsonDecoder(keySchema, keyString);
                    GenericDatumReader<Object> datumReader = new GenericDatumReader<Object>(keySchema);
                    keyObject = datumReader.read(null, decoder);
                } else if (keySerializerName.equals(DefaultSerializerFactory.JSON_SERIALIZER_TYPE_NAME)) {
                    JsonReader jsonReader = new JsonReader(new StringReader(keyString));
                    keyObject = jsonReader.read();
                } else {
                    keyObject = keyString;
                }
                key = new ByteArray(keySerializer.toBytes(keyObject));
            } else {
                key = new ByteArray(ByteUtils.fromHexString(keyString));
            }
        } catch (SerializationException se) {
            System.err.println("Error serializing key " + keyString);
            System.err.println("If this is a JSON key, you need to include escaped quotation marks in the command line if it is a string");
            se.printStackTrace();
            return;
        } catch (DecoderException de) {
            System.err.println("Error decoding key " + keyString);
            de.printStackTrace();
            return;
        } catch (IOException io) {
            System.err.println("Error parsing avro string " + keyString);
            io.printStackTrace();
            return;
        }
        boolean printedKey = false;
        // A Map<> could have been used instead of List<Entry<>> if
        // Versioned supported correct hash codes. Read the comment in
        // Versioned about the issue
        List<Entry<List<Versioned<byte[]>>, List<Integer>>> nodeValues = new ArrayList<Entry<List<Versioned<byte[]>>, List<Integer>>>();
        for (final Integer queryNodeId : queryingNodes) {
            Iterator<QueryKeyResult> iterator;
            iterator = adminClient.streamingOps.queryKeys(queryNodeId, storeName, Arrays.asList(key).iterator());
            final StringWriter stringWriter = new StringWriter();
            QueryKeyResult queryKeyResult = iterator.next();
            if (!printedKey) {
                // de-serialize and write key
                byte[] keyBytes = queryKeyResult.getKey().get();
                Object keyObject = keySerializer.toObject((null == keyCompressionStrategy) ? keyBytes : keyCompressionStrategy.inflate(keyBytes));
                writeVoldKeyOrValueInternal(keyBytes, keySerializer, keyCompressionStrategy, "KEY", out);
                printedKey = true;
            }
            // iterate through, de-serialize and write values
            if (queryKeyResult.hasValues() && queryKeyResult.getValues().size() > 0) {
                int elementId = -1;
                for (int i = 0; i < nodeValues.size(); i++) {
                    if (Objects.equal(nodeValues.get(i).getKey(), queryKeyResult.getValues())) {
                        elementId = i;
                        break;
                    }
                }
                if (elementId == -1) {
                    ArrayList<Integer> nodes = new ArrayList<Integer>();
                    nodes.add(queryNodeId);
                    nodeValues.add(new AbstractMap.SimpleEntry<List<Versioned<byte[]>>, List<Integer>>(queryKeyResult.getValues(), nodes));
                } else {
                    nodeValues.get(elementId).getValue().add(queryNodeId);
                }
                out.write(String.format("\nQueried node %d on store %s\n", queryNodeId, storeName));
                int versionCount = 0;
                if (queryKeyResult.getValues().size() > 1) {
                    out.write("VALUE " + versionCount + "\n");
                }
                for (Versioned<byte[]> versioned : queryKeyResult.getValues()) {
                    // write version
                    VectorClock version = (VectorClock) versioned.getVersion();
                    out.write("VECTOR_CLOCK_BYTE: " + ByteUtils.toHexString(version.toBytes()) + "\n");
                    out.write("VECTOR_CLOCK_TEXT: " + version.toString() + '[' + new Date(version.getTimestamp()).toString() + "]\n");
                    // write value
                    byte[] valueBytes = versioned.getValue();
                    writeVoldKeyOrValueInternal(valueBytes, valueSerializer, valueCompressionStrategy, "VALUE", out);
                    versionCount++;
                }
            } else // exception.
            if (queryKeyResult.hasException()) {
                boolean isInvalidMetadataException = queryKeyResult.getException() instanceof InvalidMetadataException;
                // you are querying only a single node.
                if (!isInvalidMetadataException || queryingNodes.size() == 1) {
                    out.write(String.format("\nNode %d on store %s returned exception\n", queryNodeId, storeName));
                    out.write(queryKeyResult.getException().toString());
                    out.write("\n====================================\n");
                }
            } else {
                if (queryingNodes.size() == 1) {
                    out.write(String.format("\nNode %d on store %s returned NULL\n", queryNodeId, storeName));
                    out.write("\n====================================\n");
                }
            }
            out.flush();
        }
        out.write("\n====================================\n");
        for (Map.Entry<List<Versioned<byte[]>>, List<Integer>> nodeValue : nodeValues) {
            out.write("Nodes with same Value " + Arrays.toString(nodeValue.getValue().toArray()));
            out.write("\n====================================\n");
        }
        if (nodeValues.size() > 1) {
            out.write("\n*** Multiple (" + nodeValues.size() + ") versions of key/value exist for the key ***\n");
        }
        out.flush();
    }
}