Examples with ObjectMapper org.codehaus.jackson.map.ObjectMapper used on opensource projects

Example 76 with ObjectMapper

use of org.codehaus.jackson.map.ObjectMapper in project pinot by linkedin.

the class AvroSchemaToPinotSchema method execute.

@Override
public boolean execute() throws Exception {
    Schema schema = null;
    if (_avroSchemaFileName != null) {
        schema = AvroUtils.getPinotSchemaFromAvroSchemaFile(_avroSchemaFileName, buildFieldTypesMap(), _timeUnit);
    } else if (_avroDataFileName != null) {
        schema = AvroUtils.extractSchemaFromAvro(new File(_avroDataFileName));
    } else {
        LOGGER.error("Error: Missing required argument, please specify either -avroSchemaFileName, or -avroDataFileName");
        return false;
    }
    if (schema == null) {
        LOGGER.error("Error: Could not read avro schema from file.");
        return false;
    }
    String avroFileName = _avroSchemaFileName;
    if (avroFileName == null) {
        avroFileName = _avroDataFileName;
    }
    // Remove extension
    String schemaName = avroFileName.replaceAll("\\..*", "");
    schema.setSchemaName(schemaName);
    if (_pinotSchemaFileName == null) {
        _pinotSchemaFileName = schemaName + ".json";
        LOGGER.info("Using {} as the Pinot schema file name", _pinotSchemaFileName);
    }
    ObjectMapper objectMapper = new ObjectMapper();
    String schemaString = objectMapper.defaultPrettyPrintingWriter().writeValueAsString(schema);
    PrintWriter printWriter = new PrintWriter(_pinotSchemaFileName);
    printWriter.println(schemaString);
    printWriter.close();
    return true;
}

Example 77 with ObjectMapper

use of org.codehaus.jackson.map.ObjectMapper in project pinot by linkedin.

the class CreateSegmentCommand method execute.

@Override
public boolean execute() throws Exception {
    LOGGER.info("Executing command: {}", toString());
    // Load generator config if exist.
    final SegmentGeneratorConfig segmentGeneratorConfig;
    if (_generatorConfigFile != null) {
        segmentGeneratorConfig = new ObjectMapper().readValue(new File(_generatorConfigFile), SegmentGeneratorConfig.class);
    } else {
        segmentGeneratorConfig = new SegmentGeneratorConfig();
    }
    // Load config from segment generator config.
    String configDataDir = segmentGeneratorConfig.getDataDir();
    if (_dataDir == null) {
        if (configDataDir == null) {
            throw new RuntimeException("Must specify dataDir.");
        }
        _dataDir = configDataDir;
    } else {
        if (configDataDir != null && !configDataDir.equals(_dataDir)) {
            LOGGER.warn("Find dataDir conflict in command line and config file, use config in command line: {}", _dataDir);
        }
    }
    FileFormat configFormat = segmentGeneratorConfig.getFormat();
    if (_format == null) {
        if (configFormat == null) {
            throw new RuntimeException("Format cannot be null in config file.");
        }
        _format = configFormat;
    } else {
        if (configFormat != _format && configFormat != FileFormat.AVRO) {
            LOGGER.warn("Find format conflict in command line and config file, use config in command line: {}", _format);
        }
    }
    String configOutDir = segmentGeneratorConfig.getOutDir();
    if (_outDir == null) {
        if (configOutDir == null) {
            throw new RuntimeException("Must specify outDir.");
        }
        _outDir = configOutDir;
    } else {
        if (configOutDir != null && !configOutDir.equals(_outDir)) {
            LOGGER.warn("Find outDir conflict in command line and config file, use config in command line: {}", _outDir);
        }
    }
    if (segmentGeneratorConfig.isOverwrite()) {
        _overwrite = true;
    }
    String configTableName = segmentGeneratorConfig.getTableName();
    if (_tableName == null) {
        if (configTableName == null) {
            throw new RuntimeException("Must specify tableName.");
        }
        _tableName = configTableName;
    } else {
        if (configTableName != null && !configTableName.equals(_tableName)) {
            LOGGER.warn("Find tableName conflict in command line and config file, use config in command line: {}", _tableName);
        }
    }
    String configSegmentName = segmentGeneratorConfig.getSegmentName();
    if (_segmentName == null) {
        if (configSegmentName == null) {
            throw new RuntimeException("Must specify segmentName.");
        }
        _segmentName = configSegmentName;
    } else {
        if (configSegmentName != null && !configSegmentName.equals(_segmentName)) {
            LOGGER.warn("Find segmentName conflict in command line and config file, use config in command line: {}", _segmentName);
        }
    }
    // Filter out all input files.
    File dir = new File(_dataDir);
    if (!dir.exists() || !dir.isDirectory()) {
        throw new RuntimeException("Data directory " + _dataDir + " not found.");
    }
    File[] files = dir.listFiles(new FilenameFilter() {

        @Override
        public boolean accept(File dir, String name) {
            return name.toLowerCase().endsWith(_format.toString().toLowerCase());
        }
    });
    if ((files == null) || (files.length == 0)) {
        throw new RuntimeException("Data directory " + _dataDir + " does not contain " + _format.toString().toUpperCase() + " files.");
    }
    // Make sure output directory does not already exist, or can be overwritten.
    File outDir = new File(_outDir);
    if (outDir.exists()) {
        if (!_overwrite) {
            throw new IOException("Output directory " + _outDir + " already exists.");
        } else {
            FileUtils.deleteDirectory(outDir);
        }
    }
    // Set other generator configs from command line.
    segmentGeneratorConfig.setDataDir(_dataDir);
    segmentGeneratorConfig.setFormat(_format);
    segmentGeneratorConfig.setOutDir(_outDir);
    segmentGeneratorConfig.setOverwrite(_overwrite);
    segmentGeneratorConfig.setTableName(_tableName);
    segmentGeneratorConfig.setSegmentName(_segmentName);
    if (_schemaFile != null) {
        if (segmentGeneratorConfig.getSchemaFile() != null && !segmentGeneratorConfig.getSchemaFile().equals(_schemaFile)) {
            LOGGER.warn("Find schemaFile conflict in command line and config file, use config in command line: {}", _schemaFile);
        }
        segmentGeneratorConfig.setSchemaFile(_schemaFile);
    }
    if (_readerConfigFile != null) {
        if (segmentGeneratorConfig.getReaderConfigFile() != null && !segmentGeneratorConfig.getReaderConfigFile().equals(_readerConfigFile)) {
            LOGGER.warn("Find readerConfigFile conflict in command line and config file, use config in command line: {}", _readerConfigFile);
        }
        segmentGeneratorConfig.setReaderConfigFile(_readerConfigFile);
    }
    if (_enableStarTreeIndex) {
        segmentGeneratorConfig.setEnableStarTreeIndex(true);
    }
    if (_starTreeIndexSpecFile != null) {
        if (segmentGeneratorConfig.getStarTreeIndexSpecFile() != null && !segmentGeneratorConfig.getStarTreeIndexSpecFile().equals(_starTreeIndexSpecFile)) {
            LOGGER.warn("Find starTreeIndexSpecFile conflict in command line and config file, use config in command line: {}", _starTreeIndexSpecFile);
        }
        segmentGeneratorConfig.setStarTreeIndexSpecFile(_starTreeIndexSpecFile);
    }
    ExecutorService executor = Executors.newFixedThreadPool(_numThreads);
    int cnt = 0;
    for (final File file : files) {
        final int segCnt = cnt;
        executor.execute(new Runnable() {

            @Override
            public void run() {
                try {
                    SegmentGeneratorConfig config = new SegmentGeneratorConfig(segmentGeneratorConfig);
                    config.setInputFilePath(file.getAbsolutePath());
                    config.setSegmentName(_segmentName + "_" + segCnt);
                    config.loadConfigFiles();
                    final SegmentIndexCreationDriverImpl driver = new SegmentIndexCreationDriverImpl();
                    driver.init(config);
                    driver.build();
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        });
        cnt += 1;
    }
    executor.shutdown();
    return executor.awaitTermination(1, TimeUnit.HOURS);
}

Example 78 with ObjectMapper

use of org.codehaus.jackson.map.ObjectMapper in project pinot by linkedin.

the class GenerateDataCommand method buildCardinalityRangeMaps.

private void buildCardinalityRangeMaps(String file, HashMap<String, Integer> cardinality, HashMap<String, IntRange> range) throws JsonParseException, JsonMappingException, IOException {
    List<SchemaAnnotation> saList;
    if (file == null) {
        // Nothing to do here.
        return;
    }
    ObjectMapper objectMapper = new ObjectMapper();
    saList = objectMapper.readValue(new File(file), new TypeReference<List<SchemaAnnotation>>() {
    });
    for (SchemaAnnotation sa : saList) {
        String column = sa.getColumn();
        if (sa.isRange()) {
            range.put(column, new IntRange(sa.getRangeStart(), sa.getRangeEnd()));
        } else {
            cardinality.put(column, sa.getCardinality());
        }
    }
}

Example 79 with ObjectMapper

use of org.codehaus.jackson.map.ObjectMapper in project pinot by linkedin.

the class QueryComparison method runFunctionMode.

private void runFunctionMode() throws Exception {
    BufferedReader resultReader = null;
    ScanBasedQueryProcessor scanBasedQueryProcessor = null;
    try (BufferedReader queryReader = new BufferedReader(new InputStreamReader(new FileInputStream(_queryFile), "UTF8"))) {
        if (_resultFile == null) {
            scanBasedQueryProcessor = new ScanBasedQueryProcessor(_segmentsDir.getAbsolutePath());
        } else {
            resultReader = new BufferedReader(new InputStreamReader(new FileInputStream(_resultFile), "UTF8"));
        }
        int passed = 0;
        int total = 0;
        String query;
        while ((query = queryReader.readLine()) != null) {
            if (query.isEmpty() || query.startsWith("#")) {
                continue;
            }
            JSONObject expectedJson = null;
            try {
                if (resultReader != null) {
                    expectedJson = new JSONObject(resultReader.readLine());
                } else {
                    QueryResponse expectedResponse = scanBasedQueryProcessor.processQuery(query);
                    expectedJson = new JSONObject(new ObjectMapper().writeValueAsString(expectedResponse));
                }
            } catch (Exception e) {
                LOGGER.error("Comparison FAILED: Id: {} Exception caught while getting expected response for query: '{}'", total, query, e);
            }
            JSONObject actualJson = null;
            if (expectedJson != null) {
                try {
                    actualJson = new JSONObject(_clusterStarter.query(query));
                } catch (Exception e) {
                    LOGGER.error("Comparison FAILED: Id: {} Exception caught while running query: '{}'", total, query, e);
                }
            }
            if (expectedJson != null && actualJson != null) {
                try {
                    if (compare(actualJson, expectedJson)) {
                        passed++;
                        LOGGER.info("Comparison PASSED: Id: {} actual Time: {} ms expected Time: {} ms Docs Scanned: {}", total, actualJson.get(TIME_USED_MS), expectedJson.get(TIME_USED_MS), actualJson.get(NUM_DOCS_SCANNED));
                        LOGGER.debug("actual Response: {}", actualJson);
                        LOGGER.debug("expected Response: {}", expectedJson);
                    } else {
                        LOGGER.error("Comparison FAILED: Id: {} query: {}", query);
                        LOGGER.info("actual Response: {}", actualJson);
                        LOGGER.info("expected Response: {}", expectedJson);
                    }
                } catch (Exception e) {
                    LOGGER.error("Comparison FAILED: Id: {} Exception caught while comparing query: '{}' actual response: {}, expected response: {}", total, query, actualJson, expectedJson, e);
                }
            }
            total++;
        }
        LOGGER.info("Total {} out of {} queries passed.", passed, total);
    } finally {
        if (resultReader != null) {
            resultReader.close();
        }
    }
}

Example 80 with ObjectMapper

use of org.codehaus.jackson.map.ObjectMapper in project pinot by linkedin.

the class ScanBasedQueryProcessor method printResult.

public static void printResult(QueryResponse queryResponse) throws IOException {
    ObjectMapper objectMapper = new ObjectMapper();
    LOGGER.info(objectMapper.writeValueAsString(queryResponse));
}