Examples with PinotSegmentRecordReader com.linkedin.pinot.core.data.readers.PinotSegmentRecordReader used on opensource projects

Example 1 with PinotSegmentRecordReader

use of com.linkedin.pinot.core.data.readers.PinotSegmentRecordReader in project pinot by linkedin.

the class ColumnarToStarTreeConverter method convertSegment.

/**
   * Helper method to perform the conversion.
   * @param columnarSegment Columnar segment directory to convert
   * @throws Exception
   */
private void convertSegment(File columnarSegment) throws Exception {
    PinotSegmentRecordReader pinotSegmentRecordReader = new PinotSegmentRecordReader(columnarSegment);
    SegmentGeneratorConfig config = new SegmentGeneratorConfig(pinotSegmentRecordReader.getSchema());
    config.setDataDir(_inputDirName);
    config.setInputFilePath(columnarSegment.getAbsolutePath());
    config.setFormat(FileFormat.PINOT);
    config.setEnableStarTreeIndex(true);
    config.setOutDir(_outputDirName);
    config.setStarTreeIndexSpecFile(_starTreeConfigFileName);
    config.setOverwrite(_overwrite);
    config.setSegmentName(columnarSegment.getName());
    SegmentIndexCreationDriver indexCreator = new SegmentIndexCreationDriverImpl();
    indexCreator.init(config);
    indexCreator.build();
}

Example 2 with PinotSegmentRecordReader

use of com.linkedin.pinot.core.data.readers.PinotSegmentRecordReader in project pinot by linkedin.

the class PinotSegmentToCsvConverter method convert.

@Override
public void convert() throws Exception {
    PinotSegmentRecordReader recordReader = new PinotSegmentRecordReader(new File(_segmentDir));
    try {
        recordReader.init();
        try (BufferedWriter recordWriter = new BufferedWriter(new FileWriter(_outputFile))) {
            if (_withHeader) {
                GenericRow row = recordReader.next();
                recordWriter.write(StringUtils.join(row.getFieldNames(), _delimiter));
                recordWriter.newLine();
                recordReader.rewind();
            }
            while (recordReader.hasNext()) {
                GenericRow row = recordReader.next();
                String[] fields = row.getFieldNames();
                List<String> record = new ArrayList<>(fields.length);
                for (String field : fields) {
                    Object value = row.getValue(field);
                    if (value instanceof Object[]) {
                        record.add(StringUtils.join((Object[]) value, _listDelimiter));
                    } else {
                        record.add(value.toString());
                    }
                }
                recordWriter.write(StringUtils.join(record, _delimiter));
                recordWriter.newLine();
            }
        }
    } finally {
        recordReader.close();
    }
}

Example 3 with PinotSegmentRecordReader

use of com.linkedin.pinot.core.data.readers.PinotSegmentRecordReader in project pinot by linkedin.

the class PinotSegmentToJsonConverter method convert.

@Override
public void convert() throws Exception {
    PinotSegmentRecordReader recordReader = new PinotSegmentRecordReader(new File(_segmentDir));
    try {
        recordReader.init();
        try (BufferedWriter recordWriter = new BufferedWriter(new FileWriter(_outputFile))) {
            while (recordReader.hasNext()) {
                GenericRow row = recordReader.next();
                JSONObject record = new JSONObject();
                for (String field : row.getFieldNames()) {
                    Object value = row.getValue(field);
                    if (value instanceof Object[]) {
                        record.put(field, new JSONArray(value));
                    } else {
                        record.put(field, value);
                    }
                }
                recordWriter.write(record.toString());
                recordWriter.newLine();
            }
        }
    } finally {
        recordReader.close();
    }
}

Example 4 with PinotSegmentRecordReader

use of com.linkedin.pinot.core.data.readers.PinotSegmentRecordReader in project pinot by linkedin.

the class PinotSegmentToAvroConverter method convert.

@Override
public void convert() throws Exception {
    PinotSegmentRecordReader recordReader = new PinotSegmentRecordReader(new File(_segmentDir));
    try {
        recordReader.init();
        Schema avroSchema = buildAvroSchemaFromPinotSchema(recordReader.getSchema());
        try (DataFileWriter<Record> recordWriter = new DataFileWriter<>(new GenericDatumWriter<Record>(avroSchema))) {
            recordWriter.create(avroSchema, new File(_outputFile));
            while (recordReader.hasNext()) {
                GenericRow row = recordReader.next();
                Record record = new Record(avroSchema);
                for (String field : row.getFieldNames()) {
                    Object value = row.getValue(field);
                    if (value instanceof Object[]) {
                        record.put(field, Arrays.asList((Object[]) value));
                    } else {
                        record.put(field, value);
                    }
                }
                recordWriter.append(record);
            }
        }
    } finally {
        recordReader.close();
    }
}