use of org.apache.carbondata.sdk.file.CarbonWriter in project carbondata by apache.
the class SDKS3Example method main.
public static void main(String[] args) throws Exception {
Logger logger = LogServiceFactory.getLogService(SDKS3Example.class.getName());
if (args == null || args.length < 3) {
logger.error("Usage: java CarbonS3Example: <access-key> <secret-key>" + "<s3-endpoint> [table-path-on-s3] [rows] [Number of writes]");
System.exit(0);
}
String backupProperty = CarbonProperties.getInstance().getProperty(CarbonLoadOptionConstants.ENABLE_CARBON_LOAD_DIRECT_WRITE_TO_STORE_PATH, CarbonLoadOptionConstants.ENABLE_CARBON_LOAD_DIRECT_WRITE_TO_STORE_PATH_DEFAULT);
CarbonProperties.getInstance().addProperty(CarbonLoadOptionConstants.ENABLE_CARBON_LOAD_DIRECT_WRITE_TO_STORE_PATH, "true");
String path = "s3a://sdk/WriterOutput";
if (args.length > 3) {
path = args[3];
}
int rows = 3;
if (args.length > 4) {
rows = Integer.parseInt(args[4]);
}
int num = 3;
if (args.length > 5) {
num = Integer.parseInt(args[5]);
}
Configuration conf = new Configuration(true);
conf.set(Constants.ACCESS_KEY, args[0]);
conf.set(Constants.SECRET_KEY, args[1]);
conf.set(Constants.ENDPOINT, args[2]);
Field[] fields = new Field[2];
fields[0] = new Field("name", DataTypes.STRING);
fields[1] = new Field("age", DataTypes.INT);
for (int j = 0; j < num; j++) {
CarbonWriter writer = CarbonWriter.builder().outputPath(path).withHadoopConf(conf).withCsvInput(new Schema(fields)).writtenBy("SDKS3Example").build();
for (int i = 0; i < rows; i++) {
writer.write(new String[] { "robot" + (i % 10), String.valueOf(i) });
}
writer.close();
}
// Read data
EqualToExpression equalToExpression = new EqualToExpression(new ColumnExpression("name", DataTypes.STRING), new LiteralExpression("robot1", DataTypes.STRING));
CarbonReader reader = CarbonReader.builder(path, "_temp").projection(new String[] { "name", "age" }).filter(equalToExpression).withHadoopConf(conf).build();
System.out.println("\nData:");
int i = 0;
while (i < 20 && reader.hasNext()) {
Object[] row = (Object[]) reader.readNextRow();
System.out.println(row[0] + " " + row[1]);
i++;
}
System.out.println("\nFinished");
reader.close();
// Read without filter
CarbonReader reader2 = CarbonReader.builder(path, "_temp").projection(new String[] { "name", "age" }).withHadoopConf(ACCESS_KEY, args[0]).withHadoopConf(SECRET_KEY, args[1]).withHadoopConf(ENDPOINT, args[2]).build();
System.out.println("\nData:");
i = 0;
while (i < 20 && reader2.hasNext()) {
Object[] row = (Object[]) reader2.readNextRow();
System.out.println(row[0] + " " + row[1]);
i++;
}
System.out.println("\nFinished");
reader2.close();
CarbonProperties.getInstance().addProperty(CarbonLoadOptionConstants.ENABLE_CARBON_LOAD_DIRECT_WRITE_TO_STORE_PATH, backupProperty);
}
use of org.apache.carbondata.sdk.file.CarbonWriter in project carbondata by apache.
the class CarbonReaderExample method main.
public static void main(String[] args) {
String path = "./testWriteFiles";
try {
FileUtils.deleteDirectory(new File(path));
CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT).addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT);
Field[] fields = new Field[11];
fields[0] = new Field("stringField", DataTypes.STRING);
fields[1] = new Field("shortField", DataTypes.SHORT);
fields[2] = new Field("intField", DataTypes.INT);
fields[3] = new Field("longField", DataTypes.LONG);
fields[4] = new Field("doubleField", DataTypes.DOUBLE);
fields[5] = new Field("boolField", DataTypes.BOOLEAN);
fields[6] = new Field("dateField", DataTypes.DATE);
fields[7] = new Field("timeField", DataTypes.TIMESTAMP);
fields[8] = new Field("decimalField", DataTypes.createDecimalType(8, 2));
fields[9] = new Field("varcharField", DataTypes.VARCHAR);
fields[10] = new Field("arrayField", DataTypes.createArrayType(DataTypes.STRING));
CarbonWriter writer = CarbonWriter.builder().outputPath(path).withLoadOption("complex_delimiter_level_1", "#").withCsvInput(new Schema(fields)).writtenBy("CarbonReaderExample").build();
for (int i = 0; i < 10; i++) {
String[] row2 = new String[] { "robot" + (i % 10), String.valueOf(i % 10000), String.valueOf(i), String.valueOf(Long.MAX_VALUE - i), String.valueOf((double) i / 2), String.valueOf(true), "2019-03-02", "2019-02-12 03:03:34", "12.345", "varchar", "Hello#World#From#Carbon" };
writer.write(row2);
}
writer.close();
File[] dataFiles = new File(path).listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
if (name == null) {
return false;
}
return name.endsWith("carbonindex");
}
});
if (dataFiles == null || dataFiles.length < 1) {
throw new RuntimeException("Carbon index file not exists.");
}
Schema schema = CarbonSchemaReader.readSchema(dataFiles[0].getAbsolutePath()).asOriginOrder();
// Transform the schema
String[] strings = new String[schema.getFields().length];
for (int i = 0; i < schema.getFields().length; i++) {
strings[i] = (schema.getFields())[i].getFieldName();
}
// Read data
CarbonReader reader = CarbonReader.builder(path, "_temp").projection(strings).build();
System.out.println("\nData:");
int i = 0;
while (reader.hasNext()) {
Object[] row = (Object[]) reader.readNextRow();
System.out.println(String.format("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t", i, row[0], row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9]));
Object[] arr = (Object[]) row[10];
for (int j = 0; j < arr.length; j++) {
System.out.print(arr[j] + " ");
}
assert (arr[0].equals("Hello"));
assert (arr[3].equals("Carbon"));
System.out.println();
i++;
}
reader.close();
// Read data
CarbonReader reader2 = CarbonReader.builder(path, "_temp").build();
System.out.println("\nData:");
i = 0;
while (reader2.hasNext()) {
Object[] row = (Object[]) reader2.readNextRow();
System.out.print(String.format("%s\t%s\t%s\t%s\t%s\t", i, row[0], row[1], row[2], row[3]));
Object[] arr = (Object[]) row[4];
for (int j = 0; j < arr.length; j++) {
System.out.print(arr[j] + " ");
}
System.out.println(String.format("\t%s\t%s\t%s\t%s\t%s\t%s\t", row[5], row[6], row[7], row[8], row[9], row[10]));
i++;
}
reader2.close();
} catch (Throwable e) {
e.printStackTrace();
assert (false);
System.out.println(e.getMessage());
} finally {
try {
FileUtils.deleteDirectory(new File(path));
} catch (IOException e) {
e.printStackTrace();
}
}
}
use of org.apache.carbondata.sdk.file.CarbonWriter in project carbondata by apache.
the class BinaryUtil method binaryToCarbon.
public static void binaryToCarbon(String sourceImageFolder, String outputPath, String sufAnnotation, final String sufImage) throws Exception {
Field[] fields = new Field[5];
fields[0] = new Field("binaryId", DataTypes.INT);
fields[1] = new Field("binaryName", DataTypes.STRING);
fields[2] = new Field("binary", DataTypes.BINARY);
fields[3] = new Field("labelName", DataTypes.STRING);
fields[4] = new Field("labelContent", DataTypes.STRING);
CarbonWriter writer = CarbonWriter.builder().outputPath(outputPath).withCsvInput(new Schema(fields)).withBlockSize(256).writtenBy("binaryExample").withPageSizeInMb(1).build();
binaryToCarbon(sourceImageFolder, writer, sufAnnotation, sufImage);
}
use of org.apache.carbondata.sdk.file.CarbonWriter in project carbondata by apache.
the class BinaryUtil method binaryToCarbon.
public static void binaryToCarbon(String sourceImageFolder, String outputPath, String sufAnnotation, final String sufImage) throws Exception {
Field[] fields = new Field[5];
fields[0] = new Field("binaryId", DataTypes.INT);
fields[1] = new Field("binaryName", DataTypes.STRING);
fields[2] = new Field("binary", DataTypes.BINARY);
fields[3] = new Field("labelName", DataTypes.STRING);
fields[4] = new Field("labelContent", DataTypes.STRING);
CarbonWriter writer = CarbonWriter.builder().outputPath(outputPath).withCsvInput(new Schema(fields)).withBlockSize(256).writtenBy("binaryExample").withPageSizeInMb(1).build();
binaryToCarbon(sourceImageFolder, writer, sufAnnotation, sufImage);
}
Aggregations