Examples with ImportedColumnMetadata com.questdb.parser.ImportedColumnMetadata used on opensource projects

Example 6 with ImportedColumnMetadata

use of com.questdb.parser.ImportedColumnMetadata in project questdb by bluestreak01.

the class ImportManager method analyzeAndParse.

private static void analyzeAndParse(File file, PlainTextLexer parser, MetadataAwareTextParser listener, CharSequence schema, int sampleSize, boolean forceHeader) throws IOException {
    parser.clear();
    ObjList<ImportedColumnMetadata> metadata = null;
    if (schema != null) {
        BootstrapEnv env = new BootstrapEnv();
        env.dateLocaleFactory = new DateLocaleFactory(new TimeZoneRuleFactory());
        env.dateFormatFactory = new DateFormatFactory();
        JsonSchemaParser jsonSchemaParser = new JsonSchemaParser(env);
        int len = schema.length();
        long addr = Unsafe.malloc(len);
        try {
            Chars.strcpy(schema, len, addr);
            try (JsonLexer lexer = new JsonLexer(1024, 4096)) {
                lexer.parse(addr, len, jsonSchemaParser);
                lexer.parseLast();
            }
            metadata = jsonSchemaParser.getMetadata();
        } catch (JsonException e) {
            throw new IOException(e);
        } finally {
            Unsafe.free(addr, len);
        }
    }
    try (RandomAccessFile raf = new RandomAccessFile(file, "r")) {
        try (FileChannel channel = raf.getChannel()) {
            long size = channel.size();
            long bufSize = ByteBuffers.getMaxMappedBufferSize(size);
            long p = 0;
            while (p < size) {
                MappedByteBuffer buf = channel.map(FileChannel.MapMode.READ_ONLY, p, size - p < bufSize ? size - p : bufSize);
                try {
                    if (p == 0) {
                        parser.analyseStructure(ByteBuffers.getAddress(buf), buf.remaining(), sampleSize, listener, forceHeader, metadata);
                    }
                    p += buf.remaining();
                    parser.parse(ByteBuffers.getAddress(buf), buf.remaining(), Integer.MAX_VALUE, listener);
                } finally {
                    ByteBuffers.release(buf);
                }
            }
            parser.parseLast();
        }
    }
}

Example 7 with ImportedColumnMetadata

use of com.questdb.parser.ImportedColumnMetadata in project questdb by bluestreak01.

the class PlainTextMetadataParser method of.

public void of(ObjList<ImportedColumnMetadata> schema, boolean forceHeader) {
    clear();
    if (schema != null) {
        for (int i = 0, n = schema.size(); i < n; i++) {
            ImportedColumnMetadata m = schema.getQuick(i);
            schemaColumns.put(m.name, m);
        }
    }
    this.forceHeader = forceHeader;
}

Example 8 with ImportedColumnMetadata

use of com.questdb.parser.ImportedColumnMetadata in project questdb by bluestreak01.

the class PlainTextStoringParser method mapColumnsAndOpenWriter.

@SuppressWarnings("unchecked")
private JournalWriter mapColumnsAndOpenWriter() throws JournalException {
    JournalMetadata<Object> jm = factory.getMetadata(name);
    if (jm.getColumnCount() != metadata.size()) {
        throw ImportColumnCountException.INSTANCE;
    }
    for (int i = 0, n = metadata.size(); i < n; i++) {
        ImportedColumnMetadata im = metadata.getQuick(i);
        ColumnMetadata cm = jm.getColumnQuick(i);
        im.importedColumnType = cm.type;
    }
    return factory.writer(jm);
}

Example 9 with ImportedColumnMetadata

use of com.questdb.parser.ImportedColumnMetadata in project questdb by bluestreak01.

the class PlainTextStoringParser method createStructure.

private JournalStructure createStructure() {
    ObjList<ColumnMetadata> m = new ObjList<>(metadata.size());
    for (int i = 0, n = metadata.size(); i < n; i++) {
        ColumnMetadata cm = new ColumnMetadata();
        ImportedColumnMetadata im = metadata.getQuick(i);
        cm.name = im.name.toString();
        cm.type = im.importedColumnType;
        switch(cm.type) {
            case ColumnType.STRING:
                cm.size = cm.avgSize + 4;
                break;
            default:
                cm.size = ColumnType.sizeOf(cm.type);
                break;
        }
        m.add(cm);
    }
    return new JournalStructure(name, m);
}