Examples with ImportedColumnMetadata com.questdb.parser.ImportedColumnMetadata used on opensource projects

Example 1 with ImportedColumnMetadata

use of com.questdb.parser.ImportedColumnMetadata in project questdb by bluestreak01.

the class ImportHandler method resumeJson.

private static void resumeJson(ImportHandlerContext ctx, ChunkedResponse r) throws DisconnectedChannelException, SlowWritableChannelException {
    final JournalMetadata m = ctx.importer.getJournalMetadata();
    final ObjList<ImportedColumnMetadata> importedMetadata = ctx.importer.getImportedMetadata();
    final int columnCount = m.getColumnCount();
    final LongList errors = ctx.importer.getErrors();
    switch(ctx.responseState) {
        case RESPONSE_PREFIX:
            long totalRows = ctx.textParser.getLineCount();
            long importedRows = ctx.importer.getImportedRowCount();
            r.put('{').putQuoted("status").put(':').putQuoted("OK").put(',').putQuoted("location").put(':').encodeUtf8AndQuote(FileNameExtractorCharSequence.get(m.getName())).put(',').putQuoted("rowsRejected").put(':').put(totalRows - importedRows).put(',').putQuoted("rowsImported").put(':').put(importedRows).put(',').putQuoted("header").put(':').put(ctx.importer.isHeader()).put(',').putQuoted("columns").put(':').put('[');
            ctx.responseState = RESPONSE_COLUMN;
        // fall through
        case RESPONSE_COLUMN:
            for (; ctx.columnIndex < columnCount; ctx.columnIndex++) {
                RecordColumnMetadata cm = m.getColumnQuick(ctx.columnIndex);
                ImportedColumnMetadata im = importedMetadata.getQuick(ctx.columnIndex);
                r.bookmark();
                if (ctx.columnIndex > 0) {
                    r.put(',');
                }
                r.put('{').putQuoted("name").put(':').putQuoted(cm.getName()).put(',').putQuoted("type").put(':').putQuoted(ColumnType.nameOf(cm.getType())).put(',').putQuoted("size").put(':').put(ColumnType.sizeOf(cm.getType())).put(',').putQuoted("errors").put(':').put(errors.getQuick(ctx.columnIndex));
                if (im.pattern != null) {
                    r.put(',').putQuoted("pattern").put(':').putQuoted(im.pattern);
                }
                if (im.dateLocale != null) {
                    r.put(',').putQuoted("locale").put(':').putQuoted(im.dateLocale.getId());
                }
                r.put('}');
            }
            ctx.responseState = RESPONSE_SUFFIX;
        // fall through
        case RESPONSE_SUFFIX:
            r.bookmark();
            r.put(']').put('}');
            r.sendChunk();
            r.done();
            break;
        default:
            break;
    }
}

Example 2 with ImportedColumnMetadata

use of com.questdb.parser.ImportedColumnMetadata in project questdb by bluestreak01.

the class ImportHandler method resumeText.

private static void resumeText(ImportHandlerContext h, ChunkedResponse r) throws IOException {
    JournalMetadata m = h.importer.getJournalMetadata();
    LongList errors = h.importer.getErrors();
    ObjList<ImportedColumnMetadata> importedMetadata = h.importer.getImportedMetadata();
    final int columnCount = m.getColumnCount();
    switch(h.responseState) {
        case RESPONSE_PREFIX:
            sep(r);
            r.put('|');
            pad(r, TO_STRING_COL1_PAD, "Location:");
            pad(r, TO_STRING_COL2_PAD, m.getName());
            pad(r, TO_STRING_COL3_PAD, "Pattern");
            pad(r, TO_STRING_COL4_PAD, "Locale");
            pad(r, TO_STRING_COL5_PAD, "Errors").put(Misc.EOL);
            r.put('|');
            pad(r, TO_STRING_COL1_PAD, "Partition by");
            pad(r, TO_STRING_COL2_PAD, PartitionBy.toString(m.getPartitionBy()));
            pad(r, TO_STRING_COL3_PAD, "");
            pad(r, TO_STRING_COL4_PAD, "");
            pad(r, TO_STRING_COL5_PAD, "").put(Misc.EOL);
            sep(r);
            r.put('|');
            pad(r, TO_STRING_COL1_PAD, "Rows handled");
            pad(r, TO_STRING_COL2_PAD, h.textParser.getLineCount());
            pad(r, TO_STRING_COL3_PAD, "");
            pad(r, TO_STRING_COL4_PAD, "");
            pad(r, TO_STRING_COL5_PAD, "").put(Misc.EOL);
            r.put('|');
            pad(r, TO_STRING_COL1_PAD, "Rows imported");
            pad(r, TO_STRING_COL2_PAD, h.importer.getImportedRowCount());
            pad(r, TO_STRING_COL3_PAD, "");
            pad(r, TO_STRING_COL4_PAD, "");
            pad(r, TO_STRING_COL5_PAD, "").put(Misc.EOL);
            sep(r);
            h.responseState = RESPONSE_COLUMN;
        // fall through
        case RESPONSE_COLUMN:
            for (; h.columnIndex < columnCount; h.columnIndex++) {
                r.bookmark();
                r.put('|');
                pad(r, TO_STRING_COL1_PAD, h.columnIndex);
                col(r, m.getColumnQuick(h.columnIndex), importedMetadata.getQuick(h.columnIndex));
                pad(r, TO_STRING_COL5_PAD, errors.getQuick(h.columnIndex));
                r.put(Misc.EOL);
            }
            h.responseState = RESPONSE_SUFFIX;
        // fall through
        case RESPONSE_SUFFIX:
            r.bookmark();
            sep(r);
            r.sendChunk();
            r.done();
            break;
        default:
            break;
    }
}

Example 3 with ImportedColumnMetadata

use of com.questdb.parser.ImportedColumnMetadata in project questdb by bluestreak01.

the class PlainTextMetadataParser method calcTypes.

/**
 * Histogram contains counts for every probe that validates field. It is possible for multiple probes to validate same field.
 * It can happen because of two reasons.
 * <p>
 * probes are compatible, for example INT is compatible wth DOUBLE in a sense that DOUBLE probe will positively
 * validate every INT. If this the case we will use order of probes as priority. First probe wins
 * <p>
 * it is possible to have mixed types in same column, in which case column has to become string.
 * to establish if we have mixed column we check if probe count + blank values add up to total number of rows.
 */
private boolean calcTypes(int count, boolean setDefault) {
    boolean allStrings = true;
    int probeCount = typeProbeCollection.getProbeCount();
    for (int i = 0; i < fieldCount; i++) {
        int offset = i * probeCount;
        int blanks = _blanks.getQuick(i);
        boolean unprobed = true;
        ImportedColumnMetadata m = _metadata.getQuick(i);
        for (int k = 0; k < probeCount; k++) {
            if (_histogram.getQuick(k + offset) + blanks == count && blanks < count) {
                unprobed = false;
                TypeProbe probe = typeProbeCollection.getProbe(k);
                m.importedColumnType = probe.getType();
                m.pattern = probe.getFormat();
                m.dateFormat = probe.getDateFormat();
                m.dateLocale = probe.getDateLocale();
                if (allStrings) {
                    allStrings = false;
                }
                break;
            }
        }
        if (setDefault && unprobed) {
            m.importedColumnType = ColumnType.STRING;
        }
    }
    return allStrings;
}

Example 4 with ImportedColumnMetadata

use of com.questdb.parser.ImportedColumnMetadata in project questdb by bluestreak01.

the class PlainTextMetadataParser method onLineCount.

@Override
public void onLineCount(int count) {
    // if some fields come up as non-string after subtracting row - we have a header
    if ((calcTypes(count, true) && !calcTypes(count - 1, false)) || forceHeader) {
        // copy headers
        for (int i = 0; i < fieldCount; i++) {
            _metadata.getQuick(i).name = _headers.getQuick(i);
        }
        header = true;
    }
    // make up field names if there is no header
    if (!header) {
        for (int i = 0; i < fieldCount; i++) {
            tempSink.clear();
            tempSink.put('f').put(i);
            _metadata.getQuick(i).name = tempSink.toString();
        }
    }
    // override calculated types with user-supplied information
    if (schemaColumns.size() > 0) {
        for (int i = 0, k = _metadata.size(); i < k; i++) {
            ImportedColumnMetadata _m = _metadata.getQuick(i);
            ImportedColumnMetadata m = schemaColumns.get(_m.name);
            if (m != null) {
                m.copyTo(_m);
            }
        }
    }
}

Example 5 with ImportedColumnMetadata

use of com.questdb.parser.ImportedColumnMetadata in project questdb by bluestreak01.

the class PlainTextStoringParser method onFields.

@Override
public void onFields(int line, ObjList<DirectByteCharSequence> values, int hi) {
    boolean append = true;
    try {
        JournalEntryWriter w = writer.entryWriter();
        for (int i = 0; i < hi; i++) {
            if (values.getQuick(i).length() == 0) {
                continue;
            }
            try {
                DirectByteCharSequence charField;
                ImportedColumnMetadata m = metadata.getQuick(i);
                switch(m.importedColumnType) {
                    case ColumnType.STRING:
                        utf8Sink.clear();
                        charField = values.getQuick(i);
                        Chars.utf8Decode(charField.getLo(), charField.getHi(), utf8Sink);
                        w.putStr(i, (DirectBytes) utf8Sink);
                        break;
                    case ColumnType.DOUBLE:
                        w.putDouble(i, Numbers.parseDouble(values.getQuick(i)));
                        break;
                    case ColumnType.INT:
                        w.putInt(i, Numbers.parseInt(values.getQuick(i)));
                        break;
                    case ColumnType.FLOAT:
                        w.putFloat(i, Numbers.parseFloat(values.getQuick(i)));
                        break;
                    case ColumnType.DATE:
                        if (m.dateFormat != null && m.dateLocale != null) {
                            w.putDate(i, m.dateFormat.parse(values.getQuick(i), m.dateLocale));
                        } else {
                            throw NumericException.INSTANCE;
                        }
                        break;
                    case ColumnType.SYMBOL:
                        utf8Sink.clear();
                        charField = values.getQuick(i);
                        Chars.utf8Decode(charField.getLo(), charField.getHi(), utf8Sink);
                        w.putSym(i, utf8Sink);
                        break;
                    case ColumnType.LONG:
                        w.putLong(i, Numbers.parseLong(values.getQuick(i)));
                        break;
                    case ColumnType.BOOLEAN:
                        w.putBool(i, Chars.equalsIgnoreCase(values.getQuick(i), "true"));
                        break;
                    default:
                        break;
                }
            } catch (Exception e) {
                switch(atomicity) {
                    case ATOMICITY_STRICT:
                        LOG.info().$("Error at (").$(line).$(',').$(i).$(')').$();
                        throw new JournalRuntimeException("Error on line: " + line + ", col: " + i);
                    default:
                        errors.increment(i);
                        LOG.debug().$("Error at (").$(line).$(',').$(i).$(") as ").$(metadata.getQuick(i).importedColumnType).$(": ").$(e.getMessage()).$();
                        append = false;
                }
                break;
            }
        }
        if (append) {
            w.append();
        }
    } catch (JournalException e) {
        throw new JournalRuntimeException(e);
    }
}