Examples with TypeProbe com.questdb.parser.typeprobe.TypeProbe used on opensource projects

Example 1 with TypeProbe

use of com.questdb.parser.typeprobe.TypeProbe in project questdb by bluestreak01.

the class PlainTextMetadataParser method calcTypes.

/**
 * Histogram contains counts for every probe that validates field. It is possible for multiple probes to validate same field.
 * It can happen because of two reasons.
 * <p>
 * probes are compatible, for example INT is compatible wth DOUBLE in a sense that DOUBLE probe will positively
 * validate every INT. If this the case we will use order of probes as priority. First probe wins
 * <p>
 * it is possible to have mixed types in same column, in which case column has to become string.
 * to establish if we have mixed column we check if probe count + blank values add up to total number of rows.
 */
private boolean calcTypes(int count, boolean setDefault) {
    boolean allStrings = true;
    int probeCount = typeProbeCollection.getProbeCount();
    for (int i = 0; i < fieldCount; i++) {
        int offset = i * probeCount;
        int blanks = _blanks.getQuick(i);
        boolean unprobed = true;
        ImportedColumnMetadata m = _metadata.getQuick(i);
        for (int k = 0; k < probeCount; k++) {
            if (_histogram.getQuick(k + offset) + blanks == count && blanks < count) {
                unprobed = false;
                TypeProbe probe = typeProbeCollection.getProbe(k);
                m.importedColumnType = probe.getType();
                m.pattern = probe.getFormat();
                m.dateFormat = probe.getDateFormat();
                m.dateLocale = probe.getDateLocale();
                if (allStrings) {
                    allStrings = false;
                }
                break;
            }
        }
        if (setDefault && unprobed) {
            m.importedColumnType = ColumnType.STRING;
        }
    }
    return allStrings;
}

Example 2 with TypeProbe

use of com.questdb.parser.typeprobe.TypeProbe in project questdb by bluestreak01.

the class PlainTextMetadataParser method onFields.

@Override
public void onFields(int line, ObjList<DirectByteCharSequence> values, int hi) {
    // keep first line in case its a header
    if (line == 0) {
        stashPossibleHeader(values, hi);
    }
    int count = typeProbeCollection.getProbeCount();
    for (int i = 0; i < hi; i++) {
        DirectByteCharSequence cs = values.getQuick(i);
        if (cs.length() == 0) {
            _blanks.increment(i);
        }
        int offset = i * count;
        for (int k = 0; k < count; k++) {
            TypeProbe probe = typeProbeCollection.getProbe(k);
            if (probe.probe(cs)) {
                _histogram.increment(k + offset);
            }
        }
    }
}