Examples with JournalMetadata com.questdb.store.factory.configuration.JournalMetadata used on opensource projects

Example 1 with JournalMetadata

use of com.questdb.store.factory.configuration.JournalMetadata in project questdb by bluestreak01.

the class PlainTextLexerTest method testImport.

@Test
public void testImport() throws Exception {
    String file = this.getClass().getResource("/csv/test-import.csv").getFile();
    ImportManager.importFile(getFactory(), file, PlainTextDelimiter.CSV, null, false);
    String location = "test-import.csv";
    Assert.assertEquals(JournalConfiguration.EXISTS, getFactory().getConfiguration().exists(location));
    try (Journal r = getFactory().reader(location)) {
        JournalMetadata m = r.getMetadata();
        Assert.assertEquals(10, m.getColumnCount());
        Assert.assertEquals(ColumnType.STRING, m.getColumnQuick(0).type);
        Assert.assertEquals(ColumnType.INT, m.getColumnQuick(1).type);
        Assert.assertEquals(ColumnType.INT, m.getColumnQuick(2).type);
        Assert.assertEquals(ColumnType.DOUBLE, m.getColumnQuick(3).type);
        Assert.assertEquals(ColumnType.DATE, m.getColumnQuick(4).type);
        Assert.assertEquals(ColumnType.DATE, m.getColumnQuick(5).type);
        Assert.assertEquals(ColumnType.DATE, m.getColumnQuick(6).type);
        Assert.assertEquals(ColumnType.STRING, m.getColumnQuick(7).type);
        Assert.assertEquals(ColumnType.BOOLEAN, m.getColumnQuick(8).type);
        Assert.assertEquals(ColumnType.LONG, m.getColumnQuick(9).type);
    }
    File actual = new File(getFactory().getConfiguration().getJournalBase(), "exp.csv");
    File expected = new File(this.getClass().getResource("/csv/test-export-expected.csv").getFile());
    try (RecordSource rs = compile("'" + location + "'")) {
        ExportManager.export(rs, getFactory(), actual, PlainTextDelimiter.CSV);
        TestUtils.assertEquals(expected, actual);
    }
}

Example 2 with JournalMetadata

use of com.questdb.store.factory.configuration.JournalMetadata in project questdb by bluestreak01.

the class PlainTextLexerTest method testImportMalformedQuote.

@Test
public void testImportMalformedQuote() throws Exception {
    String file = this.getClass().getResource("/csv/test-import-malformed.csv").getFile();
    ImportManager.importFile(getFactory(), file, PlainTextDelimiter.CSV, null, false);
    String location = "test-import-malformed.csv";
    Assert.assertEquals(JournalConfiguration.EXISTS, factoryContainer.getConfiguration().exists(location));
    try (Journal r = getFactory().reader(location)) {
        JournalMetadata m = r.getMetadata();
        Assert.assertEquals(10, m.getColumnCount());
        Assert.assertEquals(ColumnType.STRING, m.getColumnQuick(0).type);
        Assert.assertEquals(ColumnType.STRING, m.getColumnQuick(1).type);
        Assert.assertEquals(ColumnType.INT, m.getColumnQuick(2).type);
        Assert.assertEquals(ColumnType.DOUBLE, m.getColumnQuick(3).type);
        Assert.assertEquals(ColumnType.DATE, m.getColumnQuick(4).type);
        Assert.assertEquals(ColumnType.DATE, m.getColumnQuick(5).type);
        Assert.assertEquals(ColumnType.DATE, m.getColumnQuick(6).type);
        Assert.assertEquals(ColumnType.STRING, m.getColumnQuick(7).type);
        Assert.assertEquals(ColumnType.BOOLEAN, m.getColumnQuick(8).type);
        Assert.assertEquals(ColumnType.LONG, m.getColumnQuick(9).type);
    }
    File actual = new File(factoryContainer.getConfiguration().getJournalBase(), "exp.csv");
    File expected = new File(this.getClass().getResource("/csv/test-import-malformed-expected.csv").getFile());
    try (RecordSource rs = compile("'" + location + "'")) {
        ExportManager.export(rs, getFactory(), actual, PlainTextDelimiter.CSV);
        TestUtils.assertEquals(expected, actual);
    }
}

Example 3 with JournalMetadata

use of com.questdb.store.factory.configuration.JournalMetadata in project questdb by bluestreak01.

the class ImportHandler method resumeJson.

private static void resumeJson(ImportHandlerContext ctx, ChunkedResponse r) throws DisconnectedChannelException, SlowWritableChannelException {
    final JournalMetadata m = ctx.importer.getJournalMetadata();
    final ObjList<ImportedColumnMetadata> importedMetadata = ctx.importer.getImportedMetadata();
    final int columnCount = m.getColumnCount();
    final LongList errors = ctx.importer.getErrors();
    switch(ctx.responseState) {
        case RESPONSE_PREFIX:
            long totalRows = ctx.textParser.getLineCount();
            long importedRows = ctx.importer.getImportedRowCount();
            r.put('{').putQuoted("status").put(':').putQuoted("OK").put(',').putQuoted("location").put(':').encodeUtf8AndQuote(FileNameExtractorCharSequence.get(m.getName())).put(',').putQuoted("rowsRejected").put(':').put(totalRows - importedRows).put(',').putQuoted("rowsImported").put(':').put(importedRows).put(',').putQuoted("header").put(':').put(ctx.importer.isHeader()).put(',').putQuoted("columns").put(':').put('[');
            ctx.responseState = RESPONSE_COLUMN;
        // fall through
        case RESPONSE_COLUMN:
            for (; ctx.columnIndex < columnCount; ctx.columnIndex++) {
                RecordColumnMetadata cm = m.getColumnQuick(ctx.columnIndex);
                ImportedColumnMetadata im = importedMetadata.getQuick(ctx.columnIndex);
                r.bookmark();
                if (ctx.columnIndex > 0) {
                    r.put(',');
                }
                r.put('{').putQuoted("name").put(':').putQuoted(cm.getName()).put(',').putQuoted("type").put(':').putQuoted(ColumnType.nameOf(cm.getType())).put(',').putQuoted("size").put(':').put(ColumnType.sizeOf(cm.getType())).put(',').putQuoted("errors").put(':').put(errors.getQuick(ctx.columnIndex));
                if (im.pattern != null) {
                    r.put(',').putQuoted("pattern").put(':').putQuoted(im.pattern);
                }
                if (im.dateLocale != null) {
                    r.put(',').putQuoted("locale").put(':').putQuoted(im.dateLocale.getId());
                }
                r.put('}');
            }
            ctx.responseState = RESPONSE_SUFFIX;
        // fall through
        case RESPONSE_SUFFIX:
            r.bookmark();
            r.put(']').put('}');
            r.sendChunk();
            r.done();
            break;
        default:
            break;
    }
}

Example 4 with JournalMetadata

use of com.questdb.store.factory.configuration.JournalMetadata in project questdb by bluestreak01.

the class ImportHandler method resumeText.

private static void resumeText(ImportHandlerContext h, ChunkedResponse r) throws IOException {
    JournalMetadata m = h.importer.getJournalMetadata();
    LongList errors = h.importer.getErrors();
    ObjList<ImportedColumnMetadata> importedMetadata = h.importer.getImportedMetadata();
    final int columnCount = m.getColumnCount();
    switch(h.responseState) {
        case RESPONSE_PREFIX:
            sep(r);
            r.put('|');
            pad(r, TO_STRING_COL1_PAD, "Location:");
            pad(r, TO_STRING_COL2_PAD, m.getName());
            pad(r, TO_STRING_COL3_PAD, "Pattern");
            pad(r, TO_STRING_COL4_PAD, "Locale");
            pad(r, TO_STRING_COL5_PAD, "Errors").put(Misc.EOL);
            r.put('|');
            pad(r, TO_STRING_COL1_PAD, "Partition by");
            pad(r, TO_STRING_COL2_PAD, PartitionBy.toString(m.getPartitionBy()));
            pad(r, TO_STRING_COL3_PAD, "");
            pad(r, TO_STRING_COL4_PAD, "");
            pad(r, TO_STRING_COL5_PAD, "").put(Misc.EOL);
            sep(r);
            r.put('|');
            pad(r, TO_STRING_COL1_PAD, "Rows handled");
            pad(r, TO_STRING_COL2_PAD, h.textParser.getLineCount());
            pad(r, TO_STRING_COL3_PAD, "");
            pad(r, TO_STRING_COL4_PAD, "");
            pad(r, TO_STRING_COL5_PAD, "").put(Misc.EOL);
            r.put('|');
            pad(r, TO_STRING_COL1_PAD, "Rows imported");
            pad(r, TO_STRING_COL2_PAD, h.importer.getImportedRowCount());
            pad(r, TO_STRING_COL3_PAD, "");
            pad(r, TO_STRING_COL4_PAD, "");
            pad(r, TO_STRING_COL5_PAD, "").put(Misc.EOL);
            sep(r);
            h.responseState = RESPONSE_COLUMN;
        // fall through
        case RESPONSE_COLUMN:
            for (; h.columnIndex < columnCount; h.columnIndex++) {
                r.bookmark();
                r.put('|');
                pad(r, TO_STRING_COL1_PAD, h.columnIndex);
                col(r, m.getColumnQuick(h.columnIndex), importedMetadata.getQuick(h.columnIndex));
                pad(r, TO_STRING_COL5_PAD, errors.getQuick(h.columnIndex));
                r.put(Misc.EOL);
            }
            h.responseState = RESPONSE_SUFFIX;
        // fall through
        case RESPONSE_SUFFIX:
            r.bookmark();
            sep(r);
            r.sendChunk();
            r.done();
            break;
        default:
            break;
    }
}

Example 5 with JournalMetadata

use of com.questdb.store.factory.configuration.JournalMetadata in project questdb by bluestreak01.

the class QueryCompiler method compileJournal0.

@SuppressWarnings("ConstantConditions")
private RecordSource compileJournal0(QueryModel model, Factory factory) throws ParserException {
    applyLimit(model);
    RecordMetadata metadata = model.getMetadata();
    JournalMetadata journalMetadata;
    if (metadata == null) {
        metadata = model.collectJournalMetadata(factory);
    }
    if (metadata instanceof JournalMetadata) {
        journalMetadata = (JournalMetadata) metadata;
    } else {
        throw QueryError.$(0, "Internal error: invalid metadata");
    }
    if (model.getAlias() != null) {
        journalMetadata.setAlias(model.getAlias().token);
    } else {
        journalMetadata.setAlias(QueryModel.stripMarker(model.getJournalName().token));
    }
    PartitionSource ps = new JournalPartitionSource(journalMetadata, true);
    RowSource rs = null;
    String latestByCol = null;
    RecordColumnMetadata latestByMetadata = null;
    ExprNode latestByNode = null;
    if (model.getLatestBy() != null) {
        latestByNode = model.getLatestBy();
        if (latestByNode.type != ExprNode.LITERAL) {
            throw QueryError.$(latestByNode.position, "Column name expected");
        }
        latestByCol = model.translateAlias(latestByNode.token).toString();
        int colIndex = journalMetadata.getColumnIndexQuiet(latestByCol);
        if (colIndex == -1) {
            throw QueryError.invalidColumn(latestByNode.position, latestByNode.token);
        }
        latestByMetadata = journalMetadata.getColumnQuick(colIndex);
        int type = latestByMetadata.getType();
        if (type != ColumnType.SYMBOL && type != ColumnType.STRING && type != ColumnType.INT && type != ColumnType.LONG) {
            throw QueryError.position(latestByNode.position).$("Expected symbol, string, int or long column, found: ").$(ColumnType.nameOf(type)).$();
        }
        if (!latestByMetadata.isIndexed()) {
            throw QueryError.position(latestByNode.position).$("Column is not indexed").$();
        }
    }
    ExprNode where = model.getWhereClause();
    if (where != null) {
        IntrinsicModel im = queryFilterAnalyser.extract(model, where, journalMetadata, latestByCol, getTimestampIndexQuiet(model.getTimestamp(), journalMetadata));
        VirtualColumn filter = im.filter != null ? virtualColumnBuilder.createVirtualColumn(model, im.filter, journalMetadata) : null;
        if (filter != null) {
            if (filter.getType() != ColumnType.BOOLEAN) {
                throw QueryError.$(im.filter.position, "Boolean expression expected");
            }
            if (filter.isConstant()) {
                if (filter.getBool(null)) {
                    // constant TRUE, no filtering needed
                    filter = null;
                } else {
                    im.intrinsicValue = IntrinsicValue.FALSE;
                }
            }
        }
        if (im.intrinsicValue == IntrinsicValue.FALSE) {
            ps = new NoOpJournalPartitionSource(journalMetadata);
        } else {
            if (im.intervals != null) {
                ps = new MultiIntervalPartitionSource(ps, im.intervals);
            }
            if (latestByCol == null) {
                if (im.keyColumn != null) {
                    switch(journalMetadata.getColumn(im.keyColumn).getType()) {
                        case ColumnType.SYMBOL:
                            rs = buildRowSourceForSym(im);
                            break;
                        case ColumnType.STRING:
                            rs = buildRowSourceForStr(im);
                            break;
                        case ColumnType.INT:
                            rs = buildRowSourceForInt(im);
                            break;
                        case ColumnType.LONG:
                            rs = buildRowSourceForLong(im);
                            break;
                        default:
                            break;
                    }
                }
                if (filter != null) {
                    rs = new FilteredRowSource(rs == null ? new AllRowSource() : rs, filter);
                }
            } else {
                if (im.keyColumn != null && im.keyValuesIsLambda) {
                    int lambdaColIndex;
                    RecordSource lambda = compileSourceInternal(factory, im.keyValues.get(0));
                    RecordMetadata m = lambda.getMetadata();
                    switch(m.getColumnCount()) {
                        case 0:
                            Misc.free(lambda);
                            throw QueryError.$(im.keyValuePositions.getQuick(0), "Query must select at least one column");
                        case 1:
                            lambdaColIndex = 0;
                            break;
                        default:
                            lambdaColIndex = m.getColumnIndexQuiet(latestByCol);
                            if (lambdaColIndex == -1) {
                                Misc.free(lambda);
                                throw QueryError.$(im.keyValuePositions.getQuick(0), "Ambiguous column names in lambda query. Specify select clause");
                            }
                            break;
                    }
                    int lambdaColType = m.getColumnQuick(lambdaColIndex).getType();
                    mutableSig.setParamCount(2).setName("").paramType(0, latestByMetadata.getType(), true).paramType(1, lambdaColType, false);
                    LatestByLambdaRowSourceFactory fact = LAMBDA_ROW_SOURCE_FACTORIES.get(mutableSig);
                    if (fact != null) {
                        rs = fact.newInstance(latestByCol, lambda, lambdaColIndex, filter);
                    } else {
                        Misc.free(lambda);
                        throw QueryError.$(im.keyValuePositions.getQuick(0), "Mismatched types");
                    }
                } else {
                    switch(latestByMetadata.getType()) {
                        case ColumnType.SYMBOL:
                            if (im.keyColumn != null) {
                                rs = new KvIndexSymListHeadRowSource(latestByCol, new CharSequenceHashSet(im.keyValues), filter);
                            } else {
                                rs = new KvIndexSymAllHeadRowSource(latestByCol, filter);
                            }
                            break;
                        case ColumnType.STRING:
                            if (im.keyColumn != null) {
                                rs = new KvIndexStrListHeadRowSource(latestByCol, new CharSequenceHashSet(im.keyValues), filter);
                            } else {
                                Misc.free(rs);
                                throw QueryError.$(latestByNode.position, "Filter on string column expected");
                            }
                            break;
                        case ColumnType.LONG:
                            if (im.keyColumn != null) {
                                rs = new KvIndexLongListHeadRowSource(latestByCol, toLongHashSet(im), filter);
                            } else {
                                Misc.free(rs);
                                throw QueryError.$(latestByNode.position, "Filter on long column expected");
                            }
                            break;
                        case ColumnType.INT:
                            if (im.keyColumn != null) {
                                rs = new KvIndexIntListHeadRowSource(latestByCol, toIntHashSet(im), filter);
                            } else {
                                Misc.free(rs);
                                throw QueryError.$(latestByNode.position, "Filter on int column expected");
                            }
                            break;
                        default:
                            break;
                    }
                }
            }
        }
    } else if (latestByCol != null) {
        switch(latestByMetadata.getType()) {
            case ColumnType.SYMBOL:
                rs = new KvIndexSymAllHeadRowSource(latestByCol, null);
                break;
            default:
                Misc.free(rs);
                throw QueryError.$(latestByNode.position, "Only SYM columns can be used here without filter");
        }
    }
    // check for case of simple "select count() from tab"
    if (rs == null && model.getColumns().size() == 1) {
        QueryColumn qc = model.getColumns().getQuick(0);
        if ("count".equals(qc.getAst().token) && qc.getAst().paramCount == 0) {
            // remove order clause
            model.getOrderBy().clear();
            // remove columns so that there is no wrapping of result source
            model.getColumns().clear();
            return new CountRecordSource(qc.getAlias() == null ? "count" : qc.getAlias(), ps);
        }
    }
    RecordSource recordSource = new JournalRecordSource(ps, rs == null ? new AllRowSource() : rs);
    if (QueryModel.hasMarker(model.getJournalName().token)) {
        return new NoRowIdRecordSource().of(recordSource);
    }
    return recordSource;
}