Examples with InternalDatasetDetails org.apache.asterix.metadata.entities.InternalDatasetDetails used on opensource projects

Example 1 with InternalDatasetDetails

use of org.apache.asterix.metadata.entities.InternalDatasetDetails in project asterixdb by apache.

the class LangExpressionToPlanTranslator method translate.

public ILogicalPlan translate(Query expr, String outputDatasetName, ICompiledDmlStatement stmt, ILogicalOperator baseOp) throws AlgebricksException {
    MutableObject<ILogicalOperator> base = new MutableObject<>(new EmptyTupleSourceOperator());
    if (baseOp != null) {
        base = new MutableObject<>(baseOp);
    }
    Pair<ILogicalOperator, LogicalVariable> p = expr.accept(this, base);
    ArrayList<Mutable<ILogicalOperator>> globalPlanRoots = new ArrayList<>();
    ILogicalOperator topOp = p.first;
    List<LogicalVariable> liveVars = new ArrayList<>();
    VariableUtilities.getLiveVariables(topOp, liveVars);
    LogicalVariable unnestVar = liveVars.get(0);
    LogicalVariable resVar = unnestVar;
    if (outputDatasetName == null) {
        FileSplit outputFileSplit = metadataProvider.getOutputFile();
        if (outputFileSplit == null) {
            outputFileSplit = getDefaultOutputFileLocation(metadataProvider.getApplicationContext());
        }
        metadataProvider.setOutputFile(outputFileSplit);
        List<Mutable<ILogicalExpression>> writeExprList = new ArrayList<>(1);
        writeExprList.add(new MutableObject<>(new VariableReferenceExpression(resVar)));
        ResultSetSinkId rssId = new ResultSetSinkId(metadataProvider.getResultSetId());
        ResultSetDataSink sink = new ResultSetDataSink(rssId, null);
        DistributeResultOperator newTop = new DistributeResultOperator(writeExprList, sink);
        newTop.getInputs().add(new MutableObject<>(topOp));
        topOp = newTop;
        // Retrieve the Output RecordType (if any) and store it on
        // the DistributeResultOperator
        IAType outputRecordType = metadataProvider.findOutputRecordType();
        if (outputRecordType != null) {
            topOp.getAnnotations().put("output-record-type", outputRecordType);
        }
    } else {
        /**
             * add the collection-to-sequence right before the project,
             * because dataset only accept non-collection records
             */
        LogicalVariable seqVar = context.newVar();
        /**
             * This assign adds a marker function collection-to-sequence: if the input is a singleton collection, unnest
             * it; otherwise do nothing.
             */
        AssignOperator assignCollectionToSequence = new AssignOperator(seqVar, new MutableObject<>(new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.COLLECTION_TO_SEQUENCE), new MutableObject<>(new VariableReferenceExpression(resVar)))));
        assignCollectionToSequence.getInputs().add(new MutableObject<>(topOp.getInputs().get(0).getValue()));
        topOp.getInputs().get(0).setValue(assignCollectionToSequence);
        ProjectOperator projectOperator = (ProjectOperator) topOp;
        projectOperator.getVariables().set(0, seqVar);
        resVar = seqVar;
        DatasetDataSource targetDatasource = validateDatasetInfo(metadataProvider, stmt.getDataverseName(), stmt.getDatasetName());
        List<Integer> keySourceIndicator = ((InternalDatasetDetails) targetDatasource.getDataset().getDatasetDetails()).getKeySourceIndicator();
        ArrayList<LogicalVariable> vars = new ArrayList<>();
        ArrayList<Mutable<ILogicalExpression>> exprs = new ArrayList<>();
        List<Mutable<ILogicalExpression>> varRefsForLoading = new ArrayList<>();
        List<List<String>> partitionKeys = targetDatasource.getDataset().getPrimaryKeys();
        int numOfPrimaryKeys = partitionKeys.size();
        for (int i = 0; i < numOfPrimaryKeys; i++) {
            if (keySourceIndicator == null || keySourceIndicator.get(i).intValue() == 0) {
                // record part
                PlanTranslationUtil.prepareVarAndExpression(partitionKeys.get(i), resVar, vars, exprs, varRefsForLoading, context);
            } else {
                // meta part
                PlanTranslationUtil.prepareMetaKeyAccessExpression(partitionKeys.get(i), unnestVar, exprs, vars, varRefsForLoading, context);
            }
        }
        AssignOperator assign = new AssignOperator(vars, exprs);
        List<String> additionalFilteringField = DatasetUtil.getFilterField(targetDatasource.getDataset());
        List<LogicalVariable> additionalFilteringVars;
        List<Mutable<ILogicalExpression>> additionalFilteringAssignExpressions;
        List<Mutable<ILogicalExpression>> additionalFilteringExpressions = null;
        AssignOperator additionalFilteringAssign = null;
        if (additionalFilteringField != null) {
            additionalFilteringVars = new ArrayList<>();
            additionalFilteringAssignExpressions = new ArrayList<>();
            additionalFilteringExpressions = new ArrayList<>();
            PlanTranslationUtil.prepareVarAndExpression(additionalFilteringField, resVar, additionalFilteringVars, additionalFilteringAssignExpressions, additionalFilteringExpressions, context);
            additionalFilteringAssign = new AssignOperator(additionalFilteringVars, additionalFilteringAssignExpressions);
            additionalFilteringAssign.getInputs().add(new MutableObject<>(topOp));
            assign.getInputs().add(new MutableObject<>(additionalFilteringAssign));
        } else {
            assign.getInputs().add(new MutableObject<>(topOp));
        }
        Mutable<ILogicalExpression> varRef = new MutableObject<>(new VariableReferenceExpression(resVar));
        ILogicalOperator leafOperator;
        switch(stmt.getKind()) {
            case Statement.Kind.INSERT:
                leafOperator = translateInsert(targetDatasource, varRef, varRefsForLoading, additionalFilteringExpressions, assign, stmt);
                break;
            case Statement.Kind.UPSERT:
                leafOperator = translateUpsert(targetDatasource, varRef, varRefsForLoading, additionalFilteringExpressions, assign, additionalFilteringField, unnestVar, topOp, exprs, resVar, additionalFilteringAssign, stmt);
                break;
            case Statement.Kind.DELETE:
                leafOperator = translateDelete(targetDatasource, varRef, varRefsForLoading, additionalFilteringExpressions, assign);
                break;
            case Statement.Kind.CONNECT_FEED:
                leafOperator = translateConnectFeed(targetDatasource, varRef, varRefsForLoading, additionalFilteringExpressions, assign);
                break;
            case Statement.Kind.SUBSCRIBE_FEED:
                leafOperator = translateSubscribeFeed((CompiledSubscribeFeedStatement) stmt, targetDatasource, unnestVar, topOp, exprs, resVar, varRefsForLoading, varRef, assign, additionalFilteringField, additionalFilteringAssign, additionalFilteringExpressions);
                break;
            default:
                throw new AlgebricksException("Unsupported statement kind " + stmt.getKind());
        }
        topOp = leafOperator;
    }
    globalPlanRoots.add(new MutableObject<>(topOp));
    ILogicalPlan plan = new ALogicalPlanImpl(globalPlanRoots);
    eliminateSharedOperatorReferenceForPlan(plan);
    return plan;
}

Example 2 with InternalDatasetDetails

use of org.apache.asterix.metadata.entities.InternalDatasetDetails in project asterixdb by apache.

the class EquivalenceClassUtils method addEquivalenceClassesForPrimaryIndexAccess.

/**
     * Adds equivalent classes for primary index accesses, including unnest-map for
     * primary index access and data source scan through primary index ---
     * one equivalent class between a primary key variable and a record field-access expression.
     *
     * @param operator
     *            , the primary index access operator.
     * @param indexSearchVars
     *            , the returned variables from primary index access. The last variable
     *            is the record variable.
     * @param recordType
     *            , the record type of an index payload record.
     * @param metaRecordType
     *            , the type of a meta record associated with an index payload record.
     * @param dataset
     *            , the accessed dataset.
     * @param context
     *            , the optimization context.
     * @throws AlgebricksException
     */
@SuppressWarnings("unchecked")
public static void addEquivalenceClassesForPrimaryIndexAccess(ILogicalOperator operator, List<LogicalVariable> indexSearchVars, ARecordType recordType, ARecordType metaRecordType, Dataset dataset, IOptimizationContext context) throws AlgebricksException {
    if (dataset.getDatasetDetails().getDatasetType() != DatasetType.INTERNAL) {
        return;
    }
    InternalDatasetDetails datasetDetails = (InternalDatasetDetails) dataset.getDatasetDetails();
    List<List<String>> primaryKey = datasetDetails.getPrimaryKey();
    Map<String, Integer> fieldNameToIndexMap = new HashMap<String, Integer>();
    String[] fieldNames = recordType.getFieldNames();
    for (int fieldIndex = 0; fieldIndex < fieldNames.length; ++fieldIndex) {
        fieldNameToIndexMap.put(fieldNames[fieldIndex], fieldIndex);
    }
    boolean hasMeta = dataset.hasMetaPart();
    Map<String, Integer> metaFieldNameToIndexMap = new HashMap<>();
    if (hasMeta) {
        String[] metaFieldNames = metaRecordType.getFieldNames();
        for (int metaFieldIndex = 0; metaFieldIndex < metaFieldNames.length; ++metaFieldIndex) {
            metaFieldNameToIndexMap.put(metaFieldNames[metaFieldIndex], metaFieldIndex);
        }
    }
    List<Integer> keySourceIndicators = datasetDetails.getKeySourceIndicator();
    LogicalVariable recordVar = hasMeta ? indexSearchVars.get(indexSearchVars.size() - 2) : indexSearchVars.get(indexSearchVars.size() - 1);
    LogicalVariable metaRecordVar = hasMeta ? indexSearchVars.get(indexSearchVars.size() - 1) : null;
    for (int pkIndex = 0; pkIndex < primaryKey.size(); ++pkIndex) {
        LogicalVariable referredRecordVar = recordVar;
        String pkFieldName = primaryKey.get(pkIndex).get(0);
        int source = keySourceIndicators.get(pkIndex);
        Integer fieldIndexInRecord;
        if (source == 0) {
            // The field is from the main record.
            fieldIndexInRecord = fieldNameToIndexMap.get(pkFieldName);
        } else {
            // The field is from the auxiliary meta record.
            referredRecordVar = metaRecordVar;
            fieldIndexInRecord = metaFieldNameToIndexMap.get(pkFieldName);
        }
        LogicalVariable var = indexSearchVars.get(pkIndex);
        ILogicalExpression expr = new ScalarFunctionCallExpression(FunctionUtil.getFunctionInfo(BuiltinFunctions.FIELD_ACCESS_BY_INDEX), new MutableObject<ILogicalExpression>(new VariableReferenceExpression(referredRecordVar)), new MutableObject<ILogicalExpression>(new ConstantExpression(new AsterixConstantValue(new AInt32(fieldIndexInRecord)))));
        EquivalenceClass equivClass = new EquivalenceClass(Collections.singletonList(var), var, Collections.singletonList(expr));
        Map<LogicalVariable, EquivalenceClass> equivalenceMap = context.getEquivalenceClassMap(operator);
        if (equivalenceMap == null) {
            equivalenceMap = new HashMap<LogicalVariable, EquivalenceClass>();
            context.putEquivalenceClassMap(operator, equivalenceMap);
        }
        equivalenceMap.put(var, equivClass);
    }
}

Example 3 with InternalDatasetDetails

use of org.apache.asterix.metadata.entities.InternalDatasetDetails in project asterixdb by apache.

the class IndexTupleTranslatorTest method test.

@Test
public void test() throws MetadataException, IOException {
    Integer[] indicators = { 0, 1, null };
    for (Integer indicator : indicators) {
        Map<String, String> compactionPolicyProperties = new HashMap<>();
        compactionPolicyProperties.put("max-mergable-component-size", "1073741824");
        compactionPolicyProperties.put("max-tolerance-component-count", "3");
        InternalDatasetDetails details = new InternalDatasetDetails(FileStructure.BTREE, PartitioningStrategy.HASH, Collections.singletonList(Collections.singletonList("row_id")), Collections.singletonList(Collections.singletonList("row_id")), indicator == null ? null : Collections.singletonList(indicator), Collections.singletonList(BuiltinType.AINT64), false, Collections.emptyList(), false);
        Dataset dataset = new Dataset("test", "d1", "foo", "LogType", "CB", "MetaType", "DEFAULT_NG_ALL_NODES", "prefix", compactionPolicyProperties, details, Collections.emptyMap(), DatasetType.INTERNAL, 115, 0);
        Index index = new Index("test", "d1", "i1", IndexType.BTREE, Collections.singletonList(Collections.singletonList("row_id")), indicator == null ? null : Collections.singletonList(indicator), Collections.singletonList(BuiltinType.AINT64), -1, false, false, 0);
        MetadataNode mockMetadataNode = mock(MetadataNode.class);
        when(mockMetadataNode.getDatatype(any(), anyString(), anyString())).thenReturn(new Datatype("test", "d1", new ARecordType("", new String[] { "row_id" }, new IAType[] { BuiltinType.AINT64 }, true), true));
        when(mockMetadataNode.getDataset(any(), anyString(), anyString())).thenReturn(dataset);
        IndexTupleTranslator idxTranslator = new IndexTupleTranslator(null, mockMetadataNode, true);
        ITupleReference tuple = idxTranslator.getTupleFromMetadataEntity(index);
        Index deserializedIndex = idxTranslator.getMetadataEntityFromTuple(tuple);
        if (indicator == null) {
            Assert.assertEquals(Collections.singletonList(new Integer(0)), deserializedIndex.getKeyFieldSourceIndicators());
        } else {
            Assert.assertEquals(index.getKeyFieldSourceIndicators(), deserializedIndex.getKeyFieldSourceIndicators());
        }
    }
}

Example 4 with InternalDatasetDetails

use of org.apache.asterix.metadata.entities.InternalDatasetDetails in project asterixdb by apache.

the class MetadataNode method addDataset.

@Override
public void addDataset(JobId jobId, Dataset dataset) throws MetadataException, RemoteException {
    try {
        // Insert into the 'dataset' dataset.
        DatasetTupleTranslator tupleReaderWriter = tupleTranslatorProvider.getDatasetTupleTranslator(true);
        ITupleReference datasetTuple = tupleReaderWriter.getTupleFromMetadataEntity(dataset);
        insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.DATASET_DATASET, datasetTuple);
        if (dataset.getDatasetType() == DatasetType.INTERNAL) {
            // Add the primary index for the dataset.
            InternalDatasetDetails id = (InternalDatasetDetails) dataset.getDatasetDetails();
            Index primaryIndex = new Index(dataset.getDataverseName(), dataset.getDatasetName(), dataset.getDatasetName(), IndexType.BTREE, id.getPrimaryKey(), id.getKeySourceIndicator(), id.getPrimaryKeyType(), false, true, dataset.getPendingOp());
            addIndex(jobId, primaryIndex);
        }
    } catch (HyracksDataException e) {
        if (e.getComponent().equals(ErrorCode.HYRACKS) && e.getErrorCode() == ErrorCode.DUPLICATE_KEY) {
            throw new MetadataException("A dataset with this name " + dataset.getDatasetName() + " already exists in dataverse '" + dataset.getDataverseName() + "'.", e);
        } else {
            throw new MetadataException(e);
        }
    } catch (ACIDException e) {
        throw new MetadataException(e);
    }
}

Example 5 with InternalDatasetDetails

use of org.apache.asterix.metadata.entities.InternalDatasetDetails in project asterixdb by apache.

the class LogMarkerTest method testInsertWithSnapshot.

@Test
public void testInsertWithSnapshot() {
    try {
        TestNodeController nc = new TestNodeController(null, false);
        nc.init();
        StorageComponentProvider storageManager = new StorageComponentProvider();
        List<List<String>> partitioningKeys = new ArrayList<>();
        partitioningKeys.add(Collections.singletonList("key"));
        Dataset dataset = new Dataset(DATAVERSE_NAME, DATASET_NAME, DATAVERSE_NAME, DATA_TYPE_NAME, NODE_GROUP_NAME, null, null, new InternalDatasetDetails(null, PartitioningStrategy.HASH, partitioningKeys, null, null, null, false, null, false), null, DatasetType.INTERNAL, DATASET_ID, 0);
        try {
            nc.createPrimaryIndex(dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, storageManager, KEY_INDEXES, KEY_INDICATORS_LIST);
            IHyracksTaskContext ctx = nc.createTestContext(true);
            nc.newJobId();
            ITransactionContext txnCtx = nc.getTransactionManager().getTransactionContext(nc.getTxnJobId(), true);
            LSMInsertDeleteOperatorNodePushable insertOp = nc.getInsertPipeline(ctx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, KEY_INDEXES, KEY_INDICATORS_LIST, storageManager).getLeft();
            insertOp.open();
            TupleGenerator tupleGenerator = new TupleGenerator(RECORD_TYPE, META_TYPE, KEY_INDEXES, KEY_INDICATORS, RECORD_GEN_FUNCTION, UNIQUE_RECORD_FIELDS, META_GEN_FUNCTION, UNIQUE_META_FIELDS);
            VSizeFrame frame = new VSizeFrame(ctx);
            VSizeFrame marker = new VSizeFrame(ctx);
            FrameTupleAppender tupleAppender = new FrameTupleAppender(frame);
            long markerId = 0L;
            for (int j = 0; j < NUM_OF_RECORDS; j++) {
                if (j % SNAPSHOT_SIZE == 0) {
                    marker.reset();
                    marker.getBuffer().put(MessagingFrameTupleAppender.MARKER_MESSAGE);
                    marker.getBuffer().putLong(markerId);
                    marker.getBuffer().flip();
                    markerId++;
                    TaskUtil.putInSharedMap(HyracksConstants.KEY_MESSAGE, marker, ctx);
                    tupleAppender.flush(insertOp);
                }
                ITupleReference tuple = tupleGenerator.next();
                DataflowUtils.addTupleToFrame(tupleAppender, tuple, insertOp);
            }
            if (tupleAppender.getTupleCount() > 0) {
                tupleAppender.write(insertOp, true);
            }
            insertOp.close();
            nc.getTransactionManager().completedTransaction(txnCtx, DatasetId.NULL, -1, true);
            IIndexDataflowHelper dataflowHelper = nc.getPrimaryIndexDataflowHelper(dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, storageManager, KEY_INDEXES, KEY_INDICATORS_LIST);
            dataflowHelper.open();
            LSMBTree btree = (LSMBTree) dataflowHelper.getIndexInstance();
            LongPointable longPointable = LongPointable.FACTORY.createPointable();
            ComponentMetadataUtil.get(btree, ComponentMetadataUtil.MARKER_LSN_KEY, longPointable);
            long lsn = longPointable.getLong();
            int numOfMarkers = 0;
            LogReader logReader = (LogReader) nc.getTransactionSubsystem().getLogManager().getLogReader(false);
            long expectedMarkerId = markerId - 1;
            while (lsn >= 0) {
                numOfMarkers++;
                ILogRecord logRecord = logReader.read(lsn);
                lsn = logRecord.getPreviousMarkerLSN();
                long logMarkerId = logRecord.getMarker().getLong();
                Assert.assertEquals(expectedMarkerId, logMarkerId);
                expectedMarkerId--;
            }
            logReader.close();
            dataflowHelper.close();
            Assert.assertEquals(markerId, numOfMarkers);
            nc.newJobId();
            TestTupleCounterFrameWriter countOp = create(nc.getSearchOutputDesc(KEY_TYPES, RECORD_TYPE, META_TYPE), Collections.emptyList(), Collections.emptyList(), false);
            IPushRuntime emptyTupleOp = nc.getFullScanPipeline(countOp, ctx, dataset, KEY_TYPES, RECORD_TYPE, META_TYPE, new NoMergePolicyFactory(), null, null, KEY_INDEXES, KEY_INDICATORS_LIST, storageManager);
            emptyTupleOp.open();
            emptyTupleOp.close();
            Assert.assertEquals(NUM_OF_RECORDS, countOp.getCount());
        } finally {
            nc.deInit();
        }
    } catch (Throwable e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}