Examples with ConnectorManager org.teiid.dqp.internal.datamgr.ConnectorManager used on opensource projects

Example 6 with ConnectorManager

use of org.teiid.dqp.internal.datamgr.ConnectorManager in project teiid by teiid.

the class TestCompositeVDB method testRoleInherit.

@Test
public void testRoleInherit() throws Exception {
    VDBRepository repo = new VDBRepository();
    repo.setSystemStore(RealMetadataFactory.example1Cached().getMetadataStore());
    repo.setSystemFunctionManager(RealMetadataFactory.SFM);
    MetadataStore metadataStore = new MetadataStore();
    RealMetadataFactory.createPhysicalModel("x", metadataStore);
    VDBMetaData vdb = createVDBMetadata(metadataStore, "bqt");
    DataPolicyMetadata dpm = new DataPolicyMetadata();
    dpm.setName("x");
    dpm.setGrantAll(true);
    vdb.addDataPolicy(dpm);
    ConnectorManagerRepository cmr = new ConnectorManagerRepository();
    cmr.addConnectorManager("x", new ConnectorManager("y", "z"));
    repo.addVDB(vdb, metadataStore, null, null, cmr);
    metadataStore = new MetadataStore();
    RealMetadataFactory.createPhysicalModel("y", metadataStore);
    vdb = createVDBMetadata(metadataStore, "ex");
    VDBImportMetadata vdbImport = new VDBImportMetadata();
    vdbImport.setName("bqt");
    vdbImport.setVersion("1");
    vdbImport.setImportDataPolicies(true);
    vdb.getVDBImports().add(vdbImport);
    repo.addVDB(vdb, metadataStore, null, null, new ConnectorManagerRepository());
    vdb = repo.getLiveVDB("ex");
    assertEquals(1, vdb.getDataPolicyMap().get("x").getSchemas().size());
}

Example 7 with ConnectorManager

use of org.teiid.dqp.internal.datamgr.ConnectorManager in project teiid by teiid.

the class TestCompositeVDB method testDeepNesting.

@Test
public void testDeepNesting() throws Exception {
    VDBRepository repo = new VDBRepository();
    repo.setSystemStore(RealMetadataFactory.example1Cached().getMetadataStore());
    repo.setSystemFunctionManager(RealMetadataFactory.SFM);
    MetadataStore metadataStore = new MetadataStore();
    RealMetadataFactory.createPhysicalModel("x", metadataStore);
    VDBMetaData vdb = createVDBMetadata(metadataStore, "bqt");
    ConnectorManagerRepository cmr = new ConnectorManagerRepository();
    cmr.addConnectorManager("x", new ConnectorManager("y", "z"));
    repo.addVDB(vdb, metadataStore, null, null, cmr);
    metadataStore = new MetadataStore();
    RealMetadataFactory.createPhysicalModel("y", metadataStore);
    vdb = createVDBMetadata(metadataStore, "ex");
    VDBImportMetadata vdbImport = new VDBImportMetadata();
    vdbImport.setName("bqt");
    vdbImport.setVersion("1");
    vdb.getVDBImports().add(vdbImport);
    repo.addVDB(vdb, metadataStore, null, null, new ConnectorManagerRepository());
    VDBMetaData vdbInstance = repo.getVDB("ex", "1");
    assertTrue(!vdbInstance.getAttachment(ConnectorManagerRepository.class).getConnectorManagers().isEmpty());
    metadataStore = new MetadataStore();
    RealMetadataFactory.createPhysicalModel("z", metadataStore);
    vdb = createVDBMetadata(metadataStore, "ex1");
    vdbImport = new VDBImportMetadata();
    vdbImport.setName("ex");
    vdbImport.setVersion("1");
    vdb.getVDBImports().add(vdbImport);
    repo.addVDB(vdb, metadataStore, null, null, new ConnectorManagerRepository());
    vdbInstance = repo.getVDB("ex1", "1");
    assertTrue(!vdbInstance.getAttachment(ConnectorManagerRepository.class).getConnectorManagers().isEmpty());
}

Example 8 with ConnectorManager

use of org.teiid.dqp.internal.datamgr.ConnectorManager in project teiid by teiid.

the class DataTierManagerImpl method registerRequest.

public TupleSource registerRequest(CommandContext context, Command command, String modelName, final RegisterRequestParameter parameterObject) throws TeiidComponentException, TeiidProcessingException {
    RequestWorkItem workItem = context.getWorkItem();
    Assertion.isNotNull(workItem);
    if (CoreConstants.SYSTEM_MODEL.equals(modelName) || CoreConstants.SYSTEM_ADMIN_MODEL.equals(modelName)) {
        return processSystemQuery(context, command, workItem.getDqpWorkContext());
    }
    AtomicRequestMessage aqr = createRequest(workItem, command, modelName, parameterObject.connectorBindingId, parameterObject.nodeID);
    aqr.setCommandContext(context);
    if (parameterObject.fetchSize > 0) {
        aqr.setFetchSize(2 * parameterObject.fetchSize);
    }
    if (parameterObject.limit > 0) {
        aqr.setFetchSize(Math.min(parameterObject.limit, aqr.getFetchSize()));
    }
    aqr.setCopyStreamingLobs(parameterObject.copyStreamingLobs);
    Collection<GroupSymbol> accessedGroups = null;
    if (context.getDataObjects() != null) {
        QueryMetadataInterface metadata = context.getMetadata();
        accessedGroups = GroupCollectorVisitor.getGroupsIgnoreInlineViews(command, false);
        boolean usedModel = false;
        for (GroupSymbol gs : accessedGroups) {
            context.accessedDataObject(gs.getMetadataID());
            // check the source/tables/procs for determinism level
            Object mid = gs.getMetadataID();
            if (mid instanceof TempMetadataID) {
                TempMetadataID tid = (TempMetadataID) mid;
                if (tid.getOriginalMetadataID() != null) {
                    mid = tid.getOriginalMetadataID();
                }
            }
            String specificProp = metadata.getExtensionProperty(mid, AbstractMetadataRecord.RELATIONAL_URI + DDLConstants.DETERMINISM, false);
            if (specificProp == null) {
                if (!usedModel) {
                    Object modelId = metadata.getModelID(mid);
                    String prop = metadata.getExtensionProperty(modelId, AbstractMetadataRecord.RELATIONAL_URI + DDLConstants.DETERMINISM, false);
                    if (prop != null) {
                        usedModel = true;
                        // set model property
                        context.setDeterminismLevel(Determinism.valueOf(prop.toUpperCase()));
                    }
                }
                continue;
            }
            context.setDeterminismLevel(Determinism.valueOf(specificProp.toUpperCase()));
        }
    }
    ConnectorManagerRepository cmr = workItem.getDqpWorkContext().getVDB().getAttachment(ConnectorManagerRepository.class);
    ConnectorManager connectorManager = cmr.getConnectorManager(aqr.getConnectorName());
    if (connectorManager == null) {
        // can happen if sources are removed
        if (RelationalNodeUtil.hasOutputParams(command)) {
            // $NON-NLS-1$
            throw new AssertionError("A source is required to execute a procedure returning parameters");
        }
        // $NON-NLS-1$ //$NON-NLS-2$
        LogManager.logDetail(LogConstants.CTX_DQP, "source", aqr.getConnectorName(), "no longer exists, returning dummy results");
        return CollectionTupleSource.createNullTupleSource();
    }
    ConnectorWork work = connectorManager.registerRequest(aqr);
    if (!work.isForkable()) {
        aqr.setSerial(true);
    }
    CacheID cid = null;
    CacheDirective cd = null;
    if (workItem.getRsCache() != null && command.areResultsCachable()) {
        CachableVisitor cv = new CachableVisitor();
        PreOrPostOrderNavigator.doVisit(command, cv, PreOrPostOrderNavigator.PRE_ORDER, true);
        if (cv.cacheable) {
            try {
                cd = work.getCacheDirective();
            } catch (TranslatorException e) {
                // $NON-NLS-1$
                throw new TeiidProcessingException(QueryPlugin.Event.TEIID30504, e, aqr.getConnectorName() + ": " + e.getMessage());
            }
            if (cd != null) {
                if (cd.getScope() == Scope.NONE) {
                    parameterObject.doNotCache = true;
                } else {
                    String cmdString = command.toString();
                    if (cmdString.length() < 100000) {
                        // TODO: this check won't be needed if keys aren't exclusively held in memory
                        cid = new CacheID(workItem.getDqpWorkContext(), ParseInfo.DEFAULT_INSTANCE, cmdString);
                        cid.setParameters(cv.parameters);
                        if (cd.getInvalidation() == null || cd.getInvalidation() == Invalidation.NONE) {
                            CachedResults cr = workItem.getRsCache().get(cid);
                            if (cr != null && (cr.getRowLimit() == 0 || (parameterObject.limit > 0 && cr.getRowLimit() >= parameterObject.limit))) {
                                parameterObject.doNotCache = true;
                                // $NON-NLS-1$
                                LogManager.logDetail(LogConstants.CTX_DQP, "Using cache entry for", cid);
                                work.close();
                                return cr.getResults().createIndexedTupleSource();
                            }
                        } else if (cd.getInvalidation() == Invalidation.IMMEDIATE) {
                            workItem.getRsCache().remove(cid, CachingTupleSource.getDeterminismLevel(cd.getScope()));
                        }
                    }
                }
            } else {
                // $NON-NLS-1$
                LogManager.logTrace(LogConstants.CTX_DQP, aqr.getAtomicRequestID(), "no cache directive");
            }
        } else {
            // $NON-NLS-1$
            LogManager.logTrace(LogConstants.CTX_DQP, aqr.getAtomicRequestID(), "command not cachable");
        }
    }
    DataTierTupleSource dtts = new DataTierTupleSource(aqr, workItem, work, this, parameterObject.limit);
    TupleSource result = dtts;
    TupleBuffer tb = null;
    if (cid != null) {
        tb = getBufferManager().createTupleBuffer(aqr.getCommand().getProjectedSymbols(), aqr.getCommandContext().getConnectionId(), TupleSourceType.PROCESSOR);
        result = new CachingTupleSource(this, tb, (DataTierTupleSource) result, cid, parameterObject, cd, accessedGroups, workItem);
    }
    if (work.isThreadBound()) {
        result = handleThreadBound(workItem, aqr, work, cid, result, dtts, tb);
    } else if (!aqr.isSerial()) {
        dtts.addWork();
    }
    return result;
}

Example 9 with ConnectorManager

use of org.teiid.dqp.internal.datamgr.ConnectorManager in project teiid by teiid.

the class TestConnectorCapabilitiesFinder method testFind.

@Test
public void testFind() throws Exception {
    // $NON-NLS-1$
    String modelName = "model";
    // $NON-NLS-1$
    String functionName = "fakeFunction";
    BasicSourceCapabilities caps = new BasicSourceCapabilities();
    // $NON-NLS-1$
    caps.setFunctionSupport("fakeFunction", true);
    ArrayList<String> bindings = new ArrayList<String>();
    bindings.add(modelName);
    VDBMetaData vdb = Mockito.mock(VDBMetaData.class);
    ModelMetaData model = Mockito.mock(ModelMetaData.class);
    Mockito.stub(vdb.getModel(modelName)).toReturn(model);
    Mockito.stub(model.getSourceNames()).toReturn(bindings);
    BasicSourceCapabilities basicSourceCapabilities = new BasicSourceCapabilities();
    basicSourceCapabilities.setFunctionSupport(functionName, true);
    ConnectorManagerRepository repo = Mockito.mock(ConnectorManagerRepository.class);
    ConnectorManager cm = Mockito.mock(ConnectorManager.class);
    Mockito.stub(cm.getCapabilities()).toReturn(basicSourceCapabilities);
    Mockito.stub(repo.getConnectorManager(Mockito.anyString())).toReturn(cm);
    CachedFinder finder = new CachedFinder(repo, vdb);
    // Test
    SourceCapabilities actual = finder.findCapabilities(modelName);
    // $NON-NLS-1$
    assertEquals("Did not get expected capabilities", true, actual.supportsFunction(functionName));
    assertTrue(finder.isValid(modelName));
}

Example 10 with ConnectorManager

use of org.teiid.dqp.internal.datamgr.ConnectorManager in project teiid by teiid.

the class VDBService method loadMetadata.

@Override
@SuppressWarnings({ "rawtypes", "unchecked" })
protected void loadMetadata(final VDBMetaData vdb, final ModelMetaData model, final ConnectorManagerRepository cmr, final MetadataRepository metadataRepo, final MetadataStore vdbMetadataStore, final AtomicInteger loadCount, final VDBResources vdbResources) {
    String msg = IntegrationPlugin.Util.gs(IntegrationPlugin.Event.TEIID50029, vdb.getName(), vdb.getVersion(), model.getName(), SimpleDateFormat.getInstance().format(new Date()));
    model.setMetadataStatus(Model.MetadataStatus.LOADING);
    model.addRuntimeMessage(Severity.INFO, msg);
    LogManager.logInfo(LogConstants.CTX_RUNTIME, msg);
    final Runnable job = new Runnable() {

        @Override
        public void run() {
            boolean cached = false;
            Exception ex = null;
            TranslatorException te = null;
            // if this is not the first time trying to load metadata
            if (model.getMetadataStatus() != Model.MetadataStatus.LOADING) {
                model.setMetadataStatus(Model.MetadataStatus.RETRYING);
            }
            // designer based models define data types based on their built in data types, which are system vdb data types
            Map<String, Datatype> datatypes = vdbMetadataStore.getDatatypes();
            final File cachedFile = getSerializer().buildModelFile(vdb, model.getName());
            MetadataFactory factory = getSerializer().loadSafe(cachedFile, MetadataFactory.class);
            if (factory != null) {
                factory.correctDatatypes(datatypes);
                cached = true;
                // $NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
                LogManager.logDetail(LogConstants.CTX_RUNTIME, "Model ", model.getName(), "in VDB ", vdb.getName(), " was loaded from cached metadata");
            } else {
                factory = createMetadataFactory(vdb, vdbMetadataStore, model, vdbResources.getEntriesPlusVisibilities());
                ExecutionFactory ef = null;
                Object cf = null;
                for (ConnectorManager cm : getConnectorManagers(model, cmr)) {
                    if (ex != null) {
                        // $NON-NLS-1$
                        LogManager.logDetail(LogConstants.CTX_RUNTIME, ex, "Failed to get metadata, trying next source.");
                        ex = null;
                        te = null;
                    }
                    try {
                        if (cm != null) {
                            ef = cm.getExecutionFactory();
                            cf = cm.getConnectionFactory();
                        }
                    } catch (TranslatorException e) {
                        // $NON-NLS-1$
                        LogManager.logDetail(LogConstants.CTX_RUNTIME, e, "Failed to get a connection factory for metadata load.");
                        te = e;
                    }
                    ClassLoader originalCL = Thread.currentThread().getContextClassLoader();
                    try {
                        LogManager.logDetail(LogConstants.CTX_RUNTIME, IntegrationPlugin.Util.gs(IntegrationPlugin.Event.TEIID50104, vdb.getName(), vdb.getVersion(), model.getName(), cm != null ? cm.getTranslatorName() : null, cm != null ? cm.getConnectionName() : null));
                        Thread.currentThread().setContextClassLoader(metadataRepo.getClass().getClassLoader());
                        metadataRepo.loadMetadata(factory, ef, cf);
                        LogManager.logInfo(LogConstants.CTX_RUNTIME, IntegrationPlugin.Util.gs(IntegrationPlugin.Event.TEIID50030, vdb.getName(), vdb.getVersion(), model.getName(), SimpleDateFormat.getInstance().format(new Date())));
                        break;
                    } catch (Exception e) {
                        factory = createMetadataFactory(vdb, vdbMetadataStore, model, vdbResources.getEntriesPlusVisibilities());
                        ex = e;
                    } finally {
                        Thread.currentThread().setContextClassLoader(originalCL);
                    }
                }
            }
            synchronized (vdb) {
                VDBStatusChecker marked = model.removeAttachment(VDBStatusChecker.class);
                if (ex == null) {
                    if (!cached) {
                        // cache the schema to disk
                        cacheMetadataStore(model, factory);
                    }
                    metadataLoaded(vdb, model, vdbMetadataStore, loadCount, factory, true, cmr, vdbResources);
                } else {
                    String errorMsg = ex.getMessage() == null ? ex.getClass().getName() : ex.getMessage();
                    if (te != null) {
                        // $NON-NLS-1$
                        errorMsg += ": " + te.getMessage();
                    }
                    model.addRuntimeError(errorMsg);
                    model.setMetadataStatus(Model.MetadataStatus.FAILED);
                    LogManager.logWarning(LogConstants.CTX_RUNTIME, ex, IntegrationPlugin.Util.gs(IntegrationPlugin.Event.TEIID50036, vdb.getName(), vdb.getVersion(), model.getName(), errorMsg));
                    if (ex instanceof RuntimeException) {
                        metadataLoaded(vdb, model, vdbMetadataStore, loadCount, factory, false, cmr, vdbResources);
                    } else {
                        if (marked != null) {
                            getExecutor().execute(this);
                        } else {
                            // defer the load to the status checker if/when a source is available/redeployed
                            model.addAttchment(Runnable.class, this);
                        }
                    }
                }
            }
        }
    };
    Executor executor = getExecutor();
    // wrap the runnable to trap exceptions that may be caused by an asynch deployment issue
    executor.execute(new Runnable() {

        @Override
        public void run() {
            try {
                job.run();
            } catch (IllegalStateException e) {
                if (vdb.getStatus() != Status.FAILED && vdb.getStatus() != Status.REMOVED) {
                    throw e;
                }
                // $NON-NLS-1$
                LogManager.logDetail(LogConstants.CTX_RUNTIME, e, "Could not load metadata for a removed or failed deployment.");
            }
        }
    });
}