Examples with ZkSolrResourceLoader org.apache.solr.cloud.ZkSolrResourceLoader used on opensource projects

Example 11 with ZkSolrResourceLoader

use of org.apache.solr.cloud.ZkSolrResourceLoader in project lucene-solr by apache.

the class ManagedIndexSchema method persistManagedSchemaToZooKeeper.

/**
   * Persists the managed schema to ZooKeeper using optimistic concurrency.
   * <p/>
   * If createOnly is true, success is when the schema is created or if it previously existed.
   * <p/>
   * If createOnly is false, success is when the schema is persisted - this will only happen
   * if schemaZkVersion matches the version in ZooKeeper.
   * 
   * @return true on success 
   */
boolean persistManagedSchemaToZooKeeper(boolean createOnly) {
    final ZkSolrResourceLoader zkLoader = (ZkSolrResourceLoader) loader;
    final ZkController zkController = zkLoader.getZkController();
    final SolrZkClient zkClient = zkController.getZkClient();
    final String managedSchemaPath = zkLoader.getConfigSetZkPath() + "/" + managedSchemaResourceName;
    boolean success = true;
    boolean schemaChangedInZk = false;
    try {
        // Persist the managed schema
        StringWriter writer = new StringWriter();
        persist(writer);
        final byte[] data = writer.toString().getBytes(StandardCharsets.UTF_8);
        if (createOnly) {
            try {
                zkClient.create(managedSchemaPath, data, CreateMode.PERSISTENT, true);
                schemaZkVersion = 0;
                log.info("Created and persisted managed schema znode at " + managedSchemaPath);
            } catch (KeeperException.NodeExistsException e) {
                // This is okay - do nothing and fall through
                log.info("Managed schema znode at " + managedSchemaPath + " already exists - no need to create it");
            }
        } else {
            try {
                // Assumption: the path exists
                Stat stat = zkClient.setData(managedSchemaPath, data, schemaZkVersion, true);
                schemaZkVersion = stat.getVersion();
                log.info("Persisted managed schema version " + schemaZkVersion + " at " + managedSchemaPath);
            } catch (KeeperException.BadVersionException e) {
                log.error("Bad version when trying to persist schema using " + schemaZkVersion + " due to: " + e);
                success = false;
                schemaChangedInZk = true;
            }
        }
    } catch (Exception e) {
        if (e instanceof InterruptedException) {
            // Restore the interrupted status
            Thread.currentThread().interrupt();
        }
        final String msg = "Error persisting managed schema at " + managedSchemaPath;
        log.error(msg, e);
        throw new SolrException(ErrorCode.SERVER_ERROR, msg, e);
    }
    if (schemaChangedInZk) {
        String msg = "Failed to persist managed schema at " + managedSchemaPath + " - version mismatch";
        log.info(msg);
        throw new SchemaChangedInZkException(ErrorCode.CONFLICT, msg + ", retry.");
    }
    return success;
}

Example 12 with ZkSolrResourceLoader

use of org.apache.solr.cloud.ZkSolrResourceLoader in project lucene-solr by apache.

the class ManagedIndexSchema method persistManagedSchema.

/** Persist the schema to local storage or to ZooKeeper */
boolean persistManagedSchema(boolean createOnly) {
    if (loader instanceof ZkSolrResourceLoader) {
        return persistManagedSchemaToZooKeeper(createOnly);
    }
    // Persist locally
    File managedSchemaFile = new File(loader.getConfigDir(), managedSchemaResourceName);
    OutputStreamWriter writer = null;
    try {
        File parentDir = managedSchemaFile.getParentFile();
        if (!parentDir.isDirectory()) {
            if (!parentDir.mkdirs()) {
                final String msg = "Can't create managed schema directory " + parentDir.getAbsolutePath();
                log.error(msg);
                throw new SolrException(ErrorCode.SERVER_ERROR, msg);
            }
        }
        final FileOutputStream out = new FileOutputStream(managedSchemaFile);
        writer = new OutputStreamWriter(out, StandardCharsets.UTF_8);
        persist(writer);
        log.info("Upgraded to managed schema at " + managedSchemaFile.getPath());
    } catch (IOException e) {
        final String msg = "Error persisting managed schema " + managedSchemaFile;
        log.error(msg, e);
        throw new SolrException(ErrorCode.SERVER_ERROR, msg, e);
    } finally {
        IOUtils.closeQuietly(writer);
        try {
            FileUtils.sync(managedSchemaFile);
        } catch (IOException e) {
            final String msg = "Error syncing the managed schema file " + managedSchemaFile;
            log.error(msg, e);
        }
    }
    return true;
}

Example 13 with ZkSolrResourceLoader

use of org.apache.solr.cloud.ZkSolrResourceLoader in project lucene-solr by apache.

the class ManagedIndexSchemaFactory method create.

/**
   * First, try to locate the managed schema file named in the managedSchemaResourceName
   * param. If the managed schema file exists and is accessible, it is used to instantiate
   * an IndexSchema.
   *
   * If the managed schema file can't be found, the resource named by the resourceName
   * parameter is used to instantiate an IndexSchema.
   *
   * Once the IndexSchema is instantiated, if the managed schema file does not exist,
   * the instantiated IndexSchema is persisted to the managed schema file named in the
   * managedSchemaResourceName param, in the directory given by 
   * {@link org.apache.solr.core.SolrResourceLoader#getConfigDir()}, or if configs are
   * in ZooKeeper, under {@link org.apache.solr.cloud.ZkSolrResourceLoader#getConfigSetZkPath()}.
   *
   * After the managed schema file is persisted, the original schema file is
   * renamed by appending the extension named in {@link #UPGRADED_SCHEMA_EXTENSION}.
   */
@Override
public ManagedIndexSchema create(String resourceName, SolrConfig config) {
    this.resourceName = resourceName;
    this.config = config;
    this.loader = config.getResourceLoader();
    InputStream schemaInputStream = null;
    if (null == resourceName) {
        resourceName = IndexSchema.DEFAULT_SCHEMA_FILE;
    }
    int schemaZkVersion = -1;
    if (!(loader instanceof ZkSolrResourceLoader)) {
        schemaInputStream = readSchemaLocally();
    } else {
        // ZooKeeper
        final ZkSolrResourceLoader zkLoader = (ZkSolrResourceLoader) loader;
        final SolrZkClient zkClient = zkLoader.getZkController().getZkClient();
        final String managedSchemaPath = zkLoader.getConfigSetZkPath() + "/" + managedSchemaResourceName;
        Stat stat = new Stat();
        try {
            // Attempt to load the managed schema
            byte[] data = zkClient.getData(managedSchemaPath, null, stat, true);
            schemaZkVersion = stat.getVersion();
            schemaInputStream = new ByteArrayInputStream(data);
            loadedResource = managedSchemaResourceName;
            warnIfNonManagedSchemaExists();
        } catch (InterruptedException e) {
            // Restore the interrupted status
            Thread.currentThread().interrupt();
            log.warn("", e);
        } catch (KeeperException.NoNodeException e) {
            log.info("The schema is configured as managed, but managed schema resource " + managedSchemaResourceName + " not found - loading non-managed schema " + resourceName + " instead");
        } catch (KeeperException e) {
            String msg = "Error attempting to access " + managedSchemaPath;
            log.error(msg, e);
            throw new SolrException(ErrorCode.SERVER_ERROR, msg, e);
        }
        if (null == schemaInputStream) {
            // The managed schema file could not be found - load the non-managed schema
            try {
                schemaInputStream = loader.openSchema(resourceName);
                loadedResource = resourceName;
                shouldUpgrade = true;
            } catch (Exception e) {
                try {
                    // Retry to load the managed schema, in case it was created since the first attempt
                    byte[] data = zkClient.getData(managedSchemaPath, null, stat, true);
                    schemaZkVersion = stat.getVersion();
                    schemaInputStream = new ByteArrayInputStream(data);
                    loadedResource = managedSchemaPath;
                    warnIfNonManagedSchemaExists();
                } catch (Exception e1) {
                    if (e1 instanceof InterruptedException) {
                        // Restore the interrupted status
                        Thread.currentThread().interrupt();
                    }
                    final String msg = "Error loading both non-managed schema '" + resourceName + "' and managed schema '" + managedSchemaResourceName + "'";
                    log.error(msg, e);
                    throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg, e);
                }
            }
        }
    }
    InputSource inputSource = new InputSource(schemaInputStream);
    inputSource.setSystemId(SystemIdResolver.createSystemIdFromResourceName(loadedResource));
    try {
        schema = new ManagedIndexSchema(config, loadedResource, inputSource, isMutable, managedSchemaResourceName, schemaZkVersion, getSchemaUpdateLock());
    } catch (KeeperException e) {
        final String msg = "Error instantiating ManagedIndexSchema";
        log.error(msg, e);
        throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, msg, e);
    } catch (InterruptedException e) {
        // Restore the interrupted status
        Thread.currentThread().interrupt();
        log.warn("", e);
    }
    if (shouldUpgrade) {
        // Persist the managed schema if it doesn't already exist
        upgradeToManagedSchema();
    }
    return schema;
}

Example 14 with ZkSolrResourceLoader

use of org.apache.solr.cloud.ZkSolrResourceLoader in project lucene-solr by apache.

the class ManagedResourceStorage method newStorageIO.

/**
   * Creates a new StorageIO instance for a Solr core, taking into account
   * whether the core is running in cloud mode as well as initArgs. 
   */
public static StorageIO newStorageIO(String collection, SolrResourceLoader resourceLoader, NamedList<String> initArgs) {
    StorageIO storageIO;
    SolrZkClient zkClient = null;
    String zkConfigName = null;
    if (resourceLoader instanceof ZkSolrResourceLoader) {
        zkClient = ((ZkSolrResourceLoader) resourceLoader).getZkController().getZkClient();
        try {
            zkConfigName = ((ZkSolrResourceLoader) resourceLoader).getZkController().getZkStateReader().readConfigName(collection);
        } catch (Exception e) {
            log.error("Failed to get config name due to", e);
            throw new SolrException(ErrorCode.SERVER_ERROR, "Failed to load config name for collection:" + collection + " due to: ", e);
        }
        if (zkConfigName == null) {
            throw new SolrException(ErrorCode.SERVER_ERROR, "Could not find config name for collection:" + collection);
        }
    }
    if (initArgs.get(STORAGE_IO_CLASS_INIT_ARG) != null) {
        storageIO = resourceLoader.newInstance(initArgs.get(STORAGE_IO_CLASS_INIT_ARG), StorageIO.class);
    } else {
        if (zkClient != null) {
            String znodeBase = "/configs/" + zkConfigName;
            log.debug("Setting up ZooKeeper-based storage for the RestManager with znodeBase: " + znodeBase);
            storageIO = new ManagedResourceStorage.ZooKeeperStorageIO(zkClient, znodeBase);
        } else {
            storageIO = new FileStorageIO();
        }
    }
    if (storageIO instanceof FileStorageIO) {
        // using local fs, if storageDir is not set in the solrconfig.xml, assume the configDir for the core
        if (initArgs.get(STORAGE_DIR_INIT_ARG) == null) {
            File configDir = new File(resourceLoader.getConfigDir());
            boolean hasAccess = false;
            try {
                hasAccess = configDir.isDirectory() && configDir.canWrite();
            } catch (java.security.AccessControlException ace) {
            }
            if (hasAccess) {
                initArgs.add(STORAGE_DIR_INIT_ARG, configDir.getAbsolutePath());
            } else {
                // most likely this is because of a unit test 
                // that doesn't have write-access to the config dir
                // while this failover approach is not ideal, it's better
                // than causing the core to fail esp. if managed resources aren't being used
                log.warn("Cannot write to config directory " + configDir.getAbsolutePath() + "; switching to use InMemory storage instead.");
                storageIO = new ManagedResourceStorage.InMemoryStorageIO();
            }
        }
    }
    storageIO.configure(resourceLoader, initArgs);
    return storageIO;
}