Search in sources :

Example 76 with Tuple

use of org.opensearch.common.collect.Tuple in project geospatial by opensearch-project.

the class RestUploadGeoJSONAction method prepareRequest.

@Override
protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) {
    Tuple<XContentType, BytesReference> sourceTuple = restRequest.contentOrSourceParam();
    RestRequest.Method method = restRequest.getHttpRequest().method();
    UploadGeoJSONRequest request = new UploadGeoJSONRequest(method, sourceTuple.v2());
    return channel -> client.execute(UploadGeoJSONAction.INSTANCE, request, new RestToXContentListener<>(channel));
}
Also used : BytesReference(org.opensearch.common.bytes.BytesReference) POST(org.opensearch.rest.RestRequest.Method.POST) NodeClient(org.opensearch.client.node.NodeClient) BytesReference(org.opensearch.common.bytes.BytesReference) RestRequest(org.opensearch.rest.RestRequest) GeospatialPlugin(org.opensearch.geospatial.plugin.GeospatialPlugin) Tuple(org.opensearch.common.collect.Tuple) UploadGeoJSONAction(org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONAction) List(java.util.List) RestToXContentListener(org.opensearch.rest.action.RestToXContentListener) URL_DELIMITER(org.opensearch.geospatial.plugin.GeospatialPlugin.URL_DELIMITER) UploadGeoJSONRequest(org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONRequest) XContentType(org.opensearch.common.xcontent.XContentType) PUT(org.opensearch.rest.RestRequest.Method.PUT) BaseRestHandler(org.opensearch.rest.BaseRestHandler) XContentType(org.opensearch.common.xcontent.XContentType) RestRequest(org.opensearch.rest.RestRequest) UploadGeoJSONRequest(org.opensearch.geospatial.action.upload.geojson.UploadGeoJSONRequest)

Example 77 with Tuple

use of org.opensearch.common.collect.Tuple in project OpenSearch by opensearch-project.

the class InstallPluginCommand method checkMisspelledPlugin.

/**
 * Returns all the official plugin names that look similar to pluginId. *
 */
private List<String> checkMisspelledPlugin(String pluginId) {
    LevenshteinDistance ld = new LevenshteinDistance();
    List<Tuple<Float, String>> scoredKeys = new ArrayList<>();
    for (String officialPlugin : OFFICIAL_PLUGINS) {
        float distance = ld.getDistance(pluginId, officialPlugin);
        if (distance > 0.7f) {
            scoredKeys.add(new Tuple<>(distance, officialPlugin));
        }
    }
    CollectionUtil.timSort(scoredKeys, (a, b) -> b.v1().compareTo(a.v1()));
    return scoredKeys.stream().map((a) -> a.v2()).collect(Collectors.toList());
}
Also used : HttpURLConnection(java.net.HttpURLConnection) UserException(org.opensearch.cli.UserException) Arrays(java.util.Arrays) URLDecoder(java.net.URLDecoder) URL(java.net.URL) URISyntaxException(java.net.URISyntaxException) Version(org.opensearch.Version) Build(org.opensearch.Build) DirectoryStream(java.nio.file.DirectoryStream) Locale(java.util.Locale) Map(java.util.Map) URI(java.net.URI) PGPException(org.bouncycastle.openpgp.PGPException) Path(java.nio.file.Path) ZipEntry(java.util.zip.ZipEntry) OptionSet(joptsimple.OptionSet) SimpleFileVisitor(java.nio.file.SimpleFileVisitor) OptionSpec(joptsimple.OptionSpec) JcaKeyFingerprintCalculator(org.bouncycastle.openpgp.operator.jcajce.JcaKeyFingerprintCalculator) SuppressForbidden(org.opensearch.common.SuppressForbidden) PosixFileAttributes(java.nio.file.attribute.PosixFileAttributes) PosixFilePermission(java.nio.file.attribute.PosixFilePermission) Set(java.util.Set) BouncyCastleFipsProvider(org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider) PGPPublicKey(org.bouncycastle.openpgp.PGPPublicKey) Collectors(java.util.stream.Collectors) StandardCharsets(java.nio.charset.StandardCharsets) Tuple(org.opensearch.common.collect.Tuple) Objects(java.util.Objects) FileVisitResult(java.nio.file.FileVisitResult) List(java.util.List) PGPPublicKeyRingCollection(org.bouncycastle.openpgp.PGPPublicKeyRingCollection) EnvironmentAwareCommand(org.opensearch.cli.EnvironmentAwareCommand) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) LevenshteinDistance(org.apache.lucene.search.spell.LevenshteinDistance) JarHell(org.opensearch.bootstrap.JarHell) ZipInputStream(java.util.zip.ZipInputStream) MessageDigest(java.security.MessageDigest) PGPUtil(org.bouncycastle.openpgp.PGPUtil) PosixFileAttributeView(java.nio.file.attribute.PosixFileAttributeView) HashMap(java.util.HashMap) VERBOSE(org.opensearch.cli.Terminal.Verbosity.VERBOSE) TreeSet(java.util.TreeSet) StandardCopyOption(java.nio.file.StandardCopyOption) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) LinkedHashMap(java.util.LinkedHashMap) PosixFilePermissions(java.nio.file.attribute.PosixFilePermissions) URLConnection(java.net.URLConnection) ArmoredInputStream(org.bouncycastle.bcpg.ArmoredInputStream) JcaPGPContentVerifierBuilderProvider(org.bouncycastle.openpgp.operator.jcajce.JcaPGPContentVerifierBuilderProvider) ExitCodes(org.opensearch.cli.ExitCodes) Environment(org.opensearch.env.Environment) OutputStream(java.io.OutputStream) PGPSignatureList(org.bouncycastle.openpgp.PGPSignatureList) Terminal(org.opensearch.cli.Terminal) Files(java.nio.file.Files) PGPSignature(org.bouncycastle.openpgp.PGPSignature) IOException(java.io.IOException) BasicFileAttributes(java.nio.file.attribute.BasicFileAttributes) InputStreamReader(java.io.InputStreamReader) CollectionUtil(org.apache.lucene.util.CollectionUtil) IOUtils(org.opensearch.core.internal.io.IOUtils) MessageDigests(org.opensearch.common.hash.MessageDigests) Constants(org.apache.lucene.util.Constants) JcaPGPObjectFactory(org.bouncycastle.openpgp.jcajce.JcaPGPObjectFactory) BufferedReader(java.io.BufferedReader) Collections(java.util.Collections) InputStream(java.io.InputStream) ArrayList(java.util.ArrayList) LevenshteinDistance(org.apache.lucene.search.spell.LevenshteinDistance) Tuple(org.opensearch.common.collect.Tuple)

Example 78 with Tuple

use of org.opensearch.common.collect.Tuple in project OpenSearch by opensearch-project.

the class TransportUpgradeAction method newResponse.

@Override
protected UpgradeResponse newResponse(UpgradeRequest request, int totalShards, int successfulShards, int failedShards, List<ShardUpgradeResult> shardUpgradeResults, List<DefaultShardOperationFailedException> shardFailures, ClusterState clusterState) {
    Map<String, Integer> successfulPrimaryShards = new HashMap<>();
    Map<String, Tuple<Version, org.apache.lucene.util.Version>> versions = new HashMap<>();
    for (ShardUpgradeResult result : shardUpgradeResults) {
        successfulShards++;
        String index = result.getShardId().getIndex().getName();
        if (result.primary()) {
            Integer count = successfulPrimaryShards.get(index);
            successfulPrimaryShards.put(index, count == null ? 1 : count + 1);
        }
        Tuple<Version, org.apache.lucene.util.Version> versionTuple = versions.get(index);
        if (versionTuple == null) {
            versions.put(index, new Tuple<>(result.upgradeVersion(), result.oldestLuceneSegment()));
        } else {
            // We already have versions for this index - let's see if we need to update them based on the current shard
            Version version = versionTuple.v1();
            org.apache.lucene.util.Version luceneVersion = versionTuple.v2();
            // Since we rewrite the mapping during upgrade the metadata is always rewritten by the latest version
            if (result.upgradeVersion().after(versionTuple.v1())) {
                version = result.upgradeVersion();
            }
            // oldest version that we need to support
            if (result.oldestLuceneSegment().onOrAfter(versionTuple.v2()) == false) {
                luceneVersion = result.oldestLuceneSegment();
            }
            versions.put(index, new Tuple<>(version, luceneVersion));
        }
    }
    Map<String, Tuple<Version, String>> updatedVersions = new HashMap<>();
    Metadata metadata = clusterState.metadata();
    for (Map.Entry<String, Tuple<Version, org.apache.lucene.util.Version>> versionEntry : versions.entrySet()) {
        String index = versionEntry.getKey();
        Integer primaryCount = successfulPrimaryShards.get(index);
        int expectedPrimaryCount = metadata.index(index).getNumberOfShards();
        if (primaryCount == metadata.index(index).getNumberOfShards()) {
            updatedVersions.put(index, new Tuple<>(versionEntry.getValue().v1(), versionEntry.getValue().v2().toString()));
        } else {
            logger.warn("Not updating settings for the index [{}] because upgraded of some primary shards failed - " + "expected[{}], received[{}]", index, expectedPrimaryCount, primaryCount == null ? 0 : primaryCount);
        }
    }
    return new UpgradeResponse(updatedVersions, totalShards, successfulShards, failedShards, shardFailures);
}
Also used : HashMap(java.util.HashMap) Metadata(org.opensearch.cluster.metadata.Metadata) Version(org.opensearch.Version) HashMap(java.util.HashMap) Map(java.util.Map) Tuple(org.opensearch.common.collect.Tuple)

Example 79 with Tuple

use of org.opensearch.common.collect.Tuple in project OpenSearch by opensearch-project.

the class ClusterBootstrapService method checkRequirements.

private Tuple<Set<DiscoveryNode>, List<String>> checkRequirements(Set<DiscoveryNode> nodes) {
    final Set<DiscoveryNode> selectedNodes = new HashSet<>();
    final List<String> unmatchedRequirements = new ArrayList<>();
    for (final String bootstrapRequirement : bootstrapRequirements) {
        final Set<DiscoveryNode> matchingNodes = nodes.stream().filter(n -> matchesRequirement(n, bootstrapRequirement)).collect(Collectors.toSet());
        if (matchingNodes.size() == 0) {
            unmatchedRequirements.add(bootstrapRequirement);
        }
        if (matchingNodes.size() > 1) {
            throw new IllegalStateException("requirement [" + bootstrapRequirement + "] matches multiple nodes: " + matchingNodes);
        }
        for (final DiscoveryNode matchingNode : matchingNodes) {
            if (selectedNodes.add(matchingNode) == false) {
                throw new IllegalStateException("node [" + matchingNode + "] matches multiple requirements: " + bootstrapRequirements.stream().filter(r -> matchesRequirement(matchingNode, r)).collect(Collectors.toList()));
            }
        }
    }
    return Tuple.tuple(selectedNodes, unmatchedRequirements);
}
Also used : AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Node(org.opensearch.node.Node) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) Function(java.util.function.Function) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) BooleanSupplier(java.util.function.BooleanSupplier) HashSet(java.util.HashSet) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) Property(org.opensearch.common.settings.Setting.Property) LEGACY_DISCOVERY_HOSTS_PROVIDER_SETTING(org.opensearch.discovery.DiscoveryModule.LEGACY_DISCOVERY_HOSTS_PROVIDER_SETTING) StreamSupport(java.util.stream.StreamSupport) LinkedHashSet(java.util.LinkedHashSet) Setting(org.opensearch.common.settings.Setting) TimeValue(org.opensearch.common.unit.TimeValue) VotingConfiguration(org.opensearch.cluster.coordination.CoordinationMetadata.VotingConfiguration) Collections.emptyList(java.util.Collections.emptyList) Set(java.util.Set) DISCOVERY_SEED_PROVIDERS_SETTING(org.opensearch.discovery.DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING) Settings(org.opensearch.common.settings.Settings) LEGACY_DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING(org.opensearch.discovery.SettingsBasedSeedHostsProvider.LEGACY_DISCOVERY_ZEN_PING_UNICAST_HOSTS_SETTING) TransportService(org.opensearch.transport.TransportService) Collectors(java.util.stream.Collectors) Nullable(org.opensearch.common.Nullable) DISCOVERY_SEED_HOSTS_SETTING(org.opensearch.discovery.SettingsBasedSeedHostsProvider.DISCOVERY_SEED_HOSTS_SETTING) Tuple(org.opensearch.common.collect.Tuple) Consumer(java.util.function.Consumer) List(java.util.List) Logger(org.apache.logging.log4j.Logger) Stream(java.util.stream.Stream) Collections.unmodifiableSet(java.util.Collections.unmodifiableSet) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) DiscoveryModule(org.opensearch.discovery.DiscoveryModule) Names(org.opensearch.threadpool.ThreadPool.Names) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) LinkedHashSet(java.util.LinkedHashSet)

Example 80 with Tuple

use of org.opensearch.common.collect.Tuple in project OpenSearch by opensearch-project.

the class ChildrenToParentAggregatorTests method testParentChildTerms.

public void testParentChildTerms() throws IOException {
    Directory directory = newDirectory();
    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
    final Map<String, Tuple<Integer, Integer>> expectedParentChildRelations = setupIndex(indexWriter);
    indexWriter.close();
    SortedMap<Integer, Long> entries = new TreeMap<>();
    for (Tuple<Integer, Integer> value : expectedParentChildRelations.values()) {
        Long l = entries.computeIfAbsent(value.v2(), integer -> 0L);
        entries.put(value.v2(), l + 1);
    }
    List<Map.Entry<Integer, Long>> sortedValues = new ArrayList<>(entries.entrySet());
    sortedValues.sort((o1, o2) -> {
        // sort larger values first
        int ret = o2.getValue().compareTo(o1.getValue());
        if (ret != 0) {
            return ret;
        }
        // on equal value, sort by key
        return o1.getKey().compareTo(o2.getKey());
    });
    IndexReader indexReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(directory), new ShardId(new Index("foo", "_na_"), 1));
    // TODO set "maybeWrap" to true for IndexSearcher once #23338 is resolved
    IndexSearcher indexSearcher = newSearcher(indexReader, false, true);
    // verify a terms-aggregation inside the parent-aggregation
    testCaseTerms(new MatchAllDocsQuery(), indexSearcher, parent -> {
        assertNotNull(parent);
        assertTrue(JoinAggregationInspectionHelper.hasValue(parent));
        LongTerms valueTerms = parent.getAggregations().get("value_terms");
        assertNotNull(valueTerms);
        List<LongTerms.Bucket> valueTermsBuckets = valueTerms.getBuckets();
        assertNotNull(valueTermsBuckets);
        assertEquals("Had: " + parent, sortedValues.size(), valueTermsBuckets.size());
        int i = 0;
        for (Map.Entry<Integer, Long> entry : sortedValues) {
            LongTerms.Bucket bucket = valueTermsBuckets.get(i);
            assertEquals(entry.getKey().longValue(), bucket.getKeyAsNumber());
            assertEquals(entry.getValue(), (Long) bucket.getDocCount());
            i++;
        }
    });
    indexReader.close();
    directory.close();
}
Also used : IndexSearcher(org.apache.lucene.search.IndexSearcher) ArrayList(java.util.ArrayList) Index(org.opensearch.index.Index) ShardId(org.opensearch.index.shard.ShardId) Directory(org.apache.lucene.store.Directory) TreeMap(java.util.TreeMap) MatchAllDocsQuery(org.apache.lucene.search.MatchAllDocsQuery) IndexReader(org.apache.lucene.index.IndexReader) LongTerms(org.opensearch.search.aggregations.bucket.terms.LongTerms) Map(java.util.Map) SortedMap(java.util.SortedMap) HashMap(java.util.HashMap) TreeMap(java.util.TreeMap) RandomIndexWriter(org.apache.lucene.index.RandomIndexWriter) Tuple(org.opensearch.common.collect.Tuple)

Aggregations

Tuple (org.opensearch.common.collect.Tuple)151 ArrayList (java.util.ArrayList)65 List (java.util.List)49 IOException (java.io.IOException)45 Collections (java.util.Collections)44 HashMap (java.util.HashMap)40 Map (java.util.Map)40 Settings (org.opensearch.common.settings.Settings)38 ClusterState (org.opensearch.cluster.ClusterState)34 HashSet (java.util.HashSet)28 ShardId (org.opensearch.index.shard.ShardId)28 Arrays (java.util.Arrays)27 Collectors (java.util.stream.Collectors)26 Set (java.util.Set)25 Index (org.opensearch.index.Index)25 BytesReference (org.opensearch.common.bytes.BytesReference)24 OpenSearchTestCase (org.opensearch.test.OpenSearchTestCase)24 CountDownLatch (java.util.concurrent.CountDownLatch)22 Version (org.opensearch.Version)21 Strings (org.opensearch.common.Strings)21