Nest for ElasticSearch работает необычно медленно

Недавно мы обновили ElasticSearch 1.5 до 5.6, решив ряд вещей, таких как взрыв поля и другие проблемы. Однако теперь, когда мы находимся на более новой версии, мы видим неприемлемые (и откровенно нелепые) проблемы с производительностью.

Hits | Took | Nest 1.5 | Nest 5.6
---------------------------------
0    | 1ms  | 100ms    | 1190ms
1    | 1ms  | 100ms    | 720ms
2    | 4ms  | 100ms    | 350ms
42   | 10ms | 1100ms   | 3270ms
63   | 9ms  | 1700ms   | 4700ms
100  | 25ms | 2800ms   | 7400ms

У нас есть статический клиент Nest, использующий SingleNodeConnectionPool. Запросы очень простые, и мы выводим более крупные результаты (обычно не более 100). Когда мы были на 1.5, все эти запросы возвращались в течение 3 секунд. Почему запросы Nest теперь в 3-4 раза медленнее, чем раньше?

Построить индекс

PatternAnalyzer alphanumericAnalyzer = new PatternAnalyzer();
alphanumericAnalyzer.Lowercase = true;
alphanumericAnalyzer.Pattern = "[^a-zA-Z0-9áéíñóúüÁÉÍÑÓÚÜàâäôéèëêïîçùûüÿæœÀÂÄÔÉÈËÊÏΟÇÙÛÜÆŒäöüßÄÖÜẞàèéìíîòóùúÀÈÉÌÍÎÒÓÙÚ]";

CustomAnalyzer lowercaseKeywordAnalyzer = new CustomAnalyzer();
lowercaseKeywordAnalyzer.Tokenizer = "keyword";
lowercaseKeywordAnalyzer.Filter = new List<string>() { "lowercase" };

IndexSettings indexSettings = new IndexSettings();

indexSettings.NumberOfReplicas = NestClient.Config.Replicas;
indexSettings.NumberOfShards = NestClient.Config.Shards;

indexSettings.Analysis = new Analysis();
indexSettings.Analysis.Analyzers = new Analyzers();
indexSettings.Analysis.Tokenizers = new Tokenizers();

indexSettings.Analysis.Analyzers.Add("alphanumeric_analyzer", alphanumericAnalyzer);
indexSettings.Analysis.Analyzers.Add("keyword_analyzer", lowercaseKeywordAnalyzer);

indexSettings.Analysis.Tokenizers.Add("ngrams_tokenizer", new EdgeNGramTokenizer()
{
    MaxGram = NestClient.Config.MaxGram,
    MinGram = NestClient.Config.MinGram,
    TokenChars = new List<TokenChar>()
    {
        TokenChar.Letter,
        TokenChar.Digit
    }
});

indexSettings.Analysis.Analyzers.Add("ngrams_analyzer", new CustomAnalyzer()
{
    Filter = new List<string>()
    {
        "lowercase"
    },
    Tokenizer = "ngrams_tokenizer"
});

var createResponse = Client.CreateIndex(new CreateIndexRequest(IndexName)
{
    Settings = indexSettings
}).Log(isIndexRebuild: true);

var mapResult = Client.Map<SearchAsset>(m => m
    .AllField(x => x.Enabled(false))
    .AutoMap()
).Log(isIndexRebuild: true);

Поиск объекта

[ElasticsearchType(IdProperty = "assetID")]
public class SearchAsset
{
    public SearchAsset()
    {
        Extensions = new List<string>();
        Metadata = new List<MetadataValue>();
        Notes_Alphanumeric = new List<string>();
        Notes_Ngrams = new List<string>();
        UserFlags = new List<long>();
        AssetTypes = new List<string>();
    }

    private string filename;

    public long AssetID { get; set; }
    public long JobID { get; set; }
    public long JobFolderID { get; set; }
    public int Width { get; set; }
    public int Height { get; set; }

    [Text(Analyzer = "keyword_analyzer", Fielddata = true)]
    public string StorageFolderPath { get; set; }

    public bool Selected { get; set; }
    public long? SelectUserID { get; set; }
    public DateTime? SelectDateTime { get; set; }

    [Text(Analyzer = "keyword_analyzer", Fielddata = true)]
    public string JobFolderName { get; set; }

    [Text(Analyzer = "keyword_analyzer", Fielddata = true)]
    public string Filename
    {
        get { return filename; }
        set { filename = Filename_Alphanumeric = Filename_Ngrams = value; }
    }

    [Text(Analyzer = "alphanumeric_analyzer", SearchAnalyzer = "alphanumeric_analyzer")]
    public string Filename_Alphanumeric { get; private set; }

    [Text(Analyzer = "ngrams_analyzer", SearchAnalyzer = "alphanumeric_analyzer")]
    public string Filename_Ngrams { get; private set; }

    [Text(Analyzer = "keyword_analyzer", Fielddata = true)]
    public string OriginalTypeCd { get; set; }
    public int NoteCount { get; set; }
    public int PageCount { get; set; }

    public long Color { get; set; }
    public bool HasMarkup { get; set; }
    public long Status { get; set; }
    public int TotalGalleryCount { get; set; }
    public int ClosedGalleryCount { get; set; }

    //HACK: We would ideally script these in ES, but Nest/Painless has poor documentation, and we have yet to get something working within that framework.
    //Doing it here actually works, so relying on that instead.
    public bool NoStatus { get { return TotalGalleryCount == 0; } }
    public bool Flagged { get { return UserFlags.Count > 0; } }
    public bool NotPending { get { return TotalGalleryCount > 0 && TotalGalleryCount == ClosedGalleryCount; } }
    public bool Pending { get { return TotalGalleryCount > ClosedGalleryCount; } }
    public bool Notes { get { return NoteCount > 0; } }

    public long ByteCount { get; set; }
    public DateTime AddedOn { get; set; }

    [Object(Ignore = true)]
    public IndexItemType IndexItemType { get; set; }

    [Text(Analyzer = "keyword_analyzer", Fielddata = true)]
    public List<string> Extensions { get; set; }

    [Number]
    public List<long> UserFlags { get; set; }

    [Nested]
    public List<MetadataValue> Metadata { get; private set; }

    [Text]
    public List<string> AssetTypes { get; set; }

    [Text(Analyzer = "ngrams_analyzer", SearchAnalyzer = "alphanumeric_analyzer")]
    public List<string> Notes_Ngrams { get; private set; }

    [Text(Analyzer = "alphanumeric_analyzer", SearchAnalyzer = "alphanumeric_analyzer")]
    public List<string> Notes_Alphanumeric { get; private set; }
}

public class MetadataValue
{
    public long MetadataID { get; set; }
    [Text(Analyzer = "ngrams_analyzer", SearchAnalyzer = "alphanumeric_analyzer")]
    public string Ngrams { get; set; }
    [Text(Analyzer = "alphanumeric_analyzer", SearchAnalyzer = "alphanumeric_analyzer")]
    public string Alphanumeric { get; set; }
    public DateTime Date { get; set; }
}

Код вложенного запроса

public SearchResult RunSearch(IUser user, AssetCollection collection, Aggregations aggregations = null, FieldSelectors selectors = null)
{
    var elasticSearchManager = new ElasticSearchManager();

    var query = ElasticSearchHelper.BuildWhereExpression(user, collection);

    var sorts = ElasticSearchHelper.BuildOrderExpression<SearchAsset>(user, collection);

    //We want to specify the SearchAsset type so that we can both specify an index on the request, and also rely on the type mapping in the settings
    var request = new SearchRequest<SearchAsset>(elasticSearchManager.IndexName);
    request.Sort = sorts;
    request.Query = query;
    request.From = collection.FirstIndex;
    request.DocvalueFields = selectors.ElasticSearchFields.ToArray();
    request.Size = collection.LastIndex - collection.FirstIndex;
    request.Aggregations = new AggregationDictionary();

    //INFO: This allows us to log the NEST request body
    request.RequestConfiguration = new RequestConfiguration();
    request.RequestConfiguration.DisableDirectStreaming = true;

    var searchResponse = elasticSearchManager.Client.Search<SearchAsset>(request);
    ESLogger.LogElasticSearchResponse(searchResponse);

    Logger.Instance.LogInfo(new LogMessage(LogMessageAction.Search,
        new SearchContextLogData<SearchAsset>(searchResponse, collection),
        new UserContextLogData(user)
    ));

    SearchResult result = new SearchResult()
    {
        TotalCount = aggregations.CalculateTotalCount ? searchResponse.Total : 0
    };

    foreach (var searchAsset in searchResponse.Hits)
    {
        var asset = selectors.ApplyElasticSearchToAssetSelectors(searchAsset);
        result.Assets.Add(asset);
    }

    return result;
}

ElasticSearchManager

public class ElasticSearchManager
{
    public IElasticClient Client { get; private set; }
    public virtual string IndexName
    {
        get { return NestClient.IndexName; }
    }

    public ElasticSearchManager()
    {
        Client = NestClient.GetClient(null);
    }

    public IEnumerable<string> Tokenize(string field, string input)
    {
        var key = field + "_" + input;
        var tokens = GetCachedTokens(key);

        if (tokens != null)
            return tokens;

        tokens = new List<string>();

        var response = Client.Analyze(x => x.Field(field).Index(IndexName).Text(input)).Log();

        if (response.IsValid)
        {
            foreach (var token in response.Tokens)
            {
                tokens.Add(token.Token);
            }
        }

        CacheTokens(key, tokens);

        return tokens;
    }

    public IEnumerable<string> Tokenize<T>(Expression<Func<T, object>> field, string input)
        where T : class
    {
        var fieldName = field.Body.ToString();
        return Tokenize(fieldName, input);
    }

    private void CacheTokens(string input, IEnumerable<string> tokens)
    {
        if (HttpContext.Current != null)
        {
            HttpContext.Current.Items.Add(input, tokens);
        }
    }

    private List<string> GetCachedTokens(string input)
    {
        if (HttpContext.Current != null)
        {
            return HttpContext.Current.Items[input] as List<string>;
        }

        return null;
    }

    private void LoadMappings()
    {
        var mapResult = Client.Map<SearchAsset>(m => m
            .AllField(x => x.Enabled(false))
            .AutoMap()
        ).Log(isIndexRebuild: true);
    }
}

NestClient

//INFO: This class is a singleton for a reason
//Under the hood, Nest instantiate the Newtonsoft deserializer fresh for each instantiation of the client
//Therefore, if you instantiate the client fresh every request, then your deserializer gets 3x-4x slower
//Preserving the client as a singleton mitigates this cost
public static class NestClient
{
    public static string IndexName { get; private set; }

    public static ElasticSearchConfig Config
    {
        get
        {
            return SettingsManager.ElasticSearchConfig<ElasticSearchConfig>();
        }
    }

    private static object syncRoot = new object();
    private static IElasticClient instance = null;

    public static IElasticClient GetClient(string indexName = null)
    {
        if (instance == null)
        {
            IndexName = indexName;

            //if name != null, it will override the current index name in the db
            //use to create a new index, then update db when index is done building
            if (string.IsNullOrEmpty(indexName))
            {
                IndexName = Config.IndexName;
            }

            var uri = new Uri(Config.Url);
            var pool = new SingleNodeConnectionPool(uri);
            var settings = new ConnectionSettings(pool);
            settings.DefaultIndex(IndexName);

            //INFO: We want the SearchAsset object to be hard-bound to the index
            settings.InferMappingFor<SearchAsset>(m => m.IndexName(IndexName));

            instance = new ElasticClient(settings);
        }

        return instance;
    }
}

Пример вывода запроса

Здесь в это суть