Skip to content

Instantly share code, notes, and snippets.

@dalenewman
Created August 12, 2015 15:27
Show Gist options
  • Save dalenewman/9d863247ce2d8b5e79ae to your computer and use it in GitHub Desktop.
Save dalenewman/9d863247ce2d8b5e79ae to your computer and use it in GitHub Desktop.
An Extension Method for SolrNet to stream CSV data from SOLR
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using HttpWebAdapters;
using SolrNet;
using SolrNet.Commands.Parameters;
using SolrNet.Impl;
using SolrNet.Impl.FieldSerializers;
using SolrNet.Impl.QuerySerializers;
using SolrNet.Utils;
namespace YourNameSpace {
public static class SolrNetExtensions {
public static Stream Csv<T>(this ISolrReadOnlyOperations<T> operations, string core, ISolrQuery query, QueryOptions options) {
options.ExtraParams = ExtraParamsWithCsv(options);
var param = GetAllParameters(query, options, new DefaultQuerySerializer(new DefaultFieldSerializer()));
return Get(core, param);
}
private static IEnumerable<KeyValuePair<string, string>> ExtraParamsWithCsv(CommonQueryOptions options) {
var csv = new List<KeyValuePair<string, string>>() { new KeyValuePair<string, string>("wt", "csv") };
var preserve = options.ExtraParams ?? new List<KeyValuePair<string, string>>();
csv.AddRange(preserve.Where(kv => kv.Key != "wt"));
return csv;
}
private static string GetQuery(IEnumerable<KeyValuePair<string, string>> parameters) {
var param = new List<KeyValuePair<string, string>>();
if (parameters != null)
param.AddRange(parameters);
param.Add(KV.Create("version", "2.2"));
return string.Join("&", param
.Select(kv => KV.Create(HttpUtility.UrlEncode(kv.Key), HttpUtility.UrlEncode(kv.Value)))
.Select(kv => string.Format("{0}={1}", kv.Key, kv.Value))
.ToArray());
}
private static Stream Get(string serverUrl, IEnumerable<KeyValuePair<string, string>> parameters) {
var u = new UriBuilder(serverUrl);
u.Path += "/select";
u.Query = GetQuery(parameters);
var request = new HttpWebRequestFactory().Create(u.Uri);
request.Method = HttpWebRequestMethod.GET;
request.KeepAlive = true;
request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
return request.GetResponse().GetResponseStream();
}
private static IEnumerable<KeyValuePair<string, string>> GetFilterQueries(ICollection<ISolrQuery> filterQueries, ISolrQuerySerializer querySerializer) {
if (filterQueries == null || filterQueries.Count == 0)
yield break;
foreach (var fq in filterQueries) {
yield return new KeyValuePair<string, string>("fq", querySerializer.Serialize(fq));
}
}
private static IEnumerable<KeyValuePair<string, string>> GetCommonParameters(CommonQueryOptions options, ISolrQuerySerializer querySerializer) {
if (options == null)
yield break;
if (options.Start.HasValue)
yield return KV.Create("start", options.Start.ToString());
var rows = options.Rows.HasValue ? options.Rows.Value : 100000000;
yield return KV.Create("rows", rows.ToString(CultureInfo.InvariantCulture));
if (options.Fields != null && options.Fields.Count > 0)
yield return KV.Create("fl", string.Join(",", options.Fields.ToArray()));
foreach (var p in GetFilterQueries(options.FilterQueries, querySerializer))
yield return p;
if (options.ExtraParams != null)
foreach (var p in options.ExtraParams)
yield return p;
}
private static IEnumerable<KeyValuePair<string, string>> GetAllParameters(ISolrQuery query, QueryOptions options, ISolrQuerySerializer querySerializer) {
yield return KV.Create("q", querySerializer.Serialize(query));
if (options == null)
yield break;
foreach (var p in GetCommonParameters(options, querySerializer))
yield return p;
if (options.OrderBy != null && options.OrderBy.Count > 0)
yield return KV.Create("sort", string.Join(",", options.OrderBy.Select(x => x.ToString()).ToArray()));
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment