MapReduce方法主体：
public static IDictionary<TKey, TResult> MapReduce<TInput, TKey, TValue, TResult>(this IList<TInput> inputList,
Func<MapReduceData<TInput>, KeyValueClass<TKey, TValue>> map, Func<TKey, IList<TValue>, TResult> reduce)
{
object locker = new object();
ConcurrentDictionary<TKey, TResult> result = new ConcurrentDictionary<TKey, TResult>();
//保存map出来的结果
ConcurrentDictionary<TKey, IList<TValue>> mapDic = new ConcurrentDictionary<TKey, IList<TValue>>();
var parallelOptions = new ParallelOptions();
parallelOptions.MaxDegreeOfParallelism = Environment.ProcessorCount;
//并行map
Parallel.For(, inputList.Count(), parallelOptions, t =>
{
MapReduceData<TInput> data = new MapReduceData<TInput>
{
Data = inputList[t],
Index = t,
List = inputList,
};
var pair = map(data);
if (pair != null && pair.Valid)
{
//锁住防止并发操作list造成数据缺失
lock (locker)
{
//将匹配出来的结果加入结果集放入字典
IList<TValue> list = null;
if (mapDic.ContainsKey(pair.Key))
{
list = mapDic[pair.Key];
}
else
{
list = new List<TValue>();
mapDic[pair.Key] = list;
}
list.Add(pair.Value);
}
}
}); //并行reduce
Parallel.For(, mapDic.Keys.Count, parallelOptions, t =>
{
KeyValuePair<TKey, IList<TValue>> pair = mapDic.ElementAt(t);
result[pair.Key] = reduce(pair.Key, pair.Value);
});
return result;
}
KeyValueClass定义：
public class KeyValueClass<K, V>
{
public KeyValueClass(K key, V value)
{
Key = key;
Value = value;
} public KeyValueClass()
{ } public K Key { get; set; } public V Value { get; set; }
}
Console测试：
List<TestClass> listTestClass = new List<TestClass>();
listTestClass.Add(new TestClass { a = "a", g = });
listTestClass.Add(new TestClass { a = "b", g = });
listTestClass.Add(new TestClass { a = "c", g = });
listTestClass.Add(new TestClass { a = "d", g = });
listTestClass.Add(new TestClass { a = "e", g = });
listTestClass.Add(new TestClass { a = "f", g = });
listTestClass.Add(new TestClass { a = "g", g = });
listTestClass.Add(new TestClass { a = "h", g = });
IDictionary<int, string> dic = listTestClass.MapReduce(t =>
{
if (t.g < )
{
return new KeyValueClass<int, string>(t.g, t.a);
}
return null;
}, (key, values) =>
{
return string.Join(",", values);
});
TestClass定义：
public class TestClass
{
public string a { get; set; }
public string b { get; set; } public string d { get; set; } //public DateTime f { get; set; } public int g { get; set; } public List<TestClass> test { get; set; } public Dictionary<string, string> dic { get; set; }
}
结果：
1：a,e
2：d,f
3：b
4：c
词频性能测试