代码之家  ›  专栏  ›  技术社区  ›  Firestrand

反射。发出动态类型内存放大

  •  11
  • Firestrand  · 技术社区  · 14 年前

    使用C 3.5,我尝试在运行时使用反射发射生成动态类型。我用过 Dynamic Query Library 从Microsoft创建类生成器的示例。一切正常,我的问题是100个生成的类型将内存使用量增加了大约25MB。这是一个完全不可接受的内存配置文件,因为我最终希望支持在内存中生成几十万个类型。

    内存分析显示内存显然是由各种System.Reflection.Emit类型和方法持有的,尽管我不知道原因。我还没有发现其他人在谈论这个问题,所以我希望这个社区的人知道我做错了什么,或者这是预期的行为。

    下面的人为示例:

    using System;
    using System.Collections.Generic;
    using System.Text;
    using System.Reflection;
    using System.Reflection.Emit;
    
    namespace SmallRelfectExample
    {
        class Program
        {
            static void Main(string[] args)
            {
                int typeCount = 100;
                int propCount = 100;
                Random rand = new Random();
                Type dynType = null;
                SlimClassFactory scf = new SlimClassFactory();
                for (int i = 0; i < typeCount; i++)
                {
                    List<DynamicProperty> dpl = new List<DynamicProperty>(propCount);
                    for (int j = 0; j < propCount; j++)
                    {
                        dpl.Add(new DynamicProperty("Key" + rand.Next().ToString(), typeof(String)));
                    }
                    dynType = scf.CreateDynamicClass(dpl.ToArray(), i);
                    //Optionally do something with the type here
                }
                Console.WriteLine("SmallRelfectExample: {0} Types generated.", typeCount);
                Console.ReadLine();
            }
        }
        public class SlimClassFactory
        {
            private readonly ModuleBuilder module;
            public SlimClassFactory()
            {
                AssemblyName name = new AssemblyName("DynamicClasses");
                AssemblyBuilder assembly = AppDomain.CurrentDomain.DefineDynamicAssembly(name, AssemblyBuilderAccess.Run);
                module = assembly.DefineDynamicModule("Module");
    
            }
            public Type CreateDynamicClass(DynamicProperty[] properties, int Id)
            {
                string typeName = "DynamicClass" + Id.ToString();
                TypeBuilder tb = module.DefineType(typeName, TypeAttributes.Class |
                    TypeAttributes.Public, typeof(DynamicClass));
                FieldInfo[] fields = GenerateProperties(tb, properties);
                GenerateEquals(tb, fields);
                GenerateGetHashCode(tb, fields);
                Type result = tb.CreateType();
                return result;
            }
            static FieldInfo[] GenerateProperties(TypeBuilder tb, DynamicProperty[] properties)
            {
                FieldInfo[] fields = new FieldBuilder[properties.Length];
                for (int i = 0; i < properties.Length; i++)
                {
                    DynamicProperty dp = properties[i];
                    FieldBuilder fb = tb.DefineField("_" + dp.Name, dp.Type, FieldAttributes.Private);
                    PropertyBuilder pb = tb.DefineProperty(dp.Name, PropertyAttributes.HasDefault, dp.Type, null);
                    MethodBuilder mbGet = tb.DefineMethod("get_" + dp.Name,
                        MethodAttributes.Public | MethodAttributes.SpecialName | MethodAttributes.HideBySig,
                        dp.Type, Type.EmptyTypes);
                    ILGenerator genGet = mbGet.GetILGenerator();
                    genGet.Emit(OpCodes.Ldarg_0);
                    genGet.Emit(OpCodes.Ldfld, fb);
                    genGet.Emit(OpCodes.Ret);
                    MethodBuilder mbSet = tb.DefineMethod("set_" + dp.Name,
                        MethodAttributes.Public | MethodAttributes.SpecialName | MethodAttributes.HideBySig,
                        null, new Type[] { dp.Type });
                    ILGenerator genSet = mbSet.GetILGenerator();
                    genSet.Emit(OpCodes.Ldarg_0);
                    genSet.Emit(OpCodes.Ldarg_1);
                    genSet.Emit(OpCodes.Stfld, fb);
                    genSet.Emit(OpCodes.Ret);
                    pb.SetGetMethod(mbGet);
                    pb.SetSetMethod(mbSet);
                    fields[i] = fb;
                }
                return fields;
            }
            static void GenerateEquals(TypeBuilder tb, FieldInfo[] fields)
            {
                MethodBuilder mb = tb.DefineMethod("Equals",
                    MethodAttributes.Public | MethodAttributes.ReuseSlot |
                    MethodAttributes.Virtual | MethodAttributes.HideBySig,
                    typeof(bool), new Type[] { typeof(object) });
                ILGenerator gen = mb.GetILGenerator();
                LocalBuilder other = gen.DeclareLocal(tb);
                Label next = gen.DefineLabel();
                gen.Emit(OpCodes.Ldarg_1);
                gen.Emit(OpCodes.Isinst, tb);
                gen.Emit(OpCodes.Stloc, other);
                gen.Emit(OpCodes.Ldloc, other);
                gen.Emit(OpCodes.Brtrue_S, next);
                gen.Emit(OpCodes.Ldc_I4_0);
                gen.Emit(OpCodes.Ret);
                gen.MarkLabel(next);
                foreach (FieldInfo field in fields)
                {
                    Type ft = field.FieldType;
                    Type ct = typeof(EqualityComparer<>).MakeGenericType(ft);
                    next = gen.DefineLabel();
                    gen.EmitCall(OpCodes.Call, ct.GetMethod("get_Default"), null);
                    gen.Emit(OpCodes.Ldarg_0);
                    gen.Emit(OpCodes.Ldfld, field);
                    gen.Emit(OpCodes.Ldloc, other);
                    gen.Emit(OpCodes.Ldfld, field);
                    gen.EmitCall(OpCodes.Callvirt, ct.GetMethod("Equals", new Type[] { ft, ft }), null);
                    gen.Emit(OpCodes.Brtrue_S, next);
                    gen.Emit(OpCodes.Ldc_I4_0);
                    gen.Emit(OpCodes.Ret);
                    gen.MarkLabel(next);
                }
                gen.Emit(OpCodes.Ldc_I4_1);
                gen.Emit(OpCodes.Ret);
            }
            static void GenerateGetHashCode(TypeBuilder tb, FieldInfo[] fields)
            {
                MethodBuilder mb = tb.DefineMethod("GetHashCode",
                    MethodAttributes.Public | MethodAttributes.ReuseSlot |
                    MethodAttributes.Virtual | MethodAttributes.HideBySig,
                    typeof(int), Type.EmptyTypes);
                ILGenerator gen = mb.GetILGenerator();
                gen.Emit(OpCodes.Ldc_I4_0);
                foreach (FieldInfo field in fields)
                {
                    Type ft = field.FieldType;
                    Type ct = typeof(EqualityComparer<>).MakeGenericType(ft);
                    gen.EmitCall(OpCodes.Call, ct.GetMethod("get_Default"), null);
                    gen.Emit(OpCodes.Ldarg_0);
                    gen.Emit(OpCodes.Ldfld, field);
                    gen.EmitCall(OpCodes.Callvirt, ct.GetMethod("GetHashCode", new Type[] { ft }), null);
                    gen.Emit(OpCodes.Xor);
                }
                gen.Emit(OpCodes.Ret);
            }
        }
        public abstract class DynamicClass
        {
            public override string ToString()
            {
                PropertyInfo[] props = GetType().GetProperties(BindingFlags.Instance | BindingFlags.Public);
                StringBuilder sb = new StringBuilder();
                sb.Append("{");
                for (int i = 0; i < props.Length; i++)
                {
                    if (i > 0) sb.Append(", ");
                    sb.Append(props[i].Name);
                    sb.Append("=");
                    sb.Append(props[i].GetValue(this, null));
                }
                sb.Append("}");
                return sb.ToString();
            }
        }
        public class DynamicProperty
        {
            private readonly string name;
            private readonly Type type;
    
            public DynamicProperty(string name, Type type)
            {
                if (name == null) throw new ArgumentNullException("name");
                if (type == null) throw new ArgumentNullException("type");
                this.name = name;
                this.type = type;
            }
    
            public string Name
            {
                get { return name; }
            }
    
            public Type Type
            {
                get { return type; }
            }
        }
    }
    
    4 回复  |  直到 14 年前
        1
  •  4
  •   leppie    14 年前

    不幸的是,在 ModuleBuilder 抓住内存,这永远不会得到gc'd。我记不起哪个字段和它现在包含的内容,但这可以从windbg中的sos中看到。

    好消息是.NET 4支持GC-able动态程序集:)

        2
  •  4
  •   Firestrand    14 年前

    这似乎是System.Reflection.Emit中的实际内存泄漏。 下面的新解决方案 通过使用反射和手动处理过程,我可以消除大部分已用完的内存。我使用扩展方法在某些类型上添加了Dispose方法。这并不是全部清除,但代码显示了如何清除。我正以另一种方式取得我需要的结果。这里的代码是为那些对如何做感兴趣的人准备的。

    在原始样本中,您将调用 tb.Dispose() 在生成类型后在typebuilder实例上执行。请记住,扩展方法如下 这不能把一切都清理干净 但是会释放大部分内存。此代码也没有针对速度进行优化。有很多方法可以加速反射的使用,这只是一个例子。 自担风险使用。

      public static void Dispose(this TypeBuilder tb)
            {
                if (tb == null)
                    return;
                Type tbType = typeof(TypeBuilder);
                FieldInfo tbMbList = tbType.GetField("m_listMethods", BindingFlags.Instance | BindingFlags.NonPublic); //List<MethodBuilder>
                FieldInfo tbDecType = tbType.GetField("m_DeclaringType", BindingFlags.Instance | BindingFlags.NonPublic);//TypeBuilder
                FieldInfo tbGenType = tbType.GetField("m_genTypeDef", BindingFlags.Instance | BindingFlags.NonPublic);//TypeBuilder
                FieldInfo tbDeclMeth = tbType.GetField("m_declMeth", BindingFlags.Instance | BindingFlags.NonPublic);//MethodBuilder
                FieldInfo tbMbCurMeth = tbType.GetField("m_currentMethod", BindingFlags.Instance | BindingFlags.NonPublic);//MethodBuilder
                FieldInfo tbMod = tbType.GetField("m_module", BindingFlags.Instance | BindingFlags.NonPublic);//ModuleBuilder
                FieldInfo tbGenTypeParArr = tbType.GetField("m_inst", BindingFlags.Instance | BindingFlags.NonPublic); //GenericTypeParameterBuilder[] 
    
                TypeBuilder tempDecType = tbDecType.GetValue(tb) as TypeBuilder;
                tempDecType.Dispose();
                tbDecType.SetValue(tb, null);
                tempDecType = tbGenType.GetValue(tb) as TypeBuilder;
                tempDecType.Dispose();
                tbDecType.SetValue(tb, null);
    
                MethodBuilder tempMeth = tbDeclMeth.GetValue(tb) as MethodBuilder;
                tempMeth.Dispose();
                tbDeclMeth.SetValue(tb,null);
                tempMeth = tbMbCurMeth.GetValue(tb) as MethodBuilder;
                tempMeth.Dispose();
                tbMbCurMeth.SetValue(tb, null);
    
                ArrayList mbList = tbMbList.GetValue(tb) as ArrayList;
                for (int i = 0; i < mbList.Count; i++)
                {
                    tempMeth = mbList[i] as MethodBuilder;
                    tempMeth.Dispose();
                    mbList[i] = null;
                }
                tbMbList.SetValue(tb, null);
    
                ModuleBuilder tempMod = tbMod.GetValue(tb) as ModuleBuilder;
                tempMod.Dispose();
                tbMod.SetValue(tb, null);
    
                tbGenTypeParArr.SetValue(tb, null);
            }
            public static void Dispose(this MethodBuilder mb)
            {
                if (mb == null)
                    return;
                Type mbType = typeof(MethodBuilder);
                FieldInfo mbILGen = mbType.GetField("m_ilGenerator", BindingFlags.Instance | BindingFlags.NonPublic);
                //FieldInfo mbIAttr = mbType.GetField("m_iAttributes", BindingFlags.Instance | BindingFlags.NonPublic);
                FieldInfo mbMod = mbType.GetField("m_module", BindingFlags.Instance | BindingFlags.NonPublic); //ModuleBuilder 
                FieldInfo mbContType = mbType.GetField("m_containingType", BindingFlags.Instance | BindingFlags.NonPublic);
                FieldInfo mbLocSigHelp = mbType.GetField("m_localSignature", BindingFlags.Instance | BindingFlags.NonPublic);//SignatureHelper
                FieldInfo mbSigHelp = mbType.GetField("m_signature", BindingFlags.Instance | BindingFlags.NonPublic);//SignatureHelper
    
                ILGenerator tempIlGen = mbILGen.GetValue(mb) as ILGenerator;
                tempIlGen.Dispose();
                SignatureHelper tempmbSigHelp = mbLocSigHelp.GetValue(mb) as SignatureHelper;
                tempmbSigHelp.Dispose();
                tempmbSigHelp = mbSigHelp.GetValue(mb) as SignatureHelper;
                tempmbSigHelp.Dispose();
    
                ModuleBuilder tempMod = mbMod.GetValue(mb) as ModuleBuilder;
                tempMod.Dispose();
                mbMod.SetValue(mb, null);
    
                mbILGen.SetValue(mb, null);
                mbContType.SetValue(mb, null);
                mbLocSigHelp.SetValue(mb, null);
                mbSigHelp.SetValue(mb, null);
                mbMod.SetValue(mb, null);
            }
            public static void Dispose(this SignatureHelper sh)
            {
                if (sh == null)
                    return;
                Type shType = typeof(SignatureHelper);
                FieldInfo shModule = shType.GetField("m_module", BindingFlags.Instance | BindingFlags.NonPublic);
                //FieldInfo shSig = shType.GetField("m_signature", BindingFlags.Instance | BindingFlags.NonPublic);
                shModule.SetValue(sh, null);
                //shSig.SetValue(sh, null);
            }
            public static void Dispose(this ILGenerator ilGen)
            {
                if (ilGen == null)
                    return;
                Type ilGenType = typeof(ILGenerator);
                FieldInfo ilSigHelp = ilGenType.GetField("m_localSignature", BindingFlags.Instance | BindingFlags.NonPublic);//SignatureHelper
                SignatureHelper sigTemp = ilSigHelp.GetValue(ilGen) as SignatureHelper;
                sigTemp.Dispose();
                ilSigHelp.SetValue(ilGen, null);
            }
            public static void Dispose(this ModuleBuilder modBuild)
            {
                if (modBuild == null)
                    return;
                Type modBuildType = typeof(ModuleBuilder);
                FieldInfo modBuildModData = modBuildType.GetField("m__moduleData", BindingFlags.Instance | BindingFlags.NonPublic |BindingFlags.FlattenHierarchy );
                FieldInfo modTypeBuildList = modBuildType.GetField("m__TypeBuilderList", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.FlattenHierarchy);
    
                ArrayList modTypeList = modTypeBuildList.GetValue(modBuild) as ArrayList;
                if(modTypeList != null)
                {
                    for (int i = 0; i < modTypeList.Count; i++)
                    {
                        TypeBuilder tb = modTypeList[i] as TypeBuilder;
                        tb.Dispose();
                        modTypeList = null;
                    }
                    modTypeBuildList.SetValue(modBuild, null);
                }
                modBuildModData.SetValue(modBuild, null);
            }
    

    编辑 找到实际原因:在动态程序集中创建的每个类型似乎都包含对 ModuleBuilder (在 Type.Module )它依次保存着 TypeBuilder 物体。每次添加类型以检查名称冲突时都会扫描此列表。如果你保持 HashSet 在类型生成例程之外,以确保没有任何名称冲突,可以在 模块构建器 私有变量 m__TypeBuilderList 之后 Type 不会产生任何负面影响(目前为止)

        3
  •  1
  •   Marc Gravell    14 年前

    好吧,我注意到的第一件事是你正在创建一个新工厂,因此是新的 AssemblyBuilder ,每次迭代。是否可以重新使用工厂(在同一动态部件中创建多个类型)?

        4
  •  1
  •   kvb    14 年前

    不管你现在看到的实际问题是什么,我强烈建议你不要采用目前的方法。reflection.emit的设计不支持创建数十万种类型(例如,请参见 this connect issue 尽管这个特定的问题可能只适用于将它们全部放入单个动态部件中的情况)。为什么需要创建这么多类型?