无论是深度学习还是机器学习，大多情况下训练中都会遇到这几个参数，今天依据我自己的理解具体的总结一下，可能会存在错误，还请指正.
learning_rate , weight_decay , momentum这三个参数的含义. 并附上demo.
我们会使用一个例子来说明一下：
比如我们有一堆数据aaarticlea/gif;base64,R0lGODlhggEUALMAAP///wAAALq6unZ2dqqqqmZmZlRUVMzMzO7u7tzc3CIiIpiYmBAQEDIyMoiIiERERCH5BAEAAAAALAAAAACCARQAAAT+EEggBpmzXsy7/2AojmRpnmYloJjGvnAsZ8Ma2nM+q15h9D+dcEgslnwvpHHJJCk/j6YU2AkcPNapdsvFIAIIVrZLHn7D0NlAESgMfAX0cN1+G+JecD7R+fLLgIEzfiiEgoeFAX8eUTMPBRhrRo+RCnuXHIaIm5wemiWfnaKYjDkBGxIEY0OnGKpXEp+hALOjtmW1Ibm3iLuNMQJ6EwYMRcFyAMSktIp9za4LNQQLBosm09LU1iTYAtPVLN3f21Pi2iO7H58HBJCpQRPs7gAE8CTyE/Vd+O8ivtDZwKUaYCFSAwwFGMD6wI8ePAIEUQEYcHBCwoXMrCUQhmEjMgH+fBIwuKAQBUgAIkliHHEyJYCSJ1qOfLlSikyVJFaV8DgBwQKaEgwM8PITplATPoEmG7olqVGmIXRy+HVT6QOoGBoY+NbAgb+isI4CuOpBK1evVTDe8bDWlQQBxTAccLAAbYgNcDsk+AUCb9x8Dt6M8MuBAAEH85YQPjHAHom2b1EGmMAABwXJlG3sFbFiY2Z6gRMbwyyh8ojGIH4tnkDxw6oCP2+QfrmiNZaFsD2gpqBANG/frB1XXBD7tONolkrsjoVWrHEOk8cWb7LcxICZLHvrDnIgOofd3SUgJwF+MoLmWI2UV459aofqL68OmB4MwwLvIdbHJ0gf//0P123+8ARCjk3QAH3c8RXCgXolRwKDGQxX0QgQTvDHA9MxUaFyCo4woAQKbODAhBiEKMGIFjoYgokAoCiAhFKwiOJpHU7Q4YZf1DQRiQXgB4KMB+WoG48+SjAAX2c4g0wsz6zgQBAikWAIDgmoqEuTEvwhGDpY9tTAkkZMSUtgAixQADlScbmkMGRVgUabKFnp2ptMaQkcK3SOWYOZ26RpYyZdclQiVg3IiUWeAAg6gQKEGqrTLJrk9R8AB/CBWJbeJWDAR8VMWmmKHW3aQqfRfVriIppyCoCnixDwACqpchArSqLSisasb5EqwacXZDEfoOTY6skzE+ixkWXFhnEsqBb+1oqBsQEgq8AfuGZZ66zYOpusZCv0esWvPRHbQSOSlvpHARIxaVkDUSTQmbYSQGsDus6s2y4OsojL5CIHoMuOBTY8uetfBzAwXb8E/EuAZVXKZbBc/kK0MAcGWFbwwRED3IEBaF3MQcGwgDxwyA/Hk/HEuzZiwEq1eOzMNtFEo6h48w3AUcPxlGxfzYJWnHOGIlMKU9Au7ywzGgeojNE/lJ5s2XzThEFRAFtFWNfAGa7KsxxQLyB1A1Sj8uLV4VoDqb4fCECugumKgLNbJjhwxTZtf7DAcCrWfU3caP1Vdl/DBttijRg4wNfb+YBgeOFzww2D3oNP4EDfwKamBeT+i+uQLznpZHVihpB7gHgq1xCAQAJ2kU6CAGhVF/pgJjxwhQDVWLZL3Z+YBoACOAiAhu68dyRnusDjYNjpqb8uAvE2BC+B7BTU/nca5XBQvBCh+OmnBwfUzIEDYIIgQI8OLAL+TgGkHwBW55dQ1xtytP+C/CM0goABGSbpAf3PLoSAO1t6nlf+Z6TEjC8A5ZMcMgg4kXlsRH1Q4d8J6MfAAI4lFvjLQ/ieNwUKAvBOL9BfcLYzA+XBLgcmXJ4MUuiEAiUOQPYgDrjgJsOseaBtNeTGCjuQwyO48E9SwGHMiPChtxSEAy7ghRK1kMQXNFEdLFyiE48Igh9K0QNPvKIKFrdYQi560YsRAAA7" alt="" />，我们只知道这对数据是从一个aaarticlea/png;base64,iVBORw0KGgoAAAANSUhEUgAAARIAAAAVBAMAAAB4a3wcAAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAEJl2u1Tdic1EZu8yqyJi4sUJAAAACXBIWXMAAA7EAAAOxAGVKw4bAAADSUlEQVRIDbWWPWgUQRTH/5vNfezeZbNoEwzEs7AJiJc0gZDINWmsTmMQg0UsUtjIETCgKJyVHyAcSRRJUBYjXgyKsUgX5IiKhSIHKdQisqDYpLgLFnbGN/Nm4+4mt2A8B+b2zfxm3vvP7My+A/5zGZ6ICjDSG0WbyrRsWyHCYb3VCVONe0w7DP6xbRTNYoQLd6eSFh6uZSKm7Qm5iUiXHYGl76OXNaDCTO8pXNSkuBtBY0MBOHQSKKqeZIA0o/E50klb1oe1TcB0VIee95GGpllqiAg88UO94vqbQWoWzTEfjVGj1VUd6aqPNDT/QslIl3/V5NGv08gmfPG0yeslpGjI6KXRL8BSIPxsBt3aC9EVK9955npMKWlAOZbxEC1OZ9fWVkF5VpP91J6fsH1UXCRSYrgb+RWg7kUTT81ZwQpuCvOBtZyqCEMUVtKIcqwDqUqqkizRaOVZzvT2ZFcqAnynRWNOjD1HtWVdlAL5sOpWHT3p/izyBu3Wq3kxxFOiKL6BqZWTUO1/qYYadJu6pOdYXyaKDgt6kCopAa6KH6HkT0lUzRwKSSwC4o0W5ck+sv7p9vpHGsTUnJQmUB4jg1ZxQ66C3D1CgnqokOf9EDMa0XYcJ3qXaidV3BI/tHJfiefpmmXv4zWQdGBU0zkJ1TlhKpdBFNaYhN6ZHMcadMi8Q57vYWEXKvMOUUsqGaQR4sRqv1BiJdtvB+12PJ+uDGKDts419KrFQpUSplIJ0bCSTYzjvcw7wrOFiYASSSHyjoyrXSO6RpVu8ZxZT7sI3eKU25k30YuasfQcH/QMnRpRlBKmpETSsJKX2jIckXfYMy7Iqd6OSQrKO0xP5YmeF64drJ49fQYIfdmM7qfdPRjEu/ThNxdLejUd2BOmpETSsJKpxdV5G5R32HNbKaCEKTpspnI/xDK1HA/Drl/7GcxIbOQMHqf2RM2Rx53s0DlRNO6yMWDxk++4gl7eWcBjaOz8ikKz6hl4JNHP7aLuSEML4LASO0BV3omd+MrdQaryzjRdr1RZ7lrkv4L05QJ7eVsOBOGGcfQYG1M/3J3Yyzu1rZ874XbeMfsOIcEfK82R45r+Twk78o5fTzDv+EnzbUvkncZF5B1VfgM+rw2BYCJoKgAAAABJRU5ErkJggg==" alt="" />黑盒中得到的，我们现在要寻找到那个具体的函数f(x),我们定义为目标函数T.
我们现在假定有存在这个函数并且这个函数为：
aaarticlea/png;base64,iVBORw0KGgoAAAANSUhEUgAAARIAAAAVBAMAAAB4a3wcAAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAEJl2u1Tdic1EZu8yqyJi4sUJAAAACXBIWXMAAA7EAAAOxAGVKw4bAAADSUlEQVRIDbWWPWgUQRTH/5vNfezeZbNoEwzEs7AJiJc0gZDINWmsTmMQg0UsUtjIETCgKJyVHyAcSRRJUBYjXgyKsUgX5IiKhSIHKdQisqDYpLgLFnbGN/Nm4+4mt2A8B+b2zfxm3vvP7My+A/5zGZ6ICjDSG0WbyrRsWyHCYb3VCVONe0w7DP6xbRTNYoQLd6eSFh6uZSKm7Qm5iUiXHYGl76OXNaDCTO8pXNSkuBtBY0MBOHQSKKqeZIA0o/E50klb1oe1TcB0VIee95GGpllqiAg88UO94vqbQWoWzTEfjVGj1VUd6aqPNDT/QslIl3/V5NGv08gmfPG0yeslpGjI6KXRL8BSIPxsBt3aC9EVK9955npMKWlAOZbxEC1OZ9fWVkF5VpP91J6fsH1UXCRSYrgb+RWg7kUTT81ZwQpuCvOBtZyqCEMUVtKIcqwDqUqqkizRaOVZzvT2ZFcqAnynRWNOjD1HtWVdlAL5sOpWHT3p/izyBu3Wq3kxxFOiKL6BqZWTUO1/qYYadJu6pOdYXyaKDgt6kCopAa6KH6HkT0lUzRwKSSwC4o0W5ck+sv7p9vpHGsTUnJQmUB4jg1ZxQ66C3D1CgnqokOf9EDMa0XYcJ3qXaidV3BI/tHJfiefpmmXv4zWQdGBU0zkJ1TlhKpdBFNaYhN6ZHMcadMi8Q57vYWEXKvMOUUsqGaQR4sRqv1BiJdtvB+12PJ+uDGKDts419KrFQpUSplIJ0bCSTYzjvcw7wrOFiYASSSHyjoyrXSO6RpVu8ZxZT7sI3eKU25k30YuasfQcH/QMnRpRlBKmpETSsJKX2jIckXfYMy7Iqd6OSQrKO0xP5YmeF64drJ49fQYIfdmM7qfdPRjEu/ThNxdLejUd2BOmpETSsJKpxdV5G5R32HNbKaCEKTpspnI/xDK1HA/Drl/7GcxIbOQMHqf2RM2Rx53s0DlRNO6yMWDxk++4gl7eWcBjaOz8ikKz6hl4JNHP7aLuSEML4LASO0BV3omd+MrdQaryzjRdr1RZ7lrkv4L05QJ7eVsOBOGGcfQYG1M/3J3Yyzu1rZ874XbeMfsOIcEfK82R45r+Twk78o5fTzDv+EnzbUvkncZF5B1VfgM+rw2BYCJoKgAAAABJRU5ErkJggg==" alt="" />
我们现在要使用这对数据来训练目标函数. 我们可以设想如果存在一个这个函数，必定满足{x,y}所有的关系，也就是说：
aaarticlea/gif;base64,R0lGODlhTQASALMAAP///wAAAO7u7mZmZoiIiERERKqqqiIiInZ2djIyMrq6upiYmBAQEMzMzFRUVNzc3CH5BAEAAAAALAAAAABNABIAAAT+EMhJq1g15yW0/2AYDmImkGWqUieBSkSzTgox32FhGC+Q4JIfcDgRBDqUBQZIMBCJhkOm8Bg2HE+cAHFIIBQURsaw+JIdVY84K2nwJgasp7AMVxTVB8PJkHkCbAAXAH0SDggfhRQPUhROAAprjogUB0gVAw6am5yUFGAPgBIMYAAGnoKii40ZCHIsGZaBAK5tqpcSCqxFkhUJdSC9WQePBEIaBK92JgFppckZwhOZnNWoFEcSBZTQFA42U2mQYguiDVUYshNXswDZoWDpuAelFUrrAwZuCAb1D8cwHgUqUyYbgH8VVFkokAIBMB/tKBBgKMGhhA4KKGpA4CeEpTooCh4+ISWB3oSPhMrU09DjAwICaU60k0mrx8sqDsF9GLSCQ0QlFjVEAAA7" alt="" />
那么最理想的情况下 : aaarticlea/gif;base64,R0lGODlhbQASALMAAP///wAAAO7u7mZmZoiIiERERKqqqiIiInZ2djIyMrq6upiYmBAQEMzMzFRUVNzc3CH5BAEAAAAALAAAAABtABIAAAT+EMhJq1g15yW0/2AojiQ1lJNwomzreiqxSkTjKsSr75JBLASFp2AwzAAJXZLHRCkKk4TNEuhQFpgXwdDsioiThSNjOGQKD11j7G1rAlOAgkERIA4JhHBCrxgWen8OaR59TA1GEwZsXQ8BhAANVRUFWRSGEgppDwxcDHEZAU0XAJ8SDghtkpCrFaYUD2YUXHKYPakUB1Y7Qo58ewa4Jg7ExcbCE60SyhICohWxHgiMKRm6GQ0F2tvcORrTy88AuzuOcQqTEwqydbZRliDuOge0BEttAXsABrYE1HwanBHa4y+DvBfpCuAqmGGAsYeoNFSawHCCA2+UIM0BsOBZgzRyGK4l+7djkiMhIcnpeBIFkoQD+q5YajDAACIEBvQ9uEeDVhNAgNLt9OJjwYCYEsRZgDICATwkbmgwBeA0qgU5UzMgAPVBlyUFT3cw2ANzglerfAAhrXDkAwIChFR4kUv1yFuXUZ1ihBE2BIc2WKqi9RABADs=" alt="" /> ，那么我们不妨定义这样一个优化目标函数：
aaarticlea/gif;base64,R0lGODlhwgAlALMAAP///wAAAHZ2diIiIlRUVDIyMkRERJiYmLq6utzc3BAQEGZmZu7u7qqqqszMzIiIiCH5BAEAAAAALAAAAADCACUAAAT+EMhJq7046837ZAbijWRpnmiqrlQiHIHIznRt37gU53zv/7UdcEgsGgHCo3LJRCWb0Kh08pxar8Qqdsu1abu5xkOwABu/5hrjISEIjowD6cDwfWUSvCfOoQ/1MggFEgeDRmUkDIg3DAIGAQUCDhUGFJUkixqKQ5cSnQkSZBQCAwFuOQ+TJQhsR50ArxSKD5mpHqxAr7EAA6AUBbs1hibDRboYBg0NmcUczTnHFAR6OnI5B9YmDw2ulhcMAXUU2CTbP9EStr55Aes3Bu4kDgTdE8ENAxbw8vQ+6MoMErQKpSCDmAOSKDTAdmDdQmy+ClJAcGDRAjkMFIibINGYN1n+pCLp6ThhoQAECwm4I5nCwbKS/aIlCEAzwJsJBTJRWNAq4wQENwW0AhqqFUk+ClQllVCAWoB6ni4YyMaxAgJQCRRwW0qFBVJVp2B9zACOm4VCFISIArBxrbgE+X4CmCmBbrqNEgbgnbCAgN+/gG+eQEeB6wS4FcwiYDlB7woRdgEokEH4QoNwFUApyAZOlQOaBlQB+BwgdN24o/od6CS48V4VNWPTHCu2AjgLiC8I6GfBcQUHBoILHz4wtYTP9mhfWID6uIOZDp8qJMCyAXUJPisMMEtgYHEJjM8pB4CgOfbwAApQLUxje7phlXvrLBM5fatkVWGZlZw/rTj3o4n+1l8FfQFmYGslxPcAb+xZAI4v1KBnAmawCBafLAHsBwBPefmyoD3iOICIASEusk9hoGRk1nqjMcjJeN1Jtc5iAMBwnDvz0IDZTHhcGEoBpgjgiAIUJnDRA1RRxNBPDFFFzkQXHSDiAQ+8Zo4R8Q1ADSHZiNiASwI0QM2VLCCEEIW1RQUFCCg8k8t40jkYTAZurvAAYXMakdAqLM5A5QIh6DNWHQjkuScHFNEwWV56+HiEThxsgoMfc2VIyVibnaQBpN9weoKkayUnahR8jEDpDQOIRoB56LzwnYN9VnBqmWYKOmoaRWw2gQBxtiENrhsw6CKwRqxK7LHI2mZpsswqEmvAq81Gi8UD0EprbRRiXKutFYmms+23SoiIwLgvgWvuEETGNuy5HUQAADs=" alt="" />
对于这堆数据，我们认为当Loss(W)对于所有的pair{x,y}都满足 Loss(W)趋近于或者等于0时，我们认为我们找到这个理想的目标函数T. 也就是此时 aaarticlea/gif;base64,R0lGODlhSAASALMAAP///wAAAO7u7mZmZoiIiERERKqqqiIiInZ2djIyMrq6upiYmBAQEMzMzFRUVNzc3CH5BAEAAAAALAAAAABIABIAAAT+EMhJq1g1gyW0/2AYDiIgkGWqUieBSkSTKsQqLoyDIAyzO75MwWB4ARIrpA3k6EiCEgWGFXBOFtMUwbD0PLiSR0AxaZAphoPwsWo4uhqwZBGgKKwCxCGBOD8zBgt9gQ5sEwxwIQ5KHwVZhxUKbA8MXAwyE3WJHww1IJcVD2poUYgaB1YUAw6sra4IIGKYHgKaFKIeCG+nqZsTBLYeCqMsphkJjxTGvhQFBSEEuxXLE7WGfn8aq67csB8Bnh8O4c2GAAqIdBIN5gBuzBUNAbMeB9hXWQ0DBg0GCAb3tsCDMeBAgAII2lUIxuKZCkYDRXRQ4FADAnogpERMwUDQvQoRRj6c2JgCwQJyGi7c6EVSQwQAOw==" alt="" />.
以上，我们发现寻找的目标函数的问题，已经成功的转移为求解：
aaarticlea/gif;base64,R0lGODlhfAASALMAAP///wAAAJiYmIiIiO7u7lRUVLq6uszMzNzc3KqqqiIiIkRERHZ2dmZmZhAQEDIyMiH5BAEAAAAALAAAAAB8ABIAAAT+EMhJq70YEJG7/9vXCURFmqWofs3qsq9FtNRc2fGUOOtwWAxFoMDIfXwTRmCI0BQCDhpAEFAkJoZBJbvVGglN1QPzWBhFY8oiPQkUKW8KezKX1M8eAecS2OMxehUNCjUBUgAGYRQDV4uNE4x/IguKFAYBlZJqmQw8EwIPBRR+FAeipacTphICAwMbAgUGsAU/EgYCUgmutAYUnkDAFrsCDLYSCXoCisl6YcISAwFYCAtmrBnQANo8BIl9yAp7AtchDj/eCOAACXPadocUDV4EwAZvDF73EvkS2lSfABQgBACBLwzTKiSkMM1XAoJTIDI45UvdLXYQB1wrCLHQo1H+cwIcZEAjBQCSElIg6HgL0xR+nkhZUGBSAs0KNwE0iENkwgJSKCfsnFDAC8cLCQLULLjND4EAtg4sWXBMagCqElZakHrAYLRpCZbiXJpzQk4HB7cdfFrp3K+0LlO+G1ThQNe47BYiKwAtAV+5FtTh+qT04wVuFjxZzLoQFKJwjwsufBh52wUF8VostuNlwSNPnicA0/a0gMmkcbIlXj2FDTmhWrwwKOLFsYShRindCPBxns0wA1ItSHGAxnAJxX1mAhDAKACuHlahqnuKgR/rqvIRb3DipJ8D2j8BfTBk9gIHSrNyH0AKlzIsykgFqrAxa2oMkRxVyK8pA4H6/YEwAOAZdxQYoAfGHKhCgn8MZolMDiqIQTwSTvgHDhNgmBKFFWYoU4c3fOhCd58sRWIEADs=" alt="" />
也就是Loss 越小， f(x)越接近我们寻找的目标函数T.
那么说了这么多，这个和我们说的学习率learning_rate有什么关系呢？
既然我们知道了我们当前的f(x)和目标函数的T的误差，那么我们可以将这个误差转移到每一个参数上，也就是变成每一个参数w和目标函数T的参数w_t的误差. 然后我们就以一定的幅度stride来缩小和真实值的距离，我们称这个stride为学习率learning_rate 而且我们就是这么做的.
我们用公式表述就是：
我们的误差(损失)Loss:
aaarticlea/png;base64,iVBORw0KGgoAAAANSUhEUgAAAUYAAABxBAMAAABb8RdiAAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAid2rzbtmRCLvmRBUMnZ/MwGQAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAIn0lEQVR4Ae1bXYwTVRQ+7XZn2p22u4ouCVHZB6PGrLGICT6RGv8eTEx9UDQa7QquRuJSw58QYirGB3xw+wQmQhwlwoMGSyIoPLhVIiaIa40hQYJSghKDP6lBhWTFes69d+bOTGf7O7OtoXczM2fO33w9c2c6O18PQBtDW9ZG8NyEhocvzM2B2jpKD2Nb5TODe3U0S9GW0KtjW+Uzg5uvY/S0bkbPjdA8xk9gkR/Q9htJVxqCuSWMKkDGUAjXakcgVQZ918AKw9nLrW4kO2gIYqtd/ee1AP0AKcOgc8HpiFpSpdA3DAu4TxvrwK3OcxEvGunUkiFZtwKjtkcHw9XFkVSEEeAmXNoc9zjjQ1KRlKKUBMb+ozqYri6OqGIY+8yJIVM0K/3tDBiXil+lKCWBcSoPYLq6OKKKYfwtLENblNSLzsBHpOJbKUpJYPwMNaariyOqCGP0y9dlaItSIEeB02NFiI4/yxZUrNNhl/YmQMQtKce49/r5AOgaHdu0PGt3jB2BaIJUhHGy8odbkqZ0IR3dB0paDl6A69iSAy29A3bAfQCDbqlEHR9GG2J8Mbw9UrQ77o8U+4ZIxc61W4omdVMFDMBb2AW8R+RpCY+AGi6Hy/ALQDCPxp2LcVgufoFxBwC6QkFFoNwRt2xkJiGSIZVXGL9GGNmtoF2CU38VaaEDBxJKCoYchwao4CjT/SQFoJU5RvRFYPRhYg/RKOHePDjFVF5hfAeTH78IUTzkoRwteNYhWOgrAR4N51T14BjVEcSJrhBK48ruOAo/MpVXGG/G2YgnNlTqL0COFppkg/lgIVYUh3Y/1/EEQiOMU1nViXEGHvQSo/YPwB78KjgD/fmYTgvdUCLZAwUFj+9ySzG+C7HQdO9Rc6vguNNxJozf6+Leg17tjqvOX/HoDLw2kQZ1fJwtdGNWdz2/Cy8ZcLk1GxiDGbSPQ+z9k9egZHccG5thKq/OdfWH3CdVSSlKic9HdlsyXR2O8RQAqvzDGCgaeNS0IYntxLtYM4Yxv5ZUhqvdMZLBIpPKP4yg0+FpOB+5lHwYpy/DuBW/ZXDobO1wPEiPY8azGXfwfD3rM26wCLcJjMMldljXZ1xlLA3sGdfHOs76oSNpWAjQB5CY1cVmSJDv3A+sY7cPup12++gvdTtCgN+7H2L0f1DGdRDv9kLGdq8/0u0YI5XK+W7H2BF8+/ARocvHy6u6HyMM9jB6Mo16dfSkjL356E0Zm6jjdPtvjFvE3PA1o5XwTVhnRsMY1aSS7AzEvU/eWeQMRa3j4+ucDGQDei0fv231/sNk/09DMOs3jlr5G8O4pFYK320NYYwXs74DqXGAhjAe/bhUI4XvpkYwhiuVTt172OcnjOpzMJA+kOdEhqwKYzH4NSOVnZAII9IVkWIow4kMCYKxGB3AOEBUxmI6c4PEE/C3hEhXTEI8X2BEhsTIWIwOYJQIhMTm4zx4GgJEepDSzmJ0DcZRWEr/3zMiw/IpiMXoAEZ5rq11nIFR+F4QGRaMxGJ0AKMFARfZuX5D2w5pQWRYPIjF6BqMG8dPTOQFkWHBSCxGZzEq55YhHlZHCy6bGGfWlE03pzsrYbJQEyNjMTpbx1ugb6QmRsZidBbjVgiUa2JkLEZnMSKPgXWsx1A0znX4M1HZrwD8Se1Z1ic8y+RbonjJt9SeJZ72LJNviZC/LfmW3KPEJ9Zv0j1K5VuatyoV9rzo2wEul8Q9XsGbM93wezNvDtdSlh7GlspWFdSrY1VJWlJclnWs3Z0Qndbrl9KZwlbHcLF+gnoetbsTBoF+xMvHR4bg3DpT2DC+4vRuYb92d0LYgjFQmCW9I4WdV3hslqBm1HW6E7T7ZTK3w1EThmsK8Q5AGZLxbUg1uxM+L8jMq6VoShyKSwqB0Rtyo053wlITjiuZwqC4pRAY18jwNqSa3Qkr4CmZOuBy3hgUtxQC49sy3CEpGYfCtvuMda+qO8FmfQl/OW4OxytjpicoVSnIIjCOmtFOoQmMVd0JNozK6fdkbg1/V+wcBKUqBTkJjJfw58ycXSDl4fmH76AtDYFxnWi0MPoouBE4CkuojZyoYd0mEvDNSR0OGVBsBrbDMWoXwWAXfpgANbulsCN2tsQcOEaj0eJV0UcRTvFUHIUgJkjFyAmD73a3Hj2HfjfweL6m7ItMjPJds4VXwN4DLL1gF5JKOgob8OW46OHhGEWjhXKl6KMYG+HZOQoRSioiJ0y+29UK5f40wAM8nq8xO2wzMVotTOZ1jJVxh7ELaiKGJcLZvRqOoW7n4tvvYm0fotFiM+qIfgiPoIAf+G5OpHBigpUPrZzvdlpZ+Sg2a2I8Rn0Y9EYnkEAEYtrhrmNwA9UR39sjuxBPhHMA9wL2B2xhrmI+ikYLwhhKC4wo80rxUF4+shp8t9XKy0dW+DDvONcQLMSTJkZ5rtGXhpyPwNiFuI51x+mZOYe8Eg2BUTRaEEbqo2B1RFmgYKG8fKzLIphFm93Ky0fW6ONo2s4cjNVgvn/IxGgoza0oMF1mjF2IJ2K5DUo5lj0D3zEngVE0WmwGTj84MLJQKp8gJ5bw/OITcCuWT1j7SgB4kVrGYH4yUxcjTQrGLqgpNXXiq0+/gLX4R0NgFI0Wm0UfhQMjC8XOmywnJwy+W2DkViwfsypJZQRiOKEsQxle6Nq4ZOMVlpsByXiaySE4y7YCo7DTuabhwMiVIMoHBt8tMAorlQ+HWsILR0kxWa5+MqadVKFk4xUOmKZvxrgYGx5igmZaUFA/+Jnvbvw3ywSc/XIY5TP5bquVlY/5TizIQygjw1DaBje6YrTxCgMFW0xrO0b5XKNZ+UyLcT64QtPphy8u9x4br6ClzPCWBbN87hmofObYbUpMGB/GjQtGO68wty91sSmrerhSHFMF6Wg0AUuNnxL11zQ2erxCY3Wq4zW3U7AOGHdzj1dwr0uz2m7lFf4D3ABZdMksEzwAAAAASUVORK5CYII=" alt="" />
我们这一个凸函数. 我们先对这个函数进行各个分量求偏导.
aaarticlea/png;base64,iVBORw0KGgoAAAANSUhEUgAAAacAAAAqBAMAAADlmIkZAAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAdmaJqyKZVES7EN0y780nl6jHAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAGgklEQVRoBe1YXYhbVRD+cvN39242m/7Yh1Iku6kFUWr8oQqKpFihTxJBBZGy6Vao6IPZWqwV0dvaVnxoiYqIghALlvpQW4RaEcEUbaWgNAhSkT5E8UH7FH+KdSmsM3POuX+bzf4lpaVOuOfOnW9mzpl75p5zMsCCKNlAsrYgyxmN0ssqM2JAeqIbqg0n1uFwFx+zQY8AhWOzKc0P/wAPdDEYxptdUAU5uUQxXp1VbUaF94AkB+XCZh0nz+3i6ADWdnGQDARl1acr0jBcu2W1QCPrRgc12MGHVdRB1TDAWqlufuaIJbGum6ZzwUOdisd6DA2jhnK8gj2eqCNT0dIOPgbL4aBWdXQwX+HxbgY7qz66y2cNJ0GBxsVDY1qaA+JfRjM61hCQmuk+DpBU0k/PVMuoLuaecLtar/fRrM8aTgW1kQIpiiiOB+n+u4HN3bf0OYNtISYQlJU3QPRuuVFJ8DmMrkoGMSCErsUtPhoLzJqWSlCxRhl2UyQWttL9T99GcaOeYLqPE4QFghooe7oRJjSwCBYZdvrFh8MKIdtduMtHMyWf15wEtePDHJxJkdi8sNgXo3pve4LpPi4Ttu8PCkWl3xDZr0Eqf6TONtvu2cZTL6QHtruCcecnlqQLr632XkEIHZv6qwtqLXs6gKrZUF0g8zzSJQ4qOTVVBH4ksbUE9S8oFUVvotBAevQmuUhgxhLyQTaO9wa8oA4ONYYaWffkCOzypur5DH+n7N3l1smfx3n8hseBvclzQw2zb4dRUhSUtlChzuiOFQS2lYZqqetEUWZKnj+h9vAmPPoPkK0Qn8o5TbyK2+VqemMJ+yC9pJpi4lRQT9AmMYYxxOotK5/GfnIHlb1qYHaynWxjpXUvUOWcN/t2CCVvgvIWyrx+IdqWBWLbHsgDn/OjIep6yPWD+pnk7suItYCjVeLXApdot6jzRZ+M5y/kg/Qy9KaGp6amAkFRyt+MuF3K1MDZvwdnSO/JDQ/9uoFX1XjJqqEICp5Y/TlGUTkmESpb6DRbOSaxbdkEdeZdojfII/V3lrdL6pmJg8IWZPPAK3XS/x7OZZz9t8EXrwN6LOEXQybRmTpCshNYj1islGwCvwDLsYlk5m1jsJrIISdBcWeyxkRROSYxWo5XCIvYyjGJUXxcB6iTAJ3AjYGgOP3wmfT/DL3/UxeRbgPbyWR7kxLRjCXiQ39TkvreN4VJCuvbWIXSjL44dwUlI5NKMAzXB6uZhgR1tGxHg1KoHJMYNftnyFaOSYym3yG34aAm8VYgqHPc7yTGqf2UvigaUDY3UEWTLzbUY4n4IG1aYVTqq6B4Sf/BOYd8rJRp7rfamfJyfMe+TVBD5SNVCxSU3bwNp6JBKZSOSQrFRjGN2DoXNEpzHlmOJ5OXAkHxIua0WcWhtWIzjlPaDNQzFb5AS7oeS8QHGVEqq9RXQfHmu2/09Ejdrtm10y899wJepx+Tftv2+K3jKzmozNZv7najQWkUxxXKW+h0251VQa2WdQyxqiiYplCY9IPK0GQAa5ZQc//fd74/icdG8rBHR+Xi5Uv3FvFB6qvpKscrZqFwavQs1Irl5Z7FUrnroITnoBRFviktTbiK4S2UKWxLnyyTnaPVIius38Tk3apBWN5YfNznPvLYqA+AVwZJfTVTuM/oflVQXOaOojCOAfhuP3uDepR9m9gQCn1MUltoFPWOSSPr6tit3Oh2yB10/ZnKuiEw8hBvGEHYB0tTVWo49XVQKRYukqYdk0L+gsckpxKCDvF/Fm9JN9kQUvEfjGnEBytwvknq66B68SeRj0kzEx+TDEX+4FmFfCCop4xW5/tBLY74EOkEIKlfQqKz8ZWW0jBK3CetcAumWMNL/QX76Ivhob54vbacpjYwqUXtah85nUvnQKEo+Cx7NVM7NNrr7+FaSr/rb3b+j/gqfgO6YnLFRhgb6X9XqrLZ/368HoqJssf3i1GVzX557+A3Lyf+DkAPRVLZ7KG/WV25A/lZdRanoCqbu7166uK8zcF6JOcMNgLV3TmYzFtFKpumnjpv6/kbOHubybGqru7O33xuFlLZ9GqYc7NZjJadblHNTld3F+Ooq61UNk0Ns6tmb8BUkUufUt3tjcNOXqSyqeupnfBeywZdLulKdbfXrn1/Utk0NUxf3DduuJ7Kcz2Vqrt960NVNnUNs2+9BBwfxdf0JNXdgLTHrFQ2TcW0x747ubM250gs1d1O8AJl/wGuqnKTq02j6QAAAABJRU5ErkJggg==" alt="" />
对于w0的偏导数：
aaarticlea/png;base64,iVBORw0KGgoAAAANSUhEUgAAAcsAAABFBAMAAADX4dBoAAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAdmaJqyKZVES7EN0y780nl6jHAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAJ5ElEQVRoBe1bXYgkVxU+1T39MzXd081uDBgW6ZkJkqDEZl02ShQ77Aj7orTgi2iYyipEfLF3FaMi2LskkTwkjoqEiA/tgsE8rLMIIeJTi7ohkOC8yIos2AkiwQcdUVEGYfzOOfdW1a2q7pnpnwqLe+BW3Trfuefer0793jpFNCcpDam0PSdfxk3lZDDBYeXiJNQ0vHiWfjLBx/GhTxJtXD9+s0ktfkTnJsAN+u4EVCG/udQu9g41O4bBD4hKoFkl6nOrq8doOs70WXpgHAR9KUazMEgb8kiqo8KIMLKji9+KbDO8FtpKc5lomw2DyHrqWonOTmrr/yVE/SCshhUZSbcY0BOh6giVcswmw+tK16HpDWPm01dfmtT0a70IvRxVbU13OMbFQ0uLtwbdQ/9sJ5BT8e2012cB80Grzmk1bj11fak/senDEZrRn47kPFExSUWatZfAfnU7cqG1UVyR9vpZwDGa63Frp17oO5uJjfuc7VMlZ5Mc9AF6T4R6vahuakLTG3apupvCoGiV0aTRSkAFR5H2+jLsYzS/n2gebR6DZuXrn4jacc2heZnORGitE9VNTWg+/uMm+fspDIr+covo290EtOwo0l7/C/un/t61By124JWALvh/YDeVjW/dGzY3NMegSqT6biq3rg22Dv7htnXQwskvxFB0GMkrAX1ZRlI6OGgT/T5CbG2t6a8MiT7M25UTHyK6tN7lUse9QjsXS8crNP6/RB2em7vkt27RLXqrdqJJT5Zu1oe/XFMLpWlQ+hQpap8slMjV+rA+XO1zA2mLm7xINvr43QD31EKW7Pyc3eHQ/CyGadV/cre01SP6M28+Steo2i42uYBmrHPHK5uW7IGhJz4O32ppr7RH96zSOvX47BiZw15pGrTwEClqnyyUSH+LtsgbsGNuyzd5rpuDNoHS3nKL6BdioAs4RwR1JKx5I4YZi8qI3oXg/BubuLM2+sXAJy6fxtGsnUtY4l5hSjXszcbBwYFxzmdpsVPYpvYT9CpXqdqpbcPuc5sf/9PmOVQUZbeMymlN5c3Nv25uMqMzdD8VSYLIbfkmn0IliNy2a2m++jzkO2he7GA8k2hSuU1v4nBFcEr1Fu0Mawc/JC48Hu1cwpKkmYimv0u00ltqUvMuegTX7RZ5nRJ0EHNuKipugSpNoBpNepkeJk+DyGi3GABzUQ0io/TTAZE6xwbLSs8bxWimD1pa6fNwyteJXtvq0jfB+GNSrqG1dK5hcbwCknOT977Zh8Abg5VebXg3Dj/a6Va9AEcSi6GpqNAEmqS5j75e0yAyau/wZicIqkFktPIc3DoDagyW23YkwG5yv640BuUW3AZEAcb3Bj1HdSk4N0nca1gcr+wBVzPZ+4Ymbij17rVege6i16u776PfeJ2atjE0FQVNQZM0f+ffpBYHUVE6z12E0VQUQTQojhr30t8YbPVjNO3lUZ3Icod+jfVOj8rNla7XwXPvDSl8QxH3EhbXKzfEKSF739DE40H1wnsv3ENP09O1x155sF/drm5LB4amoqApaJLmU+s31gYcREX5Js9ioqkogihoYVS4Tl5PDMyicPojg4im2b9xAyo82sQp/LdzH/zjoHThBO5+6y0pfJ0U9xIW1ys7uBelWwys8xdZx7JKcMIy8lpYhget1EkOWq7yNYvFENENG0Tim3waRRBZqk1ch1alGi3eGZ4+6HE70h9W862thCXplXDrgeAh2USzODQOa6fbWvvVhq59A8iq+sV36CY/WbAMZGkWNoh6k0+gEkQxXDs7oCvxdjiDfh6jqTdgx2D8Bp4UVEYe4mvq0arcQx2nkKFJQQRNXbNBzHQgQbSIH9gar/2gGEQjCY8YIIdL2ZogLK5XATjavPctzavWfPp1GMRsFxxEK4kX4PXTAOxI6PPW6ijrQ16r6SLJKbRE1DmKu8Xb2JHUgnn25Q0P2fvz7Ow4vl44jvH/sS2eQSHmSnpb7Qd+/D6aOLSO3uxozudtteeM9s5G1h64fQ/aLDZ3dHf2wG21B7y1vIebf49gKHPTuTLNv0fQk7npXGnm3yPoydx0rjTz75Fkbjo+C71wwvn3iNdTmZuOz0Ivmmb+PYJRtcJz0/FZ6EXTzL9HZqRz07FZ6EXTfBt6BCWdm47NQi+cZv49gpLOTcdmoRdOM/8eQUnnpqNZ6IWzfBt6BCeZm47NQi+eZv49upzCjwOueoFb+ffIZEZeC8s8ZeS18uxO+7IfB/LrOf8ep+dm0wayPcgX52woH+1n0A2XGSVMG8j2s7fcsoDft7VonaWL0DnUSh18jEOZVfSL81gv3YgmPZhhlaXLMJtatTzAtxaUmaVbDCb54LQBI1ndsS7xGciaz2XNHyPNB8kZ/WUnBhqnkjZg6tVOuifWZXx+SxtOqXkJ7bjMLucnujBfnMUmqz/WXZ7oYRawOuJUglk82Lb2i7PddtbRF2dWv9/BdIN16U/jGYZTqfAGxu9E4+S+cQDrTQ6FMUl+cXZQ54szPZPhlXXpRIcMw6lUKz28haGMkaPTTH3zdGhS/Isz7QzS3bGu1knrZ9DEEiUbQ7yFoVzJzqJUmrFJoksfuPRR27MhEjWlI6L1vnXB69pXqNIh0e3G9TPXY4mSO11OzRqbRak0zSQRkier3Ud6tzibkkVp2hRLKAStnAwYHIdeBLrSYgMr8L7UVt2e1c2wjj5rxRIltwZEKOOyKJWmmSTauE4VPq84m5JFaZqmrBDU/myRjcrPFqtNtrYC7wil6JIJgtZkynWxQyZR8rfwwCUrizKZJymZS5yozNmLyRRLDSJQ/dkikYCpf8wAlZ8thFKNsyifZ8JneASLoLkaJkqaaOIylJlFaS5BMkmkCVpvWpo2mqapBhGo/dnCnLnqWP+Y4bb8s4V70CI5ERmvotuFwawSHbSxRElzbtKYLEpDUzMhOZpIwOzbNCNDRJtKEAU1TxsOKkEUlH+2qKPEZJ++Z3TzoBk6jidK8gVOSnYWpaGpmZCg+Uxhr9ZN0NQUSw6ioktwCTE0TQImgqgo8mep0RULu9gv/Ud1872hxBMll5t4ckcZk0VpaGomJGje+MaXvhqmGRoiJsUSQVTU/GzhogiioPKzxeuWoK43NvaJWLeIx4OR14LnQkcLqpBUFqWhqahcgrgql6AwXgrSUl8r9mcLQ1OVxEFkkZ8t7td6uPS2Cbngi3nY05kJTuLmopLKohxYhNc2edJmU/pxkOwfM/yzBYuDhn/M8M8W9JbTsN5f6avuiqOf68Zj8MZlVrFBzPbj/DETHhVq+wL/B8c6P8huPA/ti3DCZVaxQcz2I0G0ULlva7IubLTw+gDdIl+rK008vaDkKDcy+srSZZhNrzqFplxyk6xjM0uX24Bux47+BxyNnK0hUBBIAAAAAElFTkSuQmCC" alt="" />
那么对于分量w0承担的误差为：
aaarticlea/png;base64,iVBORw0KGgoAAAANSUhEUgAAAKUAAAAqBAMAAAAg44bkAAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAdmaJqyKZVES7EN0y780nl6jHAAAACXBIWXMAAA7EAAAOxAGVKw4bAAACq0lEQVRIDa1WMYgTQRR92Vy8vc3e3t4RFFLFiIIiEhUsJaD212h3uMbCQosI1ipqc50IlsJiIV4hHohicWAKLSzEYClXLNh4XRAP9RD0/9lsMrNyMxmzP8zO/3/+vNmdnX15wK72WBo5IvnTuJE0+YnkT+QuhUD5zZlcbaknJVwqsbEyzlH5Vn5KoCQSJTIGDq5Qzbd8XVNJ7FMiY+DiAeBu5+seKom3SmQInEXEG7Sffa67fqCH2eZB0ShxJ0LH+ywA5g0wyvDTFZz/AQQRZfeEXh93cVy0PrzGJjbxVZQvKJMMwe2bKCXA8y7VHQN+4hRibpV1uJVBZYC6z+eiGhtwlOHLCBrALZqz/Aneb3z81eNGmCi3nftoBaDXZYf5Giu0xlXAf7eN2QFwo8+NdgHV7kyI8B7eA1b7iR10CPMV7SY9aBDOddHnRj8sxNWu36vxolaY3sBvAx69pkt4iRrmYj/iBjpL88vPug724iJgOksjchDMcGiRbvP095OPdnBhfwNusykab6LbOdqp0zofANOZjwhEmJYZ1rKqVawCSRZxryMHLTOUe0OYAEtwG8Mg7baUiIJgnEjG7r9eNEz5J1rIPZGOHLS7NNp2ws44uSXW0ZKD6W0O73TcrQtXSw5Wpw6YabUPM2gQ0WU3crBiBsKpna3TVUcO4iv+M6kNCKz0ImZMDTnYMgOcjWtfGFNDDpZfMX1XSFzG1JCDNSaYWPTkYGYGRlAs5khHDmZmUPB0wdp4MBm7mcd/FPY2Ioc8MzBUKiDsQaNsSo4ZOJ0KiKxg8n5EDhkzSFOFgJDi6d1UQEhSYXpICAEhS4UCMIWAkKVCAZgQAkKSCkVgCgEhSYUiMIWAkKRCAZipgJCkQgGYEAJCkgpFYEoYQipIcREuS4WijaXC/9lfNjLtKGPwYP0AAAAASUVORK5CYII=" alt="" /> 并且这个误差带方向.
那么我们需要使我们当前的w0更加接近目标函数的T的w0_t参数.我们需要做运算：
aaarticlea/png;base64,iVBORw0KGgoAAAANSUhEUgAAAXQAAAAqCAMAAACUTlBSAAAAM1BMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADxgEwMAAAAEHRSTlMARGaZEO8yzSJ2iau73VRQ1C+tCQAAAAlwSFlzAAAOxAAADsQBlSsOGwAABYpJREFUaAXtWoF2qyAMRaQoWJ3//7XvJgF0FVprtdv65JwKDSGEACFcVepMRudtoE2eflJ3sEBdkGFKFQX+/5zstBML+H7s/ANjOFtiqIKUUv1Jn1kAZu5a+d9cZ/R8scmTiXqnqtzoP62Bj64HGftY8NeTZfQdDhekTNxnqWSBSqmrmLIaw4ovsYLzDoftyu3OmskC2sERG9WLo/aXVDM47Zk4aK3J0CEXjkrTqVlrZS7zmGVqnuSchYUFTNXCUG07imNoUgBS01yQDSt4fI8/MWe7Imq8YEbo12CfpDSm0lkoW6BSA51+To5AE2yvlGbCCIN6zAMt5pC3PQmrVAv70k85o2yKfvr5sifGM2UtUFOI2IhLH0YxWqsuRDAjVrIdxyu5mZCL0TEHcN+aQh00xwSF4/U0OiyyIvVYzeRiKNW8jJW1LR+og3iLoZNqztnlgLWHP+ooLneqgv2thJpBDsk60x0L0OLGgcipF5dei+NQDYx6JWcPW8acypSoGRneWuVg9DBtp9HFOg+eBqs5RnrBpdMZ2iNeIWOqK4xrMRcxjyHjpcU5C6Njuhyq2b0nQQ/6LFYvUZ2j8ZyWNuv7k64paKTkG2AA/nqhRdzW2vHyrxAwUiHm0XtXtdYWTOClybG8xl+9HMlGY2XC42A8x/PZNe+wXB7Wb+OWDsE9k1kABRUo9LsLA2xFdQ7Gc2hHr0xmPWu4aq4UvIZNrkxzTkQvvC2qcDTM61J5K6ojwWwSs28hhgX7SlWP7/VPd7hwAtZrijvvu4KtqM6rLuvu8AY6tnZP4mtvxJKFXkjL406E3R5637rZjOrwKV9AHbKDeIa3472JU4t1dW6gqyIAkFZOtME5o3VHd26ROiNgkXld127iVtLe+qb3JDeKFS0lGs9qvCfxWzfbUR2cXyXUIaftM7wKMRgCM1jIwrCUd50CMoJdSa8VpMR3dpE6IyDWwE3QjF+JO7YXQZNYUhLSOrlL5lTejXbbzXZUBzFtCXXADaNLKWysMu9ybHy78FjcxhtBPrD0gYyE5RJKjgIFDopnhHCLH78SNyMnvHXYpUex0quDmhTpHZy+d7MR1SEdGVrIow75IRR47TWlGJzzTeSC4Jc8I0ePvPQZGWHZXOKLN2NPmGSaWiZUFFlXmJ7IndpXHFtGsUFH12NeQxr3T1H0vBu1EdUhWWz0POoQu/qer+clh2JCqMHIhxyBF3LinLgkDISoYmaoigmOtgOhgIF7ak+TnsSiiKR7W9sm7EUhHfG86WYbqsOK8brJow4591JCKHKDHFvTGhINI1mGTK/BkQg337PhbGBpec4IbNpmCHdxnAqpPabSVFGsCIIAOLE0l0Lc/3nTzTZUh9WSzZpDHQpaZxGKHC8gECx1fiGgrcLbmBaLUQdYmxqwm4b7gDvy/JwRBnh6jTUvIDiYU3u8EEKDKDZ2zMOIf47LZ91sRHVINwGGsqhDQfX1vF4DPULkxz698sPQ0mnH8R7L5pL1dAIC8sBzRsCbncFhiIk7tR88vWmLYqOSyaNHwjH51M1WVIf0WlyOJtThsdrP8D6WtuBgF7Og/inCEtUh9ZcwACgCxq0Y3TO8K8TdsIR3PzfUl/+mF3EvS1ohYInq4PCB171JAXW4oeb/PsObl1CgUhjObzoL9S+QEeTEz5BekLK66QLVeYDnrBa8P+PgB3dQCIhlFj9D2l/vpcQlqnOL5yzbfB4FgR9dIM70LgvMPkN6Brh7l3of2c/sM6SngLuPNMa7BjX7DOkZ4O5d6n1qPwyfSShaAO4+deQ/OK7ZZ0jrgbsf1Pcjuib4LXyGVAL5PmKcv2kQ88+QVgN3v2kAf1KX6TMktR64+5Mj/fVKHwzc/frx/4yCxwJ3PzOmX9/rYcBdceT/ACtELoCjGB/KAAAAAElFTkSuQmCC" alt="" />（梯度下降算法）
来更新wo的值. 同理其他参数w,而这个学习率就是来控制我们每次靠近真实值的幅度，为什么要这么做呢？
因为我们表述的误差只是一种空间表述形式我们可以使用均方差也可以使用绝对值，还可以使用对数，以及交叉熵等等，所以只能大致的反映，并不精确，就想我们问路一样，别人告诉我们直走五分钟，有的人走的快，有的人走的慢，所以如果走的快的话，当再次问路的时候，就会发现走多了，而折回来，这就是我们训练过程中的loss曲线震荡严重的原因之一. 所以学习率要设置在合理的大小.
好了说了这么多，这是学习率. 那么什么是权重衰减weight_decay呢? 有什么作用呢？
我们接着看上面的这个Loss(w),我们发现如果参数过多的话，对于高位的w3,我们对其求偏导：
aaarticlea/png;base64,iVBORw0KGgoAAAANSUhEUgAAAcsAAABFBAMAAADX4dBoAAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAdmaJqyKZVES7EN0y780nl6jHAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAKHUlEQVRoBe1aX4gkRxn/emZ6prdnZme4ixHDIb17QUyUOJ7hYozixFsxiEoLvoiG7VyEqC/OnWKMCE6OJJKHnKsgweDDeGAwD+sewnHi0wTjHYEE90UuhAMnIYSQB13REAjC+v2p6q7q7pmd/+HIFfT0V9/vq++rr6q6u7b2BzCn4vbB3ZqTL+Wmcjga4bByahSqGp46Dr8f4WNy6GsAR89P3mxUi9/CiRFwA345AhXIb5Zaxc6BZhMYPAXgYpoeQJdanZug6TDTs3DbMAj1rpFmoZc1pJ54g8IAsGfjFz9IbHO8FlqS5grAFhlGifXUkgvHR7X134xRP4rFWOCehMUIHolVYwhlwybHazW00nT6hvn04oVRTR/qJOjDiaglGXDsF3UtW5w11N3131YKOWLWs17PIkyLVpzDqmk9tVzqjmx6d4LmxJOe3ANQTKfCzVolzH51K3Eh0sBUZL3ej7CR5rppbcmFrlVNVT5s1Y+4VhUs9Db4SII6nURWEqfp9EPwdjMYKoIyNmkEKahgKbJeL6K9keavUs2T6gRpVn781aQdSVaaD8PtCVprJ7KSOM0Hf9cE/50MhoruSgDw8zAFrViKrNf/of1j/w71osUBPBPBSf9lclM5+rOb4+YqzSGoJOJ9CMrBdm9z/z92WwstHP6ugWLApFyO4AfcE3d/vwXwUoJoaa3pV/sAn6F65dCnAU6vh3TV8VshwdnS8ooa/y1Wx8/mLvjBVbgKb9QONeFR90q9/+yaWEiaCoWvg6B6ZyGJnKv36/3VLjXgtviR55KPPngjgntiwb/k/IQecNT80cBE9B/ddTc7AK9T9T7YBq9VbNKFaRrBLa9k6uqFIQ8+Ll/P3XP34KZVWIcOPR0DtewlTYUW7gJB9c5CEuluwiY4PXJMbekjT7JatCkU9lYCgD+zgfygc5xB6QlpXjEwZVEZwAdxct7GKn5ZG91i5ANd38DVLMF58EyvaAo1HM3G/v6+ck5PabFd2ILWI/A8ieC1a1to962Nr7y2cQIFQcktofxYQ3lj458bG5TR7XALFIEnkdrSRz6Dcj+obajTfP7XWH6BzYtt7M+oNKHcgldxueLkuPUAdvq1/d8AXdQfCZ4ZPETSs+nvAlQ7pSY0b4B78b0dgNN2UYdFPZuCsltEJU1EZTbhItwNjkwioWExQsxGZRIJhT/0AMQ5VqhUO87ASDO7aKHape6UzwO8sBnCTzHjL/O1ja05uAye5RUhfjZp9NUYIt7oVTu1/o24/GAn9JwIVxIVlaagnCai6TTfwVgvyCQSqr/wahAYlX4QWnkS3VodavRWWroniF2huHZp9MoBuo1wu4b9ewWehDpf+GyCuM8OHnnAtxmPvkoTPyj1cLtTgBvgRW/3Y/BXp12Tnqg0BcU0GU2n+Xf/CgQ0iYLCPRQink1BsR8KxVVjv/obvc2ukaZ+PYoT/t2B5/C+04Fysxo6bdz3XuKLPijingbP9koN8ZHg0Vdp4vbAO/nRkzfB4/B47YHLd3S9LW+LA6g0BcU0GU2n+dj6pbUeTaKg9JGnomZTUOwHo4VB4Tw4HTZQP4Vjn+0laarxNQ2gcF8TH+F/nbjzHz335CH8+q0HfNF7UgXHwbO9koOb8QqLkXb+DOmorAI6oTJwAvyNFy3LwIuWRHpnUVGJSEVPItBHPotiP6h4TXwPrbKY/Hwgfnww4laiP0jytS0PXtor4KcHC26S1WwW+8ph7VhLpL8clbuvAL5533ufVGlnQaXHv+pHT6J85FMo94MN14734IzZDp+gPxlpygfYMhhewZ0CFx482yvpyx38wUdIpQkRKWcsehJz3XA/NOJHWqK7HxWjpCfxikHk4FLWJjh4tlcGaLZp9HWa57T59Pd4EvNd0CTqkvoDeP0YAron8G1tNc79gD+r4RTwI1QCaI/jbvE2uie1aJ6xnP4Boz/PYJP4enoS4/ewLe5Bsag36TU1DrT9Hq9YaY3fbDzn87bas3p7vZI3Atfuos3L5rru+ghcUyPgrC27u8uPiBny2fRSM11+REyPz6aXmubyI2J6fDa91DSXHxH4bNo8hV54wsuPiH+e8tm0eQq96DSXHxEz8ip0Nm2eQi86zeVHpIzkbNo4hV50mu9CRExJzqaNU+iFp7n8iJiSnE0bp9ALT3P5ETElOZtOTqEXnuW7EBFz4rNp4xR68WkuP6KdU3L6a+sXV1t+RMzFOv1dXG6G5+VHpODm6a/RmQWKi49YORwtpP+j/Y5FNpxnv9KkQLeP3r85e4S0X9ujSTb0uzZGtTxd1moCTZoUiPRKcNsTOBhimvZrm5lkQ7jDxriWp8sxG1uVJgU+hS1XemM3H2qY9psyNMiGueGoC7em2sxWvWA2J3olqP9PmvopZMtvur1JNvTaaRRf96ib+V8ihbXPxZ5L3VhEgTmMIztoWo+SS91RKPJHkpIXD3VeMzGZSjoGO4FuaJMCz6LaG2hslrvtN+XJIhvCJ1IoVUk3IGGG8h2ot1XzFCnwflTjH2VDiiJNjIOm/KaYCBbZEJ7IcUi69+foJ1PFs0mkQKNcRLnaMRSWOEGaKb+pNIlsmJSdXiJriXTP6crU9ztN+qHBmyR6ZaNvgqc/efrzOoxK88w0jMsRbetdHYDul4lUCaQjhtJM5YdfNOmHzH1kYhxTuHZCDT67Bl54b+cqkSupSFeHcSptFM0tTmU+ylGrATlXRUiVQLqG1k10N/+t9WJfHfyQB+ZNMkGP6ZWbPX0qhFzKCj05RK6kIl0dxqm0UDK3OJW5qKLjma9UIVXCKuqqPfIySyntKvqhzW4keiX8DS8+FRIuJTKTmVw5mlOZRnlPi2wloeNl0FOR8DHDlQBHkdK0SZVzSfNLUHxL0Q95ElcxFhP09GwKKFzKVwHpeEiuxCIzgu8oYlwyC4daat6WhfKeltCwGAEVC+U9bRzVWrTo3RngTAbTPpvxonXfhtKeoh/ycMbsRv1sCshcSlR1mVyZdHUIp1IlIijvaU1OpYXynjaOWu+g77gwqRJIN+sr6FOw3VT0Q4vdKGRxesnxqRBxKZ8o7NVCJFdyL1RXh3AqLRT3tDan0kLBfzPhVEIjjHNEgUmVrJv1g+J8/AuafmixG5leCStNBRKX8tJPvv8jIleaaQ7hVKpEFAoXbE6ljT7USTiVagx1qkyqZN3s2wPtE8BkNwq9stDW6MAJWNTkStVV1mU5lTZa6oqZpuPZqNrTyvHILcqhviGpEqnhs2/2tD98MRBFlYscVGyjTLxuKZpLqcmVvgbonuFUWiioPW1MCLPQeE/LUd8w3QqpElDnBZZ+lkr6tInplQ/M4lG1zexpLZ/Wnla/qcVCSJWke9pqMlslddrkb6G7Z2Zzya0ze1rLp7WnLXctjEmVpLvVUs+3gvRKqDTn6/MAb5dy8DxdjtnUKqePTY9M3XyKhn6UbZSny1q9lzX/B2wwvASWxSMGAAAAAElFTkSuQmCC" alt="" />
我们发现w3开始大于1的时候，w3会调节的很快，幅度很大，从而使得特征x3变为异常敏感.从而出现过拟合(overfitting).
这个时候，我们需要约束一下w2,w3等高阶参数的大小，于是我们对Loss增加一个惩罚项，使得Loss的正反方向，不应该只由f(x) -y 决定，而还应该加上一个aaarticlea/png;base64,iVBORw0KGgoAAAANSUhEUgAAADsAAAAVBAMAAAAKrYvGAAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAVGYiRIl2mRAyu6vvzd17cpdrAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAA+UlEQVQoFb3Qra7CMBQH8D9rR/ZBxq65BrMMhdsjEBL8ggBbiVz2BEsu8oqa625yCU+AQCAnmUOgeQYSHAbWlkKTgb01Pae/09MP4P34nL03wGF+ajgFhjoVMd1be53XswcsdSrjxM5UTvgrRpAo9qKXvFGKrihrNG/z++5fUdbgjy8GtOL+z3GieZQhd3aylFyvKTDGFKta5ZITVahwMDrNUOCsmZITOWFu8N+F01Iz7LW1RGoweqW9fnAQ+gzsyV6I0osWtcubd4sgdLnBhZsFnJGtWuok36FlPJLGMVqTxK0U03yQzw0Wp6rR+BYN/8A+IN8vjtLxDcF2PxB1SbBWAAAAAElFTkSuQmCC" alt="" />;于是Loss变成了：
aaarticlea/png;base64,iVBORw0KGgoAAAANSUhEUgAAAiUAAABxBAMAAAD4w84QAAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAid2rzbtmRCLvmRBUMnZ/MwGQAAAACXBIWXMAAA7EAAAOxAGVKw4bAAANqUlEQVR4Ae1dXYgkVxU+89fdsz3dsxrdQFAzD6IiK04cQZ+WEf8eBBkfNIpBe0yciMHNSP5MCNJGfIgP7jztCu5ia3DzoMQOmOjmwW0NRojrOiKBuETTYTXI+sNI1CyscTz3nHN/6/ZfVU317DAXuuvUOeee+91vb1dXTX1dC5ChVW/M0Fm65lEjO4rcKlQOvZy5Vh41MoPItUB2TgDyqJHrpDIWy2M+edTIOI1cu+cxnzxq5DqpjMXymE8eNTJOI9fuecwnjxq5TipjsTzmk0eNjNPItfvo85l7oRUgiNVIZgWddvFubD794f4UbggSYjWSWUGnce8+rgHcpg2zVfMpA6xrh6QmE0G51jH3TrhFJ8tW1VAR3eJZOrpLti2N4wltyLb62n+9HmAGYEUHWmyEiehVrhXMrcB1nCPvXENFdItl6dh4thNvD9d2fVMjKXe15W6Fk+ojLdCpkUTl4pm/xe0sdsgJQCwr0rEY1/vDYaatY9ma1hJOZs61wKRGEtFFM58yHzRbQthiR+8sp0Ox5n/C4das42/WtJZwcrYNYFIjieii2f69wl1vtRX0CmKPl+Umjc0uXw6H/rh1/M6a1hJOfo4ekxpJRJea7dyvvsFd+3JisuwwY7QmNtTg51c3YW7tc/RCx90tOF39FsBsDBhz8ugbrwXA1LnV+29u+om1p2FuUbkUJ8e2/8lFEpyUPw8HGmfaflZswMJ90y0c8kC3ugFfhDfQawOqjVNwCj4IMB+DI+vkYxhDTr5UOTm76Sc+Prs5dVC56FOhSyQ4wbTZzel1P0tnj3V7toPD4ynEy/id2VavygKUK1uVLfgrwGQbgw8tYXO+nISTUwCYCp0yEsOJuKW2fgxm15WrPyeYdgzqQZauMdbtb3DazeNQfQWe//emeqmJTiyWVuBgMFWAbWxb6sxiBaC6xZxgLuJX5NU+qloX966B58llOTmwtPS+paWDGOMaFLkGPgsTwWrCjPG37+Jknr0MczjFJzfUCz9FMNmZ6kK37/GkvIC8YCpMN/ANDx5OOwx/IpflBIOJzw4chiNQ342cvBWPJvhBme7OdGBDvdRBYr492altylTjn536Is5TcXK2WQ45uQIfGYaTK8jKH3YhJ9X/AjyCJ5EXYaZda6mX+oKdbZ7plHC+ka9YfW6PC0l9F5c3bodnw8QrFbymke9izOKWXCffrJ6Exi7k5DUvveoTV+DrRxtQXlujlzoRK5/+wmk8xELkVExzMrmO8TWo/eC516HlJ66uXiHXgM/OfWsXjrZ3ISf8b+i9P2b3lq1pLT7G0te0SQ0S6ysA6BrAiZT0suwwu8qa2NRwyg1tyfbo93BNECftu5RLp/qJs+u4iJTLmy2uCGpUw424tuTsvk1LQwr/BFBqV/DwQ5wcx7NYbC16pz8MsKXen1B/HtB/K7ButriGy4Nrh9m7Zr/n35QmN+EdwsmhLsGN/k2ptNoA+ptSbLZcw4249q7hYGggsw24HmAKYHG4LosqN2hcw424dpB8deziOsnc8qiRGUR+BdTpTNaWR42sGPLsP9PNXi2PGtlR5FfhHzmUyqNGDjDyKjHXzV4pjxrZUeRX4W51HZux5VEjI4Q8u9cevufprPXyqJEVQ579Z7e3X8paL48aWTHs9x+Rgcfwki5d22PqRUvCV25Py8neUy8aVubTcrL31Iv7nBgGksb+OtnnJMlA0rO/TvY5STKQ9Ay/Ts77Cqws3zuJUgrXuUuCLhpNIt85z9CcVLuoFPCaUuqlaslSqszWTIOqxaMUKuhtaE7Ky6VlH1NqTpKlVOGmcBKP+kPv6N6jn37Ppq/LjA2Ht4PXoTnRcmOsXqSI6x7KDkpJnx+1lTw0MdBQFfNPGnQvge/7TTaTI3Mk6R/giZWau8ncdI0MNKBg/uHhOHlXZOCUnMRKwVRXhC3RaGTwHXUNxUl9s5kEkY6TWKnScmmBOYlFkyPvtGcoTs79pEs4Su5VY4QTL66R+05d6lYdxm25i3InKqejTnAM5jCcVLa3+bvYm14qTkwplxM4el2bODHRMRDhDKk4EV0mCzttjFSd3syTnGjdKPeS+BBOEXNpRSh29wayIMZhKU5El8nCTguCVJ0e1AQnohutvdilbhzXYtKvcSnfqcvzOtGKUPR6A+msQrYoQcSmPgnzSoPIKhHRZXZI2GlhkKrTgyqcsHpRRUQ3Oi06c46Ls/RqLuU5TXXmRCtC0e0NZNLGYtDxRHSZJOwMVJ0O1IeW3vleRypLEdaN3gHPIHgTFzHpA2pCgZN/0mXFoTyyut5xBhoLEc6gxInoMqcbTgBNper0oMo64SyKsG70IjxIPomLmJQ4AfCc5iddcozlkdX1jjeQj2OH9+xnRwYiTkSXScJOB4BSdXpQk5ywbvQSaqJVk7iISX1O2Gl++CWc8MjqescbyAExBpM4IV2mCDsdDErV6UFNcsK60Yvwe+omcRGT+pyw0/zwSzgRRShe73gDOSDGYBInpMsUYaeDQak6PahJTlg3eheQ6E+vExGT+pxohan8pEs4YUWovd5xRh+HWbp0Iw5LnPQavk7RFRPWnFgFJIem4UUydJy9PidSZEpOhYUT7e365Iu78M1tcKzTlxNSdXrrpMogHQUkO2qH+DxX4uws//AvZHhO0D/8anMSvdvrHcc5FvNtMLXQlxNSdXqcCE5HATkq8uhPuuz1zqjl8s4/DhNbfTkhVWeME1YvxiIDMZoffnmZ+nrHc45nR62TQVrMHtpPVC/2iKSdSs7l0sKgX3ml6rzX1IuWhE9Zc0Rrpjtih6slvZ5+YntMvWj/xc5bc0Rrr6kXzfTx9yRdszOascfUi3byF+65v2X3RrH2mnrRzv3b29uLdm8Ua1+9OApbV31uet1j8VMvCGt63WPxlBSGdWhdQfEcJEYsCmtR4yQmmMJRFNaixklBQaJLUViLGicxwRSOorAWNU4KChJdisJa1DiJCaZwFIU193H6P7dx7nxrMBm9SqTHGq1osIRCS2+cyuZgwIMyzE2+aOI8qIcwcfuxNsJtrxIe1rBT3/1oRY0lIbT0xvlq38LDBc1Nvmh6xeFkohNNgcgDIjnRw9qjb9wdBaWxhEJLX/f4yXjFkbzmJl+8V/VD1h8bTj0Os0cJH6stM4QVr4hYyj2FlnLPq3RwiPKDU+QmXzzxFx3rv8OaxmIofUr0vT9nyiSMWEXEwvcdYjJMGScWShQf6NA3+XokHrH+2HgEpV+JdJzEKx4RTmJCSxnnTgs3g6Vv8kVL3AKfsf6JyLokKP1KpOMkVlFhoXUSFVrKON+xcAPLvwccBP1njCVu8nk3hb+MT24zLZBEkV9BMSVMV2NE7885UVM7Dsqbh8JCnESFlsLJYVsxsLxaQSx47lriJp+HuPTC9233Kj6HKmwKiilhuhojJSe6ojcPhUVxEhdaCievoEpVnkqJUJ+69ql3a8RSy2oZPXkkI3a6Dhs9oevT9rkWPOlLHAwVxmBOnKFUz8T4PZ2JefAxloYP3piT6mXQ6sc/HoVy88HOqZ5aRpJHVla4DmNy5IsU1WeH8Sj9WudNLg6llLxhOE5kKERJzRuhJyiVypxoTSY6DCdWu+XoHvHZjbiURYO4XGrMwb341MJeWkaSR64uYFVsjMmRL6qoOTuMRvnXOh/m/vyOSkk40YcTB6tFSV29EVYXuJznZJfmxDzg0+FEZ5gtr5PaFjpIg1herOESwKNhby3jxKI8ADOUL9LywCifHYZRWh6qL/1ahzh5Rj03U92hnVhEBPIxxl3T1RiElxIclDbK8snKgtuX5wO8ZkP5JV57mXVCtd03BqLWCT+Vsr5Y2QD4AODzDntpGacbwgn24X8S7srLQ0Wb8nsfN8rLQ0UB1WvgfXZgslNfdjixhc0I6OIEUkoySpvG8kniJHCaNSvHE9ZkqusdwwlSK5phNQg25kQdT4A0iPUWrmPcXe+tZVTyyGB46srLg8STckomnFhxIz0zU6nX4KQa3LT59szBITmxKFVvb4TeoDBTOGH5pbreMZwYFNqQBau+BkiDWF+sbdxb2qo1e2kZWR4ZDM/yRVweIp6Us0NBbMSNEsVf64D6R7Btvn1sfUhODErq7Y3QCxRlCieiycTrnYGcqA81aRDLK+WVC7/+2S9RyRjXMrI8MhheHmg52eSoPjsUxEbcSFFSr9XwA+q00qHrg0cOS1ezElQy/fsZlNTdG6EXKMoUTkR+aa53KGbePN3jzca9XG+Q3VfLGH52pLe+eNBnh2ZiFFfLAxup10orZNu3P+uPsbhMVzI8rCpDozRpyhlwolzYBJRwwj7AB/rG1omnezyjc+G3q2z21TLCff9rUhoeLW3Ty8OcHbpRWh6Uq9Rr0+u2G1on4M0BJ6YrGR5W1VOjNGnK2RdUVWXoZq53tEO2nu7xQCeIptnVyyPal5aHiTxgLGVUW+qLSg5tXkR2PKyxhN6+KChzvRP083SP1ZUgmmLXLI94X7U8THvYWGSsHcJNH048rH7XAXtxUPp6J9nZ1T2mFy0l6w721FqRnL4STBdrpG8K11T0aZ5nO7aU/k9SrGcnLfV82dGai3W0nqNlp9c9jjZOHtkFYa138wBbTI2isBZ7CMnGXUFYy+l1j9nml6J3UVjT6x5TTCpjl6Kwptc9Zpxgiu47ifX/l5glCSrHJWMAAAAASUVORK5CYII=" alt="" />
我们继续对Loss求解偏导数：
对wo求偏导：
aaarticlea/png;base64,iVBORw0KGgoAAAANSUhEUgAAApgAAABDBAMAAADKeWg/AAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAdmaJqyKZVES7EN0y780nl6jHAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAMCklEQVR4Ae1cTYhkVxU+VdXV9bqqursyiRFDkOruEGKUWAxDVKJY0i30RikXLkSGrrSBgBt7ohgV0cpgIllEG10ExUUpOGQzzmwkQVy0qBOEiL2RBBmwDCLiQktUIr1pz8+9991733k1PdVVmReZC/X63XPuOee73/upd/udUwCzb9X92fucjcfiIsud38bVXNXNKmp39ieY1C5M0qJhbD49sifvVnC4+BceVrQzENXbUCUyB5DMwNsPYXOCl1X4Tr6WAMTm0yMbL7WzoSg+znJQby10ysOs3kqQE6UZKRk+qKhJtAhC5j4s5Yy4GfG34KEJw6uTyCQAsTmTORWynkYmxcdZ7iej8qjezweKnCjNSMnwkqI+0wK4d6ZkVmHiBVT/m4LCiIiz2Hx6MuEnQyUUxicyoVfpw1OKHkprJEVO4JF/d6IBJKWGhgkSF7UKfBRgNFMyAX4aBQm6X94LukFHTsDQfHoya88Hzk0H4zOZ0OjBijags9BD8Qg/K/uxnqTUyHBEO0Erw+NQbt+YzPIgMIs6oXYhGhtq4SORsddlMq35A6LII9OoPWvcDYULcvKEQowvZG4DlLQD215EKXECq7TxG0tJQIZv9zW8n9AtpDdbMu+thmECMh+Cd4dav8dkWnPDwdRklkflq+w8IJPiM5mlgx40u350sz9YagNzAt9GYoJGTHEjw1+ZffunfAcMfw7L2H3mnziO5pLcD4vty0Ma8cT7n8B7gDRDyMU+7Nb/QKLaxjfvs64h0Na+8glNaxw/BedCrQnwch++wADInJvhQENGelF7cGPhC61Kl2ThSIrPZD754xbAIQ/wN2uteuMAmBP4EClqZz6IVKz36ENMCStoSB2/vXAePvlGKiUyf7R8sHywMsBRSe/83nU7Wuiqt6/DdfgrCZ+uvrp8oGp3jv+laY3j8p2fDbXigzxvMplkzs2QyfshMk+dws0K1x4eZoUUn8isHh93AMY8wNvUnz6s7uwZpv5CikfhMiSdSos+yJ9hBQ1XPTPaHXwNSiP4tJUS5MEO7EBp+Is1qMFzAE36tscmZCbVcXUM98CnAPaSQ7w57ItpqEUZa+1jsWiNYzJgbfhIjZ7hNSZTPOI2JjNFxkNEbYQWSCC0njJCuWeS+md2jP2b1EbwTuwQJ/U3cFMeweqg0q8DfVDKrLBhY4hqv30GVto0ZPX4+Fguc7wK3gUVGNG99hx9ba3j+Me2Pv7nrU3cqXTL+9ApP8K7ABtXcSfWkggHAj0W+1pxbLXAj9TN71FrsUFz7JG5uLX1960ttNeRObWBy0BiIYfShBPIhMUOvI6GRGbtCE+X5TZcOWge/wDoQ1JiRSfzJTgPeBqbRmcmvIhfd6Wk29wH8voN+A0r5eyCxt5CC1rsFI+CPOqjPtDyehC19Fjs27Jj4BOSbKNH6sZeaeSRiYZyQrEHUXjIUrV45W+pWGivDOMojZ+SeSjuvW1jUCUhcbJ4FeCVnR58HXn9GH+YKeKabrbL9NdvR7DrSZnMI6TzlVIXXdb/A4O7iG1shq7VYWOvecBkXuklMZmi5fUgaXuVPplaW3YsJyRpo0fq1eFS5wZkpsjIq6FIvEZkstBcGdHI8VKb75nkQvsCWh0utlFBTDX6AP0dgD/B89inD0mJFZXM+pi+4+kLn48ik/n7+qvQLvWr4+fK42bvbkBv2AyZy73Le2W6CJLD98KvYzJFi+tB0dJjsWfLjumEFG30SL063BmoZCrI2KshU7xGZLLQXRnBSIyfkqk8Gl2RRx7i5MoeLLYavVIXvg/X+INSZoWfqeJHI7j/DiKqbe5vTOYz69fWhqVu8/DaVz//JbgLfusTkuy+Z/ceIrP5+MvvG8RkihbXg6KFbTa1B4Id0wlptAstUZtBZz881MiU8ytExhaGIvEakSnCXqWfHUkXhLvM6dk7auVHGRZyUvnH5gf+OKzungG4uN7mD0qZlZyHdnJV3zf3N4bMzpP9ZJ93noVn+a85M3lfbs+0a+cQau1ykh6LqQVae0K6R2oeQpt3mG9AKxC25PxSkBkyZbQFEgjtlREIKb4jc8XGyvxFTpTmpGQ4UgagCJ9Ke5V+MJdRqQ3UVgAPDLY6b+2Gv9WwY+cQahcGMo4fi3E30rZYm9hHahmLz0UvBQAAhqLJQ2bUMsgCCYT2ygiFeEE4Mi+KtbZFTpRmpWiYtBU9ihbxQ/e39PjDLzdkaPNsJ2uTfO5tIuTVSUZv1oPyWBxp0xPSPlLLgHq/0g8AiJi2OcjSAbinArFXhj+S41sy631fFe4TJ9lmpGR4KatlSb0NfBQ9MnNGnkTs1oPa4PiEdGPWz+KuCmA7T+Fsc3bsleGrOb4lszz0VeE+cZJtRkqGD2a1VsJHsQsLtn+Kv249qPoIT8hoiAZgamTqlQEUH2fZjSLPtKsdxZkGmNpZcZHlTUk/inmj30x5cZG9mSy8NWPRvwJO3MY4x8e2sG3SZE9s9v878K15zG+jvs3AjRhIL/Mbjbytv83AbQaKyoB5b1NUeIJLEieKjRHRcZZG4VFK4kThYVKWRvEbJ04UHyZlaRS/ceJE0WFKlkaaelFMvJw4EaVjFBApZ2l4qRcFhIj/fOfEiSgdo4BIOUvDpF7YfIzCwUxqlDgR5WgUDiUC4iwNSb2QfIwigpTECS97pIggERNnaUjqhc3HKB5SSZwIczSKhxKAszQkucLmYxQPpiROhDkaxUMJkqUhyRU2H6N4MCVxwmWPFA+gQcRZGpJcYfMxiodVEidc9kjxAGYR2XyMrKZAEpc9UiBMGhSbj6HpiiMbldrFAZOPRM3HyB9+izQ2e+QWhZ9bWJsfqgcIM6f1MaeTqvFrF/qn83prrF1+qB6eEkXn2vT4E8st54rnVM5dfqjuhRJF59r0+BPLLeeBJ8HSiRn45fy9fD+YKJrXYgBxP88ulOvxsdxxOneh8xP3bGLXiQ1yBprMaV1rE1U1bQwg7ms2ikyN78odFYN5iKbEnoGynZH4AkwUzWsxgLifZxfJ9fi23DEaPK/ulNhjOFp+qBuTJqo6UboTA4j76chJe2p8V+44yXKWuknYH5gYKNBm8vd8bW6iKvl3AMoDDuf63JON78yJA6GNHwhduaOzmfOOht2GDJBZofvra7P5e76WE0WdXbTjAJyGTBc/CGvLHaOIM++62lqeS/rix8kpoiDzXrj45bqn0sp8ml+EWlclMwXEIzNAVHSq0LmXkHPZutpaCua9+HHyFFn6wiUo1zVkTqeVOaHtQsfN1jszPUA8UshMQ92E0LmXkDPcYnEltg565PpYrMDlW5b34sfJKarMIa1/lcJUrMzlFmhRwlqufyS1rg3Xd+h5eeBm65HpAcqGmrYQlz3NZ1PpYnUPFk7xXLwXP1YOcVUr179iYSpX5ma0vDZGrfweTmzLa3PUyvourcg9B78TAF6FbwwoE0otxOX4Wlh3rOZDo3il6lmqwOVg3osfK6dRcnZJua7UQb2OUlsY5WtlbYxa93s4vlaKSFEbr+9exDeNbrbemWmKgN2bqMCZrYPyhW5t7gs5rHMv057hNr3MAetjuQKXg3kvfqycohpkaf0rF6ZGZLKW18akdb+HE9jS2pxto5/TOYLv6mR6gFQgsdCtzTNh50cmYeDG9bFUgStzcS9+UjkNM8hc/asUpkZkihbXxqK19Y+BLRZxijb6OZ2j6n91Mh0ggRs4i85MF5+HBiOD2lHxNI8t18dSBa7Mxb34SeUU1SBz9a9SmBqRKVqsrRPtwoAsI1tcm4s2+jmdjY0jnUwHSHEWkeniq2FT9+Jpblt+h+JdBvbFj3u3YsgUAHYO8T3TwNs2f6PfwzFSszbP/JxOydy0aZh/zxQzC8geVZFaIAE6WzsaCjGsNz8xn9d2VDLf5hLAvfghObUhb83GzsGRGWjt2tjVP/patzan9Z3XlgeNQTpbU8Przd4B8p25iuBAaOP7wqB21As7n116h+Jhdy9+1Hcrtv7VVeYGmOzaWK1/dGtzWt957RL9+JsHgFVe3wHybHIKcW18f2RQO+or5rbvYT9FDLc21n3kFJGWN9oTydSdaVI9PoWdzfy0mIps3oWqSshQFAOI++Hom+7N2F0a/3+0ivtrVGsLZAAAAABJRU5ErkJggg==" alt="" />
对w3求偏导：
aaarticlea/png;base64,iVBORw0KGgoAAAANSUhEUgAAApgAAABDBAMAAADKeWg/AAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAdmaJqyKZVES7EN0y780nl6jHAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAMjElEQVR4Ae1cXYgkVxU+/Tfd09Mz09nEiCFIza6EGCWOS4jGKHbcERfR0D74ILJMZw0EfXE2ilER7Swmkofo6EtQfGgFQ17W3ZdgEB9GNLsEEpgXiciCbRARH7RFw8K+jOfn3qp7bt2q7amp7NbKXOjqW+fvfvXVT9ede84AlN9a2+XHLCdidZFlHt+xC5mq/Srat45yXNpn8rTo6LsXR/bE7QEc8fhn7g9oSxB1I2gRmWPolBDtZ3AiJ8oq/ChbSwB89+LIZotReigaH49y3O031+uTtN5KkJNAM1JyvCegJtECCJnbsJhhsR/x9+HeHPNWHpkEwHdnMgshG4bIpPHxKLc70/q0O8oGipwEmpGS4/MB9ZE+wJ2lktmC3Buo+48ACiMiznz34mTCLyeBoXB8IhOGjRE8GdBDbY2kyAk8+N91z4Ck1NCxg8R5rQEfB5iWSibAi94gavcbW2pX7cgFqN2Lk9l+TgU3Ozg+kwlLQ1gJGaw3hyie4mdl29eTlBo5TqmjWh0eg3p0bTLrY+Xm7Wht07PVWnjIc3Z2mUzrfrcossg0ascbu1rYlItHC3F8IfMkQC10YqMFlBInsEobt7GUBOT4dlfD/Q49QoblknlnSw+jyLwX3qO17h6Tad0NB4XJrE/rFzi4IpPGZzJrO0PoDdzRTX+8GAFzAj9AYlQjpriR4+9N337Vb4HJb2AZd5/+N9rRsXTugoXo3IQsHv/g4/gMkGYIOTuC090/kah97HvvsqFBadvf/ExIawI/CfdprRng0gi+ygDInZvhIISM9KJ24PrCF/qNAcm0JY3PZD7xiz7ALhu4m7V+d2kHmBP4CCnaRz6MVBwd0oeYElbQkXbc9sIp+OyVREpk/nx5Z3lnZYxWneGprcvWWujqRpfhMvydhE+1Xl/eCWo39/4T0prA9Vu/pLUSgyKfYDLJnZshk/samaNO4KaFa/dP0kIan8hs7e2tA8zYwNl0n9ptbW4Zpv5GikfgHHTWG336IH+GFXRcddyoO/421KbweSslyONN2ITa5Ldr0IZnAXr0a49NyOy0Zq0Z3AGfA9jq7OLDYVtctRZlrLWvxaI1gcmBtfqVGiPDH5lMiYhbn8wEGZuI2ggtECW0kVJCeWaS+tfWxn532lN4J+4QJ90ruKlPYXXcGHWBPihlVthxaYJqt30BViIyWd3b25PbHO+Cd0MDpvSsvY9+to6i/aMbD/914wR2GoP6NqzXH+QuwLEL2PG1JEJDoNdiVyuBrRb4lbr3Y2p9dujNHDIXNjb+ubGB/mFksdrAZSC+kIcKCXPIhIV1eAMdicz2VbxcliM4v9Pb+ynQh6TESpjMl+AU4GVsGl2Z8Cv8uat1Br1toKjfhVdYKVcXLG01+9DnoHgW5FUf9UrL80HU0mux68uBgS9I8vVeqZe2alOHTHSUC4ojiMJBlqglKv9K+UJ7Z5hAyfgJmbsS3tkujVskJE4WLgC8ujmE7yCvn+YPM0Vc08N2mb7ddhVOO1Im8yrS+WptgCG7b8L4NmIbm6FrdbK01dthMs8POz6ZouX5IGmHjRG5Wl8OLBckab1X6tXJ4vo1yEyQUVRDkUT1yGShuTM8y9lixM9MChH6AVqdLESoIKaWRgCjTYC/wHO4Tx+SEitBMrsz+o2nH3w+i0zmH7qvQ1QbtWbP1me94e2A0bAZMpeH57bqdBN0dt8HL/tkihbng6Kl12LHlwPTBSla75V6dbI5DpIZQMZRDZkS1SOThfGdoSxx/ITMwKvReXnlIU7Ob8FCf2lYG8BP4CJ/UMqs8DuV/2oEd91CREXm+cZkPn304tqkNujtXvzWV74Ot8FrLiGd0+89fQeR2Xvs0gfGPpmixfmgaOEku9oTwYHpgjTaZl/Uxuj4RychMuX60sjYw1AkUT0yRThsjNKWdEPEtzm9e3ut/gjDQk4a/zrxwJ8nrdNHAM4ejfiDUmYl46WdQnW3zfONIXPwznZnmzvPwDP8ba5M7svjmbr2GLTWTifptZia0toLMn6lZhPavMP8AlqBsCXXVwCZIVOsLRAltHeGEtL4MZkrdqzUN3ISaLGUHKcBAxThW+mwMVLHMq1FQG0F8MRg6/LWbvhXDXfsMWhtcyx2/FqMXU/bZ23HvlKLLb4XvaQAAExEk4XMqMXIAlFCe2doId4QMZlnxTu0RU4CzUrRsRMF9ChawA8935LzD787Jqa94+tpn86X3yZCnp2k9GY+KK/Fnja5IO0rtRh0R42RAiBi2mYgSwywFwRi7wzXkse3ZHZHrkr3iZN0M1JyfD6tZUk3Aj6LDpkZlvOI4/lgyNi/IGObo8exGwRwMksR+2Z07J3hqnl8S2Z94qp0nzhJNyMlx3vSWivhsziApt0/wHc8HwzG0BekZxICUBhZ8M4AGh+PcuCNXOpu6CyWOkDhYNVFlnVI4bOYZX095dVFdj1ZuDnHoj8FzN1meIyPbmA7QQc7t9v/r+HNec4PUR8ycC0Gktv8WpaH+kMGDhmoKgNm3aaq8ASXJE5UGyOi4yyNyqOUxInKw6Qsjeo3TpyoPkzK0qh+48SJqsOULI0k9aKaeDlxwkvHqCBSztJwUi8qCBH/+M6JE146RgWRcpaGSb1onxlVECFC6rQpccKkY+jcjmoB5iwNSb3IzXm+oaglcUKSPGgpu6qNszQk9SI35/mG4pfECUmyoKXsqjbO0jCpF3k5zzcUvyROSDqGl9txQ3F5g0uWhkm9yMt59vyu764kTkg6hl1Kv74I5huNszRM6kVezvN80d4iK0mcMEkeuJR9E7TcnOeq4E+W0quCKIyDco4r3zKX0iuGnHKOq99yl9JvDHzOMS136PyQRWcENgd131iDcIqiyB/drznMt55Lmx+y4IwgzkGdC4JrFIRTEIUbN9D3aw4DJvsV5YcsOCOIc1D3iyZVVckBCqK4xuCq5nBMlasHbipkOlrejACHH6c9WDJsjFBXAF8YDqEoFC4DnhE7NYfBdLJ876DWCZnW580IbKJZ2otzBAviC8EhFAXDeeDqax+LJc1x3C0pOrghk+Bx76G4l+r4ZLpATxY9+uY4NQ4KEEU5ZB6H85EdwNYc0n450cENaYeJv3NnBD6ZDlDKESyGLwSHURQLFx+J6XwRlgemq3JM86Lf7Qdx9+tjZ0+FJLnS5s4IYjKNiwOUcgQD+FRsC0IJLRwlZBSBcDbCvr7jK1PlmOZFn59MFZJQqcPInRH4ZGIhSCTHxTmCAXwqtph6A1o4ypJRBMLZCPv6fsCtw41LbSm6s8Li1ufqIlcay9UaoM4S0vxag/sS1eemyVRAPXzkmR55fmFJZH7tk04dblIuS9GTFRZVn2vIzNDKITlLSMo3XytkSn1uikwNVONjx1Rsks4pPAiZWFyJbZ0xvLZjVlJoLy6XpehG7tfnGjKNFsthpU4VC3WpCXpbvYsC1tqJXIZWrTRJfW6KTFBANb7AyFlwghhDj2COud9Nc9cUtvKyVGOANdSLkUQP1ef6Ra5cDot/WuJCXb8+VzhErUzkvNpemRWjVmZ0r1B97g8RfmOA9bmGzNgFQAHlS8nB54+cAycOKctcjILD7Ze5lP2noPGm1OFKhactl+XoThXsG0l9rq3gdMphUcu1pfgl156p3hUOUWvnlUorHKLWm9Fxfa5/ZXpAfXz+yDlwPIyM4iBkxrd56wo0ZyArKbwsZctlOTrLU/W5lkzxotIurlM1RWeGLllCYg5JaydySsscsq+eV3J9rkemD9TDRxeJiu2R6cDxLRnFQciksaV9CM71QVZScFkqKZfl6CxP1edaMsULyZQ6VU2mLCERh6I15ZDmgM0CE3IoWj2v5Ppcj0zQQOUxFOPjY9Gxs+CQqbakM1kOmbX3fwJr3agOF3BZKimX5egsT9XnWjLFC8mUOlWN3i4hvWi0zTEdhT0Mo0UOxVevNNWpPtcnUwOVo4/xBWJnwQmiKIlMhmE2zb6Aov9Ak5yqVH2uJVO8bAWnd1/ZkGPTMRM5c00YoeUwNa/E+lyfTOtC3wg0gE/H1mQa7+aYO9qyvLm5GQW/4mUpXlJJwMK0FqE6qc+11bIsjOtyLZldEZutnQzbiZzSxhx680quz43JVC4UloEG8GlDQ6YWGjhKWOLcPDn0ZFmKllQcsLn1uXE5bFyom0TEd0z7L4bsRM5V4v9DM6t1el4p9bkxmcqFdvyTbfEpw3w4rmmpc/M4sFqWcsiMDfbfCXIYh9EcxmLg+txsMrkUtBC+IJxS5+bJQbi9UHmsq3/L+838wtmS8R0g3P8A1DtKuCbXPMcAAAAASUVORK5CYII=" alt="" />
我们发现当x3值过大时,会改变Loss的导数的方向.而来抑制w2,w3等高阶函数的继续增长. 然而这样抑制并不是很灵活，所以我们在前面加入一个系数aaarticlea/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAsAAAAMBAMAAABGh1qtAAAAKlBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADmU0mKAAAADXRSTlMAVHbNMhBEIpndZom7PT/yAwAAAAlwSFlzAAAOxAAADsQBlSsOGwAAAEpJREFUCB1jYFR2YGBNYGDnMGBgYmBgYL/OoACkGCZxBYCozSIgksFXAEzVLgBRrCsOgKggtgsgSoHhIpAEam1mZWAoZWAI3sUAAJNqCsiGl5EyAAAAAElFTkSuQmCC" alt="" />，这个系数在数学上称之为拉格朗日乘子系数，也就是我们用到的weight_decay. 这样我们可以通过调节weight_decay系数，来调节w3,w2等高阶的增长程度。加入weight_decay后的公式：
aaarticlea/png;base64,iVBORw0KGgoAAAANSUhEUgAAApgAAABZBAMAAABoNEnCAAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAdmaJqyKZVES7EN0y780nl6jHAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAPIklEQVR4Ae1cXWxkVR3/z1dnemfazsIKAYm52xKCgDg2BBQUZt2a7As6Rn0wuHaoG0l8sQvqilEcVsGgQUd8IBIfrhI3PIjdFwIxPtSgu5CgVCJiyCaOaJRgovVjg+lL/X+cc+85Z86dmZZpd6pzkrn33P/3+d3P0/P/F2D4rdAevs3hWBzdyFLHN3cqlbVVRvHCZg+V4rFeXFR01bcf2fGLPHHE/o9d7+EOgRSEUCAwW1AagrXvw6EeVmbgoXQuBeCqbz+y9cmw2xX5x1G2gmq+lo26+ZqCmHiaopLiVR42kSZAwGzDZIrEVsjfhGt7iBd6gUkBuOoM5rYia/jAJP84ynapk+0EzfRAERNPU1RSPOlhX1AFuGyoYBag5w0UvOaJQpEIM1d9+2DCjyOPK/RPYEIj14R7PXzIHCAqYgI3/bvmCBCVGiqWEDin5eB9AJ2hggnwhOPEOrx72Tq0DuQCtNW3D2bxYcu4OkD/DCaUGzDtE6jlG0ju4G+67fKJSo0UO9SxWhbugGzYH8xsy1JzDmxu3pG1uXDQUTYOGUytfqUw0sBUbEMbuzYxLxePTUT/AuZhgIzvxIYTSCVMYIY2ZmMqEUjxYpPD/RI9QhrDBfOygu3GAvNauNrmmkcMplZXGGwbzGwne4qNW2CSfwYzs9qASt30rvqtyRAYE/gWAmM1QoobKf5c9fUuuw+in8IUHt7/D5SjsZSugInw8Ygk7nznnfgMkKYAOdGEpeBlIhXnvna5Ng0Wt/iFD/q4yvC9cJ3NVQ7ONOEzHACpc1MY+CIjvrCNcF3iY9VcnWi2JPlnMI//sAqwxgLm5kA1KK8CYwLvIUbxgncjFLMN+hFSggoq0oHZHjsCH3k9oRKYP5hanVqdbqFUqXFk+ayWFriC8CychVeJeF/hpalVL3dx818+rjKcvfBTNldskOVDDCapc1Ngct+OzGAn4XYTD1wfdRPJP4FZ2NysAayzgLEJ7lsrLC4rpP5MjNvhcSjVclX6IX4KFVScMdSo27oHMh24TVMp5NYiLEIm+tkBKMKDABV622MTMEuF9cI6XAofBVgureHDoS2qNhdpzNWfxcJVhkmBufYnNVqG3zGYYhG3LphJZCwibEXUgVhEbamLKM9MYv9Ey+h9qdiBt+ABYRK8jptsB2ZauWYA9EMqo8KK5QjZZvsETIckMrO5uSm3Od4Fb4UcdOhZex29tmZR/ujC+/+0cAg7uXq2DbXsTdwFmDuFHZdLJBQE+iw2uWJYc4E/qSvfpVZlhcq6AebEwsLfFhZQ3x9ZzFbhciAukV35iD3AhIkavIKKBGZxAy+XqRBWViub3wP6EZVQ8YP5FBwBvIxVoysTnsTXXaZUr7SBrH4VnmWmXF1QXs5XocpG8SzIpz7yLS7PB5FLn8WmLhsGviBJ1/mkLi9nOgaYqCgXFFsQhhFZwhar/JZyifrOUIYS/wmYa2Le2JZbBSISJhOnAJ5bbMCXEddb+cdIEdb0sJ2ivdk2YMmgMpgbCOdzmTqaDM5Baz+hjU3BNROVlyurDOZKo+SCKVyeDxK3kWuSqtZlw3JBEtf5pJ6JJmt9wEwiI6sKIrHqgMlEdWc4kuuTIT8zyYTvBTQTTYTIIKTKTYDmIsAf4GE8ph9RCRUvmME6vePphc9nkcF8MXgJwkyzsP5gdr3SuAjQGjYF5lTj8eUs3QSltbfDL1wwhYvzQeHSZ7Ghy4bpghSu80k9Ey22vGB6ImOrCkyx6oDJxPjOsCTRfwKm59NoRT55CJOVZZiolhuZOjwCp/mHVEaFv6ncTyO4Yh8BFarnG4N5/+zpA1GmXlk7/cW7Pg/74ZcmIKWla5YuJTArd5y5oeWCKVycDwoXDrOqPhFsmC5Ixc1Xha2E5m+JfGDK9WVHxhoKIrHqgCnERq7ZLUk3RHyb07e307K3c1iISe7vh971+6iwdAHAidmQf0hlVFI+2slU0FbPNw6ZjZfapTZ3HoAHeK+uTO7L45m6egw2V08n6bOYmsXVF2T8Sc0itLlEvQE1QdCS68sTmQJTpHUgFlHfGRaR/MdgTmtfXXvExNNiKil2PAJIwq/SRq5pjaWTCYHaNOCJwRbwVm/4rYYHegw2N98SOf4sxq7DrTK3pD+pRRa/i56yAgCIhJMWmWKLkA7EIuo7wybiDRGDeUK0fVvExNM0FRVLoYePpAn80fMtOf/w9JyIVuZr3TqlT79JiDw76eKr+aB8Fjvc5ILUn9QiEDRzTSsAIdM2JbJEAHveQPSdYUqyfw1m0DRZdp8w6W6KSoonu7lMCULgs2iAmSI5CDmeD/qE3Qsylpmdx643gMNpjFg3paPvDJPN/jWY2chk2X3CpLspKile1c3VFD6Ldcjr4zewj+eDXhv2BemI+ALYdmTeOwPIP46y7nge6qHvLA7VwbaNjW5kaUPyn8U06d2kj25ku4nC3vRFfwoYuK3jGI8uYDtEgx1Y7X9XcG+e83HUYwT6IZDc5v0kx/wxAmMERhUBtW4zquFJXJI4MdoxYnScpTHyUUrixMiHSVkao984cWL0w6QsjdFvnDgx6mFKlkaSejGa8XLihJOOMYKRcpaGkXoxgiHiH985ccJJxxjBSDlLQ6VeFI81RzBCDKlUpMQJlY5h53aMVsCcpSGpFz1zns9r1JI4IUketJQ9qo2zNCT1omfO83mNXxInJMmClrJHtXGWhkq96JXzfF7jl8QJScdwcjvOa1yOc8nSUKkXvXKeHb3dPZTECUnH0EvpuxvBYN44S0OlXvTKeR7M2g5JSeKESvLApew90HrmPI9K/MlS+qhE5I+Dco5HvqUupY9Y5JRzPPqt51L6+Qmfc0yH7Fqni/rN7vqUYCeG6B+aW3Pol9oSNU4X9Wvt+pRgB4boH1lXzWGK2FbIcbqoX2nXpwRuWaU/rGFQrZrDElZSDMFoI9fsZaXXlKBFtbNDbjTEIY2sb2RGzaHO8+qr00dAJVKnSPWaEngT2mw7hUf0cSEu3tKUlP0TSW5lisQbIc/uD7V6vqV7Rm5sQtpW73BvrYPp7B5gFtdE7VnaF1Zxwz0hdm/tIQ7rMun2A7kw+Kcm65pDOh6SS1+6qHYHWFZ+dXLg9nqAGTREuFjH/WSIG+7h3tecIQ5pZD5PE234jaJbOaa9XF7pMxTTLG5XOp/F7TkliMHMtmLbTicfImGGkJ2gXkpzhugbmRWVNuMlambKPr4yrRxTn0ttoLcXk9udzmdyoeeUYAAwpwhHLuZIeTTnJGRriL6RWVENNkwtZe9zbUjWe+JSW3bpoZOuuDbWX8zq3W5ubNOra3FVZGd0gS4fqytTBZOdraFnCOY/WYUjZy6Hp/86D5mLvwQwi0n0dx5vwL7jncrn1H1fFoPWEJ2RsUTXiIjqRVgMpm2LRyNjvScutSWXPnriJVl/sap3VQwJ1y7BlQjTuBJkUqDLxwKmDuYheAa14bZopQVvW50MaSH1IVjAR2elNVGbrk2G+ToKcNXloXy2jSbsITojYxddURF1UDCxuBJbjQ1VzupSWzyMy2XJpaJTXS7TsVCXm3hJymGlThULdbu5RGFdPa+0dGOuvYiDfs0CXQFTBZPrBDeiZ6zKnIngO1iKCecg24H3ws3waHQtVPCllg9RYKqFxjMLByPyYQ3RGRnxFW7JiAwidwff3KpKbXmmnKtjDfVkKG9zWQfiulyid7IhPup1taxZDot/WuJC3S4uY4i6Mq90dRlD7RH8BbpOPfDU0mxEX94b8A1AHGciLLTGutkX4bezV8ELOOgXAJ+k18Hz2AV48428A3OIBKYasVQcu1E5Bb5iYaBtfhVuach6D8+UdbksuxQ61+VSkW2dCnWxqevfKId9Bam6TsrkCoak28HLh5rJlZpS7ZHZsrELdNUzU4JZoXfOWcicQkQra3APwgKLjcoaVndCQX74RnqSS77h6tO/wrvPGaI7MnJpRaXrqxRRQuqxTW5zLNr/AJ5fKrXlmbIul2WXQue6XKRzoa7hOimH5TpVB0zmCoZmga6KUHR5IUx7NMK1C3QVmBLMSgRB8VQLIfwLZOrw7dJk+NjzkGl/fQPgR+sAH1uHS0qwgU8AbK1pmhM7Q3RHRoJWVFsFkwxIy0XBr/HG4FLb4LWkXJZdCh3rcplOhbqspFzH5bBSp+qAKdxGrmkX6Fq6uBCWeNQR4d4u0FVgSjDTLbgmV6sW29lz9G8Dzj1TXq1ORR+vha9CdvUJyDReLvwxCxuF/7A5qkgGe4jyAItHxmIazHhETFWhisRg2wv3L4Na77l7OSmXZTCFjnW5TKdCXdNLXA4rdaoOmGp1ptywC3RVhMLFhbDEoxFu1irQVWBKMMH8bFRZiuAdx9aojG8+LM43Sjd89mD0KD4zP4S/rxz/8KUwN4cXKraAt9YQBcx4ZCygwYxHxNRtgCnWZHuQd1Z1IFJ0XW5cqGt50TeF+8xUdg+rvZ5XWrqgFsK6F3HMAl0FpjKkd/TeSm+ZdgpP/08iZuuRaTBFR4/IDjXFXhpZz5St6kAU1nW50MmEeAgQ8VZttOsYTIurMQQ9rzS5MYbuIo5doCuXl+mT+vTeSmtTrXLLy4v/J5Fw45FFprQekUU0BQbpxzNlWlLh21y04rpcXahrGdPlsHGhrsXVGHbPK1EswdBexOlRoKuN34zv8fR2Mu2fycX/k0h045FZpvSILOJWD6yZsgHmVu0Y8l4ME76NYUJPL9DVMncdq+muZ5+dCz1UJOn/SeTn7hg1v8N1q/0D9xXo9tfqL7FTI/Mu1u36kmH/8e8JCe9i3a4vGe4JqPoH6V2s2/Ulw/5x7gkJaz0yjhiXDEvDWZuMTf5/dIz1yHjA8X8ujSnjziAI5Fs+KWuW4BMY03wImOuRmu/MEjR5vO+DgF6PtGa/ziyhj4kxWyOg1yMtMM/TLEHHtPf2xsoiBa/APBH/P+DhTCb3Hi7bithYHUzA1Ot/SBmDuQVY1Tqc1pAr01ikHIOpoRlkL5Vdsv7qrP/R3HwM5iAgahleWYzzetUzM1mkHIOpgRpkb6wdorgCM1mkHIM5CIhaJl47ZIICM16kHN/mGqeB9vHaoQlmvEg5BnMgEB0htXaorkzNHM/NNRJb2eu1Q2v9bzw33wqEiaxeO0wo2BvPzS04Bj3wrx2O5+aD4jeg3E6t4A3ofk+J/Rcuw7iKGiDwQwAAAABJRU5ErkJggg==" alt="" />
从公式可以看出 ，weight_decay越大，抑制越大，w2,w3等系数越小，weight_decay越小，抑制越小，w2,w3等系数越大
那么冲量momentum又是啥？
我们在使用梯度下降法，来调整w时公式是这样的：
aaarticlea/png;base64,iVBORw0KGgoAAAANSUhEUgAAAXQAAAAqCAMAAACUTlBSAAAAM1BMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADxgEwMAAAAEHRSTlMARGaZEO8yzSJ2iau73VRQ1C+tCQAAAAlwSFlzAAAOxAAADsQBlSsOGwAABYpJREFUaAXtWoF2qyAMRaQoWJ3//7XvJgF0FVprtdv65JwKDSGEACFcVepMRudtoE2eflJ3sEBdkGFKFQX+/5zstBML+H7s/ANjOFtiqIKUUv1Jn1kAZu5a+d9cZ/R8scmTiXqnqtzoP62Bj64HGftY8NeTZfQdDhekTNxnqWSBSqmrmLIaw4ovsYLzDoftyu3OmskC2sERG9WLo/aXVDM47Zk4aK3J0CEXjkrTqVlrZS7zmGVqnuSchYUFTNXCUG07imNoUgBS01yQDSt4fI8/MWe7Imq8YEbo12CfpDSm0lkoW6BSA51+To5AE2yvlGbCCIN6zAMt5pC3PQmrVAv70k85o2yKfvr5sifGM2UtUFOI2IhLH0YxWqsuRDAjVrIdxyu5mZCL0TEHcN+aQh00xwSF4/U0OiyyIvVYzeRiKNW8jJW1LR+og3iLoZNqztnlgLWHP+ooLneqgv2thJpBDsk60x0L0OLGgcipF5dei+NQDYx6JWcPW8acypSoGRneWuVg9DBtp9HFOg+eBqs5RnrBpdMZ2iNeIWOqK4xrMRcxjyHjpcU5C6Njuhyq2b0nQQ/6LFYvUZ2j8ZyWNuv7k64paKTkG2AA/nqhRdzW2vHyrxAwUiHm0XtXtdYWTOClybG8xl+9HMlGY2XC42A8x/PZNe+wXB7Wb+OWDsE9k1kABRUo9LsLA2xFdQ7Gc2hHr0xmPWu4aq4UvIZNrkxzTkQvvC2qcDTM61J5K6ojwWwSs28hhgX7SlWP7/VPd7hwAtZrijvvu4KtqM6rLuvu8AY6tnZP4mtvxJKFXkjL406E3R5637rZjOrwKV9AHbKDeIa3472JU4t1dW6gqyIAkFZOtME5o3VHd26ROiNgkXld127iVtLe+qb3JDeKFS0lGs9qvCfxWzfbUR2cXyXUIaftM7wKMRgCM1jIwrCUd50CMoJdSa8VpMR3dpE6IyDWwE3QjF+JO7YXQZNYUhLSOrlL5lTejXbbzXZUBzFtCXXADaNLKWysMu9ybHy78FjcxhtBPrD0gYyE5RJKjgIFDopnhHCLH78SNyMnvHXYpUex0quDmhTpHZy+d7MR1SEdGVrIow75IRR47TWlGJzzTeSC4Jc8I0ePvPQZGWHZXOKLN2NPmGSaWiZUFFlXmJ7IndpXHFtGsUFH12NeQxr3T1H0vBu1EdUhWWz0POoQu/qer+clh2JCqMHIhxyBF3LinLgkDISoYmaoigmOtgOhgIF7ak+TnsSiiKR7W9sm7EUhHfG86WYbqsOK8brJow4591JCKHKDHFvTGhINI1mGTK/BkQg337PhbGBpec4IbNpmCHdxnAqpPabSVFGsCIIAOLE0l0Lc/3nTzTZUh9WSzZpDHQpaZxGKHC8gECx1fiGgrcLbmBaLUQdYmxqwm4b7gDvy/JwRBnh6jTUvIDiYU3u8EEKDKDZ2zMOIf47LZ91sRHVINwGGsqhDQfX1vF4DPULkxz698sPQ0mnH8R7L5pL1dAIC8sBzRsCbncFhiIk7tR88vWmLYqOSyaNHwjH51M1WVIf0WlyOJtThsdrP8D6WtuBgF7Og/inCEtUh9ZcwACgCxq0Y3TO8K8TdsIR3PzfUl/+mF3EvS1ohYInq4PCB171JAXW4oeb/PsObl1CgUhjObzoL9S+QEeTEz5BekLK66QLVeYDnrBa8P+PgB3dQCIhlFj9D2l/vpcQlqnOL5yzbfB4FgR9dIM70LgvMPkN6Brh7l3of2c/sM6SngLuPNMa7BjX7DOkZ4O5d6n1qPwyfSShaAO4+deQ/OK7ZZ0jrgbsf1Pcjuib4LXyGVAL5PmKcv2kQ88+QVgN3v2kAf1KX6TMktR64+5Mj/fVKHwzc/frx/4yCxwJ3PzOmX9/rYcBdceT/ACtELoCjGB/KAAAAAElFTkSuQmCC" alt="" />
我们每一次都是计算当前的梯度：
aaarticlea/png;base64,iVBORw0KGgoAAAANSUhEUgAAANYAAAAqBAMAAAA9nbh/AAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAEIlmmXYi3UTNq1S77zJPBy8JAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAED0lEQVRYCb1WT4jcZBT/ZTKTyUzmT6AeenCZYC1aWyV23Eo9aHRcPPSSuuxBK5KDWCwsuwtSPOnY0vUg1dEqCAoOlXooCKHLiN2KHRG8SCWl4EVY51B6sGhnKQzSivW9bzKzacjWTbrxwcu87/e97/cy35/fF2CDtiUyz0cfBWQ9MiERaEWO8tF7ACk6IXLU+mBukvpyPSDz1slQFqNsqg0cG4Z39zSKDlBgjs/CRAJlsOVnhBPits2sC9R51PXwUIEyeID+OWXdtTVLJvAj0aiDMBejwn4HKm0/Tv4zaUv5HuARQ4Yf6OzuQavvEU6AtvvdJxyUqUP0ckZSkxY9Zc4dEhUsYsnakocj2C6c6BeVX8s9VKmnn7TGaJyqtfACoMwTUKOa2Av8hfuhszPqqh6heR1YGI1J/Js1cMKvdZ4InUuQ/sG1v3vs4g0y7U2rlW8q9OI0b8BztAGuDKD1gWVqL3sCLZjUtSnrhaqeZTLixuu0WkofBbvkwmMXaM1RN6tWDbSfxZ6XbgJTOIwXUdIrFjudBNV7ClfAOevteYlfdR29POSIvtFDnrI5pFO748Z9F1bxyqQJtV4XTmhl39WdzTue5eyQKUovKxeGfaHnvaH2sDlGWyONeuCGEcp82W9H6eVyKHfYzPSi4BGq0jwdFQmFbjiPXkPYWDsDCY1AHAitQLwW+ujanVKlqreZPAKi9PLwbanjxpZxFAx8dO2ufN8JdlNcGgG8d6Rtz9qdPbTiuQNngZlzXig5XvNhTtemHwGW6g47n7ylB5e+FMfmoF57e/F60cAPOI2SWWxzdmL7iEdOYQKqkbHZqZbqzLorQi9PoLpVm6cD0MIXpHZFnt/saTaDx8UzPoPMVG1mLAnsB+lv4jizLEBdxTsg0UPZxC+4hLLD+Niqt2JYH9oqaScx1XqVWz+DnWoBD/FjAbl5UuxyE5hzKp4ywB8MJ7YsTdCfcw7O07/5SvgEc5F683qRTH+KOQqvIdd9r4+XSN2Sz2HeAixi+xg/kR6z03pBGqDJtbSuPOBLCWX9NcO8qHwoU5zY6IrL2nkn18b3uCyc9vxxuV9xhF5u73jYRuTqzq9P6UfOHdqfuBB9JXx+ctdvutKYJhmpm8LpLF/+7sw3/rcPzWKKJnV9cqFR4rLbeLVpe+O5nEnHWhjp5Q6QWsSwDEgC4ljgTjnTMeKMhIx9sfJxp7vyP5hU0rL/x+QZ6G/iqIWGdDHtiltn8epNyVzBCj5Ju1bzLHItVenT59R+rWOlW+4Z0BHJtOUujGraK/cGZumL2i3asJW0a62iAfo+zbuVHqQPUp1Dqc+fDWVnwpWBb91Ua+GxGeJXG082WKJPpVsrwL4Xjwda6YbHhld7ukV8dvn5p+PV+Rf3yhhhcm8BPgAAAABJRU5ErkJggg==" alt="" />
这样会发现对于那些梯度比较小的地方，参数w更新的幅度比较小，训练变得漫长，或者收敛慢.有时候遇到非最优的凸点，会出现冲不出去的现象.
而冲量加进来是一种快速效应.借助上一次的势能来和当前的梯度来调节当前的参数w.
公式表达为：
aaarticlea/png;base64,iVBORw0KGgoAAAANSUhEUgAAAUMAAAB3BAMAAABrgT87AAAAMFBMVEX///8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAv3aB7AAAAD3RSTlMAEIlmmXYi3UTNq1S77zJPBy8JAAAACXBIWXMAAA7EAAAOxAGVKw4bAAAJr0lEQVR4Ae1bfWgcxxX/7X1pT6s7LVYpDtToYkWklptw0VlO7YTksCtKGyjnqCY0TsuWlqYNuKekpRQC7jp23UIxOeKmH7RQ1cahqDQcdhXXdtuoGPpP2nAlxQQM6TWI0C8nEqHCpKXp783eru9O96FdKdL9sQ/N3MybN2+e3szu7O5vHhCEtGyrXjVu1ATuatW+obxYy9FqXC0PvK+lwAYxpwoc6DNMt7+TaRpSuELfBHSR2iSK4DmOXGJKzjebIFyhJJNbVoyNzaJ4AIhmOeigZPWkuMKIl4FD9S0bW9ZxBRiwOOjTktWT4gojPQP8o75lI8vRgzCPACkZ8w7JjKk7gfM5S5Jwz3/wPBdCpSYiEhtNWw/jC/+pjf9dGXwS26BnIgVJNFG3DpdfAxa5DDbaNHc8+yLiJeBR1jWaimgJg3Ykr0ESuQZOkovHgT5TCptB+5DMOiYaS0AilcVwNf3uy5AkhmNMsk018es4TBO2McWmgTeLFl6k736hknBxSrLKJq5FLGE/TeCqQ18eyBeBZ/ES65KEqy3D3lwTtUW5oaibznAZsUKfFZ/B77CgEm86J6OLaavjTWeI3UnruIcnZC+po10HpcKbdOQnpz/0FzOxfwo4lsuqRO7Cb8/9qvOtO+8oW8c9/F9cVitJm1/J4yS7XG8DjHzjdJNgvOow1nEPN2aaxnCqvF+3IJd7jDdIXvekHzdLifWKSm5hzb8xZ6hmPat9GHu7uWPOZazfHt5nuTobflf5SKsvN/Ri5fcuYw17+JZdk8bU53gRHLxcQvzQb1yVgX4jaiXPjVdh5HarRIYx/p29Fu9Qae6UQSjxuHZjazF7GgPZ/hk+1MwGUeL1SeZZjBW0Cp7CqEo08UTiWqrKPTzxU0/OV4Hb25J9gV32oD/LHfiM9P74LKn52lyNWt47RRNu4DaYkhLTQFmnnWvYw5829Wnso4pXkLK4A19ttORdPwS8aALWK9D+h7f+W5UkJiIywyz4Y8ZDiOTxKh8QlnEdRSst/3Bw+igfb99YhrEIXKKmSxVOOd8bssxktwxGY0iWE9SYXsQj+luIz9vUE3iiv8qVSG3JwkAZFUnygIFhS1+Lib/G/RaXIPCnxDPRlPmljPzHQUkeKSfxNTyMATOdl8Sbjl65H2/Ii0PQyyVy6NOIiFlPXX7sgL7zhbNmUPvYb8c72/+4hM9PZKHnciohh/QD/9xpy+tX+lpQ1eqZNGjnrv0+4Epwpgy37Pf3lK8O8Qlf4ohUHXnZw4/661qTjpQM/ns+KNNv+ZCmaN4Rl4exslP0mRs7H/TXIxvzOdCQo38dH2m7GWwPcMZ6mSYKWl9V34VYdpvZm3ZqJyqJYnkoVU1Vk3Zvmqjz2voU7CKKiJuXP9mTRsYy8t48hnsRwWJvrso+O1EBnsRZxGH1pomDZowX9BKNfBO4pSevmGH1GfGqdg1ZGC/15FqMThZo1/HcwgQ92C/lXqZoKTrdy/bRNr2gnvd72sqJ2zjZIYUeCD0QeiD0QOiB0AOhB0IPhB7YUA80Y5CtBk9UW3GD8AJpao1BNg6/fm/HgTS1wSAbbBy0GqprqHTR9OGWquU9vRtd7yag2iOrkOqiqbWJCoOMHhzaezT3PD8988jNxIHcz4b2VoHcOOZYxuvfH+HHOWPGk2q05Vgel4TTJ5knE0QTaiZqIx8pzO3mF3wXg7xeBRE/4yv8cCRHbrQTb+NCNjpPBDB9lOWLIAzIj3P9BU9KbPFIy77G7znA6X72WZMmuCY+ag5z5P6Mh0E+ZoKIH1/Sx0C0ZfDbxjTuFgBsB7ZsZfkRgMcxikhVXSn8lRbFBBOczfDlmbiP4KHxWQE/XBkfmtg3Mc+MVPPiKQzKyA0YJBG//gydpY7c8GucgKj4c+4usHyWQI58nONBkpoUvucorOWRGRpJeuhWyQNpGqc98k//0Pmnl/AtGbkRg3wVKZtnV9SRm5RNAHDYYpUQmE0XyieGJ2UZ1KQUQsbGGvWVZYZx98LfM/wJpCkhJpIcL8anCbhy5AYMkm4ocpY5nTxyUwSre3SNJ5nMIoxpeyC7lR/nTsuMKqnjz5p1E41Bsz9jc4qTYO7K+NBE0xpNpEt+xJEaMUiunHsJaztHbvaAXpmM4weIVvfQ4YW+agFLiWtfdKXiBWq9SYPm4RdsVhUu6sqowzur1tRoojEfXRYE+yYGycuFiN8I0qPOkZsRqb5+AFu4FhXXGLEwcc8vs67UQJXdb1J06rKcfoImmSujDu+sRtOD27ePNnkRo3MVjlyHQV4vi24fpNzlQ769qKup0Ys1QL2+m18Tb4nW915L2dXkmmjWlCWzjVp1t6GR3b4Wu699m78WV9Pxf1t1HXfIdtFARkOtByrn5jI9YEVoQuiB0AOhB0IPhB4IPRB6IPRA6IHQA6EHQg+EHgg9EHog9MB77IFE4T0eYO3q72j62udobB009mVfw/mT7qBaAVwr2t+/giOMKy257Zj+pNtpIV8FFHdor28SJGT15E+6g16F8HVor2tKV7xKxiu1LdRJt5VxGzqHCJwh+pjLMGbxIgQ8NIhp2iA4qTBMeJgi29M7nsgKuik0LZkHeTJU0UMwqUzpEGmquXRTqkPoYucQgauyxP7GNEsIsz8jaGWJ4KTgmfwo7WKK0i64HNFNm0eTMzOEu0REQZ4SquihnFSmdFBUYZyeVNpsH4tF2EpCBOT/9sgLWePh90hJu5WgzxndoG+IRMbnCWVJhRimiylKOwiJE90URQ/PHiCESREFeRLoueEimKJM6aC0wjg9qaTljb6y4IQIkJ/+g1ChXiI+g9T+nKliFgU8FCSygEhekERimC6mqNpPQqGbFIo/IYZSREGeKlTRRTCpzNFBaYVxulIuDMOOK8kJEYiP2yubZPKGLTgxiwQPBYmks1JVB8N0MUWnfUwC9YWiRz7xDH9qkCcBR2PRRTBFmaNjjDi7YJw1Kf1jdHxbckIEfu6scUfKm+jBz1rDJjQVs0jwUJDIC7r8KQzTxRSh2q9oRwXd5BSjxFhEiijIk1my4CKYoszRcUWzFcZZk4IKsG9nowoRSE9jsYXAcBRJG/epmEUBD7no9sWxPU4kkRimhymq9iMOuilaKpK5QKWEKrKfwkJFmaOD0grjdKVe7hS6qEIEonm9JHqbKHIPbyw5U8UsjrCNSCRBzIUahulhiqr93G6FbooGuhIEGx1pCVVkSaGeoszhUlphnK5UpWvoop5vwkhlkO4kiPq6kFZirEcXTaN3dhFo2ZzMtmQHYO6vBgxd7DzWSkyxs3yX1nKX9iDNG4Qp/h/D2gSPs73VwwAAAABJRU5ErkJggg==" alt="" />
这样可以有效的避免掉入凸点无法冲出来，而且收敛速度也快很多.
附上demo: 使用mxnet编码.
//
// Created by xijun1 on 2017/12/14.
// #include <iostream>
#include <vector>
#include <string>
#include <mxnet/mxnet-cpp/MxNetCpp.h>
#include <mxnet/mxnet-cpp/op.h> namespace mlp{
class MlpNet{
public:
static mx_float OutputAccuracy(mx_float* pred, mx_float* target) {
int right = ;
for (int i = ; i < ; ++i) {
float mx_p = pred[i * + ];
float p_y = ;
for (int j = ; j < ; ++j) {
if (pred[i * + j] > mx_p) {
mx_p = pred[i * + j];
p_y = j;
}
}
if (p_y == target[i]) right++;
}
return right / 128.0;
} static void net(){
using mxnet::cpp::Symbol;
using mxnet::cpp::NDArray; Symbol x = Symbol::Variable("X");
Symbol y = Symbol::Variable("label"); std::vector<std::int32_t> shapes({ , });
//定义一个两层的网络. wx + b
Symbol weight_0 = Symbol::Variable("weight_0");
Symbol biases_0 = Symbol::Variable("biases_0"); Symbol fc_0 = mxnet::cpp::FullyConnected("fc_0",x,weight_0,biases_0
,); Symbol output_0 = mxnet::cpp::LeakyReLU("relu_0",fc_0,mxnet::cpp::LeakyReLUActType::kLeaky); Symbol weight_1 = Symbol::Variable("weight_1");
Symbol biases_1 = Symbol::Variable("biases_1");
Symbol fc_1 = mxnet::cpp::FullyConnected("fc_1",output_0,weight_1,biases_1,);
Symbol output_1 = mxnet::cpp::LeakyReLU("relu_1",fc_1,mxnet::cpp::LeakyReLUActType::kLeaky);
Symbol pred = mxnet::cpp::SoftmaxOutput("softmax",output_1,y); //目标函数，loss函数
mxnet::cpp::Context ctx = mxnet::cpp::Context::cpu( ); //定义输入数据
std::shared_ptr< mx_float > aptr_x(new mx_float[*] , [](mx_float* aptr_x){ delete [] aptr_x ;});
std::shared_ptr< mx_float > aptr_y(new mx_float[] , [](mx_float * aptr_y){ delete [] aptr_y ;}); //初始化数据
for(int i= ; i< ; i++){
for(int j=;j< ; j++){
//定义x
aptr_x.get()[i*+j]= i % +0.1f;
} //定义y
aptr_y.get()[i]= i % ;
}
std::map<std::string, mxnet::cpp::NDArray> args_map;
//导入数据
NDArray arr_x(mxnet::cpp::Shape(,),ctx, false);
NDArray arr_y(mxnet::cpp::Shape( ),ctx,false);
//将数据转换到NDArray中
arr_x.SyncCopyFromCPU(aptr_x.get(),*);
arr_x.WaitToRead(); arr_y.SyncCopyFromCPU(aptr_y.get(),);
arr_y.WaitToRead(); args_map["X"]=arr_x.Slice(,).Copy(ctx) ;
args_map["label"]=arr_y.Slice(,).Copy(ctx);
NDArray::WaitAll();
//绑定网络
mxnet::cpp::Executor *executor = pred.SimpleBind(ctx,args_map);
//选择优化器
mxnet::cpp::Optimizer *opt = mxnet::cpp::OptimizerRegistry::Find("sgd");
mx_float learning_rate = 0.0001; //学习率
mx_float weight_decay = 1e-; //权重
opt->SetParam("momentum", 0.9)
->SetParam("lr", learning_rate)
->SetParam("wd", weight_decay);
//定义各个层参数的数组
NDArray arr_w_0(mxnet::cpp::Shape(,),ctx, false);
NDArray arr_b_0(mxnet::cpp::Shape( ),ctx,false);
NDArray arr_w_1(mxnet::cpp::Shape( , ) , ctx , false);
NDArray arr_b_1(mxnet::cpp::Shape( ) , ctx , false); //初始化权重参数
arr_w_0 = 0.01f;
arr_b_1 = 0.01f;
arr_w_1 = 0.01f;
arr_b_1 = 0.01f; //初始化参数
executor->arg_dict()["weight_0"]=arr_w_0;
executor->arg_dict()["biases_0"]=arr_b_0;
executor->arg_dict()["weight_1"]=arr_w_1;
executor->arg_dict()["biases_1"]=arr_b_1; mxnet::cpp::NDArray::WaitAll();
//训练
std::cout<<" Training "<<std::endl; int max_iters = ; //最大迭代次数
//获取训练网络的参数列表
std::vector<std::string> args_name = pred.ListArguments();
for (int iter = ; iter < max_iters ; ++iter) {
executor->Forward(true);
executor->Backward(); if(iter % == ){
std::vector<NDArray> & out = executor->outputs;
std::shared_ptr<mx_float> tp_x( new mx_float[*] ,
[](mx_float * tp_x){ delete [] tp_x ;});
out[].SyncCopyToCPU(tp_x.get(),*);
NDArray::WaitAll();
std::cout<<"epoch "<<iter<<" "<<"Accuracy: "<< OutputAccuracy(tp_x.get() , aptr_y.get())<<std::endl;
}
//args_name.
for(size_t arg_ind=; arg_ind<args_name.size(); ++arg_ind){
//执行
if(args_name[arg_ind]=="X" || args_name[arg_ind]=="label")
continue; opt->Update(arg_ind,executor->arg_arrays[arg_ind],executor->grad_arrays[arg_ind]);
}
NDArray::WaitAll(); } }
};
} int main(int argc , char * argv[]){
mlp::MlpNet::net();
MXNotifyShutdown();
return EXIT_SUCCESS;
}
结果：
Training
epoch Accuracy: 0.09375
epoch Accuracy: 0.304688
epoch Accuracy: 0.195312
epoch Accuracy: 0.203125
epoch Accuracy: 0.304688
epoch Accuracy: 0.296875
epoch Accuracy: 0.304688
epoch Accuracy: 0.304688
epoch Accuracy: 0.398438
epoch Accuracy: 0.5
epoch Accuracy: 0.5
epoch Accuracy: 0.40625
epoch Accuracy: 0.5
epoch Accuracy: 0.398438
epoch Accuracy: 0.40625
epoch Accuracy: 0.703125
epoch Accuracy: 0.609375
epoch Accuracy: 0.507812
epoch Accuracy: 0.703125
epoch Accuracy: 0.703125
epoch Accuracy: 0.804688
epoch Accuracy: 0.703125
epoch Accuracy: 0.804688
epoch Accuracy: 0.804688
epoch Accuracy: 0.804688
epoch Accuracy: 0.90625
epoch Accuracy: 0.90625
epoch Accuracy: 0.90625
epoch Accuracy:
epoch Accuracy: