目前全部的訓練都是在PPO完成的

朋友很好奇為什麼沒有使用SAC，尤其在學術研究上，似乎有些問題SAC的效果遠勝PPO

因此特地以和道爾靜立3完全相同的程式，改以SAC訓練

//除了SAC部分，其餘和道爾靜立3完全相同

Doyle Stand v3 (SAC)

1.進入站立瞬間後，由於其實可能仍處於不穩定狀態，要再進入靜立狀態

2.進入站立瞬間後，可能面向並沒有瞄準目標，要轉向目標

1.任何弱點觸地皆失敗 (尾巴和劍並非弱點)

2.

if(weaknessOnGround){if(inferenceMode){brainMode = DoyleMode.GetUp;SetModel("DoyleGetUp", getUpBrain);behaviorParameters.BehaviorType = BehaviorType.InferenceOnly;}else{AddReward(-1f);judge.outLife++;judge.Reset();return;// brainMode = DoyleMode.GetUp;// SetModel("DoyleGetUp", getUpBrain);// behaviorParameters.BehaviorType = BehaviorType.InferenceOnly;}}else if(doyleRoot.localPosition.y < -10f){if(inferenceMode){brainMode = DoyleMode.GetUp;SetModel("DoyleGetUp", getUpBrain);behaviorParameters.BehaviorType = BehaviorType.InferenceOnly;}else{AddReward(-1f);judge.outY++;judge.Reset();return;// brainMode = DoyleMode.GetUp;// SetModel("DoyleGetUp", getUpBrain);// behaviorParameters.BehaviorType = BehaviorType.InferenceOnly;}}else{targetSmoothPosition = targetPositionBuffer.GetSmoothVal();headDir = targetSmoothPosition - stageBase.InverseTransformPoint(doyleHeadRb.position);rootDir = targetSmoothPosition - stageBase.InverseTransformPoint(doyleRootRb.position);flatTargetVelocity = rootDir;flatTargetVelocity.y = 0f;targetDistance = flatTargetVelocity.magnitude;Vector3 flatLeftDir = Vector3.Cross(flatTargetVelocity, Vector3.up);lookAngle = Mathf.InverseLerp(180f, 0f, Vector3.Angle(doyleHead.up, headDir));upAngle = Mathf.InverseLerp(180f, 0f, Vector3.Angle(doyleHead.right * -1f, Vector3.up));//LeanVector3 leanDir = rootAimRot * flatTargetVelocity;spineLookAngle = Mathf.InverseLerp(180f, 0f, Vector3.Angle(doyleSpine.forward, flatLeftDir));spineUpAngle = Mathf.InverseLerp(180f, 30f, Vector3.Angle(doyleSpine.right * -1f, leanDir));rootLookAngle = Mathf.InverseLerp(180f, 0f, Vector3.Angle(doyleRoot.right * -1f, flatLeftDir));rootUpAngle = Mathf.InverseLerp(180f, 20f, Vector3.Angle(doyleRoot.up, leanDir));// float velocityReward = Mathf.InverseLerp(0f, 10f, doyleRootRb.velocity.magnitude) * 0.5f + Mathf.InverseLerp(0f, 10f, doyleSpineRb.velocity.magnitude) * 0.3f + Mathf.InverseLerp(0f, 10f, doyleHeadRb.velocity.magnitude) * 0.2f;// float angularReward = Mathf.InverseLerp(0f, 6.28f, doyleRootRb.angularVelocity.magnitude) * 0.2f + Mathf.InverseLerp(0f, 6.28f, doyleSpineRb.angularVelocity.magnitude) * 0.3f + Mathf.InverseLerp(0f, 6.28f, doyleHeadRb.angularVelocity.magnitude) * 0.5f;float velocityReward = GetVelocityReward(8f);float angularReward = GetAngularVelocityReward(10f);float standReward = (doyleLeftFeetBody.isStand? 0.5f : 0f) + (doyleRightFeetBody.isStand? 0.5f : 0f);lastReward = (1f-velocityReward) * 0.015f + (1f-angularReward) * 0.015f+ (lookAngle + upAngle + spineLookAngle + spineUpAngle + rootLookAngle + rootUpAngle) * 0.008f + standReward * 0.01f+ (1f - exertionRatio) * 0.002f;totalReward += lastReward;AddReward( lastReward );if(Time.fixedTime - landingMoment > landingBufferTime){bool outVelocity = velocityReward > Mathf.Lerp(1f, 0.3f, (Time.fixedTime - landingMoment - landingBufferTime)/3f);bool outAngularVelocity = angularReward > Mathf.Lerp(1f, 0.5f, (Time.fixedTime - landingMoment - landingBufferTime)/3f);bool outSpeed = outVelocity || outAngularVelocity;float aimLimit = Mathf.Lerp(0f, 0.7f, (Time.fixedTime - landingMoment - landingBufferTime)/3f);float aimLimit2 = Mathf.Lerp(0f, 0.9f, (Time.fixedTime - landingMoment - landingBufferTime)/3f);bool outDirection = lookAngle < aimLimit2 || upAngle < aimLimit2 || spineLookAngle < aimLimit2 || rootLookAngle < aimLimit2;bool outMotion = spineUpAngle < aimLimit || rootUpAngle < aimLimit;if( outSpeed || outDirection || outMotion){AddReward(-1f);if(outSpeed){judge.outSpeed++;}if(outDirection){judge.outDirection++;}if(outMotion){judge.outMotion++;}judge.Reset();return;}}if(lookAngle > 0.9f && upAngle > 0.9f  && spineLookAngle > 0.9f  && rootLookAngle > 0.9f && velocityReward < 0.3f && angularReward < 0.5f && standReward > 0.9f){Debug.Log("Stand");totalReward += 0.01f;AddReward( 0.01f );}}

3.

for(int i=0; i<doyleBodies.Length; i++){if(doyleBodies[i].isGrounded){if(doyleBodies[i].isWeakness){weaknessOnGround = true;}else{//===Train Stand===if(doyleBodies[i].damageCoef > 0f){AddReward(-0.1f * doyleBodies[i].damageCoef);}}ConfirmLanding();// ConfirmArrived();}}