fix(gamecli): wrap angle in reward to prevent unbounded spin penalty
This commit is contained in:
@@ -53,4 +53,19 @@ public class RewardTests
|
|||||||
Assert.True(rMain < rNoop, "firing main should be worse than noop");
|
Assert.True(rMain < rNoop, "firing main should be worse than noop");
|
||||||
Assert.True(rLeft < rNoop, "firing left should be worse than noop");
|
Assert.True(rLeft < rNoop, "firing left should be worse than noop");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void AngleWrap_FullRevolutionsProduceSameReward()
|
||||||
|
{
|
||||||
|
// Physically identical poses (angle differs by 2π) must produce the same reward.
|
||||||
|
// Otherwise a spinning ship accumulates unbounded angle penalty.
|
||||||
|
var upright = new ShipState(0.5f, 0.5f, 0f, 0f, 0f, 0f);
|
||||||
|
var oneSpin = new ShipState(0.5f, 0.5f, 0f, 0f, 2f * MathF.PI, 0f);
|
||||||
|
var fiveSpins = new ShipState(0.5f, 0.5f, 0f, 0f, 10f * MathF.PI, 0f);
|
||||||
|
float r0 = Reward.Compute(upright, target: (0.5f, 0.5f), action: 0);
|
||||||
|
float r1 = Reward.Compute(oneSpin, target: (0.5f, 0.5f), action: 0);
|
||||||
|
float r5 = Reward.Compute(fiveSpins, target: (0.5f, 0.5f), action: 0);
|
||||||
|
Assert.Equal(r0, r1, precision: 4);
|
||||||
|
Assert.Equal(r0, r5, precision: 4);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,8 +3,10 @@ namespace GameCli;
|
|||||||
/// <summary>
|
/// <summary>
|
||||||
/// Reward = -distance_to_target
|
/// Reward = -distance_to_target
|
||||||
/// - λ_v · speed
|
/// - λ_v · speed
|
||||||
/// - λ_θ · |angle|
|
/// - λ_θ · |wrap(angle)|
|
||||||
/// - λ_fuel · engine_on
|
/// - λ_fuel · engine_on
|
||||||
|
/// The angle is wrapped to [-π, π] so a ship that has completed N revolutions
|
||||||
|
/// gets the same upright-penalty as a physically identical pose that hasn't.
|
||||||
/// Coefficients are public so training scripts can log/inspect them.
|
/// Coefficients are public so training scripts can log/inspect them.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public static class Reward
|
public static class Reward
|
||||||
@@ -19,11 +21,12 @@ public static class Reward
|
|||||||
float dy = target.Y - s.Y;
|
float dy = target.Y - s.Y;
|
||||||
float distance = MathF.Sqrt(dx * dx + dy * dy);
|
float distance = MathF.Sqrt(dx * dx + dy * dy);
|
||||||
float speed = MathF.Sqrt(s.VX * s.VX + s.VY * s.VY);
|
float speed = MathF.Sqrt(s.VX * s.VX + s.VY * s.VY);
|
||||||
|
float wrapped = MathF.Atan2(MathF.Sin(s.Angle), MathF.Cos(s.Angle));
|
||||||
float engineOn = action == Physics.ActionNoop ? 0f : 1f;
|
float engineOn = action == Physics.ActionNoop ? 0f : 1f;
|
||||||
|
|
||||||
return -distance
|
return -distance
|
||||||
- LambdaVelocity * speed
|
- LambdaVelocity * speed
|
||||||
- LambdaAngle * MathF.Abs(s.Angle)
|
- LambdaAngle * MathF.Abs(wrapped)
|
||||||
- LambdaFuel * engineOn;
|
- LambdaFuel * engineOn;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user