Merge pull request #15 from commaai/robust-model

Add more robust model

Author: nuwandavek

.github/workflows/main.yml

@@ -31,12 +31,12 @@ jobs:
       
       - name: Run Simple controller rollout
         run: |
-          python tinyphysics.py --model_path ./models/tinyphysics.onnx --data_path ./data/00000.csv --controller simple
+          python tinyphysics.py --model_path ./models/tinyphysics.onnx --data_path ./data/00000.csv --controller pid
       
       - name: Run batch rollouts
         run: |
-          python tinyphysics.py --model_path ./models/tinyphysics.onnx --data_path ./data --num_segs 20 --controller simple
+          python tinyphysics.py --model_path ./models/tinyphysics.onnx --data_path ./data --num_segs 20 --controller pid
 
       - name: Run report
         run: |
-          python eval.py --model_path ./models/tinyphysics.onnx --data_path ./data --num_segs 50 --test_controller open --baseline_controller simple
+          python eval.py --model_path ./models/tinyphysics.onnx --data_path ./data --num_segs 50 --test_controller zero --baseline_controller pid

README.md

@@ -16,16 +16,16 @@ bash ./download_dataset.sh
 pip install -r requirements.txt
 
 # test this works
-python tinyphysics.py --model_path ./models/tinyphysics.onnx --data_path ./data/00000.csv --debug --controller simple
+python tinyphysics.py --model_path ./models/tinyphysics.onnx --data_path ./data/00000.csv --debug --controller pid 
 ```
 
 There are some other scripts to help you get aggregate metrics: 
 ```
 # batch Metrics of a controller on lots of routes
-python tinyphysics.py --model_path ./models/tinyphysics.onnx --data_path ./data --num_segs 100 --controller simple
+python tinyphysics.py --model_path ./models/tinyphysics.onnx --data_path ./data --num_segs 100 --controller pid
 
 # generate a report comparing two controllers
-python eval.py --model_path ./models/tinyphysics.onnx --data_path ./data --num_segs 100 --test_controller simple --baseline_controller open
+python eval.py --model_path ./models/tinyphysics.onnx --data_path ./data --num_segs 100 --test_controller pid --baseline_controller zero
 
 ```
 You can also use the notebook at [`experiment.ipynb`](https://github.com/commaai/controls_challenge/blob/master/experiment.ipynb) for exploration.
@@ -44,13 +44,13 @@ Each rollout will result in 2 costs:
 
 - `jerk_cost`: $\dfrac{\Sigma((actual\\_lat\\_accel\_t - actual\\_lat\\_accel\_{t-1}) / \Delta t)^2}{steps - 1} * 100$
 
-It is important to minimize both costs. `total_cost`: $(lataccel\\_cost *5) + jerk\\_cost$
+It is important to minimize both costs. `total_cost`: $(lataccel\\_cost * 50) + jerk\\_cost$
 
 ## Submission
 Run the following command, and submit `report.html` and your code to [this form](https://forms.gle/US88Hg7UR6bBuW3BA).
 
 ```
-python eval.py --model_path ./models/tinyphysics.onnx --data_path ./data --num_segs 5000 --test_controller <insert your controller name> --baseline_controller simple
+python eval.py --model_path ./models/tinyphysics.onnx --data_path ./data --num_segs 5000 --test_controller <insert your controller name> --baseline_controller pid
 ```
 
 ## Work at comma

controllers/pid.py

@@ -0,0 +1,20 @@
+from . import BaseController
+import numpy as np
+
+class Controller(BaseController):
+  """
+  A simple PID controller
+  """
+  def __init__(self,):
+    self.p = 0.3
+    self.i = 0.05
+    self.d = -0.1
+    self.error_integral = 0
+    self.prev_error = 0
+
+  def update(self, target_lataccel, current_lataccel, state, future_plan):
+      error = (target_lataccel - current_lataccel)
+      self.error_integral += error
+      error_diff = error - self.prev_error
+      self.prev_error = error
+      return self.p * error + self.i * self.error_integral + self.d * error_diff

controllers/simple.py

@@ -3,7 +3,7 @@
 
 class Controller(BaseController):
   """
-  An open-loop controller
+  A controller that always outputs zero
   """
   def update(self, target_lataccel, current_lataccel, state, future_plan):
-    return target_lataccel
+    return 0.0

controllers/open.py renamed to controllers/zero.py

@@ -70,8 +70,8 @@ def create_report(test, baseline, sample_rollouts, costs):
   parser.add_argument("--model_path", type=str, required=True)
   parser.add_argument("--data_path", type=str, required=True)
   parser.add_argument("--num_segs", type=int, default=100)
-  parser.add_argument("--test_controller", default='simple', choices=available_controllers)
-  parser.add_argument("--baseline_controller", default='simple', choices=available_controllers)
+  parser.add_argument("--test_controller", default='pid', choices=available_controllers)
+  parser.add_argument("--baseline_controller", default='pid', choices=available_controllers)
   args = parser.parse_args()
 
   data_path = Path(args.data_path)
@@ -99,7 +99,7 @@ def create_report(test, baseline, sample_rollouts, costs):
   for controller_cat, controller_type in [('baseline', args.baseline_controller), ('test', args.test_controller)]:
     print(f"Running batch rollouts => {controller_cat} controller: {controller_type}")
     rollout_partial = partial(run_rollout, controller_type=controller_type, model_path=args.model_path, debug=False)
-    results = process_map(rollout_partial, files[SAMPLE_ROLLOUTS:], max_workers=16)
+    results = process_map(rollout_partial, files[SAMPLE_ROLLOUTS:], max_workers=16, chunksize=10)
     costs += [{'controller': controller_cat, **result[0]} for result in results]
 
   create_report(args.test_controller, args.baseline_controller, sample_rollouts, costs)

eval.py

@@ -7,7 +7,7 @@
    "outputs": [],
    "source": [
     "from tinyphysics import TinyPhysicsModel, TinyPhysicsSimulator, CONTROL_START_IDX\n",
-    "from controllers import simple\n",
+    "from controllers import pid\n",
     "from matplotlib import pyplot as plt\n",
     "import seaborn as sns\n",
     "\n",
@@ -38,7 +38,7 @@
    "outputs": [],
    "source": [
     "model = TinyPhysicsModel(\"./models/tinyphysics.onnx\", debug=True)\n",
-    "controller = simple.Controller()"
+    "controller = pid.Controller()"
    ]
   },
   {

experiment.ipynb

@@ -22,14 +22,14 @@
 ACC_G = 9.81
 FPS = 10
 CONTROL_START_IDX = 100
-COST_END_IDX = 550
+COST_END_IDX = 500
 CONTEXT_LENGTH = 20
 VOCAB_SIZE = 1024
 LATACCEL_RANGE = [-5, 5]
 STEER_RANGE = [-2, 2]
 MAX_ACC_DELTA = 0.5
 DEL_T = 0.1
-LAT_ACCEL_COST_MULTIPLIER = 5.0
+LAT_ACCEL_COST_MULTIPLIER = 50.0
 
 FUTURE_PLAN_STEPS = FPS * 5  # 5 secs
 
@@ -86,7 +86,7 @@ def get_current_lataccel(self, sim_states: List[State], actions: List[float], pa
       'states': np.expand_dims(states, axis=0).astype(np.float32),
       'tokens': np.expand_dims(tokenized_actions, axis=0).astype(np.int64)
     }
-    return self.tokenizer.decode(self.predict(input_data, temperature=1.))
+    return self.tokenizer.decode(self.predict(input_data, temperature=0.8))
 
 
 class TinyPhysicsSimulator:
@@ -148,10 +148,10 @@ def get_state_target_futureplan(self, step_idx: int) -> Tuple[State, float]:
       State(roll_lataccel=state['roll_lataccel'], v_ego=state['v_ego'], a_ego=state['a_ego']),
       state['target_lataccel'],
       FuturePlan(
-        lataccel=self.data['target_lataccel'].values[step_idx + 1 :step_idx + FUTURE_PLAN_STEPS].tolist(),
-        roll_lataccel=self.data['roll_lataccel'].values[step_idx + 1 :step_idx + FUTURE_PLAN_STEPS].tolist(),
-        v_ego=self.data['v_ego'].values[step_idx + 1 :step_idx + FUTURE_PLAN_STEPS].tolist(),
-        a_ego=self.data['a_ego'].values[step_idx + 1 :step_idx + FUTURE_PLAN_STEPS].tolist()
+        lataccel=self.data['target_lataccel'].values[step_idx + 1:step_idx + FUTURE_PLAN_STEPS].tolist(),
+        roll_lataccel=self.data['roll_lataccel'].values[step_idx + 1:step_idx + FUTURE_PLAN_STEPS].tolist(),
+        v_ego=self.data['v_ego'].values[step_idx + 1:step_idx + FUTURE_PLAN_STEPS].tolist(),
+        a_ego=self.data['a_ego'].values[step_idx + 1:step_idx + FUTURE_PLAN_STEPS].tolist()
       )
     )
 
@@ -222,7 +222,7 @@ def run_rollout(data_path, controller_type, model_path, debug=False):
   parser.add_argument("--data_path", type=str, required=True)
   parser.add_argument("--num_segs", type=int, default=100)
   parser.add_argument("--debug", action='store_true')
-  parser.add_argument("--controller", default='simple', choices=available_controllers)
+  parser.add_argument("--controller", default='pid', choices=available_controllers)
   args = parser.parse_args()
 
   data_path = Path(args.data_path)
@@ -232,7 +232,7 @@ def run_rollout(data_path, controller_type, model_path, debug=False):
   elif data_path.is_dir():
     run_rollout_partial = partial(run_rollout, controller_type=args.controller, model_path=args.model_path, debug=False)
     files = sorted(data_path.iterdir())[:args.num_segs]
-    results = process_map(run_rollout_partial, files, max_workers=16)
+    results = process_map(run_rollout_partial, files, max_workers=16, chunksize=10)
     costs = [result[0] for result in results]
     costs_df = pd.DataFrame(costs)
     print(f"\nAverage lataccel_cost: {np.mean(costs_df['lataccel_cost']):>6.4}, average jerk_cost: {np.mean(costs_df['jerk_cost']):>6.4}, average total_cost: {np.mean(costs_df['total_cost']):>6.4}")