[improve test]reboot SONiC if sanity check fail before run tset case (#443)

* [improve test]reboot sonic and trying to restore good state if sanity check fail before calling test

* revise message more descriptive and accurate follow feedbacks

Author: maggiemsft

ansible/roles/test/tasks/base_sanity.yml

@@ -1,20 +1,42 @@
    - name: Get process information in syncd docker
      shell: docker exec -i syncd ps aux | grep /usr/bin/syncd
      register: ps_out
+     ignore_errors: yes
 
    - debug: var=ps_out.stdout_lines
 
-   - name: Verify that syncd process is running
-     assert: { that: "{{ ps_out.stdout_lines | length }} > 0"}
-
-   - name: Get oragent process information
+   - name: Get orchagent process information
      shell: pgrep orchagent -a
      register: orch_out
+     ignore_errors: yes
 
    - debug: var=orch_out.stdout_lines
 
-   - name: Verify that orch process is running
-     assert: { that: "{{ orch_out.stdout_lines | length }} > 0"}
+   - block:
+     - name: reboot
+       include: common_tasks/reboot_sonic.yml
+
+     - name: Get process information in syncd docker
+       shell: docker exec -i syncd ps aux | grep /usr/bin/syncd
+       register: psnew_out
+
+     - debug: var=psnew_out.stdout_lines
+
+     - name: Verify that syncd process is running
+       assert: { that: "{{ psnew_out.stdout_lines | length }} > 0"}
+
+     - name: Get orchagent process information
+       shell: pgrep orchagent -a
+       register: orchnew_out
+
+     - debug: var=orchnew_out.stdout_lines
+
+     - name: Verify that orchagent process is running
+       assert: { that: "{{ orchnew_out.stdout_lines | length }} > 0"}
+
+     when:
+       - ({{ ps_out.stdout_lines | length }} <= 0) or ({{ orch_out.stdout_lines | length }} <= 0)
+       - recover is defined
 
    - name: Get syslog error information
      shell: cat /var/log/syslog |tail -n 5000 |grep -i error
@@ -23,5 +45,3 @@
      failed_when: false
 
    - debug: var=syslog_out.stdout_lines
-
-

ansible/roles/test/tasks/common_tasks/reboot_sonic.yml

@@ -0,0 +1,23 @@
+- name: reboot
+  become: true
+  shell: shutdown -r now "Warning! System is being rebooted remotely by reboot_sonic.yml"
+  async: 1
+  poll: 0
+  ignore_errors: true
+
+- name: pause for 1 minute before check
+  pause: minutes=1
+
+- name: Wait for switch to come back
+  local_action:
+    wait_for host={{ ansible_host }}
+    port=22
+    state=started
+    delay=10
+    timeout=180
+    search_regex="OpenSSH_[\w\.]+ Debian"
+  become: false
+  changed_when: false
+
+- name: wait for 2 minute for prcesses and interfaces to be stable
+  pause: seconds=120

ansible/roles/test/tasks/decap.yml

@@ -2,6 +2,21 @@
 # Run Decap test
 #-----------------------------------------
 
+- block:
+  - name: Set dscp_mode for decap test for broadcom
+    set_fact:
+       dscp_mode: pipe
+    when:
+       - sonic_hwsku in broadcom_hwskus
+       - dscp_mode is not defined
+
+  - name: Set dscp_mode var for decap test for mellanox
+    set_fact:
+       dscp_mode: uniform
+    when:
+       - sonic_hwsku in mellanox_hwskus
+       - dscp_mode is not defined
+
 - fail: msg="information about testbed missing."
   when: (testbed_type is not defined) or
         (dscp_mode is not defined) 

ansible/roles/test/tasks/interface.yml

@@ -1,7 +1,32 @@
 - name: Get interface facts
   interface_facts: up_ports={{minigraph_ports}}
 
-- debug: msg="Found link down ports {{ansible_interface_link_down_ports}} "
+- block:
+  - debug: msg="Found link down ports {{ansible_interface_link_down_ports}}, reload SONiC and reenable down ports"
+
+  - name: reboot
+    include: common_tasks/reboot_sonic.yml
+
+  - name: figure out fanout switch port in case it was down
+    conn_graph_facts: host={{ inventory_hostname }}
+    connection: local
+
+  - set_fact: neighbors="{{device_conn}}"
+
+  - include: resume_fanout_ports.yml
+    with_items: ansible_interface_link_down_ports
+
+  - name: pause and wait interface to be up
+    pause: seconds=30
+
+  - name: Get interface facts
+    interface_facts: up_ports={{minigraph_ports}}
+
+  when:
+    - ansible_interface_link_down_ports | length > 0
+    - recover is defined
+
+- debug: msg="Found link down ports {{ansible_interface_link_down_ports}}"
   when: ansible_interface_link_down_ports | length > 0
 
 - name: Verify interfaces are up correctly

ansible/roles/test/tasks/reboot.yml

@@ -1,26 +1,5 @@
 - name: reboot 
-  become: true
-  shell: sleep 2 && shutdown -r now "Reboot test."
-  async: 1
-  poll: 0
-  ignore_errors: true
-
-- name: pause for 1 minute before check
-  pause: minutes=1
-
-- name: Wait for switch to come back
-  local_action:
-    wait_for host={{ ansible_host }}
-    port=22
-    state=started
-    delay=10
-    timeout=180
-    search_regex="OpenSSH_[\w\.]+ Debian"
-  become: false
-  changed_when: false
-
-- name: wait again, processes and interfaces are not availabe right away
-  pause: seconds=120
+  include: common_tasks/reboot_sonic.yml 
 
 - name: sanity check to pass
   include: base_sanity.yml

ansible/roles/test/tasks/resume_fanout_ports.yml

@@ -0,0 +1,27 @@
+#This playbook is trying to bring up one fanout switch port
+- block:
+    - set_fact:
+        interface: "{{item}}"
+
+    - debug: msg={{interface}}
+
+    - set_fact:
+        peer_device: "{{neighbors[interface]['peerdevice']}}"
+        neighbor_interface: "{{neighbors[interface]['peerport']}}"
+
+    - conn_graph_facts: host={{ peer_device }}
+      connection: local
+
+    - set_fact:
+        peer_host: "{{device_info['mgmtip']}}"
+        peer_hwsku: "{{device_info['HwSku']}}"
+
+    - set_fact:
+        intfs_to_exclude: "{{interface}}"
+
+    - name: bring up neighbor interface {{neighbor_interface}} on {{peer_host}}
+      action: apswitch template=neighbor_interface_no_shut_single.j2
+      args:
+        host: "{{peer_host}}"
+        login: "{{switch_login[hwsku_map[peer_hwsku]]}}"
+      connection: switch

ansible/roles/test/tasks/test_sonic_by_testname.yml

@@ -1,41 +1,33 @@
+### Playbook that call individual testcase by name defined in roles/test/vars/testcases.yml
+#
 - debug: msg="!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
 - debug: msg="!!!!!!!!!!!!!!!!!!!! start to run test {{ testcase_name }} !!!!!!!!!!!!!!!!!!!!"
 - debug: msg="!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
 
 - name: do basic sanity check before each test
   include: base_sanity.yml
+  vars: 
+     recover: true
 
 - name: validate all interfaces is up
   include: interface.yml
+  vars:
+     recover: true
 
-########### if your test playbook requires more extra_vars than ptf_host and testbed_type, you may specify them here
-########### ptf_host and testbed_type are handled by default
-########### configure extra vars if your testcases need more vars
-########### or when you call the playbook, you have to specify your extra_vars
-- block:
-    - name: Set dscp_mode for decap test for broadcom
-      set_fact:
-          dscp_mode: pipe
-      when:
-        - sonic_hwsku in broadcom_hwskus
-        - dscp_mode is not defined
-
-    - name: Set dscp_mode var for decap test for mellanox
-      set_fact:
-          dscp_mode: uniform
-      when:
-        - sonic_hwsku in mellanox_hwskus
-        - dscp_mode is not defined
-
-- debug: var=testcases[testcase_name]['execvars']
-  when: testcases[testcase_name]['execvars'] is defined
+### by default, when calling a test case name, we pass 'testbed_type', 'ptf_host, 'dut_name(ansible_hoatname)' down to test playbook.
+### if your test playbook requires more extra vars then default, please make sure you handled them correctly within test playbook.
+- debug: var=testcases[testcase_name]['required_vars']
+  when: testcases[testcase_name]['required_vars'] is defined
 
 - name: run test case {{ testcases[testcase_name]['filename'] }} file
   include: "{{ testcases[testcase_name]['filename'] }}"
 
 - name: do basic sanity check after each test
   include: base_sanity.yml
 
+- name: validate all interfaces are up after test
+  include: interface.yml
+
 - debug: msg="!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
 - debug: msg="!!!!!!!!!!!!!!!!!!!! end running test {{ testcase_name }} !!!!!!!!!!!!!!!!!!!!"
 - debug: msg="!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"