Merge pull request #134 from sdnfv/develop

This PR releases OpenNetVM v19.05

Author: koolzz

.clang-format

@@ -0,0 +1,95 @@
+---
+Language:        Cpp
+# BasedOnStyle:  Google
+AccessModifierOffset: -1
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlinesLeft: true
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: false
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: All 
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:   
+  AfterClass:      false
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     120
+CommentPragmas:  '^ IWYU pragma:'
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DerivePointerAlignment: true
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
+IncludeCategories: 
+  - Regex:           '^<.*\.h>'
+    Priority:        1
+  - Regex:           '^<.*'
+    Priority:        2
+  - Regex:           '.*'
+    Priority:        3
+IncludeIsMainRegex: '([-_](test|unittest))?$'
+IndentCaseLabels: true
+IndentWidth:     8
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: false
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Left
+ReflowComments:  true
+SortIncludes:    true
+SpaceAfterCStyleCast: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Auto
+TabWidth:        8
+UseTab:          Never
+...
+

CPPLINT.cfg

@@ -1 +1 @@
-linelength=100
+linelength=120

ci/.gitignore

@@ -1,4 +1,9 @@
+access_log
 config
+webhook-config.json
+encrypted_secret.bin
+private.pem
+public.pem
 githubcreds
 mykeyfile
 repository

ci/README.md

@@ -3,12 +3,12 @@
 ### Setting up CI
 Run a Flask server that listens for new events from github, will get triggered when a new PR is created or when keyword `@onvm` is mentioned.  
 ```sh
-python3 webhook-receiver.py 0.0.0.0 8080 @onvm
+python3 webhook-receiver.py 0.0.0.0 8080 @onvm webhook-config.json
 ```
 
 To run CI tests manually, requires a config file, the github PR ID, request message and a response message.  
 ```sh
-./manager.sh <config file> <pr ID> <request msg>
+./manager.sh <config file> <pr ID> <repo name> <request msg>
 ```  
 
 ### Usage
@@ -34,15 +34,23 @@ The CI process can be broken into multiple steps:
     WORKER_LIST=("WORKER_1_IP WORKER_1_KEY", "WORKER_2_IP WORKER_2_KEY", ...)
     GITHUB_CREDS=path_to_creditential_file
     REPO_OWNER="OWNER_STRING"
-    REPO_NAME="NAME_STRING"
     ```
 
     Config file example:
     ```
     WORKER_LIST=("nimbnode42 nn42_key")
     GITHUB_CREDS=githubcreds
     REPO_OWNER="sdnfv"
-    REPO_NAME="openNetVM-dev"
+    ```
+
+    Webhook json config example
+    ```
+    {
+        "secret-file": "very_special_encrypted_secret_file.bin",
+        "private-key-file": "private_key.pem", 
+        "log-successful-attempts": true,
+        "authorized-users": ["puffin", "penguin", "pcoach"]
+    }
     ```
 
     GITHUB_CREDS file example:

ci/ci_busy.sh

@@ -29,43 +29,45 @@ fi
 
 if [[ -z "$3" ]]
 then
-    echo "ERROR: Missing third argument, Request body!"
+    echo "ERROR: Missing third argument, Repo name!"
     exit 1
 else
-    REQUEST=$3
+    REPO_NAME=$3
 fi
 
 if [[ -z "$4" ]]
 then
-    echo "ERROR: Missing fourth argument, POST_MSG!"
+    echo "ERROR: Missing fourth argument, Request body!"
     exit 1
 else
-    POST_MSG=$4
+    REQUEST=$4
+fi
+
+if [[ -z "$5" ]]
+then
+    echo "ERROR: Missing fifth argument, POST_MSG!"
+    exit 1
+else
+    POST_MSG=$5
 fi
 
 . $1 # source the variables from config file
 
 print_header "Checking Required Variables"
 
 
- if [[ -z "$GITHUB_CREDS" ]]
+if [[ -z "$GITHUB_CREDS" ]]
 then
     echo "ERROR: GITHUB_CREDS not provided"
     exit 1
 fi
 
- if [[ -z "$REPO_OWNER" ]]
+if [[ -z "$REPO_OWNER" ]]
 then
     echo "ERROR: REPO_OWNER not provided"
     exit 1
 fi
 
- if [[ -z "$REPO_NAME" ]]
-then
-    echo "ERROR: REPO_NAME not provided"
-    exit 1
-fi
-
 print_header "Posting Message in Comments on GitHub"
 python3 post-msg.py $GITHUB_CREDS "{\"id\": $PR_ID,\"request\":\"$REQUEST\"}" $REPO_OWNER $REPO_NAME "$POST_MSG"
 check_exit_code "ERROR: Failed to post results to GitHub"

ci/clone-and-checkout-pr.py

@@ -41,10 +41,17 @@
 cmd = "git clone " + str(repo.clone_url) + " repository"
 
 child = pexpect.spawn(cmd)
-child.expect("Username.*")
-child.sendline(username + "\n")
-child.expect("Password.*")
-child.sendline(password + "\n")
+
+if '-dev' in REPO_NAME:
+    child.expect("Username.*")
+    child.sendline(username + "\n")
+    child.expect("Password.*")
+    child.sendline(password + "\n")
+
 child.interact()
 
+# Get the latest branches for upstream (used for proper linter check)
+pexpect.run("git remote add upstream https://github.com/sdnfv/openNetVM.git", cwd="./repository")
+pexpect.run("git fetch upstream", cwd="./repository")
+
 print(pexpect.run("git checkout " + branch_name, cwd="./repository"))

ci/helper-functions.sh

@@ -87,7 +87,7 @@ obtain_core_config() {
 
 # checks if a command has failed (exited with code != 0)
 # if it does, print the error message, exit the build, and post to github
-check_exit_code(){
+check_exit_code() {
     if [ $? -ne 0 ]
     then
         echo $1
@@ -99,7 +99,46 @@ check_exit_code(){
 
 # runs the linter
 run_linter() {
-    for fn in $(git ls-files *.c | grep -v "cJSON" | grep -v "ndpi"); do
-        python style/gwclint.py --verbose=5 $fn 2>&1 | grep -v "Done" | grep -v "Total errors found: 0" >> $1
+    for fn in $(git diff --name-only upstream/develop...HEAD -- '*.c' '*.cpp' '*.h' | grep -v "cJSON" | grep -v "ndpi"); do
+        git_file_diff=$(git diff -U0 upstream/develop...HEAD -- $fn)
+        file_modifications=()
+        while read -r diff_line; do
+            # line number is denoted by @@ -lines_deleted +lines_added @@
+            # For example: @@ -1257,3 +1175,3 @@ onvm_nflib_cleanup(struct onvm_nf_context *nf_context) {
+            # We're interested in 1175, 3 as that shows the lines in the final file
+            if [[ $diff_line == @@* ]]; then
+                diff_line=$(cut -d ' ' -f 3 <<< "$diff_line")
+                diff_line=${diff_line:1}
+                line_start=$(cut -d ',' -f 1 <<< "$diff_line")
+                line_end=$(cut -d ',' -f 2 <<< "$diff_line")
+                # Expand the comment on one line diff vs multiline->therefore addition
+                [[ ! $line_end == $line_start ]] && line_end=$(($line_start + $line_end))
+                file_modifications[$line_start]=$line_end
+                # create mapping of start of chunk -> end line
+            fi
+        done <<< "$git_file_diff"
+
+        # loop through the lines changed in the current file
+        file_lint=$(python $SCRIPT_LOC/../style/gwclint.py --verbose=4 $fn 2>&1 | grep -v "Done" | grep -v "Total errors found:")
+        errors_found=0
+        while read -r line; do
+            error_line=$(cut -d ':' -f 2 <<< "$line")
+            # lint error is denoted by *:line:*
+            for start in "${!file_modifications[@]}"; do
+                # loop through the git diff line numbers
+                end=${file_modifications[$start]}
+                if [[ $error_line -ge $start && $error_line -le $end ]]; then
+                    # git diff and linter output are in range, add to lint file
+                    errors_found=$(($errors_found+1))
+                    echo "$line" >> $1
+                    break
+                fi
+            done
+        done <<< "$file_lint"
+        # loop through the linter output
+        if [[ $errors_found -gt 0 ]]; then
+            # only output to file if we had errors
+            echo "Total errors found: $errors_found" >> $1
+        fi
     done
 }

ci/manager.sh

@@ -31,10 +31,18 @@ fi
 
 if [[ -z "$3" ]]
 then
-    echo "ERROR: Missing third argument, Request body!"
+    echo "ERROR: Missing third argument, Repo name!"
     exit 1
 else
-    REQUEST=$3
+    REPO_NAME=$3
+fi
+
+if [[ -z "$4" ]]
+then
+    echo "ERROR: Missing fourth argument, Request body!"
+    exit 1
+else
+    REQUEST=$4
 fi
 
 . $1 # source the variables from config file
@@ -60,12 +68,6 @@ then
     exit 1
 fi
 
-if [[ -z "$REPO_NAME" ]]
-then
-    echo "ERROR: REPO_NAME not provided"
-    exit 1
-fi
-
 print_header "Cleaning up Old Results"
 
 sudo rm -f *.txt
@@ -188,13 +190,13 @@ do
     worker_key_file="${tuple_arr[1]}"
     scp -i $worker_key_file -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null $worker_ip:stats ./$worker_ip.stats
     check_exit_code "ERROR: Failed to fetch results from $worker_ip"
-    echo "[Results from $worker_ip]" >> results_summary.stats
-    python3 speed-tester-analysis.py ./$worker_ip.stats >> results_summary.stats
+    # TODO: this will overwrite results if we have more  than 1 worker, investigate this case
+    python3 speed-tester-analysis.py ./$worker_ip.stats $worker_ip results_summary.stats
     check_exit_code "ERROR: Failed to analyze results from $worker_ip"
 done
 
 print_header "Posting Results in Comment on GitHub"
-python3 post-msg.py $GITHUB_CREDS "{\"id\": $PR_ID,\"request\":\"$REQUEST\",\"linter\": 1,\"results\": 1}" $REPO_OWNER $REPO_NAME "Run successful see results:"
+python3 post-msg.py $GITHUB_CREDS "{\"id\": $PR_ID,\"request\":\"$REQUEST\",\"linter\": 1,\"results\": 1,\"review\": 1}" $REPO_OWNER $REPO_NAME "Run successful see results:"
 check_exit_code "ERROR: Failed to post results to GitHub"
 
 print_header "Finished Executing"