Skip to content

Commit 80b3fb4

Browse files
zelinhzelinh
authored
Fix issues for WhiteSource scan workflow (#1411)
* Fix issues for gradle dependencies scan and enable scan on OpenSearch repo Signed-off-by: Zelin Hao <[email protected]> * Modify based on reviews Signed-off-by: Zelin Hao <[email protected]> * Update the script comments and format Signed-off-by: Zelin Hao <[email protected]> * Remove unnecessary java check Signed-off-by: Zelin Hao <[email protected]> * Add absolute path for removal jar Signed-off-by: Zelin Hao <[email protected]> * Add the timeout options in jenkinsfile Signed-off-by: Zelin Hao <[email protected]>
1 parent 4d12b94 commit 80b3fb4

File tree

4 files changed

+61
-18
lines changed

4 files changed

+61
-18
lines changed

‎tools/vulnerability-scan/Jenkinsfile renamed to ‎jenkins/vulnerability-scan/whitesource-scan.jenkinsfile

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
pipeline {
22
agent none
3+
options {
4+
timeout(time: 3, unit: 'HOURS')
5+
}
36
triggers {
47
cron('H 3 * * *')
58
}
@@ -41,8 +44,13 @@ pipeline {
4144

4245
void scan() {
4346
git url: 'https://github.com/opensearch-project/opensearch-build.git', branch: 'main'
44-
sh '''
47+
sh """
48+
curl -SLO https://services.gradle.org/distributions/gradle-6.6.1-bin.zip
49+
unzip gradle-6.6.1-bin.zip
50+
GRADLE_HOME=`realpath gradle-6.6.1`
51+
export PATH=\${GRADLE_HOME}/bin:\$PATH
52+
gradle --version
4553
cd ./tools/vulnerability-scan/
4654
./wss-scan.sh
47-
'''
55+
"""
4856
}

‎tools/vulnerability-scan/wss-scan.config

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
baseDirPath=$(pwd)
22
gitBasePath=https://github.com/opensearch-project/
3-
gitRepos=OpenSearch-Dashboards,alerting,alerting-dashboards-plugin,anomaly-detection,anomaly-detection-dashboards-plugin,asynchronous-search,common-utils,cross-cluster-replication,dashboards-reports,dashboards-visualizations,data-prepper,index-management,index-management-dashboards-plugin,job-scheduler,k-NN,opensearch-java,opensearch-js,opensearch-py,opensearch-dsl-py,performance-analyzer,perftop,security,security-dashboards-plugin,sql,observability
3+
gitRepos=OpenSearch,OpenSearch-Dashboards,alerting,alerting-dashboards-plugin,anomaly-detection,anomaly-detection-dashboards-plugin,asynchronous-search,common-utils,cross-cluster-replication,dashboards-reports,dashboards-visualizations,data-prepper,index-management,index-management-dashboards-plugin,job-scheduler,k-NN,opensearch-java,opensearch-js,opensearch-py,opensearch-dsl-py,performance-analyzer,perftop,security,security-dashboards-plugin,sql,observability

‎tools/vulnerability-scan/wss-scan.sh

Lines changed: 48 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,17 +21,19 @@
2121

2222
set -e
2323

24-
java -version
25-
if [ "$?" != 0 ]
26-
then
27-
echo "Java has not been setup"
28-
exit 1
29-
fi
24+
# Generate temporary `settings.gradle` file based on the name in `build.gradle`
25+
function generate_settings_gradle() {
26+
settings_gradle_content=`cat build.gradle | grep name | grep -i $repo | head -n 1 | sed 's/name/rootProject.name =/g' | awk '{$1=$1};1'`
27+
echo $settings_gradle_content > settings.gradle
28+
cat settings.gradle
29+
}
30+
31+
java -version; gradle -v; mvn -v; node -v; npm -v; yarn -v
3032

3133
if [ ! -f "wss-unified-agent.jar" ]
3234
then
3335
# Download the WhiteSource Agent
34-
curl https://unified-agent.s3.amazonaws.com/wss-unified-agent-21.11.1.jar --output wss-unified-agent.jar
36+
curl https://unified-agent.s3.amazonaws.com/wss-unified-agent-21.11.2.1.jar --output wss-unified-agent.jar
3537
fi
3638

3739
# scan the config file for the user configurations
@@ -47,33 +49,67 @@ basepath=$baseDirPath"/repos"
4749

4850
echo "Cleaning up scan directories if already present"
4951
rm -rf $basepath
52+
echo "Cleaning up temp file that will affect scan"
53+
rm -rf /tmp/ws*
5054

5155
mkdir -p $basepath
5256

5357
# clone the desired Repos for scanning
5458
for repo in $gitRepos
5559
do
5660
echo "Cloning repo "$gitBasePath$repo
57-
git clone "$gitBasePath$repo".git $basepath"/"$repo
61+
if [ ${repo} == "OpenSearch" ]
62+
then
63+
echo "Cloning "$repo" at branch 1.x"
64+
git clone -b 1.x "$gitBasePath$repo".git ${basepath}/${repo}
65+
# /qa/ in OpenSearch repo contains all files related to the previous version of ODFE.
66+
# WhiteSource will attept to download them and take hours to build or timeout error.
67+
# Remove /qa/ because it's irrelevant to OpenSearch.
68+
rm -rf ${basepath}/${repo}/qa
69+
else
70+
echo "Cloning "$repo" at main branch"
71+
git clone "$gitBasePath$repo".git ${basepath}/${repo}
72+
fi
5873
done
5974

6075
echo -n > info.txt
6176
target='OpenSearch'
62-
mvn -v; node -v; npm -v; yarn -v
6377

6478
# scan the Repos using the WhiteSource Unified Agent
6579
for repo in $gitRepos
6680
do
6781
repo_path=$basepath"/"$repo
6882
if [ -d "$repo_path" ]
6983
then
84+
cd $repo_path
7085
echo "Scanning repo: "$gitBasePath$repo " Project: " $repo
71-
java -jar wss-unified-agent.jar -c wss-unified-agent.config -d $repo_path -apiKey $wss_apikey -product "$target" -project $repo
86+
if [ -e "build.gradle" ]
87+
then
88+
echo "build.gradle for $repo exist in root"
89+
if [ ! -e "settings.gradle" ]
90+
then
91+
echo "settings.gradle not exist in $repo, create one based on build.gradle name"
92+
generate_settings_gradle
93+
else
94+
echo "settings.gradle exist $repo"
95+
fi
96+
else
97+
# Exceptions for some repos that have `build.gradle` in sub-folder
98+
if [ ${repo} == "observability" ]
99+
then
100+
cd $repo_path/opensearch-observability
101+
generate_settings_gradle
102+
else
103+
echo "build.gradle for $repo not exist, either dashboards repo or missing necessary files"
104+
fi
105+
fi
106+
java -jar $baseDirPath/wss-unified-agent.jar -c $baseDirPath/wss-unified-agent.config -d $repo_path -apiKey $wss_apikey -product "$target" -project $repo
107+
cd $baseDirPath && pwd
72108
else
73109
echo "Scanning failed for repo: "$gitBasePath$repo " Project: " $repo
74110
fi
75111
done
76112

77113
# remove the WhiteSource unified Jar
78-
rm "wss-unified-agent.jar"
79-
echo "WhiteSource vulnerability scan completed"
114+
rm $baseDirPath/wss-unified-agent.jar
115+
echo "WhiteSource vulnerability scan completed"

‎tools/vulnerability-scan/wss-unified-agent.config

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,10 +78,10 @@ resolveAllDependencies=false
7878
resolveAllDependencies=false
7979
archiveExtractionDepth=7
8080
followSymbolicLinks=true
81-
commandTimeout=1900
8281
gradle.resolveDependencies=true
8382
gradle.aggregateModules=true
84-
gradle.preferredEnvironment=wrapper
83+
gradle.preferredEnvironment=gradle
84+
#gradle.wrapperPath=./gradlew
8585
maven.resolveDependencies=true
8686
maven.runPreStep=true
8787
maven.aggregateModules=true
@@ -176,7 +176,6 @@ ruby.overwriteGemFile=true
176176
#gradle.runPreStep=true
177177
#gradle.ignoreSourceFiles=true
178178
#gradle.aggregateModules=true
179-
#gradle.preferredEnvironment=wrapper
180179
#gradle.localRepositoryPath=
181180
#gradle.wrapperPath=
182181
#gradle.downloadMissingDependencies=false

0 commit comments

Comments
 (0)