Skip to content

Commit 9139040

Browse files
davidmcgrewGitHub Enterprise
davidmcgrew
authored and
GitHub Enterprise
committed
Merge pull request #345 from network-intelligence/dev
Merging `dev` into `trunk` for VERSION 2.6.2
2 parents bb917c7 + a820f39 commit 9139040

File tree

155 files changed

+5013
-535898
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

155 files changed

+5013
-535898
lines changed

‎.github/workflows/build-wheels.yml

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
name: Mercury wheels Build and Publish to PyPI
2+
3+
on:
4+
workflow_dispatch:
5+
push:
6+
branches:
7+
- main
8+
9+
jobs:
10+
build-mercury-wheels:
11+
name: Build wheels on ${{ matrix.os }}
12+
runs-on: ${{ matrix.os }}
13+
environment:
14+
name: pypi
15+
url: https://pypi.org/p/mercury-python
16+
permissions:
17+
id-token: write
18+
strategy:
19+
matrix:
20+
os: [ubuntu-latest]
21+
22+
steps:
23+
- uses: actions/checkout@v4
24+
with:
25+
detch-depth: 1
26+
27+
- name: Install packages
28+
run: sudo apt-get update && sudo apt-get install -y zlib1g-dev libssl-dev make
29+
30+
- name: Set up Python
31+
uses: actions/setup-python@v3
32+
with:
33+
python-version: '3.x'
34+
35+
- name: Set up QEMU
36+
if: runner.os == 'Linux'
37+
uses: docker/setup-qemu-action@v3
38+
with:
39+
platforms: all
40+
41+
- name: Install pip packages
42+
run: python -m pip install cython cibuildwheel==2.20.0
43+
44+
- name: Build Mercury
45+
run: ./configure && make && cp -r src/cython/* ./
46+
47+
- name: Build wheels
48+
uses: pypa/[email protected]
49+
env:
50+
CIBW_ARCHS_LINUX: x86_64 aarch64
51+
CIBW_ENVIRONMENT: CC='g++' CXX='g++' MERCURY_DIR='./'
52+
CIBW_SKIP: "*-musllinux_* pp*"
53+
CIBW_BEFORE_ALL: yum install -y openssl-devel make zlib-devel || apt-get install -y zlib1g-dev libssl-dev make || apk add zlib1g-dev libssl-dev make
54+
CIBW_TEST_COMMAND: cd {project} && python mercury_python_test.py
55+
56+
- name: Upload to S3
57+
env:
58+
AWS_ACCESS_KEY_ID: ${{secrets.ACCESS_KEY_ID}}
59+
AWS_SECRET_ACCESS_KEY: ${{secrets.ACCESS_KEY_SECRET}}
60+
AWS_DEFAULT_REGION: us-east-1
61+
S3_BUCKET: ${{secrets.S3_BUCKET}}
62+
run: VERSION=$(cat src/cython/_version.py | tr -d '\n' | cut -d \' -f2) && aws s3 cp wheelhouse/ "s3://$S3_BUCKET/version=$VERSION" --recursive

‎Makefile

+4-21
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ else
2121
endif
2222

2323
.PHONY: install install-no-systemd
24-
install: install-mercury install-resources install-etc-config install-systemd
25-
install-nosystemd: install-mercury install-resources install-etc-config
24+
install: install-mercury install-etc-config
25+
install-nosystemd: install-mercury install-etc-config
2626

2727
.PHONY: install-mercury
2828
install-mercury:
@@ -33,17 +33,7 @@ else
3333
$(INSTALLDATA) mercury /usr/share/bash-completion/completions/ # note: completion script has same name as binary
3434
endif
3535

36-
.PHONY: install-resources
37-
install-resources:
38-
ifneq ($(wildcard src/Makefile), src/Makefile)
39-
@echo $(COLOR_RED) "error: run ./configure before running make (src/Makefile is missing)" $(COLOR_OFF)
40-
else
41-
cd resources && $(MAKE) install
42-
endif
43-
44-
# leave this variable empty; we want to force the user to set it, as a
45-
# reminder that they should create a usable local configuration
46-
MERCURY_CFG =
36+
MERCURY_CFG = mercury.cfg
4737
.PHONY: install-etc-config
4838
install-etc-config:
4939
ifneq ($(wildcard src/Makefile), src/Makefile)
@@ -53,10 +43,7 @@ ifneq ($(MERCURY_CFG),)
5343
$(INSTALL) -d /etc/mercury
5444
$(INSTALLDATA) $(MERCURY_CFG) /etc/mercury/mercury.cfg
5545
else
56-
@echo $(COLOR_RED) "error: you must specify the configuration file; run as 'make install MERCURY_CFG=filename'" $(COLOR_OFF)
57-
@echo $(COLOR_RED) "where 'filename' is the configuration file you want to use for this installation. You can" $(COLOR_OFF)
58-
@echo $(COLOR_RED) "use mercury.cfg as a template, but you *must* change the interface line to the appropriate" $(COLOR_OFF)
59-
@echo $(COLOR_RED) "network interface for your system. (Use 'cat /proc/net/dev' to see Linux interfaces.)" $(COLOR_OFF)
46+
@echo $(COLOR_RED) "error: no configuration file specified; run as 'make install MERCURY_CFG=filename'" $(COLOR_OFF)
6047
@/bin/false
6148
endif
6249
endif
@@ -77,7 +64,6 @@ ifneq ($(wildcard src/Makefile), src/Makefile)
7764
@echo $(COLOR_RED) "error: run ./configure before running make (src/Makefile is missing)" $(COLOR_OFF)
7865
else
7966
cd src && $(MAKE) install-nonroot
80-
cd resources && $(MAKE) install-nonroot
8167
endif
8268

8369
.PHONY: install-certtools
@@ -99,7 +85,6 @@ else
9985
rm -f /etc/mercury/mercury.cfg
10086
rm -rf /etc/mercury
10187
cd src && $(MAKE) uninstall
102-
cd resources && $(MAKE) uninstall
10388
endif
10489

10590
.PHONY: uninstall-systemd
@@ -189,10 +174,8 @@ ifneq ($(wildcard src/Makefile), src/Makefile)
189174
else
190175
cd src && $(MAKE) distclean
191176
cd test && $(MAKE) distclean
192-
cd resources && $(MAKE) distclean
193177
rm -rf autom4te.cache config.log config.status Makefile_helper.mk
194178
rm -f lib/*.so
195-
-git clean -xf
196179
endif
197180

198181
.PHONY: package-deb

‎VERSION

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
2.6.1
1+
2.6.2

‎build_pkg.sh

+2-4
Original file line numberDiff line numberDiff line change
@@ -89,8 +89,7 @@ if [ "$BUILDTYPE" == "deb" ]; then
8989
--after-remove ./install_mercury/postuninstall_remove \
9090
--deb-after-purge ./install_mercury/postuninstall_purge \
9191
./src/mercury=/usr/local/bin/ mercury.cfg=/etc/mercury/ \
92-
./mercury=/usr/share/bash-completion/completions/ \
93-
./resources/resources.tgz=/usr/local/share/mercury/
92+
./mercury=/usr/share/bash-completion/completions/
9493

9594
elif [ "$BUILDTYPE" == "rpm" ]; then
9695

@@ -104,6 +103,5 @@ elif [ "$BUILDTYPE" == "rpm" ]; then
104103
--description "$DESCRIPTION" \
105104
--after-remove ./install_mercury/postuninstall_rpm \
106105
./install_mercury/mercury.service=/usr/lib/systemd/system/ \
107-
./src/mercury=/usr/local/bin/ mercury.cfg=/etc/mercury/ \
108-
./resources/pyasn.db=/usr/local/share/mercury/ ./resources/fingerprint_db.json.gz=/usr/local/share/mercury/
106+
./src/mercury=/usr/local/bin/ mercury.cfg=/etc/mercury/
109107
fi

‎configure

+1-2
Original file line numberDiff line numberDiff line change
@@ -5925,7 +5925,7 @@ else case e in #(
59255925
printf "%s\n" "$as_me: WARNING: llvm not found; make fuzz-test may not work" >&2;} ;;
59265926
esac
59275927
fi
5928-
ac_config_files="$ac_config_files src/libmerc/Makefile src/cython/Makefile src/Makefile test/Makefile unit_tests/Makefile resources/Makefile Makefile_helper.mk install_mercury/mercury.service"
5928+
ac_config_files="$ac_config_files src/libmerc/Makefile src/cython/Makefile src/Makefile test/Makefile unit_tests/Makefile Makefile_helper.mk install_mercury/mercury.service"
59295929

59305930
cat >confcache <<\_ACEOF
59315931
# This file is a shell script that caches the results of configure
@@ -6633,7 +6633,6 @@ do
66336633
"src/Makefile") CONFIG_FILES="$CONFIG_FILES src/Makefile" ;;
66346634
"test/Makefile") CONFIG_FILES="$CONFIG_FILES test/Makefile" ;;
66356635
"unit_tests/Makefile") CONFIG_FILES="$CONFIG_FILES unit_tests/Makefile" ;;
6636-
"resources/Makefile") CONFIG_FILES="$CONFIG_FILES resources/Makefile" ;;
66376636
"Makefile_helper.mk") CONFIG_FILES="$CONFIG_FILES Makefile_helper.mk" ;;
66386637
"install_mercury/mercury.service") CONFIG_FILES="$CONFIG_FILES install_mercury/mercury.service" ;;
66396638

‎configure.ac

+1-1
Original file line numberDiff line numberDiff line change
@@ -102,5 +102,5 @@ AS_IF([test "x$CLANGPP" = xyes],
102102
AS_IF([test "x$LLVM" = xyes],
103103
[],
104104
[AC_MSG_WARN([llvm not found; make fuzz-test may not work])])
105-
AC_CONFIG_FILES(src/libmerc/Makefile src/cython/Makefile src/Makefile test/Makefile unit_tests/Makefile resources/Makefile Makefile_helper.mk install_mercury/mercury.service)
105+
AC_CONFIG_FILES(src/libmerc/Makefile src/cython/Makefile src/Makefile test/Makefile unit_tests/Makefile Makefile_helper.mk install_mercury/mercury.service)
106106
AC_OUTPUT

‎doc/CHANGELOG.md

+26
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,31 @@
11
# CHANGELOG for Mercury
22

3+
## VERSION 2.6.2
4+
* Removed default interface from template configuration file
5+
`mercury.cfg` and added runtime check to require that an interface
6+
be specified when a configuration file is used.
7+
* Added `--crypto-assess=<policy>` option, which implements an
8+
assessment of the cryptographic security of TLS, DTLS, and QUIC
9+
sessions and clients. The currently implemented policies are
10+
`quantum_safe` and `quantum_safe_verbose`. The former is the
11+
default, and the latter provides human-readable names.
12+
* Integrated support for the IPSec protocols IKEv2 and ESP.
13+
* Integrated minimal LDAP support, which provides details only for
14+
`bindRequest` and `bindResponse` messages.
15+
* Added support for the `LINKTYPE_LINUX_SLL` (Linux 'Cooked Capture').
16+
* Improved the `stats` test to avoid spurious failues, by allowing
17+
mercury JSON output and mercury stats counts to differ by up to
18+
10% due to lossy stats collection.
19+
* Enabled `--raw-features` and `--reassembly` to be enabled through
20+
the configuration file.
21+
* Refactored `pmercury` to use `c++` code where possible.
22+
* Re-enabled ^C signal handler for PCAP processing.
23+
* Moved `dns.id` and `ip.id` fields to the tail end of the JSON
24+
record, to improve Parquet compressibility.
25+
* Classification improved to allow multiple fingerprints to utilize
26+
the same Weighted Naive Bayes models.
27+
* Various internal improvements and unit test tweaks.
28+
329
## VERSION 2.6.1
430
* Improved STUN implementation: added test cases, fixed fingerprint
531
feature nits, renamed variables for consistency with the RFCs, and

‎doc/sphinx/Doxyfile

+1
Original file line numberDiff line numberDiff line change
@@ -842,6 +842,7 @@ FILE_PATTERNS = *.c \
842842
*.dox \
843843
*.py \
844844
*.pyw \
845+
*.pyx \
845846
*.f90 \
846847
*.f95 \
847848
*.f03 \

‎doc/sphinx/source/conf.py

+10-3
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,17 @@
1515
# -- General configuration ---------------------------------------------------
1616
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
1717

18+
import os
1819
import sys
1920
sys.path.append("./docproj/ext/breathe/")
21+
sys.path.append(os.path.abspath('../../../src/cython/'))
2022

21-
extensions = ['breathe']
23+
extensions = [
24+
'breathe',
25+
'sphinx.ext.autodoc',
26+
'sphinx.ext.viewcode',
27+
'sphinx_autodoc_typehints'
28+
]
2229

2330
breathe_default_project = "mercury"
2431

@@ -29,7 +36,7 @@
2936
# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
3037

3138
html_theme = 'sphinx_rtd_theme' # 'alabaster'
32-
html_static_path = ['_static']
39+
#html_static_path = ['_static']
3340

3441
###################################
3542

@@ -38,5 +45,5 @@
3845
}
3946

4047
breathe_projects_source = {
41-
"mercury" : ( "../../../", ["src/libmerc/datum.h"] )
48+
"mercury" : ( "../../../", ["src/libmerc/datum.h"])
4249
}

‎doc/sphinx/source/index.rst

+6
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,12 @@ Mercury Library Documentation
7272
.. doxygengroup:: bitoperations
7373
:project: mercury
7474

75+
Mercury Python Library Documentation
76+
====================================
77+
78+
.. automodule:: mercury
79+
:members:
80+
:show-inheritance:
7581

7682

7783
Indices and tables

‎mercury.cfg

+6-6
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,10 @@
88
# ignored, as are unparseable lines.
99

1010
# network interface for packet capture
11-
capture = ens33
11+
#capture = ens33
1212

1313
# name of JSON output file or directory for fingerprints and metadata
14-
fingerprint = fingerprint.json
14+
fingerprint = metadata.json
1515

1616
# filter out packets based on protocol
1717
#select = dns,dhcp,dtls,tcp,http,tls,wireguard
@@ -23,13 +23,13 @@ fingerprint = fingerprint.json
2323
# nonselected-udp-data
2424

2525
# 'dns-json' causes DNS responses to be reported with full detail in JSON
26-
# dns-json
26+
dns-json
2727

2828
# 'certs-json' causes certificates to be reported with full detail in JSON
29-
# certs-json
29+
certs-json
3030

3131
# 'metadata' causes extensive metadata to be reported in JSON
32-
# metadata
32+
metadata
3333

3434
# after dropping root privileges, change to this user
3535
user = mercury
@@ -47,7 +47,7 @@ threads = cpu
4747
buffer = 0.05
4848

4949
# perform analysis, include results in JSON output file
50-
analysis = 1
50+
#analysis = 1
5151

5252
# set resource directory
5353
# resources = /usr/local/share/mercury/resources.tgz

‎python/MANIFEST.in

-6
This file was deleted.

0 commit comments

Comments
 (0)