Skip to content

Commit 765c16b

Browse files
jetterojettero
committed
Merge branch 'py3' into develop
* py3: make echo actually show information fix linting for hubble_status object various bad escape fixes pylint changes to make signing pass pylint changes let's make pylint non-optional not really py3 related test bugfixes logging is a built-in. It should not be in reqs still no good update jenkins test image avoid checking signatures on None type "files" minor fix for s3fs. Sometimes _init() gives None important bugfix for s3fs (binary pickles) fix minor clean-install bug with configfile permissions additional changes needed for py3 branch more python 3 changes needed — why in this merge though?? Basic signing for roots, s3fs and azurefs (untested) fix error in sqlite returner add a few more options add args, byte dumps, pretty-print dumps, and colorized dumps This seems handy. clean up test for correctness some short names should just be short names add the tests for the current problem and fixes for it clean up test for correctness clean up test for pylint fix bug in hubblestack_nova/openssl.py caused by strptime fix py3 audit bug do the 3.6.10 thing test the hec obj a little more make some minor repairs to the hec.dq py3 is pickier about comparisons in sorted platform.linux_distribution & platform.dist replaced with distro Update nova_loader.py fix error when running hubble without -v remove unnecessary conversion to list change syntax to make it compatible with python 3 do the 3.6.10 thing test the hec obj a little more make some minor repairs to the hec.dq py3 is pickier about comparisons in sorted fix audit file fix changes platform.linux_distribution & platform.dist replaced with distro Update nova_loader.py fix error when running hubble without -v remove unnecessary conversion to list change syntax to make it compatible with python 3
2 parents 9ff90be + e0e81dc commit 765c16b

File tree

162 files changed

+3047
-355
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

162 files changed

+3047
-355
lines changed

‎.gitignore

+9
Original file line numberDiff line numberDiff line change
@@ -123,3 +123,12 @@ relevant-files.txt
123123
# sometimes when I stop docker with a sigquit, hubble drops core (??)
124124
core
125125
core.*
126+
127+
# contrib/gen-pretend-certs.sh data dumptory
128+
.pretend-certs
129+
MANIFEST
130+
SIGNATURE
131+
test-*.file
132+
133+
# certificate bundles for binary distributions
134+
pre_packaged_certificates.py

‎.pipeline

+18-14
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11

2-
def imgname = 'hubblestack/jenkins:centos-v1.0.8'
2+
def imgname = 'hubblestack/jenkins:centos-v1.0.14'
33

44
pipeline {
55
agent { docker { image "${imgname}" } }
@@ -13,15 +13,21 @@ pipeline {
1313
environment {
1414
PY_COLORS = 1
1515
HS_PROFILE = 1
16+
TEST_PY_V = '3.6.10'
1617
}
1718

1819
stages {
1920
stage('setup') {
2021
steps {
22+
sh '''#!/bin/bash
23+
git clean -dfx
24+
'''
2125
sh '''#!/bin/bash
2226
source /etc/profile.d/kersplat.sh
23-
pyenv local $PY_V
24-
pyenv shell $PY_V
27+
export PY_V="$TEST_PY_V"
28+
pyenv local $TEST_PY_V
29+
pyenv shell $TEST_PY_V
30+
echo "pyenv version-name: $(pyenv version-name)"
2531
set -x -e
2632
rm -rf vlib venv .pytest_cache
2733
pip install --cache-dir ./pip.cache -t ./vlib virtualenv
@@ -37,9 +43,9 @@ pipeline {
3743
/usr/bin/git fetch --no-tags --progress https://github.com/hubblestack/hubble.git +refs/heads/develop:refs/remotes/origin/develop
3844
echo git branch -vva
3945
git branch -vva
40-
echo "LHS=$LHS RHS=$RHS"
4146
LHS="origin/${CHANGE_TARGET:-develop}"
4247
RHS="${BRANCH_NAME:+origin/}${BRANCH_NAME:-HEAD}"
48+
echo "LHS=$LHS RHS=$RHS"
4349
if [[ $(git show -s --format='%s%n%b' "${LHS}..${RHS}") =~ LINT-FULL ]]
4450
then find hubblestack -name "*.py"
4551
else find hubblestack -name "*.py" -print0 | xargs -r0 git diff --name-only "$LHS" "$RHS"
@@ -63,16 +69,14 @@ pipeline {
6369
}
6470
stage('pylint') {
6571
steps {
66-
catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
67-
sh '''#!/bin/bash
68-
source ./venv/bin/activate
69-
< relevant-files.txt xargs -r pylint --output-format=json \
70-
> tests/unittests/output/pylint.json
71-
x=$?
72-
python ./tests/automation/pylint-json-to-html tests/unittests/output/pylint.json
73-
exit $x
74-
'''
75-
}
72+
sh '''#!/bin/bash
73+
source ./venv/bin/activate
74+
< relevant-files.txt xargs -r pylint --output-format=json \
75+
> tests/unittests/output/pylint.json
76+
x=$?
77+
python ./tests/automation/pylint-json-to-html tests/unittests/output/pylint.json
78+
exit $x
79+
'''
7680
}
7781
}
7882
stage('bandit') {

‎contrib/gen-pretend-certs.py

+227
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
#!/usr/bin/env python
2+
# coding: UTF-8
3+
4+
import six
5+
import os
6+
import shutil
7+
import argparse
8+
import datetime
9+
10+
from cryptography import x509
11+
from cryptography.hazmat.backends import default_backend
12+
from cryptography.hazmat.primitives import hashes, serialization
13+
from cryptography.hazmat.primitives.asymmetric import rsa, ed448, ed25519
14+
from cryptography.x509.oid import NameOID
15+
16+
DEFAULT_PDIR = '.pretend-certs'
17+
18+
def genkey(key_type='rsa', rsa_key_size=1024, rsa_public_exponent=65537, **args):
19+
if key_type == 'rsa':
20+
return rsa.generate_private_key(
21+
public_exponent=rsa_public_exponent,
22+
key_size=rsa_key_size, backend=default_backend())
23+
elif key_type == 'ed448':
24+
return ed448.Ed448PrivateKey.generate()
25+
elif key_type == 'ed25519':
26+
return ed25519.Ed25519PrivateKey.generate()
27+
raise ValueError('Unknown key_type={}'.format(key_type))
28+
29+
def as_pem(key):
30+
if isinstance(key, (rsa.RSAPrivateKey, ed448.Ed448PrivateKey, ed25519.Ed25519PrivateKey)):
31+
return key.private_bytes(
32+
encoding=serialization.Encoding.PEM,
33+
format=serialization.PrivateFormat.PKCS8,
34+
encryption_algorithm=serialization.NoEncryption())
35+
elif isinstance(key, (rsa.RSAPublicKey, ed448.Ed448PublicKey, ed25519.Ed25519PublicKey)):
36+
return key.public_bytes(
37+
encoding=serialization.Encoding.PEM,
38+
format=serialization.PublicFormat.SubjectPublicKeyInfo)
39+
elif isinstance(key, x509.Certificate):
40+
return key.public_bytes(encoding=serialization.Encoding.PEM)
41+
raise ValueError('Unhandled key class {}'.format(type(key)))
42+
43+
class Authority:
44+
def __init__(self, key, crt):
45+
self.key = key
46+
self.crt = crt
47+
48+
def gen_CA(fname='ca-root', cn='ca-root', path_length=0, authority=None, pdir=DEFAULT_PDIR, **args):
49+
private_key = genkey(**args)
50+
public_key = private_key.public_key()
51+
52+
with open(os.path.join(pdir, fname + '.key'), 'w') as fh:
53+
fh.write( as_pem(private_key) )
54+
55+
with open(os.path.join(pdir, fname + '.unsigned'), 'w') as fh:
56+
fh.write( as_pem(public_key) )
57+
58+
ksec_100 = datetime.timedelta(0, 100e3, 0)
59+
Msec_300 = datetime.timedelta(0, 300e6, 0)
60+
61+
builder = x509.CertificateBuilder()
62+
63+
subject = issuer = x509.Name([
64+
x509.NameAttribute(NameOID.COUNTRY_NAME, u'US'),
65+
x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, u'State'),
66+
x509.NameAttribute(NameOID.LOCALITY_NAME, u'City'),
67+
x509.NameAttribute(NameOID.ORGANIZATION_NAME, u'Org'),
68+
x509.NameAttribute(NameOID.ORGANIZATIONAL_UNIT_NAME, u'Group'),
69+
x509.NameAttribute(NameOID.COMMON_NAME, six.text_type(cn)),
70+
])
71+
72+
if authority:
73+
issuer = authority.crt.subject
74+
75+
builder = builder.subject_name(subject)
76+
builder = builder.issuer_name(issuer)
77+
builder = builder.not_valid_before(datetime.datetime.today() - ksec_100)
78+
builder = builder.not_valid_after(datetime.datetime.today() + Msec_300)
79+
builder = builder.serial_number(x509.random_serial_number())
80+
builder = builder.public_key(public_key)
81+
82+
authority_public_key = authority.crt.public_key() if authority else public_key
83+
builder = builder.add_extension(
84+
x509.AuthorityKeyIdentifier.from_issuer_public_key(authority_public_key), critical=False
85+
)
86+
builder = builder.add_extension(
87+
x509.SubjectKeyIdentifier.from_public_key(public_key), critical=False
88+
)
89+
builder = builder.add_extension(
90+
x509.BasicConstraints(ca=True, path_length=path_length), critical=True,
91+
)
92+
builder = builder.add_extension(
93+
x509.KeyUsage(
94+
digital_signature=True,
95+
key_cert_sign=True,
96+
crl_sign=False,
97+
key_agreement=False,
98+
key_encipherment=False,
99+
content_commitment=False,
100+
data_encipherment=False,
101+
encipher_only=False,
102+
decipher_only=False,
103+
), critical=True
104+
)
105+
106+
signing_args = {
107+
'private_key': authority.key if authority else private_key,
108+
'backend': default_backend(),
109+
'algorithm': None,
110+
}
111+
112+
if isinstance(signing_args['private_key'], rsa.RSAPrivateKey):
113+
signing_args['algorithm'] = hashes.SHA256()
114+
115+
certificate = builder.sign(**signing_args)
116+
117+
with open(os.path.join(pdir, fname + '.crt'), 'w') as fh:
118+
fh.write( as_pem(certificate) )
119+
120+
return Authority(private_key, certificate)
121+
122+
def gen_leaf(authority, fname_template='{}', cn='Certy Cert McCertFace', pdir=DEFAULT_PDIR, **args):
123+
private_key = genkey(**args)
124+
public_key = private_key.public_key()
125+
126+
private_name = fname_template.format('private')
127+
public_name = fname_template.format('public')
128+
129+
with open(os.path.join(pdir, private_name + '.key'), 'w') as fh:
130+
fh.write( as_pem(private_key) )
131+
132+
with open(os.path.join(pdir, public_name + '.unsigned'), 'w') as fh:
133+
fh.write( as_pem(public_key) )
134+
135+
ksec_100 = datetime.timedelta(0, 100e3, 0)
136+
Msec_300 = datetime.timedelta(0, 300e6, 0)
137+
138+
builder = x509.CertificateBuilder()
139+
subject = x509.Name([
140+
x509.NameAttribute(NameOID.COUNTRY_NAME, u'US'),
141+
x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, u'State'),
142+
x509.NameAttribute(NameOID.LOCALITY_NAME, u'City'),
143+
x509.NameAttribute(NameOID.ORGANIZATION_NAME, u'Org'),
144+
x509.NameAttribute(NameOID.ORGANIZATIONAL_UNIT_NAME, u'Group'),
145+
x509.NameAttribute(NameOID.COMMON_NAME, six.text_type(cn)),
146+
])
147+
148+
builder = builder.subject_name(subject)
149+
builder = builder.issuer_name(authority.crt.subject)
150+
builder = builder.not_valid_before(datetime.datetime.today() - ksec_100)
151+
builder = builder.not_valid_after(datetime.datetime.today() + Msec_300)
152+
builder = builder.serial_number(x509.random_serial_number())
153+
builder = builder.public_key(public_key)
154+
155+
authority_public_key = authority.crt.public_key()
156+
# this would pin us to exactly one issuer; without it, any matching issuer
157+
# CN should do the trick
158+
# builder = builder.add_extension(
159+
# x509.AuthorityKeyIdentifier.from_issuer_public_key(authority_public_key), critical=False
160+
# )
161+
builder = builder.add_extension(
162+
x509.SubjectKeyIdentifier.from_public_key(public_key), critical=False
163+
)
164+
builder = builder.add_extension(
165+
x509.KeyUsage(
166+
digital_signature=True,
167+
data_encipherment=True,
168+
content_commitment=True,
169+
key_cert_sign=False,
170+
crl_sign=False,
171+
key_agreement=False,
172+
key_encipherment=False,
173+
encipher_only=False,
174+
decipher_only=False,
175+
), critical=True
176+
)
177+
178+
signing_args = {
179+
'private_key': authority.key,
180+
'backend': default_backend(),
181+
'algorithm': None,
182+
}
183+
184+
if isinstance(signing_args['private_key'], rsa.RSAPrivateKey):
185+
signing_args['algorithm'] = hashes.SHA256()
186+
187+
certificate = builder.sign(**signing_args)
188+
189+
with open(os.path.join(pdir, public_name + '.crt'), 'w') as fh:
190+
fh.write( as_pem(certificate) )
191+
192+
return Authority(private_key, certificate)
193+
194+
def main(root_cn, int1_cn, int2_cn, **args):
195+
if os.path.isdir(args['pdir']):
196+
shutil.rmtree(args['pdir'])
197+
os.mkdir(args['pdir'])
198+
199+
ca = gen_CA(cn=root_cn, fname='ca-root', path_length=1, **args)
200+
ia1 = gen_CA(cn=int1_cn, fname='intermediate-1', authority=ca, path_length=0, **args)
201+
ia2 = gen_CA(cn=int2_cn, fname='intermediate-2', authority=ca, path_length=0, **args)
202+
203+
lf1 = gen_leaf(cn='Certy Cert #1', fname_template='{}-1', authority=ia1, **args)
204+
lf2 = gen_leaf(cn='Certy Cert #2', fname_template='{}-2', authority=ia2, **args)
205+
206+
with open(os.path.join(args['pdir'], 'bundle.pem'), 'w') as ofh:
207+
for i in range(1,3):
208+
with open(os.path.join(args['pdir'], 'intermediate-{}.crt'.format(i)), 'r') as ifh:
209+
ofh.write(ifh.read())
210+
211+
if __name__ == '__main__':
212+
parser = argparse.ArgumentParser( # description='this program',
213+
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
214+
parser.add_argument('-v', '--verbose', action='store_true')
215+
parser.add_argument('-o', '--output-dir', dest='pdir', type=str, default=DEFAULT_PDIR)
216+
parser.add_argument('-R', '--root-cn', type=six.text_type, default='car.hubblestack.io')
217+
parser.add_argument('-I', '--int1-cn', type=six.text_type, default='ia1.hubblestack.io')
218+
parser.add_argument('-J', '--int2-cn', type=six.text_type, default='ia2.hubblestack.io')
219+
parser.add_argument('-t', '--key-type', type=six.text_type,
220+
choices=['rsa', 'ed448', 'ed25519'], default='rsa')
221+
parser.add_argument('-z', '--rsa-key-size', type=int, default=1024)
222+
parser.add_argument('-p', '--rsa-public-exponent', type=int, default=65537)
223+
224+
args = parser.parse_args()
225+
226+
try: main(**args.__dict__)
227+
except KeyboardInterrupt: pass

‎doc/conf.py

+7-7
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,9 @@
5050
master_doc = 'index'
5151

5252
# General information about the project.
53-
project = u'HubbleStack'
54-
copyright = u'2018, Colton Myers, Christer Edwards'
55-
author = u'Colton Myers, Christer Edwards'
53+
project = 'HubbleStack'
54+
copyright = '2018, Colton Myers, Christer Edwards'
55+
author = 'Colton Myers, Christer Edwards'
5656

5757
# The version info for the project you're documenting, acts as replacement for
5858
# |version| and |release|, also used in various other places throughout the
@@ -140,8 +140,8 @@
140140
# (source start file, target name, title,
141141
# author, documentclass [howto, manual, or own class]).
142142
latex_documents = [
143-
(master_doc, 'HubbleStack.tex', u'HubbleStack Documentation',
144-
u'Colton Myers, Christer Edwards', 'manual'),
143+
(master_doc, 'HubbleStack.tex', 'HubbleStack Documentation',
144+
'Colton Myers, Christer Edwards', 'manual'),
145145
]
146146

147147

@@ -150,7 +150,7 @@
150150
# One entry per manual page. List of tuples
151151
# (source start file, name, description, authors, manual section).
152152
man_pages = [
153-
(master_doc, 'hubblestack', u'HubbleStack Documentation',
153+
(master_doc, 'hubblestack', 'HubbleStack Documentation',
154154
[author], 1)
155155
]
156156

@@ -161,7 +161,7 @@
161161
# (source start file, target name, title, author,
162162
# dir menu entry, description, category)
163163
texinfo_documents = [
164-
(master_doc, 'HubbleStack', u'HubbleStack Documentation',
164+
(master_doc, 'HubbleStack', 'HubbleStack Documentation',
165165
author, 'HubbleStack', 'One line description of project.',
166166
'Miscellaneous'),
167167
]

‎hubble.py

100644100755
File mode changed.

0 commit comments

Comments
 (0)