Skip to content

Commit b09a9d0

Browse files
TenebrisoTenebriso
authored
Merge pull request #780 from jettero/py3-data-sign
data sign [py3 rebase] and some additional merge forward
2 parents a6c891a + fc26f55 commit b09a9d0

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

87 files changed

+2651
-12
lines changed

‎.gitignore

+9
Original file line numberDiff line numberDiff line change
@@ -123,3 +123,12 @@ relevant-files.txt
123123
# sometimes when I stop docker with a sigquit, hubble drops core (??)
124124
core
125125
core.*
126+
127+
# contrib/gen-pretend-certs.sh data dumptory
128+
.pretend-certs
129+
MANIFEST
130+
SIGNATURE
131+
test-*.file
132+
133+
# certificate bundles for binary distributions
134+
pre_packaged_certificates.py

‎.pipeline

+3
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@ pipeline {
1919
stages {
2020
stage('setup') {
2121
steps {
22+
sh '''#!/bin/bash
23+
git clean -dfx
24+
'''
2225
sh '''#!/bin/bash
2326
source /etc/profile.d/kersplat.sh
2427
pyenv local $PY_V

‎contrib/disk-queue-info.py

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#!/usr/bin/env python
2+
3+
from __future__ import print_function
4+
5+
import os
6+
import argparse
7+
import json
8+
from json.decoder import WHITESPACE
9+
from hubblestack.hec.dq import DiskQueue
10+
11+
def get_args(*a):
12+
parser = argparse.ArgumentParser(description='hubble disk queue info extractor')
13+
parser.add_argument("cachedir", nargs='+')
14+
parser.add_argument('-p', '--peek', '--print', action='store_true',
15+
help='attempt to dump all the bytes found in queue in the order they would be dequeued')
16+
parser.add_argument('-e', '--evil-decode', action='store_true',
17+
help='attempt to decode all json found in queue (implies -p) and pretty-print them')
18+
parser.add_argument('-c', '--with-color', action='store_true',
19+
help='attempt to pull in pygments and colorize the output of -e (implies -p and -e)')
20+
parser.add_argument('-m', '--show-meta', action='store_true',
21+
help='attempt to show meta-data as a comment in peek mode or a spurious _META_ field in evil mode')
22+
args = parser.parse_args(*a)
23+
if args.with_color:
24+
args.evil_decode = True
25+
if args.evil_decode:
26+
args.peek = True
27+
return args
28+
29+
def show_info(dirname):
30+
dq = DiskQueue(dirname)
31+
print("QUEUE={} ITEMS={} SIZE={}".format(dirname, dq.cn, dq.sz))
32+
33+
def evil_decode(docbytes):
34+
decoder = json.JSONDecoder()
35+
idx = WHITESPACE.match(docbytes, 0).end()
36+
while idx < len(docbytes):
37+
try:
38+
obj, end = decoder.raw_decode(docbytes, idx)
39+
yield obj
40+
idx = WHITESPACE.match(docbytes, end).end()
41+
except ValueError as e:
42+
print('docbytes:\n', docbytes)
43+
raise
44+
45+
def read_entries(dirname, evil=False, color=False, meta=False):
46+
dq = DiskQueue(dirname)
47+
for item,meta in dq.iter_peek():
48+
if evil:
49+
for obj in evil_decode(item):
50+
if meta:
51+
obj['_META_'] = meta
52+
obj = json.dumps(obj, sort_keys=True, indent=2)
53+
if color:
54+
try:
55+
from pygments import highlight, lexers, formatters
56+
obj = highlight(obj, lexers.JsonLexer(),
57+
formatters.TerminalFormatter())
58+
except ImportError:
59+
pass
60+
print(obj)
61+
else:
62+
if meta:
63+
print('# ', json.dumps(meta))
64+
print(item)
65+
66+
def main(args):
67+
for dirname in args.cachedir:
68+
try:
69+
if not os.path.isdir(dirname):
70+
raise Exception("directory does not exist")
71+
if args.peek:
72+
read_entries(dirname, evil=args.evil_decode, color=args.with_color, meta=args.show_meta)
73+
else:
74+
show_info(dirname)
75+
except Exception as e:
76+
print("# Exception while reading {}: {}".format(dirname, repr(e)))
77+
78+
if __name__ == '__main__':
79+
try:
80+
main(get_args())
81+
except KeyboardInterrupt:
82+
pass

‎contrib/gen-pretend-certs.py

+227
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
#!/usr/bin/env python
2+
# coding: UTF-8
3+
4+
import six
5+
import os
6+
import shutil
7+
import argparse
8+
import datetime
9+
10+
from cryptography import x509
11+
from cryptography.hazmat.backends import default_backend
12+
from cryptography.hazmat.primitives import hashes, serialization
13+
from cryptography.hazmat.primitives.asymmetric import rsa, ed448, ed25519
14+
from cryptography.x509.oid import NameOID
15+
16+
DEFAULT_PDIR = '.pretend-certs'
17+
18+
def genkey(key_type='rsa', rsa_key_size=1024, rsa_public_exponent=65537, **args):
19+
if key_type == 'rsa':
20+
return rsa.generate_private_key(
21+
public_exponent=rsa_public_exponent,
22+
key_size=rsa_key_size, backend=default_backend())
23+
elif key_type == 'ed448':
24+
return ed448.Ed448PrivateKey.generate()
25+
elif key_type == 'ed25519':
26+
return ed25519.Ed25519PrivateKey.generate()
27+
raise ValueError('Unknown key_type={}'.format(key_type))
28+
29+
def as_pem(key):
30+
if isinstance(key, (rsa.RSAPrivateKey, ed448.Ed448PrivateKey, ed25519.Ed25519PrivateKey)):
31+
return key.private_bytes(
32+
encoding=serialization.Encoding.PEM,
33+
format=serialization.PrivateFormat.PKCS8,
34+
encryption_algorithm=serialization.NoEncryption())
35+
elif isinstance(key, (rsa.RSAPublicKey, ed448.Ed448PublicKey, ed25519.Ed25519PublicKey)):
36+
return key.public_bytes(
37+
encoding=serialization.Encoding.PEM,
38+
format=serialization.PublicFormat.SubjectPublicKeyInfo)
39+
elif isinstance(key, x509.Certificate):
40+
return key.public_bytes(encoding=serialization.Encoding.PEM)
41+
raise ValueError('Unhandled key class {}'.format(type(key)))
42+
43+
class Authority:
44+
def __init__(self, key, crt):
45+
self.key = key
46+
self.crt = crt
47+
48+
def gen_CA(fname='ca-root', cn='ca-root', path_length=0, authority=None, pdir=DEFAULT_PDIR, **args):
49+
private_key = genkey(**args)
50+
public_key = private_key.public_key()
51+
52+
with open(os.path.join(pdir, fname + '.key'), 'w') as fh:
53+
fh.write( as_pem(private_key) )
54+
55+
with open(os.path.join(pdir, fname + '.unsigned'), 'w') as fh:
56+
fh.write( as_pem(public_key) )
57+
58+
ksec_100 = datetime.timedelta(0, 100e3, 0)
59+
Msec_300 = datetime.timedelta(0, 300e6, 0)
60+
61+
builder = x509.CertificateBuilder()
62+
63+
subject = issuer = x509.Name([
64+
x509.NameAttribute(NameOID.COUNTRY_NAME, u'US'),
65+
x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, u'State'),
66+
x509.NameAttribute(NameOID.LOCALITY_NAME, u'City'),
67+
x509.NameAttribute(NameOID.ORGANIZATION_NAME, u'Org'),
68+
x509.NameAttribute(NameOID.ORGANIZATIONAL_UNIT_NAME, u'Group'),
69+
x509.NameAttribute(NameOID.COMMON_NAME, six.text_type(cn)),
70+
])
71+
72+
if authority:
73+
issuer = authority.crt.subject
74+
75+
builder = builder.subject_name(subject)
76+
builder = builder.issuer_name(issuer)
77+
builder = builder.not_valid_before(datetime.datetime.today() - ksec_100)
78+
builder = builder.not_valid_after(datetime.datetime.today() + Msec_300)
79+
builder = builder.serial_number(x509.random_serial_number())
80+
builder = builder.public_key(public_key)
81+
82+
authority_public_key = authority.crt.public_key() if authority else public_key
83+
builder = builder.add_extension(
84+
x509.AuthorityKeyIdentifier.from_issuer_public_key(authority_public_key), critical=False
85+
)
86+
builder = builder.add_extension(
87+
x509.SubjectKeyIdentifier.from_public_key(public_key), critical=False
88+
)
89+
builder = builder.add_extension(
90+
x509.BasicConstraints(ca=True, path_length=path_length), critical=True,
91+
)
92+
builder = builder.add_extension(
93+
x509.KeyUsage(
94+
digital_signature=True,
95+
key_cert_sign=True,
96+
crl_sign=False,
97+
key_agreement=False,
98+
key_encipherment=False,
99+
content_commitment=False,
100+
data_encipherment=False,
101+
encipher_only=False,
102+
decipher_only=False,
103+
), critical=True
104+
)
105+
106+
signing_args = {
107+
'private_key': authority.key if authority else private_key,
108+
'backend': default_backend(),
109+
'algorithm': None,
110+
}
111+
112+
if isinstance(signing_args['private_key'], rsa.RSAPrivateKey):
113+
signing_args['algorithm'] = hashes.SHA256()
114+
115+
certificate = builder.sign(**signing_args)
116+
117+
with open(os.path.join(pdir, fname + '.crt'), 'w') as fh:
118+
fh.write( as_pem(certificate) )
119+
120+
return Authority(private_key, certificate)
121+
122+
def gen_leaf(authority, fname_template='{}', cn='Certy Cert McCertFace', pdir=DEFAULT_PDIR, **args):
123+
private_key = genkey(**args)
124+
public_key = private_key.public_key()
125+
126+
private_name = fname_template.format('private')
127+
public_name = fname_template.format('public')
128+
129+
with open(os.path.join(pdir, private_name + '.key'), 'w') as fh:
130+
fh.write( as_pem(private_key) )
131+
132+
with open(os.path.join(pdir, public_name + '.unsigned'), 'w') as fh:
133+
fh.write( as_pem(public_key) )
134+
135+
ksec_100 = datetime.timedelta(0, 100e3, 0)
136+
Msec_300 = datetime.timedelta(0, 300e6, 0)
137+
138+
builder = x509.CertificateBuilder()
139+
subject = x509.Name([
140+
x509.NameAttribute(NameOID.COUNTRY_NAME, u'US'),
141+
x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, u'State'),
142+
x509.NameAttribute(NameOID.LOCALITY_NAME, u'City'),
143+
x509.NameAttribute(NameOID.ORGANIZATION_NAME, u'Org'),
144+
x509.NameAttribute(NameOID.ORGANIZATIONAL_UNIT_NAME, u'Group'),
145+
x509.NameAttribute(NameOID.COMMON_NAME, six.text_type(cn)),
146+
])
147+
148+
builder = builder.subject_name(subject)
149+
builder = builder.issuer_name(authority.crt.subject)
150+
builder = builder.not_valid_before(datetime.datetime.today() - ksec_100)
151+
builder = builder.not_valid_after(datetime.datetime.today() + Msec_300)
152+
builder = builder.serial_number(x509.random_serial_number())
153+
builder = builder.public_key(public_key)
154+
155+
authority_public_key = authority.crt.public_key()
156+
# this would pin us to exactly one issuer; without it, any matching issuer
157+
# CN should do the trick
158+
# builder = builder.add_extension(
159+
# x509.AuthorityKeyIdentifier.from_issuer_public_key(authority_public_key), critical=False
160+
# )
161+
builder = builder.add_extension(
162+
x509.SubjectKeyIdentifier.from_public_key(public_key), critical=False
163+
)
164+
builder = builder.add_extension(
165+
x509.KeyUsage(
166+
digital_signature=True,
167+
data_encipherment=True,
168+
content_commitment=True,
169+
key_cert_sign=False,
170+
crl_sign=False,
171+
key_agreement=False,
172+
key_encipherment=False,
173+
encipher_only=False,
174+
decipher_only=False,
175+
), critical=True
176+
)
177+
178+
signing_args = {
179+
'private_key': authority.key,
180+
'backend': default_backend(),
181+
'algorithm': None,
182+
}
183+
184+
if isinstance(signing_args['private_key'], rsa.RSAPrivateKey):
185+
signing_args['algorithm'] = hashes.SHA256()
186+
187+
certificate = builder.sign(**signing_args)
188+
189+
with open(os.path.join(pdir, public_name + '.crt'), 'w') as fh:
190+
fh.write( as_pem(certificate) )
191+
192+
return Authority(private_key, certificate)
193+
194+
def main(root_cn, int1_cn, int2_cn, **args):
195+
if os.path.isdir(args['pdir']):
196+
shutil.rmtree(args['pdir'])
197+
os.mkdir(args['pdir'])
198+
199+
ca = gen_CA(cn=root_cn, fname='ca-root', path_length=1, **args)
200+
ia1 = gen_CA(cn=int1_cn, fname='intermediate-1', authority=ca, path_length=0, **args)
201+
ia2 = gen_CA(cn=int2_cn, fname='intermediate-2', authority=ca, path_length=0, **args)
202+
203+
lf1 = gen_leaf(cn='Certy Cert #1', fname_template='{}-1', authority=ia1, **args)
204+
lf2 = gen_leaf(cn='Certy Cert #2', fname_template='{}-2', authority=ia2, **args)
205+
206+
with open(os.path.join(args['pdir'], 'bundle.pem'), 'w') as ofh:
207+
for i in range(1,3):
208+
with open(os.path.join(args['pdir'], 'intermediate-{}.crt'.format(i)), 'r') as ifh:
209+
ofh.write(ifh.read())
210+
211+
if __name__ == '__main__':
212+
parser = argparse.ArgumentParser( # description='this program',
213+
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
214+
parser.add_argument('-v', '--verbose', action='store_true')
215+
parser.add_argument('-o', '--output-dir', dest='pdir', type=str, default=DEFAULT_PDIR)
216+
parser.add_argument('-R', '--root-cn', type=six.text_type, default='car.hubblestack.io')
217+
parser.add_argument('-I', '--int1-cn', type=six.text_type, default='ia1.hubblestack.io')
218+
parser.add_argument('-J', '--int2-cn', type=six.text_type, default='ia2.hubblestack.io')
219+
parser.add_argument('-t', '--key-type', type=six.text_type,
220+
choices=['rsa', 'ed448', 'ed25519'], default='rsa')
221+
parser.add_argument('-z', '--rsa-key-size', type=int, default=1024)
222+
parser.add_argument('-p', '--rsa-public-exponent', type=int, default=65537)
223+
224+
args = parser.parse_args()
225+
226+
try: main(**args.__dict__)
227+
except KeyboardInterrupt: pass

‎hubble.py

100644100755
File mode changed.

‎hubblestack/daemon.py

+5-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
import uuid
2121
from datetime import datetime
2222

23-
import salt.fileclient
2423
import salt.fileserver
2524
import salt.fileserver.gitfs
2625
import salt.modules.cmdmod
@@ -30,6 +29,8 @@
3029
import salt.utils.gitfs
3130
import salt.utils.path
3231
from croniter import croniter
32+
33+
import hubblestack.utils.signing
3334
import hubblestack.splunklogging
3435
import hubblestack.log
3536
import hubblestack.hec.opt
@@ -794,6 +795,9 @@ def refresh_grains(initial=False):
794795
hubblestack.status.__salt__ = __salt__
795796
hubble_status.start_sigusr1_signal_handler()
796797

798+
hubblestack.utils.signing.__opts__ = __opts__
799+
hubblestack.utils.signing.__salt__ = __salt__
800+
797801
if not initial and __salt__['config.get']('splunklogging', False):
798802
hubblestack.log.emit_to_splunk(__grains__, 'INFO', 'hubblestack.grains_report')
799803

‎hubblestack/extmods/fileserver/azurefs.py

+2
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@
7070
# Import third party libs
7171
import salt.ext.six as six
7272

73+
from hubblestack.utils.signing import find_wrapf
7374

7475
__virtualname__ = 'azurefs'
7576

@@ -95,6 +96,7 @@ def __virtual__():
9596
return True
9697

9798

99+
@find_wrapf(not_found={'path': '', 'rel': ''})
98100
def find_file(path, saltenv='base', **kwargs):
99101
"""
100102
Search the environment for the relative path

0 commit comments

Comments
 (0)