Skip to content

Commit 05b08d2

Browse files
alvarolopezalvarolopez
committed
feat: improve distribution loading and handling
- Implent dry-run and download-only options. - Do not rely anymore on stevedore and entrypoints to define what we can manage, and generate the distribution list automatically from subclasses. - Remove centos distributions, as they are outdated.
1 parent fd8cdbf commit 05b08d2

File tree

6 files changed

+273
-432
lines changed

6 files changed

+273
-432
lines changed

‎imgsync/distros/__init__.py

+49-136
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
"""Base classes and options for the distros."""
1+
"""Package to manage distribution download."""
22

33
# Copyright (c) 2016 Alvaro Lopez Garcia
44

@@ -14,32 +14,23 @@
1414
# License for the specific language governing permissions and limitations
1515
# under the License.
1616

17-
import abc
18-
import hashlib
19-
import os
20-
import tempfile
17+
import itertools
2118

2219
from oslo_config import cfg
2320
from oslo_log import log
24-
import requests
25-
import six
26-
import stevedore
27-
28-
from imgsync import exception
29-
from imgsync import glance
30-
31-
SUPPORTED_DISTROS = [
32-
"centos6",
33-
"centos7",
34-
"ubuntu14",
35-
"ubuntu16",
36-
"ubuntu18",
37-
"ubuntu20",
38-
"ubuntu22",
39-
"debian10",
40-
"debian11",
41-
"debian12",
42-
]
21+
22+
from imgsync.distros import debian
23+
from imgsync.distros import ubuntu
24+
25+
26+
_DISTRO_OBJS = {
27+
i.name: i
28+
for i in itertools.chain(
29+
ubuntu.Ubuntu.__subclasses__(), debian.Debian.__subclasses__()
30+
)
31+
}
32+
33+
SUPPORTED_DISTROS = list(_DISTRO_OBJS.keys())
4334

4435
opts = [
4536
cfg.StrOpt(
@@ -66,129 +57,51 @@
6657
),
6758
]
6859

60+
cli_opts = [
61+
cfg.BoolOpt(
62+
"download-only",
63+
default=False,
64+
help="Only download the images, do not sync them to glance. Be aware"
65+
" that images are deleted after being synced to glance, so if you use"
66+
" this option the images will be deleted after the download. Use this"
67+
" only for debugging purposes.",
68+
),
69+
cfg.BoolOpt(
70+
"dry-run",
71+
default=False,
72+
help="Do not sync the images, only show what would be done. This still"
73+
" equires authenticating with Glance, in order to check what would be done.",
74+
),
75+
]
76+
77+
6978
CONF = cfg.CONF
7079
CONF.register_opts(opts)
80+
CONF.register_cli_opts(cli_opts)
7181

7282
LOG = log.getLogger(__name__)
7383

7484

75-
@six.add_metaclass(abc.ABCMeta)
76-
class BaseDistro(object):
77-
"""Base class for all distributions."""
78-
79-
url = None
80-
81-
def __init__(self):
82-
"""Initialize the BaseDistro object."""
83-
self.glance = glance.GLANCE
84-
85-
@abc.abstractproperty
86-
def what(self):
87-
"""Get what to sync. This has to be implemented by the child class."""
88-
return None
89-
90-
def sync(self):
91-
"""Sync the images, calling the method that is needed."""
92-
if self.what == "all":
93-
self._sync_all()
94-
elif self.what == "latest":
95-
self._sync_latest()
96-
else:
97-
LOG.warn("Nothing to do")
98-
99-
def _get_file_checksum(self, path, block_size=2**20):
100-
"""Get the checksum of a file.
101-
102-
Get the checksum of a file using sha512.
103-
104-
:param path: the path to the file
105-
:param block_size: block size to use when reading the file
106-
:returns: sha512 object
107-
"""
108-
sha512 = hashlib.sha512()
109-
with open(path, "rb") as f:
110-
buf = f.read(block_size)
111-
while len(buf) > 0:
112-
sha512.update(buf)
113-
buf = f.read(block_size)
114-
return sha512
115-
116-
def verify_checksum(
117-
self,
118-
location,
119-
name,
120-
checksum,
121-
):
122-
"""Verify the image's checksum."""
123-
# TODO(aloga): not implemented yet
124-
125-
def _download_one(self, url, checksum):
126-
"""Download a file.
127-
128-
Download a file from a url and return a temporary file object.
129-
130-
:param url: the url to download
131-
:param checksum: tuple in the form (checksum_name, checksum_value)
132-
:returns: temporary file object
133-
"""
134-
with tempfile.NamedTemporaryFile(suffix=".imgsync", delete=False) as location:
135-
try:
136-
response = requests.get(url, stream=True, timeout=10)
137-
except Exception as e:
138-
os.remove(location.name)
139-
LOG.error(e)
140-
raise exception.ImageDownloadFailed(code=e.errno, reason=e.message)
141-
142-
if not response.ok:
143-
os.remove(location.name)
144-
LOG.error(
145-
"Cannot download image: (%s) %s",
146-
response.status_code,
147-
response.reason,
148-
)
149-
raise exception.ImageDownloadFailed(
150-
code=response.status_code, reason=response.reason
151-
)
152-
153-
for block in response.iter_content(1024):
154-
if block:
155-
location.write(block)
156-
location.flush()
157-
158-
checksum_map = {"sha512": hashlib.sha512, "sha256": hashlib.sha256}
159-
sha = checksum_map.get(checksum[0])()
160-
block_size = 2**20
161-
with open(location.name, "rb") as f:
162-
buf = f.read(block_size)
163-
while len(buf) > 0:
164-
sha.update(buf)
165-
buf = f.read(block_size)
166-
167-
if sha.hexdigest() != checksum[1]:
168-
os.remove(location.name)
169-
e = exception.ImageVerificationFailed(
170-
url=url, expected=checksum, obtained=sha.hexdigest()
171-
)
172-
LOG.error(e)
173-
raise e
174-
175-
LOG.info("Image '%s' downloaded", url)
176-
return location
177-
178-
17985
class DistroManager(object):
18086
"""Class to manage the distributions."""
18187

18288
def __init__(self):
18389
"""Initialize the DistroManager object."""
184-
self.distros = stevedore.NamedExtensionManager(
185-
"imgsync.distros",
186-
CONF.distributions,
187-
invoke_on_load=True,
188-
propagate_map_exceptions=True,
189-
)
90+
self.distros = []
91+
92+
for distro in CONF.distributions:
93+
if distro not in SUPPORTED_DISTROS:
94+
raise ValueError("Unsupported distribution %s" % distro)
95+
self.distros.append(_DISTRO_OBJS[distro]())
19096

19197
def sync(self):
19298
"""Sync the distributions."""
193-
LOG.info("Syncing %s", self.distros.names())
194-
self.distros.map_method("sync")
99+
if CONF.download_only:
100+
LOG.warn("Only downloading the images, not checkinf if they need sync.")
101+
102+
if CONF.dry_run:
103+
LOG.warn("Dry run, not syncing the images to glance.")
104+
105+
for distro in self.distros:
106+
LOG.info("Syncing %s", distro.name)
107+
distro.sync()

‎imgsync/distros/base.py

+166
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
"""Base class for all distributions."""
2+
3+
import abc
4+
import hashlib
5+
import os
6+
import tempfile
7+
8+
from oslo_config import cfg
9+
from oslo_log import log
10+
import requests
11+
12+
from imgsync import exception
13+
from imgsync import glance
14+
15+
CONF = cfg.CONF
16+
17+
LOG = log.getLogger(__name__)
18+
19+
20+
class BaseDistro(object, metaclass=abc.ABCMeta):
21+
"""Base class for all distributions."""
22+
23+
url = None
24+
25+
def __init__(self):
26+
"""Initialize the BaseDistro object."""
27+
self.glance = glance.GLANCE
28+
29+
@abc.abstractproperty
30+
def what(self):
31+
"""Get what to sync. This has to be implemented by the child class."""
32+
return None
33+
34+
def sync(self):
35+
"""Sync the images, calling the method that is needed."""
36+
if self.what == "all":
37+
self._sync_all()
38+
elif self.what == "latest":
39+
self._sync_latest()
40+
else:
41+
LOG.warn("Nothing to do")
42+
43+
def _get_file_checksum(self, path, block_size=2**20):
44+
"""Get the checksum of a file.
45+
46+
Get the checksum of a file using sha512.
47+
48+
:param path: the path to the file
49+
:param block_size: block size to use when reading the file
50+
:returns: sha512 object
51+
"""
52+
sha512 = hashlib.sha512()
53+
with open(path, "rb") as f:
54+
buf = f.read(block_size)
55+
while len(buf) > 0:
56+
sha512.update(buf)
57+
buf = f.read(block_size)
58+
return sha512
59+
60+
def verify_checksum(
61+
self,
62+
location,
63+
name,
64+
checksum,
65+
):
66+
"""Verify the image's checksum."""
67+
# TODO(aloga): not implemented yet
68+
69+
def _download_one(self, url, checksum):
70+
"""Download a file.
71+
72+
Download a file from a url and return a temporary file object.
73+
74+
:param url: the url to download
75+
:param checksum: tuple in the form (checksum_name, checksum_value)
76+
:returns: temporary file object
77+
"""
78+
with tempfile.NamedTemporaryFile(suffix=".imgsync", delete=False) as location:
79+
try:
80+
response = requests.get(url, stream=True, timeout=10)
81+
except Exception as e:
82+
os.remove(location.name)
83+
LOG.error(e)
84+
raise exception.ImageDownloadFailed(code=e.errno, reason=e.message)
85+
86+
if not response.ok:
87+
os.remove(location.name)
88+
LOG.error(
89+
"Cannot download image: (%s) %s",
90+
response.status_code,
91+
response.reason,
92+
)
93+
raise exception.ImageDownloadFailed(
94+
code=response.status_code, reason=response.reason
95+
)
96+
97+
for block in response.iter_content(1024):
98+
if block:
99+
location.write(block)
100+
location.flush()
101+
102+
checksum_map = {"sha512": hashlib.sha512, "sha256": hashlib.sha256}
103+
sha = checksum_map.get(checksum[0])()
104+
block_size = 2**20
105+
with open(location.name, "rb") as f:
106+
buf = f.read(block_size)
107+
while len(buf) > 0:
108+
sha.update(buf)
109+
buf = f.read(block_size)
110+
111+
if sha.hexdigest() != checksum[1]:
112+
os.remove(location.name)
113+
e = exception.ImageVerificationFailed(
114+
url=url, expected=checksum, obtained=sha.hexdigest()
115+
)
116+
LOG.error(e)
117+
raise e
118+
119+
LOG.info("Image '%s' downloaded", url)
120+
return location
121+
122+
def _needs_download(self, name, checksum_type, checksum):
123+
"""Check if the image needs to be downloaded."""
124+
if CONF.download_only:
125+
return True
126+
127+
image = self.glance.get_image_by_name(name)
128+
if image:
129+
if image.get("imgsync.%s" % checksum_type) == checksum:
130+
LOG.info("Image already downloaded and synchroniced")
131+
return False
132+
else:
133+
LOG.error(
134+
"Glance image chechsum (%s, %s) and official "
135+
"checksum %s missmatch.",
136+
image.id,
137+
image.get("imgsync.%s" % checksum_type),
138+
checksum,
139+
)
140+
return True
141+
142+
def _sync_with_glance(
143+
self, name, url, distro, checksum_type, checksum, architecture, file_format
144+
):
145+
"""Upload the image to glance."""
146+
location = None
147+
try:
148+
location = self._download_one(url, (checksum_type, checksum))
149+
if not (CONF.download_only or CONF.dry_run):
150+
self.glance.upload(
151+
location,
152+
name,
153+
architecture=architecture,
154+
file_format=file_format,
155+
container_format="bare",
156+
checksum={checksum_type: checksum},
157+
os_distro=distro,
158+
os_version=self.version,
159+
)
160+
LOG.info("Synchronized %s", name)
161+
else:
162+
LOG.info("Downloaded %s", name)
163+
finally:
164+
if location is not None:
165+
LOG.debug("Removing %s", location.name)
166+
os.remove(location.name)

0 commit comments

Comments
 (0)