Skip to content

Add support for custom build-prerequisites Dockerfile #260

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 8, 2022
Merged

Add support for custom build-prerequisites Dockerfile #260

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 32 additions & 15 deletions ue4docker/build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,14 +88,11 @@ def build():
# Create an auto-deleting temporary directory to hold our build context
with tempfile.TemporaryDirectory() as tempDir:

# Copy our Dockerfiles to the temporary directory
contextOrig = join(os.path.dirname(os.path.abspath(__file__)), "dockerfiles")
contextRoot = join(tempDir, "dockerfiles")
shutil.copytree(contextOrig, contextRoot)

# Create the builder instance to build the Docker images
builder = ImageBuilder(
contextRoot,
join(tempDir, "dockerfiles"),
config.containerPlatform,
logger,
config.rebuild,
Expand Down Expand Up @@ -383,17 +380,37 @@ def build():
"VISUAL_STUDIO_BUILD_NUMBER=" + config.visualStudioBuildNumber,
]

builder.build(
"ue4-build-prerequisites",
[config.prereqsTag],
commonArgs + config.platformArgs + prereqsArgs,
)
builtImages.append("ue4-build-prerequisites")
custom_prerequisites_dockerfile = config.args.prerequisites_dockerfile
if custom_prerequisites_dockerfile is not None:
builder.build_builtin_image(
"ue4-base-build-prerequisites",
[config.prereqsTag],
commonArgs + config.platformArgs + prereqsArgs,
builtin_name="ue4-build-prerequisites",
)
builtImages.append("ue4-base-build-prerequisites")
else:
builder.build_builtin_image(
"ue4-build-prerequisites",
[config.prereqsTag],
commonArgs + config.platformArgs + prereqsArgs,
)

prereqConsumerArgs = [
"--build-arg",
"PREREQS_TAG={}".format(config.prereqsTag),
]

if custom_prerequisites_dockerfile is not None:
builder.build(
"ue4-build-prerequisites",
[config.prereqsTag],
commonArgs + config.platformArgs + prereqConsumerArgs,
dockerfile_template=custom_prerequisites_dockerfile,
context_dir=os.path.dirname(custom_prerequisites_dockerfile),
)

builtImages.append("ue4-build-prerequisites")
else:
logger.info("Skipping ue4-build-prerequisities image build.")

Expand All @@ -418,11 +435,11 @@ def build():
"--build-arg",
"VERBOSE_OUTPUT={}".format("1" if config.verbose == True else "0"),
]
builder.build(
builder.build_builtin_image(
"ue4-source",
mainTags,
commonArgs + config.platformArgs + ue4SourceArgs + credentialArgs,
secrets,
secrets=secrets,
)
builtImages.append("ue4-source")
else:
Expand All @@ -436,7 +453,7 @@ def build():

# Build the UE4 Engine source build image, unless requested otherwise by the user
if config.buildTargets["engine"]:
builder.build(
builder.build_builtin_image(
"ue4-engine",
mainTags,
commonArgs + config.platformArgs + ue4BuildArgs,
Expand All @@ -453,7 +470,7 @@ def build():
else []
)

builder.build(
builder.build_builtin_image(
"ue4-minimal",
mainTags,
commonArgs + config.platformArgs + ue4BuildArgs + minimalArgs,
Expand Down Expand Up @@ -483,7 +500,7 @@ def build():
)

# Build the image
builder.build(
builder.build_builtin_image(
"ue4-full",
mainTags,
commonArgs
Expand Down
5 changes: 5 additions & 0 deletions ue4docker/infrastructure/BuildConfiguration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,11 @@ def addArguments(parser):
default=None,
help="Set a specific changelist number in the Unreal Engine's Build.version file",
)
parser.add_argument(
"--prerequisites-dockerfile",
default=None,
help="Specifies path to custom ue4-build-prerequisites dockerfile",
)

def __init__(self, parser, argv, logger):
"""
Expand Down
4 changes: 2 additions & 2 deletions ue4docker/infrastructure/DockerUtils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ def exists(name):
return False

@staticmethod
def build(tags, context, args):
def build(tags: [str], context: str, args: [str]) -> [str]:
"""
Returns the `docker build` command to build an image
"""
Expand All @@ -58,7 +58,7 @@ def build(tags, context, args):
)

@staticmethod
def buildx(tags, context, args, secrets):
def buildx(tags: [str], context: str, args: [str], secrets: [str]) -> [str]:
"""
Returns the `docker buildx` command to build an image with the BuildKit backend
"""
Expand Down
131 changes: 91 additions & 40 deletions ue4docker/infrastructure/ImageBuilder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,25 +5,34 @@
from .GlobalConfiguration import GlobalConfiguration
import glob, humanfriendly, os, shutil, subprocess, tempfile, time
from os.path import basename, exists, join
from jinja2 import Environment, Template
from jinja2 import Environment


class ImageBuildParams(object):
def __init__(
self, dockerfile: str, context_dir: str, env: Optional[Dict[str, str]] = None
):
self.dockerfile = dockerfile
self.context_dir = context_dir
self.env = env


class ImageBuilder(object):
def __init__(
self,
root,
platform,
tempDir: str,
platform: str,
logger,
rebuild=False,
dryRun=False,
layoutDir=None,
templateContext=None,
combine=False,
rebuild: bool = False,
dryRun: bool = False,
layoutDir: str = None,
templateContext: Dict[str, str] = None,
combine: bool = False,
):
"""
Creates an ImageBuilder for the specified build parameters
"""
self.root = root
self.tempDir = tempDir
self.platform = platform
self.logger = logger
self.rebuild = rebuild
Expand All @@ -32,17 +41,60 @@ def __init__(
self.templateContext = templateContext if templateContext is not None else {}
self.combine = combine

def build(self, name, tags, args, secrets=None):
def get_built_image_context(self, name):
"""
Resolve the full path to the build context for the specified image
"""
return os.path.normpath(
os.path.join(
os.path.dirname(os.path.abspath(__file__)),
"..",
"dockerfiles",
basename(name),
self.platform,
)
)

def build_builtin_image(
self,
name: str,
tags: [str],
args: [str],
builtin_name: str = None,
secrets: Dict[str, str] = None,
):
context_dir = self.get_built_image_context(
name if builtin_name is None else builtin_name
)
return self.build(
name, tags, args, join(context_dir, "Dockerfile"), context_dir, secrets
)

def build(
self,
name: str,
tags: [str],
args: [str],
dockerfile_template: str,
context_dir: str,
secrets: Dict[str, str] = None,
):
"""
Builds the specified image if it doesn't exist or if we're forcing a rebuild
"""

workdir = join(self.tempDir, basename(name), self.platform)
os.makedirs(workdir, exist_ok=True)

# Create a Jinja template environment and render the Dockerfile template
environment = Environment(
autoescape=False, trim_blocks=True, lstrip_blocks=True
)
dockerfile = join(self.context(name), "Dockerfile")
templateInstance = environment.from_string(FilesystemUtils.readFile(dockerfile))
dockerfile = join(workdir, "Dockerfile")

templateInstance = environment.from_string(
FilesystemUtils.readFile(dockerfile_template)
)
rendered = templateInstance.render(self.templateContext)

# Compress excess whitespace introduced during Jinja rendering and save the contents back to disk
Expand Down Expand Up @@ -70,7 +122,6 @@ def build(self, name, tags, args, secrets=None):

# Determine whether we are building using `docker buildx` with build secrets
imageTags = self._formatTags(name, tags)
command = DockerUtils.build(imageTags, self.context(name), args)

if self.platform == "linux" and secrets is not None and len(secrets) > 0:

Expand All @@ -82,9 +133,11 @@ def build(self, name, tags, args, secrets=None):
secretFlags.append("id={},src={}".format(secret, secretFile))

# Generate the `docker buildx` command to use our build secrets
command = DockerUtils.buildx(
imageTags, self.context(name), args, secretFlags
)
command = DockerUtils.buildx(imageTags, context_dir, args, secretFlags)
else:
command = DockerUtils.build(imageTags, context_dir, args)

command += ["--file", dockerfile]

env = os.environ.copy()
if self.platform == "linux":
Expand All @@ -97,41 +150,35 @@ def build(self, name, tags, args, secrets=None):
command,
"build",
"built",
env=env,
ImageBuildParams(dockerfile, context_dir, env),
)

def context(self, name):
"""
Resolve the full path to the build context for the specified image
"""
return join(self.root, basename(name), self.platform)

def pull(self, image):
def pull(self, image: str) -> None:
"""
Pulls the specified image if it doesn't exist or if we're forcing a pull of a newer version
"""
self._processImage(image, None, DockerUtils.pull(image), "pull", "pulled")

def willBuild(self, name, tags):
def willBuild(self, name: str, tags: [str]) -> bool:
"""
Determines if we will build the specified image, based on our build settings
"""
imageTags = self._formatTags(name, tags)
return self._willProcess(imageTags[0])

def _formatTags(self, name, tags):
def _formatTags(self, name: str, tags: [str]):
"""
Generates the list of fully-qualified tags that we will use when building an image
"""
return [
"{}:{}".format(GlobalConfiguration.resolveTag(name), tag) for tag in tags
]

def _willProcess(self, image):
def _willProcess(self, image: [str]) -> bool:
"""
Determines if we will build or pull the specified image, based on our build settings
"""
return self.rebuild == True or DockerUtils.exists(image) == False
return self.rebuild or not DockerUtils.exists(image)

def _processImage(
self,
Expand All @@ -140,14 +187,14 @@ def _processImage(
command: [str],
actionPresentTense: str,
actionPastTense: str,
env: Optional[Dict[str, str]] = None,
):
build_params: Optional[ImageBuildParams] = None,
) -> None:
"""
Processes the specified image by running the supplied command if it doesn't exist (use rebuild=True to force processing)
"""

# Determine if we are processing the image
if self._willProcess(image) == False:
if not self._willProcess(image):
self.logger.info(
'Image "{}" exists and rebuild not requested, skipping {}.'.format(
image, actionPresentTense
Expand All @@ -159,7 +206,7 @@ def _processImage(
self.logger.action(
'{}ing image "{}"...'.format(actionPresentTense.capitalize(), image)
)
if self.dryRun == True:
if self.dryRun:
print(command)
self.logger.action(
'Completed dry run for image "{}".'.format(image), newline=False
Expand All @@ -170,19 +217,19 @@ def _processImage(
if self.layoutDir is not None:

# Determine whether we're performing a simple copy or combining generated Dockerfiles
source = self.context(name)
if self.combine == True:
if self.combine:

# Ensure the destination directory exists
dest = join(self.layoutDir, "combined")
self.logger.action(
'Merging "{}" into "{}"...'.format(source, dest), newline=False
'Merging "{}" into "{}"...'.format(build_params.context_dir, dest),
newline=False,
)
os.makedirs(dest, exist_ok=True)

# Merge the source Dockerfile with any existing Dockerfile contents in the destination directory
# (Insert a single newline between merged file contents and ensure we have a single trailing newline)
sourceDockerfile = join(source, "Dockerfile")
sourceDockerfile = build_params.dockerfile
destDockerfile = join(dest, "Dockerfile")
dockerfileContents = (
FilesystemUtils.readFile(destDockerfile)
Expand All @@ -199,7 +246,7 @@ def _processImage(

# Copy any supplemental files from the source directory to the destination directory
# (Exclude any extraneous files which are not referenced in the Dockerfile contents)
for file in glob.glob(join(source, "*.*")):
for file in glob.glob(join(build_params.context_dir, "*.*")):
if basename(file) in dockerfileContents:
shutil.copy(file, join(dest, basename(file)))

Expand All @@ -213,9 +260,11 @@ def _processImage(
# Copy the source directory to the destination
dest = join(self.layoutDir, basename(name))
self.logger.action(
'Copying "{}" to "{}"...'.format(source, dest), newline=False
'Copying "{}" to "{}"...'.format(build_params.context_dir, dest),
newline=False,
)
shutil.copytree(source, dest)
shutil.copytree(build_params.context_dir, dest)
shutil.copy(build_params.dockerfile, dest)
self.logger.action(
'Copied Dockerfile for image "{}".'.format(image), newline=False
)
Expand All @@ -224,7 +273,9 @@ def _processImage(

# Attempt to process the image using the supplied command
startTime = time.time()
exitCode = subprocess.call(command, env=env)
exitCode = subprocess.call(
command, env=build_params.env if build_params else None
)
endTime = time.time()

# Determine if processing succeeded
Expand Down