Skip to content

Commit 4e6e64b

Browse files
adamrehnadamrehn
committed
Split engine components into separate filesystem layers
1 parent a7111bd commit 4e6e64b

File tree

8 files changed

+160
-122
lines changed

8 files changed

+160
-122
lines changed

‎ue4docker/build.py

+1-2
Original file line numberDiff line numberDiff line change
@@ -285,8 +285,7 @@ def build():
285285
# Build the minimal UE4 CI image, unless requested otherwise by the user
286286
buildUe4Minimal = config.noMinimal == False
287287
if buildUe4Minimal == True:
288-
buildGraphArg = ['--build-arg', 'BUILDGRAPH_ARGS=' + ' '.join(config.buildGraphArgs)]
289-
builder.build('ue4-minimal', mainTags, commonArgs + config.platformArgs + config.exclusionFlags + ue4BuildArgs + buildGraphArg)
288+
builder.build('ue4-minimal', mainTags, commonArgs + config.platformArgs + ue4BuildArgs)
290289
builtImages.append('ue4-minimal')
291290
else:
292291
logger.info('User specified `--no-minimal`, skipping ue4-minimal image build.')

‎ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile

+16-12
Original file line numberDiff line numberDiff line change
@@ -35,16 +35,13 @@ RUN python3 /tmp/patch-build-graph.py /home/ue4/UnrealEngine/Engine/Build/Instal
3535
RUN ./Engine/Build/BatchFiles/Linux/Build.sh UnrealHeaderTool Linux Development -SkipBuild
3636

3737
# Create an Installed Build of the Engine
38-
ARG BUILDGRAPH_ARGS
3938
WORKDIR /home/ue4/UnrealEngine
40-
RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build Linux" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true ${BUILDGRAPH_ARGS} {{ buildgraph_args }} && \
39+
RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build Linux" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} {{ buildgraph_args }} && \
4140
rm -R -f /home/ue4/UnrealEngine/LocalBuilds/InstalledDDC
4241

43-
# Determine if we are removing debug symbols and/or template projects in order to reduce the final container image size
44-
COPY exclude-components.py /tmp/exclude-components.py
45-
ARG EXCLUDE_DEBUG
46-
ARG EXCLUDE_TEMPLATES
47-
RUN python3 /tmp/exclude-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux $EXCLUDE_DEBUG $EXCLUDE_TEMPLATES
42+
# Split out components (DDC, debug symbols, template projects) so they can be copied into the final container image as separate filesystem layers
43+
COPY split-components.py /tmp/split-components.py
44+
RUN python3 /tmp/split-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux /home/ue4/UnrealEngine/Components
4845

4946
{% if (not disable_all_patches) and (not disable_target_patches) %}
5047
# Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini
@@ -73,15 +70,22 @@ FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG}
7370

7471
# Copy the Installed Build files from the builder image
7572
COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux /home/ue4/UnrealEngine
73+
{% if excluded_components.ddc == false %}
74+
COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/DDC /home/ue4/UnrealEngine
75+
{% endif %}
76+
{% if excluded_components.debug == false %}
77+
COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/DebugSymbols /home/ue4/UnrealEngine
78+
{% endif %}
79+
{% if excluded_components.templates == false %}
80+
COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/TemplatesAndSamples /home/ue4/UnrealEngine
81+
{% endif %}
7682
WORKDIR /home/ue4/UnrealEngine
7783

7884
{% if not disable_labels %}
7985
# Add labels to the built image to identify which components (if any) were excluded from the build that it contains
80-
# (Note that we need to redeclare the relevant ARG directives here because they are scoped to each individual stage in a multi-stage build)
81-
ARG EXCLUDE_DEBUG
82-
ARG EXCLUDE_TEMPLATES
83-
LABEL com.adamrehn.ue4-docker.excluded.debug=${EXCLUDE_DEBUG}
84-
LABEL com.adamrehn.ue4-docker.excluded.templates=${EXCLUDE_TEMPLATES}
86+
LABEL com.adamrehn.ue4-docker.excluded.ddc={% if excluded_components.ddc == true %}1{% else %}0{% endif %}
87+
LABEL com.adamrehn.ue4-docker.excluded.debug={% if excluded_components.debug == true %}1{% else %}0{% endif %}
88+
LABEL com.adamrehn.ue4-docker.excluded.templates={% if excluded_components.templates == true %}1{% else %}0{% endif %}
8589
{% endif %}
8690

8791
# Perform first-run setup for Mono, UnrealBuildTool and AutomationTool, which makes it possible to build Unreal projects and plugins as users other than `ue4`

‎ue4docker/dockerfiles/ue4-minimal/linux/exclude-components.py

-43
This file was deleted.

‎ue4docker/dockerfiles/ue4-minimal/linux/split-components.py

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/usr/bin/env python3
2+
import glob, os, shutil, sys
3+
from os.path import basename, dirname, exists, join
4+
5+
# Logs a message to stderr
6+
def log(message):
7+
print(message, file=sys.stderr)
8+
sys.stderr.flush()
9+
10+
# Extracts the files and directories for the specified component and moves them to a separate output directory
11+
def extractComponent(inputDir, outputDir, component, description, items):
12+
13+
# Print progress output
14+
log('\nExtracting {}...'.format(description))
15+
16+
# Create the output directory for the component if it doesn't already exist
17+
componentDir = join(outputDir, component)
18+
os.makedirs(outputDir, exist_ok=True)
19+
20+
# Move each file and directory for the component to the output directory
21+
for item in items:
22+
23+
# Verify that the item exists
24+
if not exists(item):
25+
log('Skipping non-existent item: {}'.format(item))
26+
continue
27+
28+
# Print progress output
29+
log('Moving: {}'.format(item))
30+
31+
# Ensure the parent directory for the item exists in the output directory
32+
parent = dirname(item).replace(inputDir, componentDir)
33+
os.makedirs(parent, exist_ok=True)
34+
35+
# Perform the move
36+
shutil.move(
37+
item,
38+
join(parent, basename(item))
39+
)
40+
41+
# Retrieve the path to the root directory of the Installed Build
42+
rootDir = sys.argv[1]
43+
44+
# Retrieve the path to the root output directory for extracted components and ensure it exists
45+
outputDir = sys.argv[2]
46+
os.makedirs(outputDir, exist_ok=True)
47+
48+
# Extract the DDC
49+
ddc = [join(rootDir, 'Engine', 'DerivedDataCache', 'Compressed.ddp')]
50+
extractComponent(rootDir, outputDir, 'DDC', 'Derived Data Cache (DDC)', ddc)
51+
52+
# Extract debug symbols
53+
symbolFiles = glob.glob(join(rootDir, '**', '*.debug'), recursive=True) + glob.glob(join(rootDir, '**', '*.sym'), recursive=True)
54+
extractComponent(rootDir, outputDir, 'DebugSymbols', 'debug symbols', symbolFiles)
55+
56+
# Extract template projects and samples
57+
subdirs = [join(rootDir, subdir) for subdir in ['FeaturePacks', 'Samples', 'Templates']]
58+
extractComponent(rootDir, outputDir, 'TemplatesAndSamples', 'template projects and samples', subdirs)

‎ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile

+18-11
Original file line numberDiff line numberDiff line change
@@ -28,14 +28,12 @@ RUN python C:\patch-build-graph.py C:\UnrealEngine\Engine\Build\InstalledEngineB
2828
# Create an Installed Build of the Engine
2929
ARG BUILDGRAPH_ARGS
3030
WORKDIR C:\UnrealEngine
31-
RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true %BUILDGRAPH_ARGS% {{ buildgraph_args }} && `
31+
RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} {{ buildgraph_args }} && `
3232
(if exist C:\UnrealEngine\LocalBuilds\InstalledDDC rmdir /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC)
3333

34-
# Determine if we are removing debug symbols and/or template projects in order to reduce the final container image size
35-
COPY exclude-components.py C:\exclude-components.py
36-
ARG EXCLUDE_DEBUG
37-
ARG EXCLUDE_TEMPLATES
38-
RUN python C:\exclude-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows %EXCLUDE_DEBUG% %EXCLUDE_TEMPLATES%
34+
# Split out components (DDC, debug symbols, template projects) so they can be copied into the final container image as separate filesystem layers
35+
COPY split-components.py C:\split-components.py
36+
RUN python C:\split-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows C:\UnrealEngine\Components
3937

4038
{% if (not disable_all_patches) and (not disable_target_patches) %}
4139
# Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini
@@ -50,14 +48,23 @@ FROM prerequisites as minimal
5048
ARG NAMESPACE
5149
FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG}
5250
{% endif %}
51+
52+
# Copy the Installed Build files from the builder image
5353
COPY --from=builder C:\UnrealEngine\LocalBuilds\Engine\Windows C:\UnrealEngine
54+
{% if excluded_components.ddc == false %}
55+
COPY --from=builder C:\UnrealEngine\Components\DDC C:\UnrealEngine
56+
{% endif %}
57+
{% if excluded_components.debug == false %}
58+
COPY --from=builder C:\UnrealEngine\Components\DebugSymbols C:\UnrealEngine
59+
{% endif %}
60+
{% if excluded_components.templates == false %}
61+
COPY --from=builder C:\UnrealEngine\Components\TemplatesAndSamples C:\UnrealEngine
62+
{% endif %}
5463
WORKDIR C:\UnrealEngine
5564

5665
{% if not disable_labels %}
5766
# Add labels to the built image to identify which components (if any) were excluded from the build that it contains
58-
# (Note that we need to redeclare the relevant ARG directives here because they are scoped to each individual stage in a multi-stage build)
59-
ARG EXCLUDE_DEBUG
60-
ARG EXCLUDE_TEMPLATES
61-
LABEL com.adamrehn.ue4-docker.excluded.debug=${EXCLUDE_DEBUG}
62-
LABEL com.adamrehn.ue4-docker.excluded.templates=${EXCLUDE_TEMPLATES}
67+
LABEL com.adamrehn.ue4-docker.excluded.ddc={% if excluded_components.ddc == true %}1{% else %}0{% endif %}
68+
LABEL com.adamrehn.ue4-docker.excluded.debug={% if excluded_components.debug == true %}1{% else %}0{% endif %}
69+
LABEL com.adamrehn.ue4-docker.excluded.templates={% if excluded_components.templates == true %}1{% else %}0{% endif %}
6370
{% endif %}

‎ue4docker/dockerfiles/ue4-minimal/windows/exclude-components.py

-43
This file was deleted.

‎ue4docker/dockerfiles/ue4-minimal/windows/split-components.py

+58
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/usr/bin/env python3
2+
import glob, os, shutil, sys
3+
from os.path import basename, dirname, exists, join
4+
5+
# Logs a message to stderr
6+
def log(message):
7+
print(message, file=sys.stderr)
8+
sys.stderr.flush()
9+
10+
# Extracts the files and directories for the specified component and moves them to a separate output directory
11+
def extractComponent(inputDir, outputDir, component, description, items):
12+
13+
# Print progress output
14+
log('\nExtracting {}...'.format(description))
15+
16+
# Create the output directory for the component if it doesn't already exist
17+
componentDir = join(outputDir, component)
18+
os.makedirs(outputDir, exist_ok=True)
19+
20+
# Move each file and directory for the component to the output directory
21+
for item in items:
22+
23+
# Verify that the item exists
24+
if not exists(item):
25+
log('Skipping non-existent item: {}'.format(item))
26+
continue
27+
28+
# Print progress output
29+
log('Moving: {}'.format(item))
30+
31+
# Ensure the parent directory for the item exists in the output directory
32+
parent = dirname(item).replace(inputDir, componentDir)
33+
os.makedirs(parent, exist_ok=True)
34+
35+
# Perform the move
36+
shutil.move(
37+
item,
38+
join(parent, basename(item))
39+
)
40+
41+
# Retrieve the path to the root directory of the Installed Build
42+
rootDir = sys.argv[1]
43+
44+
# Retrieve the path to the root output directory for extracted components and ensure it exists
45+
outputDir = sys.argv[2]
46+
os.makedirs(outputDir, exist_ok=True)
47+
48+
# Extract the DDC
49+
ddc = [join(rootDir, 'Engine', 'DerivedDataCache', 'Compressed.ddp')]
50+
extractComponent(rootDir, outputDir, 'DDC', 'Derived Data Cache (DDC)', ddc)
51+
52+
# Extract debug symbols
53+
symbolFiles = glob.glob(join(rootDir, '**', '*.pdb'), recursive=True)
54+
extractComponent(rootDir, outputDir, 'DebugSymbols', 'debug symbols', symbolFiles)
55+
56+
# Extract template projects and samples
57+
subdirs = [join(rootDir, subdir) for subdir in ['FeaturePacks', 'Samples', 'Templates']]
58+
extractComponent(rootDir, outputDir, 'TemplatesAndSamples', 'template projects and samples', subdirs)

‎ue4docker/infrastructure/BuildConfiguration.py

+9-11
Original file line numberDiff line numberDiff line change
@@ -210,15 +210,13 @@ def __init__(self, parser, argv):
210210
if self.opts.get('credential_mode', 'endpoint') not in validCredentialModes:
211211
raise RuntimeError('invalid value specified for the `credential_mode` option, valid values are {} when building {} containers'.format(validCredentialModes, self.containerPlatform.title()))
212212

213-
# Generate our flags for keeping or excluding components
214-
self.buildGraphArgs = [
215-
'-set:WithDDC={}'.format('false' if ExcludedComponent.DDC in self.excludedComponents else 'true')
216-
]
217-
self.exclusionFlags = [
218-
'--build-arg', 'EXCLUDE_DEBUG={}'.format(1 if ExcludedComponent.Debug in self.excludedComponents else 0),
219-
'--build-arg', 'EXCLUDE_TEMPLATES={}'.format(1 if ExcludedComponent.Templates in self.excludedComponents else 0)
220-
]
221-
213+
# Generate Jinja context values for keeping or excluding components
214+
self.opts['excluded_components'] = {
215+
'ddc': ExcludedComponent.DDC in self.excludedComponents,
216+
'debug': ExcludedComponent.Debug in self.excludedComponents,
217+
'templates': ExcludedComponent.Templates in self.excludedComponents
218+
}
219+
222220
# If we're building Windows containers, generate our Windows-specific configuration settings
223221
if self.containerPlatform == 'windows':
224222
self._generateWindowsConfig()
@@ -257,8 +255,8 @@ def _generateWindowsConfig(self):
257255
# Note: We must not pass VS2019 arg for older UE4 versions that didn't have VS2019 variable in their build graph xml.
258256
# Otherwise, UAT errors out with "Unknown argument: VS2019".
259257
if self.visualStudio != VisualStudio.VS2017:
260-
self.buildGraphArgs += [f'-set:VS{self.visualStudio}=true']
261-
258+
self.opts['buildgraph_args'] = self.opts.get('buildgraph_args', '') + f' -set:VS{self.visualStudio}=true'
259+
262260
# Determine base tag for the Windows release of the host system
263261
self.hostRelease = WindowsUtils.getWindowsRelease()
264262
self.hostBasetag = WindowsUtils.getReleaseBaseTag(self.hostRelease)

0 commit comments

Comments
 (0)