Skip to content

Commit 2f3c71e

Browse files
crazywhalecccrazywhalecc
authored
Downloader enhancement (#685)
* Quiet console output for non --debug mode * Adjust console output and PHPDoc * Allow locking different arch pre-built content * Add install-pkg and pre-built test * Fix typo * Add debug console output for Downloader * Add libc version for pre-built content name * Separate musl-dist and non-musl-dist * Add additional log output for pre-built finder * Return default version for musl and musl-wrapper * Test arm runner * Re-enable musl version detect * Add upx cmd for tests * Remove comment * Add SPC_DOCKER_DEBUG for gnu docker, remove classmap for alpine docker * Add glibc build for CI * Fix PHP warning in test-extensions.php * Remove redundant suffix, add libc version suffix * Fix redundant pre-built name calling * Fix redundant pre-built name calling * Fix CI wrong runner name * Fix end of line space * Full spell for SPC_DOWNLOAD
2 parents ee54b6d + 5c04638 commit 2f3c71e

18 files changed

+292
-115
lines changed

‎.github/workflows/build-unix.yml

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,13 @@ on:
66
os:
77
required: true
88
description: Build target OS
9+
default: 'linux-x86_64'
910
type: choice
1011
options:
1112
- 'linux-x86_64'
1213
- 'linux-aarch64'
14+
- 'linux-x86_64-glibc'
15+
- 'linux-aarch64-glibc'
1316
- 'macos-x86_64'
1417
- 'macos-aarch64'
1518
php-version:
@@ -22,7 +25,6 @@ on:
2225
- '8.3'
2326
- '8.2'
2427
- '8.1'
25-
- '8.0'
2628
extensions:
2729
description: Extensions to build (comma separated)
2830
required: true
@@ -77,9 +79,19 @@ jobs:
7779
RUNS_ON="ubuntu-latest"
7880
;;
7981
linux-aarch64)
80-
DOWN_CMD="SPC_USE_ARCH=aarch64 ./bin/spc-alpine-docker download"
81-
BUILD_CMD="SPC_USE_ARCH=aarch64 ./bin/spc-alpine-docker build"
82-
RUNS_ON="ubuntu-latest"
82+
DOWN_CMD="./bin/spc-alpine-docker download"
83+
BUILD_CMD="./bin/spc-alpine-docker build"
84+
RUNS_ON="ubuntu-24.04-arm"
85+
;;
86+
linux-x86_64-glibc)
87+
DOWN_CMD="./bin/spc-gnu-docker download"
88+
BUILD_CMD="./bin/spc-gnu-docker build"
89+
RUNS_ON="ubuntu-22.04"
90+
;;
91+
linux-aarch64-glibc)
92+
DOWN_CMD="./bin/spc-gnu-docker download"
93+
BUILD_CMD="./bin/spc-gnu-docker build"
94+
RUNS_ON="ubuntu-22.04-arm"
8395
;;
8496
macos-x86_64)
8597
DOWN_CMD="composer update --no-dev --classmap-authoritative && ./bin/spc doctor --auto-fix && ./bin/spc download"

‎.github/workflows/tests.yml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -176,18 +176,18 @@ jobs:
176176
run: composer update -q --no-ansi --no-interaction --no-scripts --no-progress --prefer-dist
177177

178178
- name: "Run Build Tests (doctor)"
179-
run: bin/spc doctor --auto-fix --debug
179+
run: php src/globals/test-extensions.php doctor_cmd ${{ matrix.os }} ${{ matrix.php }}
180180

181181
- name: "Prepare UPX for Windows"
182-
if: matrix.os == 'windows-latest'
182+
if: ${{ startsWith(matrix.os, 'windows-') }}
183183
run: |
184-
bin/spc install-pkg upx
184+
php src/globals/test-extensions.php install_upx_cmd ${{ matrix.os }} ${{ matrix.php }}
185185
echo "UPX_CMD=$(php src/globals/test-extensions.php upx)" >> $env:GITHUB_ENV
186186
187187
- name: "Prepare UPX for Linux"
188-
if: matrix.os == 'ubunut-latest'
188+
if: ${{ startsWith(matrix.os, 'ubuntu-') }}
189189
run: |
190-
bin/spc install-pkg upx
190+
php src/globals/test-extensions.php install_upx_cmd ${{ matrix.os }} ${{ matrix.php }}
191191
echo "UPX_CMD=$(php src/globals/test-extensions.php upx)" >> $GITHUB_ENV
192192
193193
- name: "Run Build Tests (download)"

‎bin/spc-alpine-docker

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ WORKDIR /app
9595
ADD ./src /app/src
9696
COPY ./composer.* /app/
9797
ADD ./bin /app/bin
98-
RUN composer install --no-dev --classmap-authoritative
98+
RUN composer install --no-dev
9999
EOF
100100
fi
101101

‎bin/spc-gnu-docker

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -145,4 +145,8 @@ echo 'SPC_CMD_VAR_PHP_MAKE_EXTRA_LIBS="-ldl -lpthread -lm -lresolv -lutil -lrt"'
145145
# shellcheck disable=SC2086
146146
# shellcheck disable=SC2090
147147

148-
$DOCKER_EXECUTABLE run --rm $INTERACT -e SPC_FIX_DEPLOY_ROOT="$(pwd)" --env-file /tmp/spc-gnu-docker.env $MOUNT_LIST cwcc-spc-gnu-$SPC_USE_ARCH bin/spc $@
148+
if [ "$SPC_DOCKER_DEBUG" = "yes" ]; then
149+
$DOCKER_EXECUTABLE run --rm -it --privileged $INTERACT -e SPC_FIX_DEPLOY_ROOT="$(pwd)" --env-file /tmp/spc-gnu-docker.env $MOUNT_LIST cwcc-spc-gnu-$SPC_USE_ARCH
150+
else
151+
$DOCKER_EXECUTABLE run --rm $INTERACT -e SPC_FIX_DEPLOY_ROOT="$(pwd)" --env-file /tmp/spc-gnu-docker.env $MOUNT_LIST cwcc-spc-gnu-$SPC_USE_ARCH bin/spc $@
152+
fi

‎config/pre-built.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"repo": "static-php/static-php-cli-hosted",
33
"prefer-stable": true,
4-
"match-pattern": "{name}-{arch}-{os}.txz",
5-
"suffix": "txz"
6-
}
4+
"match-pattern-linux": "{name}-{arch}-{os}-{libc}-{libcver}.txz",
5+
"match-pattern": "{name}-{arch}-{os}.txz"
6+
}

‎src/SPC/builder/LibraryBase.php

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
use SPC\exception\RuntimeException;
99
use SPC\exception\WrongUsageException;
1010
use SPC\store\Config;
11+
use SPC\store\Downloader;
1112
use SPC\store\FileSystem;
1213
use SPC\store\SourceManager;
1314

@@ -45,8 +46,9 @@ public function setup(bool $force = false): int
4546
$lock = json_decode(FileSystem::readFile(DOWNLOAD_PATH . '/.lock.json'), true) ?? [];
4647
$source = Config::getLib(static::NAME, 'source');
4748
// if source is locked as pre-built, we just tryInstall it
48-
if (isset($lock[$source]) && ($lock[$source]['lock_as'] ?? SPC_LOCK_SOURCE) === SPC_LOCK_PRE_BUILT) {
49-
return $this->tryInstall($lock[$source]['filename'], $force);
49+
$pre_built_name = Downloader::getPreBuiltLockName($source);
50+
if (isset($lock[$pre_built_name]) && ($lock[$pre_built_name]['lock_as'] ?? SPC_DOWNLOAD_SOURCE) === SPC_DOWNLOAD_PRE_BUILT) {
51+
return $this->tryInstall($lock[$pre_built_name]['filename'], $force);
5052
}
5153
return $this->tryBuild($force);
5254
}

‎src/SPC/builder/linux/SystemUtil.php

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,4 +182,39 @@ public static function getSupportedDistros(): array
182182
'arch', 'manjaro',
183183
];
184184
}
185+
186+
/**
187+
* Get libc version string from ldd
188+
*/
189+
public static function getLibcVersionIfExists(): ?string
190+
{
191+
if (PHP_OS_FAMILY === 'Linux' && getenv('SPC_LIBC') === 'glibc') {
192+
$result = shell()->execWithResult('ldd --version', false);
193+
if ($result[0] !== 0) {
194+
return null;
195+
}
196+
// get first line
197+
$first_line = $result[1][0];
198+
// match ldd version: "ldd (some useless text) 2.17" match 2.17
199+
$pattern = '/ldd\s+\(.*?\)\s+(\d+\.\d+)/';
200+
if (preg_match($pattern, $first_line, $matches)) {
201+
return $matches[1];
202+
}
203+
return null;
204+
}
205+
if (PHP_OS_FAMILY === 'Linux' && getenv('SPC_LIBC') === 'musl') {
206+
if (self::isMuslDist()) {
207+
$result = shell()->execWithResult('ldd 2>&1', false);
208+
} else {
209+
$result = shell()->execWithResult('/usr/local/musl/lib/libc.so 2>&1', false);
210+
}
211+
// Match Version * line
212+
// match ldd version: "Version 1.2.3" match 1.2.3
213+
$pattern = '/Version\s+(\d+\.\d+\.\d+)/';
214+
if (preg_match($pattern, $result[1][1] ?? '', $matches)) {
215+
return $matches[1];
216+
}
217+
}
218+
return null;
219+
}
185220
}

‎src/SPC/command/DeleteDownloadCommand.php

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
use SPC\exception\DownloaderException;
88
use SPC\exception\FileSystemException;
99
use SPC\exception\WrongUsageException;
10+
use SPC\store\Downloader;
1011
use SPC\store\FileSystem;
1112
use Symfony\Component\Console\Attribute\AsCommand;
1213
use Symfony\Component\Console\Input\InputArgument;
@@ -47,30 +48,35 @@ public function handle(): int
4748
$chosen_sources = $sources;
4849
$lock = json_decode(FileSystem::readFile(DOWNLOAD_PATH . '/.lock.json'), true) ?? [];
4950

51+
$deleted_sources = [];
5052
foreach ($chosen_sources as $source) {
5153
$source = trim($source);
52-
if (!isset($lock[$source])) {
53-
logger()->warning("Source/Package [{$source}] not locked or not downloaded, skipped.");
54-
continue;
54+
foreach ([$source, Downloader::getPreBuiltLockName($source)] as $name) {
55+
if (isset($lock[$name])) {
56+
$deleted_sources[] = $name;
57+
}
5558
}
59+
}
60+
61+
foreach ($deleted_sources as $lock_name) {
5662
// remove download file/dir if exists
57-
if ($lock[$source]['source_type'] === 'archive') {
58-
if (file_exists($path = FileSystem::convertPath(DOWNLOAD_PATH . '/' . $lock[$source]['filename']))) {
63+
if ($lock[$lock_name]['source_type'] === 'archive') {
64+
if (file_exists($path = FileSystem::convertPath(DOWNLOAD_PATH . '/' . $lock[$lock_name]['filename']))) {
5965
logger()->info('Deleting file ' . $path);
6066
unlink($path);
6167
} else {
62-
logger()->warning("Source/Package [{$source}] file not found, skip deleting file.");
68+
logger()->warning("Source/Package [{$lock_name}] file not found, skip deleting file.");
6369
}
6470
} else {
65-
if (is_dir($path = FileSystem::convertPath(DOWNLOAD_PATH . '/' . $lock[$source]['dirname']))) {
71+
if (is_dir($path = FileSystem::convertPath(DOWNLOAD_PATH . '/' . $lock[$lock_name]['dirname']))) {
6672
logger()->info('Deleting dir ' . $path);
6773
FileSystem::removeDir($path);
6874
} else {
69-
logger()->warning("Source/Package [{$source}] directory not found, skip deleting dir.");
75+
logger()->warning("Source/Package [{$lock_name}] directory not found, skip deleting dir.");
7076
}
7177
}
7278
// remove locked sources
73-
unset($lock[$source]);
79+
unset($lock[$lock_name]);
7480
}
7581
FileSystem::writeFile(DOWNLOAD_PATH . '/.lock.json', json_encode($lock, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE));
7682
logger()->info('Delete success!');

‎src/SPC/command/DownloadCommand.php

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
namespace SPC\command;
66

7+
use SPC\builder\linux\SystemUtil;
78
use SPC\builder\traits\UnixSystemUtilTrait;
89
use SPC\exception\DownloaderException;
910
use SPC\exception\FileSystemException;
@@ -212,7 +213,7 @@ public function handle(): int
212213
if (isset($config['filename'])) {
213214
$new_config['filename'] = $config['filename'];
214215
}
215-
logger()->info("Fetching source {$source} from custom url [{$ni}/{$cnt}]");
216+
logger()->info("[{$ni}/{$cnt}] Downloading source {$source} from custom url: {$new_config['url']}");
216217
Downloader::downloadSource($source, $new_config, true);
217218
} elseif (isset($custom_gits[$source])) {
218219
$config = Config::getSource($source);
@@ -224,23 +225,30 @@ public function handle(): int
224225
if (isset($config['path'])) {
225226
$new_config['path'] = $config['path'];
226227
}
227-
logger()->info("Fetching source {$source} from custom git [{$ni}/{$cnt}]");
228+
logger()->info("[{$ni}/{$cnt}] Downloading source {$source} from custom git: {$new_config['url']}");
228229
Downloader::downloadSource($source, $new_config, true);
229230
} else {
230231
$config = Config::getSource($source);
231232
// Prefer pre-built, we need to search pre-built library
232233
if ($this->getOption('prefer-pre-built') && ($config['provide-pre-built'] ?? false) === true) {
233234
// We need to replace pattern
234-
$find = str_replace(['{name}', '{arch}', '{os}'], [$source, arch2gnu(php_uname('m')), strtolower(PHP_OS_FAMILY)], Config::getPreBuilt('match-pattern'));
235+
$replace = [
236+
'{name}' => $source,
237+
'{arch}' => arch2gnu(php_uname('m')),
238+
'{os}' => strtolower(PHP_OS_FAMILY),
239+
'{libc}' => getenv('SPC_LIBC') ?: 'default',
240+
'{libcver}' => PHP_OS_FAMILY === 'Linux' ? (SystemUtil::getLibcVersionIfExists() ?? 'default') : 'default',
241+
];
242+
$find = str_replace(array_keys($replace), array_values($replace), Config::getPreBuilt('match-pattern'));
235243
// find filename in asset list
236244
if (($url = $this->findPreBuilt($pre_built_libs, $find)) !== null) {
237-
logger()->info("Fetching pre-built content {$source} [{$ni}/{$cnt}]");
238-
Downloader::downloadSource($source, ['type' => 'url', 'url' => $url], $force_all || in_array($source, $force_list), SPC_LOCK_PRE_BUILT);
245+
logger()->info("[{$ni}/{$cnt}] Downloading pre-built content {$source}");
246+
Downloader::downloadSource($source, ['type' => 'url', 'url' => $url], $force_all || in_array($source, $force_list), SPC_DOWNLOAD_PRE_BUILT);
239247
continue;
240248
}
241249
logger()->warning("Pre-built content not found for {$source}, fallback to source download");
242250
}
243-
logger()->info("Fetching source {$source} [{$ni}/{$cnt}]");
251+
logger()->info("[{$ni}/{$cnt}] Downloading source {$source}");
244252
Downloader::downloadSource($source, $config, $force_all || in_array($source, $force_list));
245253
}
246254
}
@@ -352,6 +360,7 @@ private function calculateSourcesByLib(array $libs, bool $include_suggests = tru
352360
*/
353361
private function findPreBuilt(array $assets, string $filename): ?string
354362
{
363+
logger()->debug("Finding pre-built asset {$filename}");
355364
foreach ($assets as $asset) {
356365
if ($asset['name'] === $filename) {
357366
return $asset['browser_download_url'];

‎src/SPC/command/dev/PackLibCommand.php

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
use SPC\builder\BuilderProvider;
88
use SPC\builder\LibraryBase;
9+
use SPC\builder\linux\SystemUtil;
910
use SPC\command\BuildCommand;
1011
use SPC\exception\ExceptionHandler;
1112
use SPC\exception\FileSystemException;
@@ -23,6 +24,7 @@ class PackLibCommand extends BuildCommand
2324
public function configure(): void
2425
{
2526
$this->addArgument('library', InputArgument::REQUIRED, 'The library will be compiled');
27+
$this->addOption('show-libc-ver', null, null);
2628
}
2729

2830
public function handle(): int
@@ -47,7 +49,7 @@ public function handle(): int
4749
// Get lock info
4850
$lock = json_decode(file_get_contents(DOWNLOAD_PATH . '/.lock.json'), true) ?? [];
4951
$source = Config::getLib($lib->getName(), 'source');
50-
if (!isset($lock[$source]) || ($lock[$source]['lock_as'] ?? SPC_LOCK_SOURCE) === SPC_LOCK_PRE_BUILT) {
52+
if (!isset($lock[$source]) || ($lock[$source]['lock_as'] ?? SPC_DOWNLOAD_SOURCE) === SPC_DOWNLOAD_PRE_BUILT) {
5153
logger()->critical("The library {$lib->getName()} is downloaded as pre-built, we need to build it instead of installing pre-built.");
5254
return static::FAILURE;
5355
}
@@ -69,7 +71,16 @@ public function handle(): int
6971
// write list to packlib_files.txt
7072
FileSystem::writeFile(WORKING_DIR . '/packlib_files.txt', implode("\n", $increase_files));
7173
// pack
72-
$filename = WORKING_DIR . '/dist/' . $lib->getName() . '-' . arch2gnu(php_uname('m')) . '-' . strtolower(PHP_OS_FAMILY) . '.' . Config::getPreBuilt('suffix');
74+
$filename = Config::getPreBuilt('match-pattern');
75+
$replace = [
76+
'{name}' => $lib->getName(),
77+
'{arch}' => arch2gnu(php_uname('m')),
78+
'{os}' => strtolower(PHP_OS_FAMILY),
79+
'{libc}' => getenv('SPC_LIBC') ?: 'default',
80+
'{libcver}' => PHP_OS_FAMILY === 'Linux' ? (SystemUtil::getLibcVersionIfExists() ?? 'default') : 'default',
81+
];
82+
$filename = str_replace(array_keys($replace), array_values($replace), $filename);
83+
$filename = WORKING_DIR . '/dist/' . $filename;
7384
f_passthru('tar -czf ' . $filename . ' -T ' . WORKING_DIR . '/packlib_files.txt');
7485
logger()->info('Pack library ' . $lib->getName() . ' to ' . $filename . ' complete.');
7586
}

‎src/SPC/store/Config.php

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,30 @@ class Config
2222

2323
public static ?array $pre_built = null;
2424

25+
/**
26+
* @throws WrongUsageException
27+
* @throws FileSystemException
28+
*/
2529
public static function getPreBuilt(string $name): mixed
2630
{
2731
if (self::$pre_built === null) {
2832
self::$pre_built = FileSystem::loadConfigArray('pre-built');
2933
}
34+
$supported_sys_based = ['match-pattern', 'prefer-stable', 'repo'];
35+
if (in_array($name, $supported_sys_based)) {
36+
$m_key = match (PHP_OS_FAMILY) {
37+
'Windows' => ['-windows', '-win', ''],
38+
'Darwin' => ['-macos', '-unix', ''],
39+
'Linux' => ['-linux', '-unix', ''],
40+
'BSD' => ['-freebsd', '-bsd', '-unix', ''],
41+
default => throw new WrongUsageException('OS ' . PHP_OS_FAMILY . ' is not supported'),
42+
};
43+
foreach ($m_key as $v) {
44+
if (isset(self::$pre_built["{$name}{$v}"])) {
45+
return self::$pre_built["{$name}{$v}"];
46+
}
47+
}
48+
}
3049
return self::$pre_built[$name] ?? null;
3150
}
3251

0 commit comments

Comments
 (0)