Skip to content

Commit 7584e96

Browse files
3XX03XX0
committed
Filter NVRM proc filesystem based on visible devices
1 parent 93c46e1 commit 7584e96

File tree

4 files changed

+148
-11
lines changed

4 files changed

+148
-11
lines changed

‎src/nvc_internal.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
#define NV_UVM_TOOLS_DEVICE_PATH _PATH_DEV "nvidia-uvm-tools"
3131
#define NV_PERSISTENCED_SOCKET _PATH_VARRUN "nvidia-persistenced/socket"
3232
#define NV_MPS_PIPE_DIR _PATH_TMP "nvidia-mps"
33+
#define NV_PROC_DRIVER "/proc/driver/nvidia"
34+
#define NV_UVM_PROC_DRIVER "/proc/driver/nvidia-uvm"
3335

3436
struct nvc_context {
3537
bool initialized;

‎src/nvc_mount.c

Lines changed: 106 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,11 @@
1919
#include "utils.h"
2020
#include "xfuncs.h"
2121

22-
#define CGROUP_DEVICE_ALLOW "devices.allow"
23-
#define CGROUP_DEVICE_STR "c %u:%u %s"
24-
2522
static char *mount_files(struct error *, const struct nvc_container *, const char *, char *[], size_t);
2623
static char *mount_device(struct error *, const struct nvc_container *, const char *);
2724
static char *mount_ipc(struct error *, const struct nvc_container *, const char *);
25+
static char *mount_procfs(struct error *, const struct nvc_container *);
26+
static char *mount_procfs_gpu(struct error *, const struct nvc_container *, const char *);
2827
static void unmount(const char *);
2928
static int setup_cgroup(struct error *, const char *, dev_t);
3029
static int symlink_library(struct error *, const char *, const char *, const char *, const char *, uid_t, gid_t);
@@ -129,10 +128,95 @@ mount_ipc(struct error *err, const struct nvc_container *cnt, const char *ipc)
129128
return (NULL);
130129
}
131130

131+
static char *
132+
mount_procfs(struct error *err, const struct nvc_container *cnt)
133+
{
134+
char path[PATH_MAX];
135+
char *ptr, *mnt, *param;
136+
mode_t mode;
137+
char *buf = NULL;
138+
const char *files[] = {
139+
NV_PROC_DRIVER "/params",
140+
NV_PROC_DRIVER "/version",
141+
NV_PROC_DRIVER "/registry",
142+
};
143+
144+
if (path_resolve(err, path, cnt->cfg.rootfs, NV_PROC_DRIVER) < 0)
145+
return (NULL);
146+
log_infof("mounting tmpfs at %s", path);
147+
if (xmount(err, "tmpfs", path, "tmpfs", 0, "mode=0555") < 0)
148+
return (NULL);
149+
150+
ptr = path + strlen(path);
151+
152+
for (size_t i = 0; i < nitems(files); ++i) {
153+
if (file_mode(err, files[i], &mode) < 0) {
154+
if (err->code == ENOENT)
155+
continue;
156+
goto fail;
157+
}
158+
if (file_read_text(err, files[i], &buf) < 0)
159+
goto fail;
160+
/* Prevent NVRM from ajusting the device nodes. */
161+
if (i == 0 && (param = strstr(buf, "ModifyDeviceFiles: 1")) != NULL)
162+
param[19] = '0';
163+
if (path_append(err, path, basename(files[i])) < 0)
164+
goto fail;
165+
if (file_create(err, path, buf, cnt->uid, cnt->gid, mode) < 0)
166+
goto fail;
167+
*ptr = '\0';
168+
free(buf);
169+
buf = NULL;
170+
}
171+
if (xmount(err, NULL, path, NULL, MS_REMOUNT | MS_NODEV|MS_NOSUID|MS_NOEXEC, NULL) < 0)
172+
goto fail;
173+
if ((mnt = xstrdup(err, path)) == NULL)
174+
goto fail;
175+
return (mnt);
176+
177+
fail:
178+
*ptr = '\0';
179+
free(buf);
180+
unmount(path);
181+
return (NULL);
182+
}
183+
184+
static char *
185+
mount_procfs_gpu(struct error *err, const struct nvc_container *cnt, const char *busid)
186+
{
187+
char path[PATH_MAX] = {0};
188+
char *gpu = NULL;
189+
char *mnt = NULL;
190+
mode_t mode;
191+
192+
if (xasprintf(err, &gpu, "%s/gpus/%s", NV_PROC_DRIVER, busid) < 0)
193+
return (NULL);
194+
if (file_mode(err, gpu, &mode) < 0)
195+
goto fail;
196+
if (path_resolve(err, path, cnt->cfg.rootfs, gpu) < 0)
197+
goto fail;
198+
if (file_create(err, path, NULL, cnt->uid, cnt->gid, mode) < 0)
199+
goto fail;
200+
201+
log_infof("mounting %s at %s", gpu, path);
202+
if (xmount(err, gpu, path, NULL, MS_BIND, NULL) < 0)
203+
goto fail;
204+
if (xmount(err, NULL, path, NULL, MS_BIND|MS_REMOUNT | MS_RDONLY|MS_NODEV|MS_NOSUID|MS_NOEXEC, NULL) < 0)
205+
goto fail;
206+
if ((mnt = xstrdup(err, path)) == NULL)
207+
goto fail;
208+
209+
fail:
210+
if (mnt == NULL)
211+
unmount(path);
212+
free(gpu);
213+
return (mnt);
214+
}
215+
132216
static void
133217
unmount(const char *path)
134218
{
135-
if (path == NULL)
219+
if (path == NULL || strempty(path))
136220
return;
137221
umount2(path, MNT_DETACH);
138222
file_remove(NULL, path);
@@ -145,14 +229,14 @@ setup_cgroup(struct error *err, const char *cgroup, dev_t id)
145229
FILE *fs;
146230
int rv = -1;
147231

148-
if (path_join(err, path, cgroup, CGROUP_DEVICE_ALLOW) < 0)
232+
if (path_join(err, path, cgroup, "devices.allow") < 0)
149233
return (-1);
150234
if ((fs = xfopen(err, path, "a")) == NULL)
151235
return (-1);
152236

153237
log_infof("whitelisting device node %u:%u", major(id), minor(id));
154238
/* XXX dprintf doesn't seem to catch the write errors, flush the stream explicitly instead. */
155-
if (fprintf(fs, CGROUP_DEVICE_STR, major(id), minor(id), "rwm") < 0 || fflush(fs) == EOF || ferror(fs)) {
239+
if (fprintf(fs, "c %u:%u rw", major(id), minor(id)) < 0 || fflush(fs) == EOF || ferror(fs)) {
156240
error_set(err, "write error: %s", path);
157241
goto fail;
158242
}
@@ -208,6 +292,7 @@ symlink_libraries(struct error *err, const struct nvc_container *cnt, const char
208292
int
209293
nvc_driver_mount(struct nvc_context *ctx, const struct nvc_container *cnt, const struct nvc_driver_info *info)
210294
{
295+
char *procfs_mnt = NULL;
211296
char **files_mnt = NULL;
212297
char **ipcs_mnt = NULL;
213298
char **devs_mnt = NULL;
@@ -225,6 +310,10 @@ nvc_driver_mount(struct nvc_context *ctx, const struct nvc_container *cnt, const
225310
if (nsenter(&ctx->err, cnt->mnt_ns, CLONE_NEWNS) < 0)
226311
return (-1);
227312

313+
/* Procfs mount */
314+
if ((procfs_mnt = mount_procfs(&ctx->err, cnt)) == NULL)
315+
goto fail;
316+
228317
/* File mounts */
229318
nfiles_mnt = 3;
230319
files_mnt = mnt = array_new(&ctx->err, nfiles_mnt);
@@ -279,6 +368,7 @@ nvc_driver_mount(struct nvc_context *ctx, const struct nvc_container *cnt, const
279368

280369
fail:
281370
if (rv < 0) {
371+
unmount(procfs_mnt);
282372
for (size_t i = 0; files_mnt != NULL && i < nfiles_mnt; ++i)
283373
unmount(files_mnt[i]);
284374
for (size_t i = 0; ipcs_mnt != NULL && i < nipcs_mnt; ++i)
@@ -290,6 +380,7 @@ nvc_driver_mount(struct nvc_context *ctx, const struct nvc_container *cnt, const
290380
rv = nsenterat(&ctx->err, ctx->mnt_ns, CLONE_NEWNS);
291381
}
292382

383+
free(procfs_mnt);
293384
array_free(files_mnt, nfiles_mnt);
294385
array_free(ipcs_mnt, nipcs_mnt);
295386
array_free(devs_mnt, ndevs_mnt);
@@ -299,7 +390,8 @@ nvc_driver_mount(struct nvc_context *ctx, const struct nvc_container *cnt, const
299390
int
300391
nvc_device_mount(struct nvc_context *ctx, const struct nvc_container *cnt, const struct nvc_device *dev)
301392
{
302-
char *mnt = NULL;
393+
char *dev_mnt = NULL;
394+
char *proc_mnt = NULL;
303395
struct stat s;
304396
int rv = -1;
305397

@@ -311,14 +403,16 @@ nvc_device_mount(struct nvc_context *ctx, const struct nvc_container *cnt, const
311403
if (nsenter(&ctx->err, cnt->mnt_ns, CLONE_NEWNS) < 0)
312404
return (-1);
313405

406+
if ((proc_mnt = mount_procfs_gpu(&ctx->err, cnt, dev->busid)) == NULL)
407+
goto fail;
314408
if (!(cnt->flags & OPT_NO_DEVBIND)) {
315409
if (xstat(&ctx->err, dev->node.path, &s) < 0)
316410
return (-1);
317411
if (s.st_rdev != dev->node.id) {
318412
error_setx(&ctx->err, "invalid device node: %s", dev->node.path);
319413
return (-1);
320414
}
321-
if ((mnt = mount_device(&ctx->err, cnt, dev->node.path)) == NULL)
415+
if ((dev_mnt = mount_device(&ctx->err, cnt, dev->node.path)) == NULL)
322416
goto fail;
323417
}
324418
if (!(cnt->flags & OPT_NO_CGROUPS)) {
@@ -329,12 +423,14 @@ nvc_device_mount(struct nvc_context *ctx, const struct nvc_container *cnt, const
329423

330424
fail:
331425
if (rv < 0) {
332-
unmount(mnt);
426+
unmount(proc_mnt);
427+
unmount(dev_mnt);
333428
assert_func(nsenterat(NULL, ctx->mnt_ns, CLONE_NEWNS));
334429
} else {
335430
rv = nsenterat(&ctx->err, ctx->mnt_ns, CLONE_NEWNS);
336431
}
337432

338-
free(mnt);
433+
free(proc_mnt);
434+
free(dev_mnt);
339435
return (rv);
340436
}

‎src/utils.c

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -436,6 +436,8 @@ file_create(struct error *err, const char *path, void *data, uid_t uid, gid_t gi
436436
gid_t egid;
437437
mode_t perm;
438438
int fd;
439+
size_t size;
440+
int flags = O_NOFOLLOW|O_CREAT;
439441
int rv = -1;
440442

441443
if ((p = xstrdup(err, path)) == NULL)
@@ -462,8 +464,16 @@ file_create(struct error *err, const char *path, void *data, uid_t uid, gid_t gi
462464
if (symlink(data, path) < 0 && errno != EEXIST)
463465
goto fail;
464466
} else {
465-
if ((fd = open(path, O_NOFOLLOW|O_CREAT, perm)) < 0)
467+
if (data != NULL) {
468+
size = strlen(data);
469+
flags |= O_WRONLY;
470+
}
471+
if ((fd = open(path, flags, perm)) < 0)
472+
goto fail;
473+
if (data != NULL && write(fd, data, size) < (ssize_t)size) {
474+
close(fd);
466475
goto fail;
476+
}
467477
close(fd);
468478
}
469479
rv = 0;
@@ -554,6 +564,34 @@ file_read_line(struct error *err, const char *path, char *buf, size_t size)
554564
return (rv);
555565
}
556566

567+
int
568+
file_read_text(struct error *err, const char *path, char **txt)
569+
{
570+
FILE *fs;
571+
size_t n;
572+
char buf[512];
573+
int rv = -1;
574+
575+
if ((fs = xfopen(err, path, "r")) == NULL)
576+
return (-1);
577+
*txt = NULL;
578+
while ((n = fread(buf, 1, sizeof(buf), fs)) > 0) {
579+
buf[n] = '\0';
580+
if (strjoin(err, txt, buf, "") < 0)
581+
goto fail;
582+
}
583+
if (feof(fs))
584+
rv = 0;
585+
else
586+
error_setx(err, "file read error: %s", path);
587+
588+
fail:
589+
if (rv < 0)
590+
free(*txt);
591+
fclose(fs);
592+
return (rv);
593+
}
594+
557595
int
558596
file_read_uint32(struct error *err, const char *path, uint32_t *v)
559597
{

‎src/utils.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ int file_remove(struct error *, const char *);
6666
int file_exists(struct error *, const char *);
6767
int file_mode(struct error *, const char *, mode_t *);
6868
int file_read_line(struct error *, const char *, char *, size_t);
69+
int file_read_text(struct error *, const char *, char **);
6970
int file_read_uint32(struct error *, const char *, uint32_t *);
7071

7172
int path_append(struct error *, char *, const char *);

0 commit comments

Comments
 (0)