Add support for device architecture requirement

3XX0 · 3XX0 · commit 6c67a195bf48 · 2017-10-25T16:08:07.000-07:00
diff --git a/src/driver.c b/src/driver.c
@@ -32,13 +32,19 @@
 #define SONAME_LIBCUDA "libcuda.so.1"
 #define SONAME_LIBNVML "libnvidia-ml.so.1"
 
+#define MAX_DEVICES     64
 #define REAP_TIMEOUT_MS 10
 
 static int reset_cuda_environment(struct error *);
 static int setup_rpc_client(struct driver *);
 static noreturn void setup_rpc_service(struct driver *, uid_t, gid_t, pid_t);
 static int reap_process(struct error *, pid_t, int, bool);
 
+static struct driver_device {
+        nvmlDevice_t nvml;
+        CUdevice cuda;
+} device_handles[MAX_DEVICES];
+
 #define call_nvml(ctx, sym, ...) __extension__ ({                                                      \
         union {void *ptr; __typeof__(&sym) fn;} u_;                                                    \
         nvmlReturn_t r_;                                                                               \
@@ -83,7 +89,7 @@ reset_cuda_environment(struct error *err)
         const struct { const char *name, *value; } env[] = {
                 {"CUDA_DISABLE_UNIFIED_MEMORY", "1"},
                 {"CUDA_CACHE_DISABLE", "1"},
-                {"CUDA_DEVICE_ORDER", "FASTEST_FIRST"},
+                {"CUDA_DEVICE_ORDER", "PCI_BUS_ID"},
                 {"CUDA_VISIBLE_DEVICES", NULL},
                 {"CUDA_MPS_PIPE_DIRECTORY", "/dev/null"},
         };
@@ -418,49 +424,46 @@ driver_get_device_count_1_svc(ptr_t ctxptr, driver_get_device_count_res *res, ma
 }
 
 int
-driver_get_device_handle(struct driver *ctx, unsigned int idx, driver_device_handle *dev, bool pci_order)
+driver_get_device(struct driver *ctx, unsigned int idx, struct driver_device **dev)
 {
-        struct driver_get_device_handle_res res = {0};
+        struct driver_get_device_res res = {0};
         int rv = -1;
 
-        if (call_rpc(ctx, &res, driver_get_device_handle_1, idx, pci_order) < 0)
+        if (call_rpc(ctx, &res, driver_get_device_1, idx) < 0)
                 goto fail;
-        *dev = (driver_device_handle)res.driver_get_device_handle_res_u.handle;
+        *dev = (struct driver_device *)res.driver_get_device_res_u.dev;
         rv = 0;
 
  fail:
-        xdr_free((xdrproc_t)xdr_driver_get_device_handle_res, (caddr_t)&res);
+        xdr_free((xdrproc_t)xdr_driver_get_device_res, (caddr_t)&res);
         return (rv);
 }
 
 bool_t
-driver_get_device_handle_1_svc(ptr_t ctxptr, u_int idx, bool_t pci_order, driver_get_device_handle_res *res, maybe_unused struct svc_req *req)
+driver_get_device_1_svc(ptr_t ctxptr, u_int idx, driver_get_device_res *res, maybe_unused struct svc_req *req)
 {
         struct driver *ctx = (struct driver *)ctxptr;
-        driver_device_handle handle;
-        CUdevice cudev;
         int domainid, deviceid, busid;
         char buf[NVML_DEVICE_PCI_BUS_ID_BUFFER_SIZE];
 
         memset(res, 0, sizeof(*res));
-        if (pci_order) {
-                if (call_nvml(ctx, nvmlDeviceGetHandleByIndex, idx, &handle) < 0)
-                        goto fail;
-        } else {
-                if (call_cuda(ctx, cuDeviceGet, &cudev, (int)idx) < 0)
-                        goto fail;
-                if (call_cuda(ctx, cuDeviceGetAttribute, &domainid, CU_DEVICE_ATTRIBUTE_PCI_DOMAIN_ID, cudev) < 0)
-                        goto fail;
-                if (call_cuda(ctx, cuDeviceGetAttribute, &busid, CU_DEVICE_ATTRIBUTE_PCI_BUS_ID, cudev) < 0)
-                        goto fail;
-                if (call_cuda(ctx, cuDeviceGetAttribute, &deviceid, CU_DEVICE_ATTRIBUTE_PCI_DEVICE_ID, cudev) < 0)
-                        goto fail;
-                snprintf(buf, sizeof(buf), "%04x:%02x:%02x.0", domainid, busid, deviceid);
-
-                if (call_nvml(ctx, nvmlDeviceGetHandleByPciBusId, buf, &handle) < 0)
-                        goto fail;
+        if (idx >= MAX_DEVICES) {
+                error_setx(ctx->err, "too many devices");
+                goto fail;
         }
-        res->driver_get_device_handle_res_u.handle = (ptr_t)handle;
+        if (call_cuda(ctx, cuDeviceGet, &device_handles[idx].cuda, (int)idx) < 0)
+                goto fail;
+        if (call_cuda(ctx, cuDeviceGetAttribute, &domainid, CU_DEVICE_ATTRIBUTE_PCI_DOMAIN_ID, device_handles[idx].cuda) < 0)
+                goto fail;
+        if (call_cuda(ctx, cuDeviceGetAttribute, &busid, CU_DEVICE_ATTRIBUTE_PCI_BUS_ID, device_handles[idx].cuda) < 0)
+                goto fail;
+        if (call_cuda(ctx, cuDeviceGetAttribute, &deviceid, CU_DEVICE_ATTRIBUTE_PCI_DEVICE_ID, device_handles[idx].cuda) < 0)
+                goto fail;
+        snprintf(buf, sizeof(buf), "%04x:%02x:%02x.0", domainid, busid, deviceid);
+        if (call_nvml(ctx, nvmlDeviceGetHandleByPciBusId, buf, &device_handles[idx].nvml) < 0)
+                goto fail;
+
+        res->driver_get_device_res_u.dev = (ptr_t)&device_handles[idx];
         return (true);
 
  fail:
@@ -469,7 +472,7 @@ driver_get_device_handle_1_svc(ptr_t ctxptr, u_int idx, bool_t pci_order, driver
 }
 
 int
-driver_get_device_minor(struct driver *ctx, driver_device_handle dev, unsigned int *minor)
+driver_get_device_minor(struct driver *ctx, struct driver_device *dev, unsigned int *minor)
 {
         struct driver_get_device_minor_res res = {0};
         int rv = -1;
@@ -488,10 +491,11 @@ bool_t
 driver_get_device_minor_1_svc(ptr_t ctxptr, ptr_t dev, driver_get_device_minor_res *res, maybe_unused struct svc_req *req)
 {
         struct driver *ctx = (struct driver *)ctxptr;
+        struct driver_device *handle = (struct driver_device *)dev;
         unsigned int minor;
 
         memset(res, 0, sizeof(*res));
-        if (call_nvml(ctx, nvmlDeviceGetMinorNumber, (nvmlDevice_t)dev, &minor) < 0)
+        if (call_nvml(ctx, nvmlDeviceGetMinorNumber, handle->nvml, &minor) < 0)
                 goto fail;
         res->driver_get_device_minor_res_u.minor = minor;
         return (true);
@@ -502,7 +506,7 @@ driver_get_device_minor_1_svc(ptr_t ctxptr, ptr_t dev, driver_get_device_minor_r
 }
 
 int
-driver_get_device_busid(struct driver *ctx, driver_device_handle dev, char **busid)
+driver_get_device_busid(struct driver *ctx, struct driver_device *dev, char **busid)
 {
         struct driver_get_device_busid_res res = {0};
         int rv = -1;
@@ -522,10 +526,11 @@ bool_t
 driver_get_device_busid_1_svc(ptr_t ctxptr, ptr_t dev, driver_get_device_busid_res *res, maybe_unused struct svc_req *req)
 {
         struct driver *ctx = (struct driver *)ctxptr;
+        struct driver_device *handle = (struct driver_device *)dev;
         nvmlPciInfo_t pci;
 
         memset(res, 0, sizeof(*res));
-        if (call_nvml(ctx, nvmlDeviceGetPciInfo_v2, (nvmlDevice_t)dev, &pci) < 0)
+        if (call_nvml(ctx, nvmlDeviceGetPciInfo_v2, handle->nvml, &pci) < 0)
                 goto fail;
         if ((res->driver_get_device_busid_res_u.busid = xstrdup(ctx->err, pci.busId)) == NULL)
                 goto fail;
@@ -537,7 +542,7 @@ driver_get_device_busid_1_svc(ptr_t ctxptr, ptr_t dev, driver_get_device_busid_r
 }
 
 int
-driver_get_device_uuid(struct driver *ctx, driver_device_handle dev, char **uuid)
+driver_get_device_uuid(struct driver *ctx, struct driver_device *dev, char **uuid)
 {
         struct driver_get_device_uuid_res res = {0};
         int rv = -1;
@@ -557,10 +562,11 @@ bool_t
 driver_get_device_uuid_1_svc(ptr_t ctxptr, ptr_t dev, driver_get_device_uuid_res *res, maybe_unused struct svc_req *req)
 {
         struct driver *ctx = (struct driver *)ctxptr;
+        struct driver_device *handle = (struct driver_device *)dev;
         char buf[NVML_DEVICE_UUID_BUFFER_SIZE];
 
         memset(res, 0, sizeof(*res));
-        if (call_nvml(ctx, nvmlDeviceGetUUID, (nvmlDevice_t)dev, buf, sizeof(buf)) < 0)
+        if (call_nvml(ctx, nvmlDeviceGetUUID, handle->nvml, buf, sizeof(buf)) < 0)
                 goto fail;
         if ((res->driver_get_device_uuid_res_u.uuid = xstrdup(ctx->err, buf)) == NULL)
                 goto fail;
@@ -570,3 +576,42 @@ driver_get_device_uuid_1_svc(ptr_t ctxptr, ptr_t dev, driver_get_device_uuid_res
         error_to_xdr(ctx->err, res);
         return (true);
 }
+
+int
+driver_get_device_arch(struct driver *ctx, struct driver_device *dev, char **arch)
+{
+        struct driver_get_device_arch_res res = {0};
+        int rv = -1;
+
+        if (call_rpc(ctx, &res, driver_get_device_arch_1, (ptr_t)dev) < 0)
+                goto fail;
+        if (xasprintf(ctx->err, arch, "%u.%u", res.driver_get_device_arch_res_u.arch.major,
+            res.driver_get_device_arch_res_u.arch.minor) < 0)
+                goto fail;
+        rv = 0;
+
+ fail:
+        xdr_free((xdrproc_t)xdr_driver_get_device_arch_res, (caddr_t)&res);
+        return (rv);
+}
+
+bool_t
+driver_get_device_arch_1_svc(ptr_t ctxptr, ptr_t dev, driver_get_device_arch_res *res, maybe_unused struct svc_req *req)
+{
+        struct driver *ctx = (struct driver *)ctxptr;
+        struct driver_device *handle = (struct driver_device *)dev;
+        int major, minor;
+
+        memset(res, 0, sizeof(*res));
+        if (call_cuda(ctx, cuDeviceGetAttribute, &major, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, handle->cuda) < 0)
+                goto fail;
+        if (call_cuda(ctx, cuDeviceGetAttribute, &minor, CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, handle->cuda) < 0)
+                goto fail;
+        res->driver_get_device_arch_res_u.arch.major = (unsigned int)major;
+        res->driver_get_device_arch_res_u.arch.minor = (unsigned int)minor;
+        return (true);
+
+ fail:
+        error_to_xdr(ctx->err, res);
+        return (true);
+}
diff --git a/src/driver.h b/src/driver.h
@@ -21,6 +21,8 @@ SVCXPRT *svcunixfd_create(int, u_int, u_int);
 #define SOCK_CLT 0
 #define SOCK_SVC 1
 
+struct driver_device;
+
 struct driver {
         struct error *err;
         void *cuda_dl;
@@ -31,18 +33,17 @@ struct driver {
         CLIENT *rpc_clt;
 };
 
-typedef struct nvmlDevice_st *driver_device_handle;
-
 void driver_program_1(struct svc_req *, register SVCXPRT *);
 
 int driver_init(struct driver *, struct error *, uid_t, gid_t);
 int driver_shutdown(struct driver *);
 int driver_get_rm_version(struct driver *, char **);
 int driver_get_cuda_version(struct driver *, char **);
 int driver_get_device_count(struct driver *, unsigned int *);
-int driver_get_device_handle(struct driver *, unsigned int, driver_device_handle *, bool);
-int driver_get_device_minor(struct driver *, driver_device_handle, unsigned int *);
-int driver_get_device_busid(struct driver *, driver_device_handle, char **);
-int driver_get_device_uuid(struct driver *, driver_device_handle, char **);
+int driver_get_device(struct driver *, unsigned int, struct driver_device **);
+int driver_get_device_minor(struct driver *, struct driver_device *, unsigned int *);
+int driver_get_device_busid(struct driver *, struct driver_device *, char **);
+int driver_get_device_uuid(struct driver *, struct driver_device *, char **);
+int driver_get_device_arch(struct driver *, struct driver_device *, char **);
 
 #endif /* HEADER_DRIVER_H */
diff --git a/src/driver_rpc.x b/src/driver_rpc.x
@@ -41,16 +41,28 @@ union driver_get_cuda_version_res switch (int errcode) {
                 string errmsg<>;
 };
 
+struct driver_device_arch {
+        unsigned int major;
+        unsigned int minor;
+};
+
+union driver_get_device_arch_res switch (int errcode) {
+        case 0:
+                driver_device_arch arch;
+        default:
+                string errmsg<>;
+};
+
 union driver_get_device_count_res switch (int errcode) {
         case 0:
                 unsigned int count;
         default:
                 string errmsg<>;
 };
 
-union driver_get_device_handle_res switch (int errcode) {
+union driver_get_device_res switch (int errcode) {
         case 0:
-                ptr_t handle;
+                ptr_t dev;
         default:
                 string errmsg<>;
 };
@@ -83,9 +95,10 @@ program DRIVER_PROGRAM {
                 driver_get_rm_version_res DRIVER_GET_RM_VERSION(ptr_t) = 3;
                 driver_get_cuda_version_res DRIVER_GET_CUDA_VERSION(ptr_t) = 4;
                 driver_get_device_count_res DRIVER_GET_DEVICE_COUNT(ptr_t) = 5;
-                driver_get_device_handle_res DRIVER_GET_DEVICE_HANDLE(ptr_t, unsigned int, bool) = 6;
+                driver_get_device_res DRIVER_GET_DEVICE(ptr_t, unsigned int) = 6;
                 driver_get_device_minor_res DRIVER_GET_DEVICE_MINOR(ptr_t, ptr_t) = 7;
                 driver_get_device_busid_res DRIVER_GET_DEVICE_BUSID(ptr_t, ptr_t) = 8;
                 driver_get_device_uuid_res DRIVER_GET_DEVICE_UUID(ptr_t, ptr_t) = 9;
+                driver_get_device_arch_res DRIVER_GET_DEVICE_ARCH(ptr_t, ptr_t) = 10;
         } = 1;
 } = 0x1;
diff --git a/src/dsl.c b/src/dsl.c
@@ -116,31 +116,39 @@ evaluate_rule(char *buf, char *expr, void *ctx, const struct dsl_rule rules[], s
 }
 
 int
-dsl_evaluate(struct error *err, char *expr, void *ctx, const struct dsl_rule rules[], size_t size)
+dsl_evaluate(struct error *err, const char *predicate, void *ctx, const struct dsl_rule rules[], size_t size)
 {
+        char *ptr, *expr = NULL;
         char *or_expr, *and_expr;
         int ret = true;
+        int rv = -1;
         char buf[EXPR_MAX];
 
-        while ((or_expr = strsep(&expr, " ")) != NULL) {
+        if ((expr = ptr = xstrdup(err, predicate)) == NULL)
+                goto fail;
+        while ((or_expr = strsep(&ptr, " ")) != NULL) {
                 if (*or_expr == '\0')
                         continue;
                 while ((and_expr = strsep(&or_expr, ",")) != NULL) {
                         if (*and_expr == '\0')
                                 continue;
                         if ((ret = evaluate_rule(buf, and_expr, ctx, rules, size)) < 0) {
                                 error_setx(err, "invalid expression");
-                                return (-1);
+                                goto fail;
                         }
                         if (!ret)
                                 break;
                 }
                 if (and_expr == NULL)
-                        return (0);
+                        break;
         }
         if (!ret) {
                 error_setx(err, "unsatisfied condition: %s", buf);
-                return (-1);
+                goto fail;
         }
-        return (0);
+        rv = 0;
+
+ fail:
+        free(expr);
+        return (rv);
 }
diff --git a/src/dsl.h b/src/dsl.h
@@ -24,6 +24,6 @@ struct dsl_rule {
 };
 
 int dsl_compare_version(const char *, enum dsl_comparator, const char *);
-int dsl_evaluate(struct error *, char *, void *, const struct dsl_rule [], size_t);
+int dsl_evaluate(struct error *, const char *, void *, const struct dsl_rule [], size_t);
 
 #endif /* HEADER_DSL_H */
diff --git a/src/nvc.h b/src/nvc.h
@@ -59,6 +59,7 @@ struct nvc_driver_info {
 struct nvc_device {
         char *uuid;
         char *busid;
+        char *arch;
         struct nvc_device_node node;
 };
 
diff --git a/src/nvc_cli.c b/src/nvc_cli.c
diff --git a/src/nvc_info.c b/src/nvc_info.c
