Fix install using remote yaml files (#3875)

Hind-M · web-flow · commit 5cef48350ad4 · 2025-04-08T10:47:42.000+02:00
diff --git a/libmamba/include/mamba/api/install.hpp b/libmamba/include/mamba/api/install.hpp
@@ -105,7 +105,8 @@ namespace mamba
 
         bool eval_selector(const std::string& selector, const std::string& platform);
 
-        yaml_file_contents read_yaml_file(fs::u8path yaml_file, const std::string platform);
+        yaml_file_contents
+        read_yaml_file(const Context& ctx, const std::string& yaml_file, const std::string& platform);
 
         inline void to_json(nlohmann::json&, const other_pkg_mgr_spec&)
         {
diff --git a/libmamba/src/api/install.cpp b/libmamba/src/api/install.cpp
@@ -82,13 +82,54 @@ namespace mamba
             return found_it->second;
         }
 
-        yaml_file_contents read_yaml_file(fs::u8path yaml_file, const std::string platform)
+        std::unique_ptr<TemporaryFile>
+        downloaded_file_from_url(const Context& ctx, const std::string& url_str)
         {
-            auto file = fs::weakly_canonical(util::expand_home(yaml_file.string()));
-            if (!fs::exists(file))
+            if (url_str.find("://") != std::string::npos)
+            {
+                LOG_INFO << "Downloading file from " << url_str;
+                auto url_parts = util::rsplit(url_str, '/');
+                std::string filename = (url_parts.size() == 1) ? "" : url_parts.back();
+                auto tmp_file = std::make_unique<TemporaryFile>("mambaf", util::concat("_", filename));
+                download::Request request(
+                    "Environment lock or yaml file",
+                    download::MirrorName(""),
+                    url_str,
+                    tmp_file->path()
+                );
+                const download::Result res = download::download(std::move(request), ctx.mirrors, ctx);
+
+                if (!res || res.value().transfer.http_status != 200)
+                {
+                    throw std::runtime_error(
+                        fmt::format("Could not download environment lock or yaml file from {}", url_str)
+                    );
+                }
+
+                return tmp_file;
+            }
+            return nullptr;
+        }
+
+        yaml_file_contents
+        read_yaml_file(const Context& ctx, const std::string& yaml_file, const std::string& platform)
+        {
+            // Download content of environment yaml file
+            auto tmp_yaml_file = downloaded_file_from_url(ctx, yaml_file);
+            fs::u8path file;
+
+            if (tmp_yaml_file)
+            {
+                file = tmp_yaml_file->path();
+            }
+            else
             {
-                LOG_ERROR << "YAML spec file '" << file.string() << "' not found";
-                throw std::runtime_error("File not found. Aborting.");
+                file = fs::weakly_canonical(util::expand_home(yaml_file));
+                if (!fs::exists(file))
+                {
+                    LOG_ERROR << "YAML spec file '" << file.string() << "' not found";
+                    throw std::runtime_error("File not found. Aborting.");
+                }
             }
 
             yaml_file_contents result;
@@ -148,8 +189,15 @@ namespace mamba
                         }
                         else if (key == "pip")
                         {
-                            const auto yaml_parent_path = fs::absolute(yaml_file).parent_path().string(
-                            );
+                            std::string yaml_parent_path;
+                            if (tmp_yaml_file)  // yaml file is fetched remotely
+                            {
+                                yaml_parent_path = yaml_file;
+                            }
+                            else
+                            {
+                                yaml_parent_path = fs::absolute(yaml_file).parent_path().string();
+                            }
                             result.others_pkg_mgrs_specs.push_back({
                                 "pip",
                                 map_el.second.as<std::vector<std::string>>(),
@@ -695,28 +743,11 @@ namespace mamba
         bool remove_prefix_on_failure
     )
     {
-        std::unique_ptr<TemporaryFile> tmp_lock_file;
         fs::u8path file;
+        auto tmp_lock_file = detail::downloaded_file_from_url(ctx, lockfile);
 
-        if (lockfile.find("://") != std::string::npos)
+        if (tmp_lock_file)
         {
-            LOG_INFO << "Downloading lockfile";
-            tmp_lock_file = std::make_unique<TemporaryFile>();
-            download::Request request(
-                "Environment Lockfile",
-                download::MirrorName(""),
-                lockfile,
-                tmp_lock_file->path()
-            );
-            const download::Result res = download::download(std::move(request), ctx.mirrors, ctx);
-
-            if (!res || res.value().transfer.http_status != 200)
-            {
-                throw std::runtime_error(
-                    fmt::format("Could not download environment lockfile from {}", lockfile)
-                );
-            }
-
             file = tmp_lock_file->path();
         }
         else
@@ -833,7 +864,7 @@ namespace mamba
                 }
                 else if (is_yaml_file_name(file))
                 {
-                    const auto parse_result = read_yaml_file(file, context.platform);
+                    const auto parse_result = read_yaml_file(context, file, context.platform);
 
                     if (parse_result.channels.size() != 0)
                     {
diff --git a/libmamba/tests/src/core/test_env_file_reading.cpp b/libmamba/tests/src/core/test_env_file_reading.cpp
@@ -48,6 +48,7 @@ namespace mamba
             const auto& context = mambatests::context();
             using V = std::vector<std::string>;
             auto res = detail::read_yaml_file(
+                context,
                 mambatests::test_data_dir / "env_file/env_1.yaml",
                 context.platform
             );
@@ -57,6 +58,7 @@ namespace mamba
             REQUIRE_FALSE(res.others_pkg_mgrs_specs.size());
 
             auto res2 = detail::read_yaml_file(
+                context,
                 mambatests::test_data_dir / "env_file/env_2.yaml",
                 context.platform
             );
@@ -77,6 +79,7 @@ namespace mamba
             const auto& context = mambatests::context();
             using V = std::vector<std::string>;
             auto res = detail::read_yaml_file(
+                context,
                 mambatests::test_data_dir / "env_file/env_3.yaml",
                 context.platform
             );
@@ -90,6 +93,66 @@ namespace mamba
             REQUIRE(o.deps == V({ "pytest", "numpy" }));
             REQUIRE(o.cwd == fs::absolute(mambatests::test_data_dir / "env_file"));
         }
+
+        TEST_CASE("remote_yaml_file")
+        {
+            SECTION("classic_env_yaml_file")
+            {
+                const auto& context = mambatests::context();
+                using V = std::vector<std::string>;
+                auto res = detail::read_yaml_file(
+                    context,
+                    "https://raw.githubusercontent.com/mamba-org/mamba/refs/heads/main/micromamba/tests/env-create-export.yaml",
+                    context.platform
+                );
+                REQUIRE(res.name == "");
+                REQUIRE(res.channels == V({ "https://conda.anaconda.org/conda-forge" }));
+                REQUIRE(res.dependencies == V({ "micromamba=0.24.0" }));
+            }
+
+            SECTION("env_yaml_file_with_pip")
+            {
+                const auto& context = mambatests::context();
+                using V = std::vector<std::string>;
+                auto res = detail::read_yaml_file(
+                    context,
+                    "https://raw.githubusercontent.com/mamba-org/mamba/refs/heads/main/libmamba/tests/data/env_file/env_3.yaml",
+                    context.platform
+                );
+                REQUIRE(res.name == "env_3");
+                REQUIRE(res.channels == V({ "conda-forge", "bioconda" }));
+                REQUIRE(res.dependencies == V({ "test1", "test2", "test3", "pip" }));
+
+                REQUIRE(res.others_pkg_mgrs_specs.size() == 1);
+                auto o = res.others_pkg_mgrs_specs[0];
+                REQUIRE(o.pkg_mgr == "pip");
+                REQUIRE(o.deps == V({ "pytest", "numpy" }));
+                REQUIRE(
+                    o.cwd == "https://raw.githubusercontent.com/mamba-org/mamba/refs/heads/main/libmamba/tests/data/env_file/env_3.yaml"
+                );
+            }
+
+            SECTION("env_yaml_file_with_specs_selection")
+            {
+                const auto& context = mambatests::context();
+                using V = std::vector<std::string>;
+                auto res = detail::read_yaml_file(
+                    context,
+                    "https://raw.githubusercontent.com/mamba-org/mamba/refs/heads/main/libmamba/tests/data/env_file/env_2.yaml",
+                    context.platform
+                );
+                REQUIRE(res.name == "env_2");
+                REQUIRE(res.channels == V({ "conda-forge", "bioconda" }));
+#ifdef __linux__
+                REQUIRE(res.dependencies == V({ "test1-unix", "test1-linux", "test2-linux", "test4" }));
+#elif __APPLE__
+                REQUIRE(res.dependencies == V({ "test1-unix", "test1-osx", "test4" }));
+#elif _WIN32
+                REQUIRE(res.dependencies == V({ "test1-win", "test4" }));
+#endif
+                REQUIRE_FALSE(res.others_pkg_mgrs_specs.size());
+            }
+        }
     }
 
 }  // namespace mamba
diff --git a/micromamba/tests/test_create.py b/micromamba/tests/test_create.py
@@ -1222,6 +1222,24 @@ def test_requires_pip_install_no_parent_dir_specified(
         os.chdir(initial_working_dir)  # Switch back to original working dir.
 
 
+@pytest.mark.parametrize("shared_pkgs_dirs", [True], indirect=True)
+def test_create_from_remote_yaml_file(tmp_home, tmp_root_prefix, tmp_path):
+    env_prefix = tmp_path / "myenv"
+    spec_file = "https://raw.githubusercontent.com/mamba-org/mamba/refs/heads/main/micromamba/tests/env-create-export.yaml"
+
+    res = helpers.create("-p", env_prefix, "-f", spec_file, "--json")
+    assert res["success"]
+
+    packages = helpers.umamba_list("-p", env_prefix, "--json")
+    assert any(
+        package["name"] == "micromamba"
+        and package["version"] == "0.24.0"
+        and package["channel"] == "conda-forge"
+        and package["base_url"] == "https://conda.anaconda.org/conda-forge"
+        for package in packages
+    )
+
+
 @pytest.mark.parametrize("shared_pkgs_dirs", [True], indirect=True)
 def test_pre_commit_compat(tmp_home, tmp_root_prefix, tmp_path):
     # We test compatibility with the downstream pre-commit package here because the pre-commit project does not currently accept any code changes related to Conda, see https://github.com/pre-commit/pre-commit/pull/2446#issuecomment-1353394177.

Original file line number	Diff line number	Diff line change
`@@ -105,7 +105,8 @@ namespace mamba`
`105`	`105`
`106`	`106`	`bool eval_selector(const std::string& selector, const std::string& platform);`
`107`	`107`
`108`		`- yaml_file_contents read_yaml_file(fs::u8path yaml_file, const std::string platform);`
	`108`	`+ yaml_file_contents`
	`109`	`+ read_yaml_file(const Context& ctx, const std::string& yaml_file, const std::string& platform);`
`109`	`110`
`110`	`111`	`inline void to_json(nlohmann::json&, const other_pkg_mgr_spec&)`
`111`	`112`	`{`