Merge pull request #2 from ties/feature/more-schemes-supported

Support more schemes

Author: hatamiarash7

src/functions/extract_domain.cpp

@@ -35,7 +35,7 @@ namespace duckdb
             Connection con (db);
 
             // Extract the host from the URL
-            std::regex host_regex (R"(^(?:https?:\/\/)?([^\/\?:]+))");
+            std::regex host_regex (R"(^(?:(?:https?|ftp|rsync):\/\/)?([^\/\?:]+))");
             std::smatch host_match;
             if (!std::regex_search (input, host_match, host_regex))
             {

src/functions/extract_host.cpp

@@ -24,12 +24,13 @@ namespace duckdb
         {
             // Regex to match the host component of a URL
             // Explanation:
-            // ^                - Start of the string
-            // (?:              - Non-capturing group for the optional protocol
-            //   https?:\/\/    - Matches "http://" or "https://"
+            // ^                        - Start of the string
+            // (?:                      - Non-capturing group for the optional protocol
+            //   (?:ftp|https?|rsync)   - Non-capturing group for the scheme
+            //   :\/\/                  - Matches "://"
             // )?
-            // ([^\/\s:?#]+)    - Capturing group for the host (any characters except '/', ':', '?', '#', or whitespace)
-            std::regex host_regex (R"(^(?:https?:\/\/)?([^\/\s:?#]+))");
+            // ([^\/\s:?#]+)            - Capturing group for the host (any characters except '/', ':', '?', '#', or whitespace)
+            std::regex host_regex (R"(^(?:(?:ftp|https?|rsync):\/\/)?([^\/\s:?#]+))");
             std::smatch host_match;
 
             // Use regex_search to find the host component in the input string

src/functions/extract_path.cpp

@@ -26,11 +26,11 @@ namespace duckdb
             // Explanation:
             // ^                - Start of the string
             // (?:              - Non-capturing group for the protocol and domain part
-            //   (?:https?:\/\/)? - Optional http:// or https://
+            //   (?:(?:ftp|https?|rsync):\/\/)? - Optional ftp://, http://, https://, or rsync://
             //   (?:[^\/\s]+)    - Domain name (any characters except '/' or whitespace)
             // )
             // (\/[^?#]*)       - Capturing group for the path (starts with '/', followed by any characters except '?' or '#')
-            std::regex path_regex (R"(^(?:(?:https?:\/\/)?(?:[^\/\s]+))(\/[^?#]*))");
+            std::regex path_regex (R"(^(?:(?:(?:ftp|https?|rsync):\/\/)?(?:[^\/\s]+))(\/[^?#]*))");
             std::smatch path_match;
 
             // Use regex_search to find the path component in the input string

src/functions/extract_schema.cpp

@@ -30,7 +30,7 @@ namespace duckdb
             // |                - OR
             // (mailto|sms|tel) - Capturing group for other protocols (mailto, sms, tel)
             // :[^/]            - Matches ":" followed by any character except "/"
-            std::regex schema_regex (R"(^(http|https|ftp):\/\/|^(mailto|sms|tel):[^/])");
+            std::regex schema_regex (R"(^(http|https|ftp|rsync):\/\/|^(mailto|sms|tel):[^/])");
             std::smatch schema_match;
 
             // Use regex_search to find the schema component in the input string

src/functions/extract_subdomain.cpp

@@ -29,7 +29,7 @@ namespace duckdb
             Connection con (db);
 
             // Extract the host from the URL
-            std::regex host_regex (R"(^(?:https?:\/\/)?([^\/\?:]+))");
+            std::regex host_regex (R"(^(?:(?:https?|ftp|rsync):\/\/)?([^\/\?:]+))");
             std::smatch host_match;
             if (!std::regex_search (input, host_match, host_regex))
             {

src/functions/extract_tld.cpp

@@ -30,7 +30,7 @@ namespace duckdb
             Connection con (db);
 
             // Extract the host from the URL
-            std::regex host_regex (R"(^(?:https?:\/\/)?([^\/\?:]+))");
+            std::regex host_regex (R"(^(?:(?:https?|ftp|rsync):\/\/)?([^\/\?:]+))");
             std::smatch host_match;
             if (!std::regex_search (input, host_match, host_regex))
             {

test/sql/extract_domain.test

@@ -144,6 +144,16 @@ SELECT extract_domain('com:443');
 ----
 com
 
+query I
+SELECT extract_domain('ftp://example.com/path');
+----
+example.com
+
+query I
+SELECT extract_domain('rsync://rpki.example.com/path');
+----
+example.com
+
 query I
 SELECT extract_domain('c');
 ----
@@ -162,4 +172,4 @@ SELECT extract_domain('http:/example.com.ac/path');
 query I
 SELECT extract_domain('http:/example.com.ac:443/path');
 ----
-(empty)
+(empty)

test/sql/extract_host.test

@@ -53,3 +53,13 @@ query I
 SELECT extract_host('example.com.ac:443/path/path');
 ----
 example.com.ac
+
+query I
+SELECT extract_host('ftp://ftp.example.com/path');
+----
+ftp.example.com
+
+query I
+SELECT extract_host('rsync://rpki.example.com/path');
+----
+rpki.example.com

test/sql/extract_path.test

@@ -53,3 +53,13 @@ query I
 SELECT extract_path('example.com.ac:443/path/path');
 ----
 /path/path
+
+query I
+SELECT extract_path('ftp://ftp.exmple.com/path/path');
+----
+/path/path
+
+query I
+SELECT extract_path('rsync://rpki.exmple.com/path/path');
+----
+/path/path

test/sql/extract_schema.test

@@ -95,6 +95,13 @@ SELECT extract_schema('ftp:/example.com');
 ----
 (empty)
 
+# ---------------------------- RSYNC ----------------------------
+
+query I
+SELECT extract_schema('rsync://rpki.example.com');
+----
+rsync
+
 # ---------------------------- TEL ----------------------------
 
 query I

test/sql/extract_subdomain.test

@@ -74,6 +74,16 @@ SELECT extract_subdomain('https://b.a.example.com.ac:443/path');
 ----
 b.a
 
+query I
+SELECT extract_subdomain('rsync://rpki.example.com/path');
+----
+rpki
+
+query I
+SELECT extract_subdomain('ftp://ftp1.example.com/path');
+----
+ftp1
+
 query I
 SELECT extract_subdomain('com.ac');
 ----

test/sql/extract_tld.test

@@ -124,6 +124,16 @@ SELECT extract_tld('https://example.com.ac:443/path/path');
 ----
 com.ac
 
+query I
+SELECT extract_tld('ftp://ftp.example.com/path/path');
+----
+com
+
+query I
+SELECT extract_tld('rsync://rpki.example.com/path/path');
+----
+com
+
 
 query I
 SELECT extract_tld('com.ac');