Merge pull request #40 from opsdisk/migrate-to-pyproject-install

Migrate from setup.py to pyproject.toml

Author: opsdisk

README.md

@@ -38,14 +38,14 @@ Google's preferred method is to use their [API](https://developers.google.com/cu
 pip install yagooglesearch
 ```
 
-## <span>setup.py</span>
+## pyproject.toml
 
 ```bash
 git clone https://github.com/opsdisk/yagooglesearch
 cd yagooglesearch
-virtualenv -p python3.7 .venv  # If using a virtual environment.
+virtualenv -p python3 .venv  # If using a virtual environment.
 source .venv/bin/activate  # If using a virtual environment.
-python setup.py install
+pip install .  # Reads from pyproject.toml
 ```
 
 ## Usage
@@ -75,6 +75,13 @@ for url in urls:
     print(url)
 ```
 
+## Max ~400 results returned
+
+Even though searching Google through the GUI will display a message like "About 13,000,000 results", that does not mean
+`yagooglesearch` will find anything close to that.  Testing shows that at most, about 400 results are returned.  If you
+set 400 < `max_search_result_urls_to_return`, a warning message will be printed to the logs.  See
+<https://github.com/opsdisk/yagooglesearch/issues/28> for the discussion.
+
 ## Google is blocking me!
 
 Low and slow is the strategy when executing Google searches using `yagooglesearch`.  If you start getting HTTP 429
@@ -297,4 +304,4 @@ Project Link: [https://github.com/opsdisk/yagooglesearch](https://github.com/ops
 
 * [KennBro](https://github.com/KennBro) - <https://github.com/opsdisk/yagooglesearch/pull/9>
 * [ArshansGithub](https://github.com/ArshansGithub) - <https://github.com/opsdisk/yagooglesearch/pull/21>
-* [pguridi](https://github.com/pguridi) - <https://github.com/opsdisk/yagooglesearch/pull/38>
+* [pguridi](https://github.com/pguridi) - <https://github.com/opsdisk/yagooglesearch/pull/38>

pyproject.toml

@@ -0,0 +1,32 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "yagooglesearch"
+dynamic = ["version"]
+dependencies = ["beautifulsoup4>=4.9.3", "requests>=2.31.0", "requests[socks]"]
+requires-python = ">=3.6"
+authors = [{ name = "Brennon Thomas", email = "[email protected]" }]
+description = "A Python library for executing intelligent, realistic-looking, and tunable Google searches."
+readme = { file = "README.md", content-type = "text/markdown" }
+license = { file = "LICENSE" }
+keywords = ["python", "google", "search", "googlesearch"]
+classifiers = ["Programming Language :: Python"]
+
+[project.urls]
+Homepage = "https://github.com/opsdisk/yagooglesearch"
+Documentation = "https://github.com/opsdisk/yagooglesearch"
+Repository = "https://github.com/opsdisk/yagooglesearch"
+
+[tool.setuptools.dynamic]
+version = { attr = "yagooglesearch.__version__" }
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+yagooglesearch = ["*.txt"]
+
+[tool.black]
+line-length = 120

setup.py

@@ -5,13 +5,16 @@
 import time
 import urllib
 
+
 # Third party Python libraries.
 from bs4 import BeautifulSoup
 import requests
 
+
 # Custom Python libraries.
 
-__version__ = "1.8.2"
+
+__version__ = "1.9.0"
 
 # Logging
 ROOT_LOGGER = logging.getLogger("yagooglesearch")
@@ -175,10 +178,17 @@ def __init__(
             ROOT_LOGGER.warning("The largest value allowed by Google for num is 100.  Setting num to 100.")
             self.num = 100
 
+        if 400 < self.max_search_result_urls_to_return:
+            ROOT_LOGGER.warning(
+                "yagooglesearch is usually only able to retrieve a maximum of ~400 results.  See README for more details."
+            )
+
         # Populate cookies with GOOGLE_ABUSE_EXEMPTION if it is provided.  Otherwise, initialize cookies to None.
         # It will be updated with each request in get_page().
         if self.google_exemption:
-            self.cookies = {"GOOGLE_ABUSE_EXEMPTION": self.google_exemption}
+            self.cookies = {
+                "GOOGLE_ABUSE_EXEMPTION": self.google_exemption,
+            }
         else:
             self.cookies = None
 
@@ -312,7 +322,8 @@ def http_429_detected(self):
         """Increase the HTTP 429 cool off period."""
 
         new_http_429_cool_off_time_in_minutes = round(
-            self.http_429_cool_off_time_in_minutes * self.http_429_cool_off_factor, 2
+            self.http_429_cool_off_time_in_minutes * self.http_429_cool_off_factor,
+            2,
         )
         ROOT_LOGGER.info(
             f"Increasing HTTP 429 cool off time by a factor of {self.http_429_cool_off_factor}, "
@@ -321,8 +332,7 @@ def http_429_detected(self):
         self.http_429_cool_off_time_in_minutes = new_http_429_cool_off_time_in_minutes
 
     def get_page(self, url):
-        """
-        Request the given URL and return the response page.
+        """Request the given URL and return the response page.
 
         :param str url: URL to retrieve.