Merge pull request #1725 from karlcow/1586/1

Fixes #1586 - Adds a data/ path logic to the code.

Author: Mike Taylor

.gitignore

@@ -13,9 +13,8 @@ uploads/
 config/secrets.py
 package-lock.json
 
-# backup folder contains the issues.db backup file with issues dump
-# for the previous versions
-backups/*.db
+# The data folder contains information that shouldn't live in version control.
+data/*
 
 # these are our 'dist' files
 # checking them into the repo is silly

config/environment.py

@@ -3,11 +3,16 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""Generic configuration for the project."""
 
 import os
 
 # Define the application base directory
 BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+DATA_PATH = os.path.join(BASE_DIR, 'data')
+# Add the data/ directory if it doesn't exist.
+if not os.path.exists(DATA_PATH):
+    os.makedirs(DATA_PATH)
 
 # the PRODUCTION and DEVELOPMENT environment variables are set in uwsgi.conf
 # on the webcompat server. If they're not set, they will default to None

tools/topsites.py

@@ -3,6 +3,7 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""GitHub Webhook module for assigning priority to sites."""
 
 import base64
 import datetime
@@ -16,12 +17,12 @@
 
 import requests
 from requests.exceptions import ConnectionError
-from sqlalchemy import create_engine
 from sqlalchemy import Column
-from sqlalchemy import Integer
-from sqlalchemy import String
+from sqlalchemy import create_engine
 from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import Integer
 from sqlalchemy.orm import sessionmaker
+from sqlalchemy import String
 
 # Add webcompat module to import path
 sys.path.append(os.path.realpath(os.pardir))
@@ -36,6 +37,9 @@
 ATS_HASH_ALGORITHM = 'HmacSHA256'
 ATS_COUNT = 100
 
+# Location of the DB and its backup.
+DB_PATH = app.config['DATA_PATH']
+
 # Regions to dump to topsites.db
 REGIONS = ['GLOBAL', 'US', 'FR', 'IN', 'DE', 'TW', 'ID', 'HK', 'SG', 'PL',
            'GB', 'RU']
@@ -48,15 +52,15 @@
 # Cache parsed sites, change priority if raised
 topsites = {}
 
-engine = create_engine('sqlite:///' + os.path.join(
-    app.config['BASE_DIR'], 'topsites-new.db'))
+engine = create_engine('sqlite:///' + os.path.join(DB_PATH, 'topsites-new.db'))
 Base = declarative_base()
 Session = sessionmaker(bind=engine)
 session = Session()
 
 
 class Site(Base):
     """SQLAchemy base object for an Alexa top site."""
+
     __tablename__ = "topsites"
 
     url = Column(String, primary_key=True)
@@ -65,6 +69,7 @@ class Site(Base):
     ranking = Column(Integer)
 
     def __init__(self, url, priority, country_code, ranking):
+        """Initialize parameters of the Alexa top site DB."""
         self.url = url
         self.priority = priority
         self.country_code = country_code
@@ -168,7 +173,7 @@ def build_query_string(country_code, start_ranking):
 
 
 def gen_sign(data):
-    """Computes RFC 2104-compliant HMAC signature."""
+    """Compute RFC 2104-compliant HMAC signature."""
     dig = hmac.new(ats_secret_key, data, hashlib.sha256).digest()
     return base64.b64encode(dig)
 
@@ -199,8 +204,8 @@ def node_text(tree, tag_name):
     # Archive topsites.db and rename topsites-new.db to topsites.db
     session.close()
     archive_date = time.strftime("%Y%m%d", time.localtime())
-    os.rename(os.path.join(app.config['BASE_DIR'], 'topsites.db'),
-              os.path.join(app.config['BASE_DIR'],
+    os.rename(os.path.join(DB_PATH, 'topsites.db'),
+              os.path.join(DB_PATH,
                            'topsites-archive-{}.db'.format(archive_date)))
-    os.rename(os.path.join(app.config['BASE_DIR'], 'topsites-new.db'),
-              os.path.join(app.config['BASE_DIR'], 'topsites.db'))
+    os.rename(os.path.join(DB_PATH, 'topsites-new.db'),
+              os.path.join(DB_PATH, 'topsites.db'))

webcompat/db/__init__.py

@@ -4,7 +4,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-"""Databases initialization."""
+"""Database initialization."""
 
 from hashlib import sha512
 import os
@@ -20,7 +20,7 @@
 from webcompat import app
 
 session_engine = create_engine('sqlite:///' + os.path.join(
-    app.config['BASE_DIR'], 'session.db'))
+    app.config['DATA_PATH'], 'session.db'))
 session_db = scoped_session(sessionmaker(autocommit=False,
                                          autoflush=False,
                                          bind=session_engine))
@@ -29,12 +29,15 @@
 
 
 class User(UsersBase):
+    """Define the user DB holding the sessions."""
+
     __tablename__ = 'users'
 
     user_id = Column(String(128), unique=True, primary_key=True)
     access_token = Column(String(128), unique=True)
 
     def __init__(self, access_token):
+        """Initialize the user db parameters."""
         self.access_token = access_token
         # We use the user_id in the session cookie to identify auth'd users.
         # Here we salt and hash the GitHub access token so you can't get