Merge 'rebased'

Author: wil93

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "isolate"]
+	path = isolate
+	url = https://github.com/cms-dev/isolate.git

AUTHORS.txt

@@ -26,6 +26,7 @@ Kento Nikaido (aka snukent) <[email protected]>
 William Pettersson <[email protected]>
 Federico Scrinzi (aka volpino) <[email protected]>
 Luca Versari <[email protected]>
+Hyungsuk Yoon <[email protected]>
 Kenneth Wong <[email protected]>
 
 And many other people that didn't write code, but provided useful

cms/__init__.py

@@ -41,13 +41,14 @@
     # log
     # Nothing intended for external use, no need to advertise anything.
     # util
-    "mkdir", "utf8_decoder", "Address", "ServiceCoord", "get_safe_shard",
-    "get_service_address", "get_service_shards", "default_argument_parser",
+    "ConfigError", "mkdir", "utf8_decoder", "Address", "ServiceCoord",
+    "get_safe_shard", "get_service_address", "get_service_shards",
+    "default_argument_parser",
     # conf
     "config",
     # plugin
     "plugin_list", "plugin_lookup",
-    ]
+]
 
 
 # Instantiate or import these objects.
@@ -102,7 +103,9 @@
     LANG_PASCAL: "lib.pas",
 }
 
-from .util import mkdir, utf8_decoder, Address, ServiceCoord, get_safe_shard, \
-    get_service_address, get_service_shards, default_argument_parser
+
+from .util import ConfigError, mkdir, utf8_decoder, Address, ServiceCoord, \
+    get_safe_shard, get_service_address, get_service_shards, \
+    default_argument_parser
 from .conf import config
 from .plugin import plugin_list, plugin_lookup

cms/db/__init__.py

@@ -74,7 +74,8 @@
     # drop
     "drop_db",
     # util
-    "get_contest_list", "is_contest_id", "ask_for_contest",
+    "test_db_connection", "get_contest_list", "is_contest_id",
+    "ask_for_contest",
     # test
     "Test", "TestQuestion", "QuestionFile", "TestScore",
     # forum
@@ -84,7 +85,7 @@
 
 # Instantiate or import these objects.
 
-version = 12
+version = 13
 
 
 engine = create_engine(config.database, echo=config.database_debug,
@@ -113,7 +114,8 @@
 from .init import init_db
 from .drop import drop_db
 
-from .util import get_contest_list, is_contest_id, ask_for_contest
+from .util import test_db_connection, get_contest_list, is_contest_id, \
+    ask_for_contest
 
 
 configure_mappers()

cms/db/filecacher.py

@@ -436,7 +436,7 @@ def __init__(self, service=None, path=None, null=False, enabled=None):
             logger.error("Cannot create necessary directories.")
             raise RuntimeError("Cannot create necessary directories.")
 
-    def load(self, digest):
+    def load(self, digest, if_needed=False):
         """Load the file with the given digest into the cache.
 
         Ask the backend to provide the file and, if it's available,
@@ -445,15 +445,20 @@ def load(self, digest):
         If caching is disabled, this function does nothing.
 
         digest (unicode): the digest of the file to load.
+        if_needed (bool): only load the file if it is not present in
+            the local cache.
 
         raise (KeyError): if the backend cannot find the file.
 
         """
         if self.enabled:
+            cache_file_path = os.path.join(self.file_dir, digest)
+            if if_needed and os.path.exists(cache_file_path):
+                return
+
             ftmp_handle, temp_file_path = tempfile.mkstemp(dir=self.temp_dir,
                                                            text=False)
             ftmp = os.fdopen(ftmp_handle, 'w')
-            cache_file_path = os.path.join(self.file_dir, digest)
 
             fobj = self.backend.get_file(digest)
 

cms/db/submission.py

@@ -7,6 +7,7 @@
 # Copyright © 2010-2012 Matteo Boscariol <[email protected]>
 # Copyright © 2012-2013 Luca Wehrstedt <[email protected]>
 # Copyright © 2013 Bernard Blackham <[email protected]>
+# Copyright © 2014 Fabian Gundlach <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -87,6 +88,17 @@ class Submission(Base):
         String,
         nullable=True)
 
+    # Comment from the administrator on the submission.
+    comment = Column(
+        Unicode,
+        nullable=False,
+        default="")
+
+    @property
+    def short_comment(self):
+        """The first line of the comment."""
+        return self.comment.split("\n", 1)[0]
+
     # Follows the description of the fields automatically added by
     # SQLAlchemy.
     # files (dict of File objects indexed by filename)

cms/db/task.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 # Contest Management System - http://cms-dev.github.io/
-# Copyright © 2010-2013 Giovanni Mascellani <[email protected]>
+# Copyright © 2010-2014 Giovanni Mascellani <[email protected]>
 # Copyright © 2010-2012 Stefano Maggiolo <[email protected]>
 # Copyright © 2010-2012 Matteo Boscariol <[email protected]>
 # Copyright © 2012-2014 Luca Wehrstedt <[email protected]>
@@ -433,6 +433,16 @@ class Dataset(Base):
     # managers (dict of Manager objects indexed by filename)
     # testcases (dict of Testcase objects indexed by codename)
 
+    @property
+    def active(self):
+        """Shorthand for detecting if the dataset is active.
+
+        return (bool): True if this dataset is the active one for its
+            task.
+
+        """
+        return self is self.task.active_dataset
+
 
 class Manager(Base):
     """Class to store additional files needed to compile or evaluate a

cms/db/util.py

@@ -30,9 +30,29 @@
 
 import sys
 
+from sqlalchemy.exc import OperationalError
+
+from cms import ConfigError
 from . import SessionGen, Contest
 
 
+def test_db_connection():
+    """Perform an operation that raises if the DB is not reachable.
+
+    raise (sqlalchemy.exc.OperationalError): if the DB cannot be
+        accessed (usually for permission problems).
+
+    """
+    try:
+        # We do not care of the specific query executed here, we just
+        # use it to ensure that the DB is accessible.
+        with SessionGen() as session:
+            session.execute("select 0;")
+    except OperationalError:
+        raise ConfigError("Operational error while talking to the DB. "
+                          "Is the connection string in cms.conf correct?")
+
+
 def get_contest_list(session=None):
     """Return all the contest objects available on the database.
 

cms/io/rpc.py

@@ -37,7 +37,7 @@
 import gevent.socket
 import gevent.event
 
-from cms import get_service_address
+from cms import Address, get_service_address
 
 
 logger = logging.getLogger(__name__)
@@ -418,6 +418,9 @@ def __init__(self, remote_service_coord, auto_retry=None):
             remote service in case the connection is lost; if not given
             no automatic reconnection attempts will occur.
 
+        raise (KeyError): if the coordinates are not specified in the
+            configuration.
+
         """
         super(RemoteServiceClient, self).__init__(
             get_service_address(remote_service_coord))
@@ -651,3 +654,46 @@ def remote_method(**data):
             return result
 
         return remote_method
+
+
+class FakeRemoteServiceClient(RemoteServiceClient):
+    """A RemoteServiceClient not actually connected to anything.
+
+    This is useful for connections to optional services, to avoid
+    having to specify different behaviors in the case the services are
+    or are not available.
+
+    In all aspects, an object of this class behaves exactly like a
+    RemoteServiceClient that will never be able to connect.
+
+    """
+    def __init__(self, remote_service_coord, auto_retry=None):
+        """Initialization.
+
+        This constructor does not call the parent constructor, because
+        it would fail (as the service coord are not in the
+        configuration). This is potentially a problem, but as this
+        client will never connect not many member variable access are
+        performed.
+
+        """
+        RemoteServiceBase.__init__(self, Address("None", 0))
+        self.remote_service_coord = remote_service_coord
+        self.pending_outgoing_requests = dict()
+        self.pending_outgoing_requests_results = dict()
+        self.auto_retry = auto_retry
+
+    def connect(self):
+        """Do nothing, as this is a fake client."""
+        pass
+
+    def disconnect(self):
+        """Do nothing, as this is a fake client."""
+        pass
+
+    def execute_rpc(self, method, data):
+        """Just return an AsyncResult encoding an error."""
+        result = gevent.event.AsyncResult()
+        result.set_exception(
+            RPCError("Called a method of a non-configured service."))
+        return result

cms/io/service.py

@@ -3,7 +3,7 @@
 
 # Contest Management System - http://cms-dev.github.io/
 # Copyright © 2010-2014 Giovanni Mascellani <[email protected]>
-# Copyright © 2010-2013 Stefano Maggiolo <[email protected]>
+# Copyright © 2010-2014 Stefano Maggiolo <[email protected]>
 # Copyright © 2010-2012 Matteo Boscariol <[email protected]>
 # Copyright © 2013 Luca Wehrstedt <[email protected]>
 #
@@ -37,7 +37,6 @@
 import signal
 import socket
 import _socket
-import sys
 import time
 
 import gevent
@@ -46,12 +45,14 @@
 from gevent.server import StreamServer
 from gevent.backdoor import BackdoorServer
 
-from cms import config, mkdir, ServiceCoord, Address, get_service_address
+from cms import ConfigError, config, mkdir, ServiceCoord, Address, \
+    get_service_address
 from cms.log import root_logger, shell_handler, ServiceFilter, \
     CustomFormatter, LogServiceHandler, FileHandler
 from cmscommon.datetime import monotonic_time
 
-from .rpc import rpc_method, RemoteServiceServer, RemoteServiceClient
+from .rpc import rpc_method, RemoteServiceServer, RemoteServiceClient, \
+    FakeRemoteServiceClient
 
 
 logger = logging.getLogger(__name__)
@@ -99,9 +100,9 @@ def __init__(self, shard=0):
         try:
             address = get_service_address(self._my_coord)
         except KeyError:
-            logger.critical("Couldn't find %r in the configuration.",
-                            self._my_coord)
-            sys.exit(1)
+            raise ConfigError("Unable to find address for service %r. "
+                              "Is it specified in core_services in cms.conf?" %
+                              (self._my_coord,))
 
         self.rpc_server = StreamServer(address, self._connection_handler)
         self.backdoor = None
@@ -172,7 +173,8 @@ def _connection_handler(self, sock, address):
         remote_service = RemoteServiceServer(self, address)
         remote_service.handle(sock)
 
-    def connect_to(self, coord, on_connect=None, on_disconnect=None):
+    def connect_to(self, coord, on_connect=None, on_disconnect=None,
+                   must_be_present=True):
         """Return a proxy to a remote service.
 
         Obtain a communication channel to the remote service at the
@@ -184,12 +186,26 @@ def connect_to(self, coord, on_connect=None, on_disconnect=None):
             connects.
         on_disconnect (function|None): to be called when it
             disconnects.
+        must_be_present (bool): if True, the coord must be present in
+            the configuration; otherwise, it can be missing and in
+            that case the return value is a fake client (that is, a
+            client that never connects and ignores all calls).
 
         return (RemoteServiceClient): a proxy to that service.
 
         """
         if coord not in self.remote_services:
-            service = RemoteServiceClient(coord, auto_retry=0.5)
+            try:
+                service = RemoteServiceClient(coord, auto_retry=0.5)
+            except KeyError:
+                # The coordinates are invalid: raise a ConfigError if
+                # the service was needed, or return a dummy client if
+                # the service was optional.
+                if must_be_present:
+                    raise ConfigError("Missing address and port for %s "
+                                      "in cms.conf." % (coord, ))
+                else:
+                    service = FakeRemoteServiceClient(coord, None)
             service.connect()
             self.remote_services[coord] = service
         else: