Merge pull request #1392 from davebx/toolshed_decouple

Use the API to install repositories instead of loading the toolshed in an iframe.

Author: bgruening

lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py

@@ -147,8 +147,8 @@ def uwsgi_app_factory():
     root = os.path.abspath(uwsgi.opt.get('galaxy_root', os.getcwd()))
     config_file = uwsgi.opt.get('galaxy_config_file', os.path.join(root, 'config', 'galaxy.ini'))
     global_conf = {
-        '__file__' : config_file if os.path.exists(__file__) else None,
-        'here' : root }
+        '__file__': config_file if os.path.exists(__file__) else None,
+        'here': root }
     parser = configparser.ConfigParser()
     parser.read(config_file)
     try:
@@ -627,6 +627,18 @@ def populate_api_routes( webapp, app ):
                             new={ 'install_repository_revision': 'POST' },
                             parent_resources=dict( member_name='tool_shed_repository', collection_name='tool_shed_repositories' ) )
 
+    webapp.mapper.connect( 'tool_shed_repository',
+                           '/api/tool_shed_repositories/:id/status',
+                           controller='tool_shed_repositories',
+                           action='status',
+                           conditions=dict( method=[ "GET" ] ) )
+
+    webapp.mapper.connect( 'install_repository',
+                           '/api/tool_shed_repositories/install',
+                           controller='tool_shed_repositories',
+                           action='install',
+                           conditions=dict( method=[ 'POST' ] ) )
+
     # ==== Trace/Metrics Logger
     # Connect logger from app
     if app.trace_logger:

lib/galaxy/webapps/galaxy/buildapp.py

@@ -4,7 +4,7 @@
 import shutil
 
 from six import string_types
-from sqlalchemy import false, or_
+from sqlalchemy import or_
 
 import tool_shed.repository_types.util as rt_util
 from admin import AdminGalaxy
@@ -192,6 +192,56 @@ def browse_tool_shed( self, trans, **kwd ):
         url = util.build_url( tool_shed_url, pathspec=[ 'repository', 'browse_valid_categories' ], params=params )
         return trans.response.send_redirect( url )
 
+    @web.expose
+    @web.require_admin
+    def browse_toolsheds( self, trans, **kwd ):
+        message = escape( kwd.get( 'message', '' ) )
+        return trans.fill_template( '/webapps/galaxy/admin/toolsheds.mako',
+                                    message=message,
+                                    status='error' )
+
+    @web.expose
+    @web.require_admin
+    def browse_toolshed( self, trans, **kwd ):
+        tool_shed_url = kwd.get( 'tool_shed_url', '' )
+        tool_shed_url = common_util.get_tool_shed_url_from_tool_shed_registry( trans.app, tool_shed_url )
+        url = common_util.url_join( tool_shed_url, pathspec=[ 'api', 'categories' ] )
+        categories = json.loads( common_util.tool_shed_get( trans.app, url ) )
+        repositories = []
+        url = common_util.url_join( tool_shed_url, pathspec=[ 'api', 'repositories' ] )
+        for repo in json.loads( common_util.tool_shed_get( trans.app, url ) ):
+            repositories.append( dict( value=repo[ 'id' ], label='%s/%s' % ( repo[ 'owner' ], repo[ 'name' ] ) ) )
+        return trans.fill_template( '/admin/tool_shed_repository/browse_categories.mako',
+                                    tool_shed_url=tool_shed_url,
+                                    categories=categories,
+                                    repositories=json.dumps( repositories ) )
+
+    @web.expose
+    @web.require_admin
+    def browse_tool_shed_category( self, trans, **kwd ):
+        tool_shed_url = kwd.get( 'tool_shed_url', '' )
+        category_id = kwd.get( 'category_id', '' )
+        url = common_util.url_join( tool_shed_url )
+        json_data = json.loads( common_util.tool_shed_get( trans.app, url, pathspec=[ 'api', 'categories', category_id, 'repositories' ] ) )
+        for idx, repository in enumerate( json_data[ 'repositories' ] ):
+            try:
+                metadata = json.loads( common_util.tool_shed_get( trans.app,
+                                                                  url,
+                                                                  pathspec=[ 'api', 'repositories', repository[ 'id' ], 'metadata' ],
+                                                                  params=dict( recursive=False ) ) )
+                json_data[ 'repositories' ][ idx ][ 'metadata' ] = metadata
+            except:
+                json_data[ 'repositories' ][ idx ][ 'metadata' ] = { 'tools_functionally_correct': True }
+
+        repositories = []
+        url = common_util.url_join( tool_shed_url, pathspec=[ 'api', 'repositories' ] )
+        for repo in json.loads( common_util.tool_shed_get( trans.app, url ) ):
+            repositories.append( dict( value=repo[ 'id' ], label='%s/%s' % ( repo[ 'owner' ], repo[ 'name' ] ) ) )
+        return trans.fill_template( '/admin/tool_shed_repository/browse_category.mako',
+                                    tool_shed_url=tool_shed_url,
+                                    category=json_data,
+                                    repositories=json.dumps( repositories ) )
+
     @web.expose
     @web.require_admin
     def browse_tool_sheds( self, trans, **kwd ):
@@ -1222,6 +1272,21 @@ def prepare_for_install( self, trans, **kwd ):
                                         message=message,
                                         status=status )
 
+    @web.expose
+    @web.require_admin
+    def preview_repository( self, trans, **kwd ):
+        tool_shed_url = kwd.get( 'tool_shed_url', '' )
+        tsr_id = kwd.get( 'tsr_id', '' )
+        toolshed_data = json.loads( common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=[ 'api', 'repositories', tsr_id ] ) )
+        toolshed_data[ 'metadata' ] = json.loads( common_util.tool_shed_get( trans.app, tool_shed_url, pathspec=[ 'api', 'repositories', tsr_id, 'metadata' ] ) )
+        shed_tool_conf_select_field = tool_util.build_shed_tool_conf_select_field( trans.app )
+        tool_panel_section_select_field = tool_util.build_tool_panel_section_select_field( trans.app )
+        return trans.fill_template( '/admin/tool_shed_repository/preview_repository.mako',
+                                    tool_shed_url=tool_shed_url,
+                                    toolshed_data=toolshed_data,
+                                    tool_panel_section_select_field=tool_panel_section_select_field,
+                                    shed_tool_conf_select_field=shed_tool_conf_select_field )
+
     @web.expose
     @web.require_admin
     def purge_repository( self, trans, **kwd ):
@@ -2008,41 +2073,8 @@ def update_to_changeset_revision( self, trans, **kwd ):
 
     @web.expose
     @web.require_admin
-    def update_tool_shed_status_for_installed_repository( self, trans, all_installed_repositories=False, **kwd ):
-        message = escape( kwd.get( 'message', '' ) )
-        status = kwd.get( 'status', 'done' )
-        if all_installed_repositories:
-            success_count = 0
-            repository_names_not_updated = []
-            updated_count = 0
-            for repository in trans.install_model.context.query( trans.install_model.ToolShedRepository ) \
-                                                         .filter( trans.install_model.ToolShedRepository.table.c.deleted == false() ):
-                ok, updated = \
-                    repository_util.check_or_update_tool_shed_status_for_installed_repository( trans.app, repository )
-                if ok:
-                    success_count += 1
-                else:
-                    repository_names_not_updated.append( '<b>%s</b>' % escape( str( repository.name ) ) )
-                if updated:
-                    updated_count += 1
-            message = "Checked the status in the tool shed for %d repositories.  " % success_count
-            message += "Updated the tool shed status for %d repositories.  " % updated_count
-            if repository_names_not_updated:
-                message += "Unable to retrieve status from the tool shed for the following repositories:\n"
-                message += ", ".join( repository_names_not_updated )
-        else:
-            repository_id = kwd.get( 'id', None )
-            repository = suc.get_tool_shed_repository_by_id( trans.app, repository_id )
-            ok, updated = \
-                repository_util.check_or_update_tool_shed_status_for_installed_repository( trans.app, repository )
-            if ok:
-                if updated:
-                    message = "The tool shed status for repository <b>%s</b> has been updated." % escape( str( repository.name ) )
-                else:
-                    message = "The status has not changed in the tool shed for repository <b>%s</b>." % escape( str( repository.name ) )
-            else:
-                message = "Unable to retrieve status from the tool shed for repository <b>%s</b>." % escape( str( repository.name ) )
-                status = 'error'
+    def update_tool_shed_status_for_installed_repository( self, trans, **kwd ):
+        message, status = repository_util.check_for_updates( trans.app, trans.install_model, kwd.get( 'id', None ) )
         return trans.response.send_redirect( web.url_for( controller='admin_toolshed',
                                                           action='browse_repositories',
                                                           message=message,

lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py

@@ -785,6 +785,8 @@ def metadata( self, trans, id, **kwd ):
                 metadata_dict[ 'tool_dependencies' ] = repository.get_tool_dependencies( changehash )
             else:
                 metadata_dict[ 'tool_dependencies' ] = {}
+            if metadata.includes_tools:
+                metadata_dict[ 'tools' ] = metadata.metadata[ 'tools' ]
             all_metadata[ '%s:%s' % ( int( changeset ), changehash ) ] = metadata_dict
         return all_metadata
 

lib/galaxy/webapps/tool_shed/api/repositories.py

@@ -494,7 +494,7 @@ def __get_install_info_from_tool_shed( self, tool_shed_url, name, owner, changes
 
     def __handle_repository_contents( self, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir,
                                       tool_shed=None, tool_section=None, shed_tool_conf=None, reinstalling=False,
-                                      tool_versions_response=None ):
+                                      tool_versions_response=None, tool_panel_section_mapping={} ):
         """
         Generate the metadata for the installed tool shed repository, among other things.
         This method is called when an administrator is installing a new repository or
@@ -572,7 +572,8 @@ def __handle_repository_contents( self, tool_shed_repository, tool_path, reposit
                                             owner=tool_shed_repository.owner,
                                             shed_tool_conf=shed_tool_conf,
                                             tool_panel_dict=tool_panel_dict,
-                                            new_install=True )
+                                            new_install=True,
+                                            tool_panel_section_mapping=tool_panel_section_mapping )
         if 'data_manager' in irmm_metadata_dict:
             dmh = data_manager.DataManagerHandler( self.app )
             dmh.install_data_managers( self.app.config.shed_data_manager_config_file,
@@ -648,13 +649,20 @@ def initiate_repository_installation( self, installation_dict ):
         status = installation_dict[ 'status' ]
         tool_panel_section_id = installation_dict[ 'tool_panel_section_id' ]
         tool_panel_section_keys = installation_dict[ 'tool_panel_section_keys' ]
+        tool_panel_section_mapping = installation_dict.get( 'tool_panel_section_mapping', {} )
         tool_path = installation_dict[ 'tool_path' ]
         tool_shed_url = installation_dict[ 'tool_shed_url' ]
         # Handle contained tools.
         if includes_tools_for_display_in_tool_panel and ( new_tool_panel_section_label or tool_panel_section_id ):
             self.tpm.handle_tool_panel_section( self.app.toolbox,
                                                 tool_panel_section_id=tool_panel_section_id,
                                                 new_tool_panel_section_label=new_tool_panel_section_label )
+        if includes_tools_for_display_in_tool_panel and ( tool_panel_section_mapping is not None ):
+            for tool_guid in tool_panel_section_mapping:
+                if tool_panel_section_mapping[ tool_guid ][ 'action' ] == 'create':
+                    new_tool_panel_section_name = tool_panel_section_mapping[ tool_guid ][ 'tool_panel_section' ]
+                    log.debug( 'Creating tool panel section "%s" for tool %s' % ( new_tool_panel_section_name, tool_guid ) )
+                    self.tpm.handle_tool_panel_section( self.app.toolbox, None, tool_panel_section_mapping[ tool_guid ][ 'tool_panel_section' ] )
         encoded_repository_ids = [ self.app.security.encode_id( tsr.id ) for tsr in created_or_updated_tool_shed_repositories ]
         new_kwd = dict( includes_tools=includes_tools,
                         includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel,
@@ -719,6 +727,7 @@ def __initiate_and_install_repositories( self, tool_shed_url, repository_revisio
         if install_tool_dependencies:
             self.__assert_can_install_dependencies()
         new_tool_panel_section_label = install_options.get( 'new_tool_panel_section_label', '' )
+        tool_panel_section_mapping = install_options.get( 'tool_panel_section_mapping', {} )
         shed_tool_conf = install_options.get( 'shed_tool_conf', None )
         if shed_tool_conf:
             # Get the tool_path setting.
@@ -737,6 +746,7 @@ def __initiate_and_install_repositories( self, tool_shed_url, repository_revisio
         # for each repository being installed.
         installation_dict = dict( install_repository_dependencies=install_repository_dependencies,
                                   new_tool_panel_section_label=new_tool_panel_section_label,
+                                  tool_panel_section_mapping=tool_panel_section_mapping,
                                   no_changes_checked=False,
                                   repo_info_dicts=repo_info_dicts,
                                   tool_panel_section_id=tool_panel_section_id,
@@ -761,6 +771,7 @@ def __initiate_and_install_repositories( self, tool_shed_url, repository_revisio
                                       status='done',
                                       tool_panel_section_id=tool_panel_section_id,
                                       tool_panel_section_keys=tool_panel_section_keys,
+                                      tool_panel_section_mapping=tool_panel_section_mapping,
                                       tool_path=tool_path,
                                       tool_shed_url=tool_shed_url )
             # Prepare the repositories for installation.  Even though this
@@ -783,13 +794,15 @@ def __initiate_and_install_repositories( self, tool_shed_url, repository_revisio
                 tool_panel_section_keys=tool_panel_section_keys,
                 repo_info_dicts=filtered_repo_info_dicts,
                 install_tool_dependencies=install_tool_dependencies,
+                tool_panel_section_mapping=tool_panel_section_mapping,
             )
             return self.install_repositories(tsr_ids, decoded_kwd, reinstalling=False)
 
     def install_repositories( self, tsr_ids, decoded_kwd, reinstalling ):
         shed_tool_conf = decoded_kwd.get( 'shed_tool_conf', '' )
         tool_path = decoded_kwd[ 'tool_path' ]
         tool_panel_section_keys = util.listify( decoded_kwd[ 'tool_panel_section_keys' ] )
+        tool_panel_section_mapping = decoded_kwd.get( 'tool_panel_section_mapping', {} )
         repo_info_dicts = util.listify( decoded_kwd[ 'repo_info_dicts' ] )
         install_tool_dependencies = decoded_kwd['install_tool_dependencies']
         filtered_repo_info_dicts = []
@@ -827,14 +840,15 @@ def install_repositories( self, tsr_ids, decoded_kwd, reinstalling ):
                                                    shed_tool_conf=shed_tool_conf,
                                                    tool_path=tool_path,
                                                    install_tool_dependencies=install_tool_dependencies,
-                                                   reinstalling=reinstalling )
+                                                   reinstalling=reinstalling,
+                                                   tool_panel_section_mapping=tool_panel_section_mapping )
                 installed_tool_shed_repositories.append( tool_shed_repository )
         else:
             raise RepositoriesInstalledException()
         return installed_tool_shed_repositories
 
     def install_tool_shed_repository( self, tool_shed_repository, repo_info_dict, tool_panel_section_key, shed_tool_conf, tool_path,
-                                      install_tool_dependencies, reinstalling=False ):
+                                      install_tool_dependencies, reinstalling=False, tool_panel_section_mapping={} ):
         self.app.install_model.context.flush()
         if tool_panel_section_key:
             _, tool_section = self.app.toolbox.get_section( tool_panel_section_key )
@@ -878,7 +892,8 @@ def install_tool_shed_repository( self, tool_shed_repository, repo_info_dict, to
                                                tool_section=tool_section,
                                                shed_tool_conf=shed_tool_conf,
                                                tool_versions_response=tool_versions_response,
-                                               reinstalling=reinstalling )
+                                               reinstalling=reinstalling,
+                                               tool_panel_section_mapping=tool_panel_section_mapping )
             self.install_model.context.refresh( tool_shed_repository )
             metadata = tool_shed_repository.metadata
             if 'tools' in metadata:

lib/tool_shed/galaxy_install/install_manager.py

@@ -37,11 +37,12 @@ def add_to_shed_tool_config( self, shed_tool_conf_dict, elem_list ):
             self.config_elems_to_xml_file( config_elems, shed_tool_conf, tool_path )
 
     def add_to_tool_panel( self, repository_name, repository_clone_url, changeset_revision, repository_tools_tups, owner,
-                           shed_tool_conf, tool_panel_dict, new_install=True ):
+                           shed_tool_conf, tool_panel_dict, new_install=True, tool_panel_section_mapping={} ):
         """A tool shed repository is being installed or updated so handle tool panel alterations accordingly."""
         # We need to change the in-memory version and the file system version of the shed_tool_conf file.
         shed_tool_conf_dict = self.get_shed_tool_conf_dict( shed_tool_conf )
         tool_path = shed_tool_conf_dict[ 'tool_path' ]
+        tool_panel_dict = self.update_tool_panel_dict( tool_panel_dict, tool_panel_section_mapping, repository_tools_tups )
         # Generate the list of ElementTree Element objects for each section or tool.
         elem_list = self.generate_tool_panel_elem_list( repository_name,
                                                         repository_clone_url,
@@ -465,3 +466,14 @@ def remove_guids( self, guids_to_remove, shed_tool_conf, uninstall ):
         # Update the config_elems of the in-memory shed_tool_conf_dict.
         shed_tool_conf_dict[ 'config_elems' ] = config_elems
         toolbox.update_shed_config( shed_tool_conf_dict, integrated_panel_changes=uninstall )
+
+    def update_tool_panel_dict( self, tool_panel_dict, tool_panel_section_mapping, repository_tools_tups ):
+        for tool_guid in tool_panel_dict:
+            if tool_guid not in tool_panel_section_mapping:
+                continue
+            for idx, tool in enumerate( tool_panel_dict[ tool_guid ] ):
+                section_name = tool_panel_section_mapping[ tool_guid ][ 'tool_panel_section' ]
+                section_id = str( tool_panel_section_mapping[ tool_guid ][ 'tool_panel_section' ].lower().replace( ' ', '_' ) )
+                tool_panel_dict[ tool_guid ][ idx ][ 'name' ] = section_name
+                tool_panel_dict[ tool_guid ][ idx ][ 'id' ] = section_id
+        return tool_panel_dict