Merge commit '86ceaf920b21c4405794c867d60ad13552fec98e' into dinsic-release-v1.10.x

* commit '86ceaf920b21c4405794c867d60ad13552fec98e':
  Update tests that assumed PL50 for room upgrade, fix others (#805)
  Test /user/devices/ on fed reader (#799)
  Update tests to match removal of Synapse sending events when creating or deleting aliases.
  Route GET /groups queries to workers (#798)
  Don't push Dendrite images to Docker Hub (#797)
  Test that remote device list is refetched on leave/rejoin (#793)
  Update build.sh
  Update dendrite.Dockerfile
  Apply suggestions from code review
  add test for device key signatures over federation

Author: anoadragon453

.gitignore

@@ -11,4 +11,5 @@
 /var
 \#*
 .vscode
-.coverage*
+.coverage*
+.idea

docker/README.md

@@ -9,7 +9,6 @@ Included currently is:
 - matrixdotorg/sytest:stretch and matrixdotorg/sytest:buster, base containers with SyTest dependencies installed
 - matrixdotorg/sytest-synapse:py35, a container which will run SyTest against Synapse on Python 3.5 + Stretch
 - matrixdotorg/sytest-synapse:py37, a container which will run SyTest against Synapse on Python 3.7 + Buster
-- matrixdotorg/sytest-dendrite:go111, a container which will run SyTest against Dendrite on Go 1.11 + Stretch
 - matrixdotorg/sytest-dendrite:go113, a container which will run SyTest against Dendrite on Go 1.13 + Buster
 
 ## Using the containers

docker/dendrite.Dockerfile

@@ -1,11 +1,9 @@
 ARG DEBIAN_VERSION=buster
-
 FROM matrixdotorg/sytest:${DEBIAN_VERSION}
 
-ARG GO_VERSION
+ARG GO_VERSION=1.13.7
 ENV GO_DOWNLOAD https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz
 
-# Install Go 1.11
 RUN mkdir -p /goroot /gopath
 RUN wget -q $GO_DOWNLOAD -O go.tar.gz
 RUN tar xf go.tar.gz -C /goroot --strip-components=1

docker/push.sh

@@ -10,6 +10,3 @@ docker push matrixdotorg/sytest:bullseye
 docker push matrixdotorg/sytest-synapse:py35
 docker push matrixdotorg/sytest-synapse:py37
 docker push matrixdotorg/sytest-synapse:py38
-docker push matrixdotorg/sytest-dendrite:latest
-docker push matrixdotorg/sytest-dendrite:go111
-docker push matrixdotorg/sytest-dendrite:go113

lib/SyTest/Federation/Client.pm

@@ -271,6 +271,8 @@ sub join_room
 
          $room->insert_event( $member_event );
 
+         $store->{rooms_by_id}{ $room->room_id } = $room;
+
          Future->done( $room );
       });
    });

lib/SyTest/Federation/Datastore.pm

@@ -301,7 +301,10 @@ sub get_auth_chain_events
    while( @event_ids ) {
       my $event = $events_by_id{shift @event_ids};
 
-      my @auth_ids = map { $_->[0] } @{ $event->{auth_events} };
+      my $room = $self->get_room( $event->{room_id} ) or
+         croak "Unknown room $event->{room_id}";
+
+      my @auth_ids = @{ $room->event_ids_from_refs( $event->{auth_events} ) };
 
       foreach my $id ( @auth_ids ) {
          next if $events_by_id{$id};

lib/SyTest/Federation/Room.pm

@@ -321,6 +321,10 @@ sub insert_outlier_event
 
    croak "Event not ref" unless ref $event;
 
+   my $event_id = $self->id_for_event( $event );
+
+   $self->{datastore}->put_event( $event_id, $event );
+
    if( defined $event->{state_key} ) {
       $self->{current_state}{ join "\0", $event->{type}, $event->{state_key} }
          = $event;

lib/SyTest/Federation/Server.pm

@@ -274,10 +274,13 @@ sub on_request_federation_v1_make_join
          404, "Not found", [ Content_length => 0 ], "",
       ) );
 
+   my $room_version = $room->room_version;
+
    Future->done( json => {
       event => $room->make_join_protoevent(
          user_id => $user_id,
       ),
+      room_version => "$room_version",
    } );
 }
 
@@ -308,7 +311,7 @@ sub on_request_federation_v2_send_join
    my $event = $req->body_from_json;
 
    my @auth_chain = $store->get_auth_chain_events(
-      map { $_->[0] } @{ $event->{auth_events} }
+      @{ $room->event_ids_from_refs( $event->{auth_events} ) }
    );
    my @state_events = $room->current_state_events;
 
@@ -477,6 +480,15 @@ sub on_request_federation_v1_send
 
    # A PDU is an event
    foreach my $event ( @{ $body->{pdus} } ) {
+      my $room = $self->{datastore}->get_room( $event->{room_id} );
+
+      if ( $room ) {
+         my $event_id = $room->id_for_event( $event );
+         $room->insert_event( $event );
+      } else {
+         warn "Unknown room ${ $event->{room_id} }"
+      }
+
       next if $self->on_event( $event );
 
       warn "TODO: Unhandled incoming event of type '$event->{type}'";
@@ -545,4 +557,41 @@ sub on_event
    return 1;
 }
 
+=head2 await_request_v1_send_join_reject_v2
+
+   $fut = $server->await_request_v1_send_join_reject_v2( $room_id );
+   my ( $request, $room_id, $event_id ) = $fut->get;
+
+Awaits inbound request for /v1/send_join endpoint while rejecting inbound
+requests to /v2/send_join. Using the C<await_request_v1_send_join> standard
+has the problem that C<SyTest::Federation::Server> will handle /v2/send_join
+appropriately unless overriden, and so remote servers that use v2 will never
+call v1 endpoint in such a case.
+
+=cut
+
+sub await_request_v1_send_join_reject_v2 {
+   my $self = shift;
+   my ( $room_id ) = @_;
+
+   my $v2_fut = $self->await_request_v2_send_join( $room_id )
+   ->then( sub {
+      my ( $req, $room_id, $event_id ) = @_;
+      $req->respond( HTTP::Response->new(
+         404, "Not found", [ Content_length => 0 ], "",
+      ) );
+
+      Future->done
+   });
+
+   $self->await_request_v1_send_join( $room_id )
+   ->then( sub {
+      my ( $req, $room_id, $event_id ) = @_;
+
+      $v2_fut->cancel();
+
+      Future->done( $req, $room_id, $event_id )
+   })
+}
+
 1;

lib/SyTest/Homeserver/Synapse.pm

@@ -1073,6 +1073,7 @@ sub start
 
       $self->{paths}{pem_file} = $self->write_file( "combined.pem", $cert . $key );
       $self->{paths}{path_map_file} = $self->write_file( "path_map_file", $self->generate_haproxy_map );
+      $self->{paths}{get_path_map_file} = $self->write_file( "get_path_map_file", $self->generate_haproxy_get_map );
 
       $self->{haproxy_config} = $self->write_file( "haproxy.conf", $self->generate_haproxy_config );
 
@@ -1132,6 +1133,9 @@ defaults
 frontend http-in
     bind ${bind_host}:$ports->{haproxy} ssl crt $self->{paths}{pem_file}
 
+    acl has_get_map path -m reg -M -f $self->{paths}{get_path_map_file}
+    use_backend %[path,map_reg($self->{paths}{get_path_map_file},synapse)] if has_get_map METH_GET
+
     use_backend %[path,map_reg($self->{paths}{path_map_file},synapse)]
 
 backend synapse
@@ -1187,6 +1191,8 @@ sub generate_haproxy_map
 ^/_matrix/federation/v1/event_auth/                   federation_reader
 ^/_matrix/federation/v1/exchange_third_party_invite/  federation_reader
 ^/_matrix/federation/v1/send/                         federation_reader
+^/_matrix/federation/v1/get_groups_publicised         federation_reader
+^/_matrix/federation/v1/user/devices/                 federation_reader
 ^/_matrix/key/v2/query                                federation_reader
 
 ^/_matrix/client/(api/v1|r0|unstable)/publicRooms$                client_reader
@@ -1202,6 +1208,8 @@ sub generate_haproxy_map
 ^/_matrix/client/(api/v1|r0|unstable)/voip/turnServer$            client_reader
 ^/_matrix/client/(r0|unstable)/register$                          client_reader
 ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/messages$          client_reader
+^/_matrix/client/(api/v1|r0|unstable)/get_groups_publicised$      client_reader
+^/_matrix/client/(api/v1|r0|unstable)/joined_groups$              client_reader
 
 ^/_matrix/client/(api/v1|r0|unstable)/keys/upload  frontend_proxy
 
@@ -1215,6 +1223,16 @@ sub generate_haproxy_map
 EOCONFIG
 }
 
+sub generate_haproxy_get_map
+{
+    return <<'EOCONFIG';
+^/_matrix/federation/v1/groups/                 federation_reader
+
+^/_matrix/client/(api/v1|r0|unstable)/groups/   client_reader
+
+EOCONFIG
+}
+
 sub secure_port
 {
    my $self = shift;

tests/30rooms/05aliases.pl

@@ -177,40 +177,13 @@ sub _test_can_create_and_delete_alias {
 
       content => { room_id => $room_id },
    )->then( sub {
-      matrix_get_room_state( $user, $room_id,
-         type      => "m.room.aliases",
-         state_key => $server_name,
-      )
-   })->then( sub {
-      my ( $body ) = @_;
-
-      log_if_fail "Aliases after adding alias", $body;
-
-      assert_json_keys( $body, qw( aliases ) );
-      assert_json_list( my $aliases = $body->{aliases} );
-
-      any { $_ eq $alias } @$aliases or die "Expected alias to be in list";
-
       do_request_json_for( $user,
         method => "DELETE",
         uri    => "/r0/directory/room/$alias",
 
         content => {},
       )
    })->then( sub {
-      matrix_get_room_state( $user, $room_id,
-         type      => "m.room.aliases",
-         state_key => $server_name,
-      )
-   })->then( sub {
-      my ( $body ) = @_;
-
-      log_if_fail "Aliases after deleting alias", $body;
-      assert_json_keys( $body, qw( aliases ) );
-      assert_json_list( my $aliases = $body->{aliases} );
-
-      none { $_ eq $alias } @$aliases or die "Expected alias to not be in list";
-
       Future->done;
    });
 }
@@ -278,37 +251,13 @@ sub _test_can_create_and_delete_alias {
             content => { alias => $room_alias }
          )
       })->then( sub {
-         matrix_get_room_state( $creator, $room_id,
-            type      => "m.room.aliases",
-            state_key => $server_name,
-         )
-      })->then( sub {
-         my ( $body ) = @_;
-
-         assert_json_keys( $body, qw( aliases ) );
-         assert_json_list( my $aliases = $body->{aliases} );
-
-         any { $_ eq $room_alias } @$aliases or die "Expected alias to be in list";
-
          do_request_json_for( $creator,
            method => "DELETE",
            uri    => "/r0/directory/room/$room_alias",
 
            content => {},
          )
       })->then( sub {
-         matrix_get_room_state( $creator, $room_id,
-            type      => "m.room.aliases",
-            state_key => $server_name,
-         )
-      })->then( sub {
-         my ( $body ) = @_;
-
-         assert_json_keys( $body, qw( aliases ) );
-         assert_json_list( my $aliases = $body->{aliases} );
-
-         none { $_ eq $room_alias } @$aliases or die "Expected alias to not be in list";
-
          matrix_get_room_state( $creator, $room_id,
             type      => "m.room.canonical_alias",
          )
@@ -340,37 +289,13 @@ sub _test_can_create_and_delete_alias {
             content => { room_id => $room_id },
          )
       })->then( sub {
-         matrix_get_room_state( $creator, $room_id,
-            type      => "m.room.aliases",
-            state_key => $server_name,
-         )
-      })->then( sub {
-         my ( $body ) = @_;
-
-         assert_json_keys( $body, qw( aliases ) );
-         assert_json_list( my $aliases = $body->{aliases} );
-
-         any { $_ eq $room_alias } @$aliases or die "Expected alias to be in list";
-
          do_request_json_for( $other_user,
            method => "DELETE",
            uri    => "/r0/directory/room/$room_alias",
 
            content => {},
          )
       })->then( sub {
-         matrix_get_room_state( $creator, $room_id,
-            type      => "m.room.aliases",
-            state_key => $server_name,
-         )
-      })->then( sub {
-         my ( $body ) = @_;
-
-         assert_json_keys( $body, qw( aliases ) );
-         assert_json_list( my $aliases = $body->{aliases} );
-
-         none { $_ eq $room_alias } @$aliases or die "Expected alias to not be in list";
-
          Future->done(1);
       })
    };
@@ -399,37 +324,13 @@ sub _test_can_create_and_delete_alias {
             content => { alias => $room_alias }
          )
       })->then( sub {
-         matrix_get_room_state( $creator, $room_id,
-            type      => "m.room.aliases",
-            state_key => $server_name,
-         )
-      })->then( sub {
-         my ( $body ) = @_;
-
-         assert_json_keys( $body, qw( aliases ) );
-         assert_json_list( my $aliases = $body->{aliases} );
-
-         any { $_ eq $room_alias } @$aliases or die "Expected alias to be in list";
-
          do_request_json_for( $other_user,
            method => "DELETE",
            uri    => "/r0/directory/room/$room_alias",
 
            content => {},
          )
       })->then( sub {
-         matrix_get_room_state( $creator, $room_id,
-            type      => "m.room.aliases",
-            state_key => $server_name,
-         )
-      })->then( sub {
-         my ( $body ) = @_;
-
-         assert_json_keys( $body, qw( aliases ) );
-         assert_json_list( my $aliases = $body->{aliases} );
-
-         none { $_ eq $room_alias } @$aliases or die "Expected alias to not be in list";
-
          Future->done(1);
       })
    };