Merge pull request #2667 from sciencehistory/cloudfront_over_s3

jrochkind · web-flow · commit fe334deea5b0 · 2024-07-10T10:56:11.000-04:00
Support Cloudfront distros in front of select S3 buckets
diff --git a/Gemfile b/Gemfile
@@ -66,6 +66,7 @@ gem "aws-sdk-cloudwatchevents", "~> 1.0"
 gem "aws-sdk-cloudwatchlogs", "~> 1.0"
 gem "aws-sdk-mediaconvert", "~> 1.0"
 gem "aws-sdk-s3", "~> 1.0"
+gem "aws-sdk-cloudfront", "~> 1.91"
 
 # Use postgresql as the database for Active Record
 gem 'pg', '>= 0.18', '< 2.0'
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -97,6 +97,9 @@ GEM
       execjs (~> 2)
     aws-eventstream (1.3.0)
     aws-partitions (1.939.0)
+    aws-sdk-cloudfront (1.91.0)
+      aws-sdk-core (~> 3, >= 3.193.0)
+      aws-sigv4 (~> 1.1)
     aws-sdk-cloudwatchevents (1.72.0)
       aws-sdk-core (~> 3, >= 3.193.0)
       aws-sigv4 (~> 1.1)
@@ -768,6 +771,7 @@ DEPENDENCIES
   activerecord-postgres_enum (~> 2.0)
   alba (~> 3.1)
   attr_json (~> 2.3)
+  aws-sdk-cloudfront (~> 1.91)
   aws-sdk-cloudwatchevents (~> 1.0)
   aws-sdk-cloudwatchlogs (~> 1.0)
   aws-sdk-mediaconvert (~> 1.0)
diff --git a/lib/scihist_digicoll/env.rb b/lib/scihist_digicoll/env.rb
@@ -2,6 +2,7 @@
 require "shrine/storage/file_system"
 require "shrine/storage/s3"
 require 'faster_s3_url/shrine/storage'
+require 'scihist_digicoll/shrine_storage/cloudfront_s3_storage'
 
 
 module ScihistDigicoll
@@ -236,15 +237,37 @@ def self.persistent_redis_connection!
     define_key :s3_bucket_derivatives
     define_key :s3_bucket_derivatives_video
 
-    # Note: the values for :s3_bucket_derivatives_host and :s3_bucket_derivatives_video_host
-    # can be obtained by running `terraform output`.
-    define_key :s3_bucket_derivatives_host      # Cloudfront hostname for regular derivs bucket
-    define_key :s3_bucket_derivatives_video_host # Ditto, for video derivs bucket
-
     define_key :s3_bucket_uploads
     define_key :s3_bucket_on_demand_derivatives
     define_key :s3_bucket_dzi
 
+    # if we are using CloudFront in front of a bucket, we set Cloudfront Distro hostname
+    # in variables corresponding to bucket name env, with `_host` on the end. These
+    # are all assumed cloudfront if set.
+    #
+    # All of these cloudfront hostnames are somewhat opaque, and can be found
+    # from `terraform output` output from existing infrastructure, either staging
+    # or prod.
+
+    define_key :s3_bucket_originals_host
+    define_key :s3_bucket_originals_video_host # not currently used as we don't provide access to originals
+    define_key :s3_bucket_derivatives_host
+    define_key :s3_bucket_derivatives_video_host
+    define_key :s3_bucket_on_demand_derivatives_host
+    define_key :s3_bucket_dzi_host
+
+    # If we are using Cloudfront with restricted buckets, the key-pair id and RSA public
+    # key need to be set.
+    define_key :cloudfront_key_pair_id # from `terraform output`
+    # private key can be found in 1password as:
+    #   `scihist-digicoll-production_private_key.pem` or
+    #   `scihist-digicoll-staging_private_key.pem`
+    #
+    #   Value needs to be exported from 1Password in `PKCS#8` format, will begin with
+    #   '-----BEGIN PRIVATE KEY-----' [NOT 'OPENSSH PRIVATE KEY']
+    define_key :cloudfront_private_key
+
+
     define_key :ingest_bucket, default: -> {
       if !Rails.env.production?
         # This bucket isn't actually mounted on workstations, but you can
@@ -361,13 +384,18 @@ def self.solr_collection_name
     # @param bucket_key [String] required. in `production` mode this is the bucket name, otherwise
     #                            it becomes part of the prefix location.
     #
+    # @param public [Boolean] (default false), if FALSE the storage requires signed URLs, and
+    #                         we will set up shrine storage to generate them -- either using
+    #                         AWS access key (direct), or Cloudfront public key (if host is set for
+    #                         Cloudfront distro in front)
+    #
     # @param prefix [String] prefix passed to shrine storage, a "directory" within the
     #                        storage. for dev_s3 and dev_file modes combined with
     #                        other pre-prefix.
     #
-    # @param host [String] used only for `production` S3 mode, passed to Shrine storage as
-    #                      'host' param, used for cloudfront CDN and/or other CNAME, alternate
-    #                      host used to access S3 bucket.
+    # @param host [String] used only for `production` S3 mode, if set we assume a CloudFront CDN distro
+    #                      on top of bucket, and set up url-generation accordingly, including with
+    #                      Cloudfront-style signing (using env for cloudfront public key which will be required)
     #
     # @param s3_storage_options [Hash] passed directly to Shrine storage for S3 modes, additional
     #                                  arbitrary options. Can override other defaults or params.
@@ -377,7 +405,7 @@ def self.solr_collection_name
     #                       file system). Normally left unset, it will default to
     #                       env key :storage_mode, which is what you want it to do.
     #
-    def self.appropriate_shrine_storage(bucket_key:, mode: lookup!(:storage_mode), prefix: nil,
+    def self.appropriate_shrine_storage(bucket_key:, public: false, mode: lookup!(:storage_mode), prefix: nil,
                                         host: nil, s3_storage_options: {} )
       unless %I{s3_bucket_uploads s3_bucket_originals s3_bucket_originals_video s3_bucket_derivatives
                 s3_bucket_derivatives_video
@@ -400,14 +428,29 @@ def self.appropriate_shrine_storage(bucket_key:, mode: lookup!(:storage_mode), p
           region:            lookup(:aws_region)
         }.merge(s3_storage_options))
       elsif mode == "production"
-        FasterS3Url::Shrine::Storage.new(**{
-          bucket:            lookup!(bucket_key),
-          host:              host,
-          prefix:            prefix,
-          access_key_id:     lookup!(:aws_access_key_id),
-          secret_access_key: lookup!(:aws_secret_access_key),
-          region:            lookup!(:aws_region)
-        }.merge(s3_storage_options))
+        if host.present?
+          # Assumed cloudfront if we have a host!
+          ScihistDigicoll::ShrineStorage::CloudfrontS3Storage.new(**{
+            bucket:            lookup!(bucket_key),
+            host:              host,
+            public:            public,
+            prefix:            prefix,
+            access_key_id:     lookup!(:aws_access_key_id),
+            secret_access_key: lookup!(:aws_secret_access_key),
+            region:            lookup!(:aws_region),
+            cloudfront_key_pair_id:     lookup(:cloudfront_key_pair_id),
+            cloudfront_private_key: lookup(:cloudfront_private_key)
+          }.merge(s3_storage_options))
+        else
+          FasterS3Url::Shrine::Storage.new(**{
+            bucket:            lookup!(bucket_key),
+            public:            public,
+            prefix:            prefix,
+            access_key_id:     lookup!(:aws_access_key_id),
+            secret_access_key: lookup!(:aws_secret_access_key),
+            region:            lookup!(:aws_region)
+          }.merge(s3_storage_options))
+        end
       else
         raise TypeError.new("unrecognized storage mode: #{mode}")
       end
@@ -423,22 +466,22 @@ def self.shrine_cache_storage
 
     def self.shrine_store_storage
       @shrine_store_storage ||=
-        appropriate_shrine_storage(bucket_key: :s3_bucket_originals)
+        appropriate_shrine_storage(bucket_key: :s3_bucket_originals, public: false, host: lookup(:s3_bucket_originals_host))
     end
 
     # we store video originals in separate location
     def self.shrine_store_video_storage
       @shrine_video_store_storage ||=
-        appropriate_shrine_storage(bucket_key: :s3_bucket_originals_video)
+        appropriate_shrine_storage(bucket_key: :s3_bucket_originals_video, public: false, host: lookup(:s3_bucket_originals_video_host))
     end
 
     # Note we set shrine S3 storage to public, to upload with public ACLs
     def self.shrine_derivatives_storage
       @shrine_derivatives_storage ||=
         appropriate_shrine_storage( bucket_key: :s3_bucket_derivatives,
                                     host: lookup(:s3_bucket_derivatives_host),
+                                    public: true,
                                     s3_storage_options: {
-                                      public: true,
                                       upload_options: {
                                         # derivatives are public and at unique random URLs, so
                                         # can be cached far-future
@@ -451,8 +494,8 @@ def self.shrine_video_derivatives_storage
       @shrine_derivatives_video_storage ||=
         appropriate_shrine_storage( bucket_key: :s3_bucket_derivatives_video,
                                     host: lookup(:s3_bucket_derivatives_video_host),
+                                    public: true,
                                     s3_storage_options: {
-                                      public: true,
                                       upload_options: {
                                         # derivatives are public and at unique random URLs, so
                                         # can be cached far-future
@@ -467,16 +510,25 @@ def self.shrine_video_derivatives_storage
     def self.shrine_restricted_derivatives_storage
       @shrine_restricted_derivatives_storage ||=
         appropriate_shrine_storage( bucket_key: :s3_bucket_originals,
+                                    host: lookup(:s3_bucket_originals_host),
+                                    public: false,
                                     prefix: "restricted_derivatives")
     end
 
     # Note we set shrine S3 storage to public, to upload with public ACLs
     def self.shrine_on_demand_derivatives_storage
       @shrine_on_demand_derivatives_storage ||=
         appropriate_shrine_storage( bucket_key: :s3_bucket_on_demand_derivatives,
+                                    host: lookup(:s3_bucket_on_demand_derivatives_host),
+                                    public: true,
                                     s3_storage_options: {
-                                      public: true
-                                    })
+                                      upload_options: {
+                                        # these have fingerprints in their URLs, so they are
+                                        # cacheable forever. only started setting this in Jul 2024
+                                        cache_control: "max-age=31536000, public"
+                                      }
+                                    }
+                                  )
     end
 
     def self.shrine_combined_audio_derivatives_storage
@@ -485,16 +537,22 @@ def self.shrine_combined_audio_derivatives_storage
         appropriate_shrine_storage( bucket_key: :s3_bucket_derivatives,
                                     host: lookup(:s3_bucket_derivatives_host),
                                     prefix: "combined_audio_derivatives",
+                                    public: true,
                                     s3_storage_options: {
-                                      public: true
+                                      upload_options: {
+                                        # these have fingerprints in their URLs, so they are
+                                        # cacheable forever. only started setting this in Jul 2024
+                                        cache_control: "max-age=31536000, public"
+                                      }
                                     })
     end
 
     def self.shrine_dzi_storage
       @shrine_dzi_storage ||=
         appropriate_shrine_storage( bucket_key: :s3_bucket_dzi,
+                                    host: lookup(:s3_bucket_dzi_host),
+                                    public: true,
                                     s3_storage_options: {
-                                      public: true,
                                       upload_options: {
                                         # our DZI's are all public right now, and at unique-to-content
                                         # URLs, cache forever.
diff --git a/lib/scihist_digicoll/shrine_storage/cloudfront_s3_storage.rb b/lib/scihist_digicoll/shrine_storage/cloudfront_s3_storage.rb
@@ -0,0 +1,125 @@
+require "shrine/storage/s3"
+require 'faster_s3_url/shrine/storage'
+require "aws-sdk-cloudfront"
+
+
+# A shrine storage for use with AWS S3, and sub-classing Shrine's S3 storage -- but will generate
+# access URLs assuming CloudFront CDN in front of S3, according to our conventions.
+#
+# A `host` is required on initialization -- the cloudfront distro hostname
+# `public` is set on initialization, and can't be over-ridden in #url, if not public then CloudFront signing will be used
+#
+# Cloudfront key id and public key are generally supplied from Env.
+#
+# https://shrinerb.com/docs/storage/s3
+#
+# For the clodufront_key_pair_id and cloudfront_private_key, see https://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/private-content-trusted-signers.html#private-content-creating-cloudfront-key-pairs
+#
+# For general write-up of how we've set things up, see https://bibwild.wordpress.com/2024/06/18/cloudfront-in-front-of-s3-using-response-content-disposition/
+module ScihistDigicoll
+  module ShrineStorage
+    class CloudfrontS3Storage < ::Shrine::Storage::S3
+      attr_reader :host, :public_mode, :cloudfront_signer, :key_pair_id
+
+      QUERY_PARAMS_PROXIED = {
+        response_cache_control: "response-cache-control",
+        response_content_disposition: "response-content-disposition",
+        response_content_encoding: "response-content-encoding",
+        response_content_language: "response-content-language",
+        response_content_type: "response-content-type"
+      }.freeze
+
+      DEFAULT_EXPIRES_IN = 1.day
+
+      # generally any option tht Shrine::Storage::S3 respects can be used, and are passed through,
+      # except `signer` is not supported and will error -- we always use CloudFront signing if public: false
+      #
+      # @param host [String] set only in initializer, the Cloudfront distro host
+      #
+      # @parma public [Boolean] (default true), if false, then Cloudfront signing will be done to urls, can not be
+      #      over-ridden in #url, since the way we set up CloudFront distros they either require signing of all urls or none.
+      #
+      # @param cloudfront_key_pair_kid [String] required if public:false, the cloudfront_key_pair id used for signing
+      #
+      # @param cloudfront_private_key [String] required if public:false, RSA publikc key corresopnding to cloudfront_key_pair_id
+      #
+      # NOTE: You will still need keys `access_key_id:` and `secret_access_key:` (or otherwise have AWS credentials
+      #       auto-discoverable), because bucket edit operations etc require them!
+      #
+      def initialize(host:, public: true, cloudfront_key_pair_id: nil, cloudfront_private_key: nil, **options)
+        if options[:signer]
+          raise ArgumentError.new("#{self.class.name} does not support :signer option of Shrine::Storage::S3.")
+        end
+
+        @public_mode = !!public
+        @host = host
+
+        @public_builder = FasterS3Url::Shrine::Storage.new(public: true, host: host, **options)
+
+        if !public_mode
+          @cloudfront_signer = Aws::CloudFront::UrlSigner.new(
+             key_pair_id: (cloudfront_key_pair_id || raise(ArgumentError.new("option `cloudfront_key_pair_id:` is required when `public:` is false"))),
+             private_key: (cloudfront_private_key || raise(ArgumentError.new("option `cloudfront_private_key:` is required when `public:` is false")))
+          )
+        end
+
+        # We always tell the underlying shrine s3 storage we are NOt public, becuase we assume
+        # the underlying S3 bucket is not public, in case it matters, we need to be signing requests
+        # etc.
+        super(public: false, **options)
+      end
+
+      # unlike base Shrine::Storage::S3, does not support `host` here, do it in
+      # initializer instead.
+      #
+      # # We IGNORE `public` option here -- our Cloudfront is either public or does not support,
+      # public, no way to change per-url. But we ignore rather than raise, to allow
+      # swap-in compatibility with code expecting to send it for normal S3.
+      #
+      # Unlike Shrine::Storage::S3, recognized S3 options (AWS ruby SDK style)
+      # *are* passed on in public mode too, becuase in some cases we have set
+      # up cloudfront to proxy them.
+      #
+      # Otherwise, same options as Shrine::S3::Storage should be supported, please
+      # see docs there. https://shrinerb.com/docs/storage/s3
+      def url(id, **options)
+        if options[:host]
+          raise ArgumentError.new("#{self.class.name}#url does not support :host option of Shrine::Storage::S3. You can only set host in initializer")
+        end
+
+        if public_mode
+          public_url(id, **options)
+        else
+          signed_url(id, **options)
+        end
+      end
+
+      def public_url(key, **options)
+        "#{@public_builder.url(key)}#{query_param_serialized(options)}"
+      end
+
+      def signed_url(key, expires_in: DEFAULT_EXPIRES_IN, **options)
+        expires = options[:expires]&.to_i || (Time.now.utc.to_i + expires_in.to_i)
+
+        cloudfront_signer.signed_url(public_url(key, **options),
+          expires: expires,
+          **options
+        )
+      end
+
+      protected
+
+      def query_param_serialized(options)
+        return nil if options.blank?
+
+        params = options.collect do |key, value|
+          [QUERY_PARAMS_PROXIED[key], value] if QUERY_PARAMS_PROXIED.has_key?(key)
+        end.compact
+
+        return nil if params.blank?
+
+        "?#{params.to_h.compact.to_param}"
+      end
+    end
+  end
+end
diff --git a/spec/lib/scihist_digicoll/shrine_storage/cloudfront_s3_storage_spec.rb b/spec/lib/scihist_digicoll/shrine_storage/cloudfront_s3_storage_spec.rb
diff --git a/spec/test_support/demo_private_key.pem b/spec/test_support/demo_private_key.pem
