Allow configure services for individual attachments

This commit is contained in:
DmitryTsepelev 2019-01-14 22:56:35 +03:00
parent eaff375de4
commit e7f798c3f5
20 changed files with 296 additions and 80 deletions

@ -2,13 +2,14 @@
class CreateActiveStorageTables < ActiveRecord::Migration[5.2]
def change
create_table :active_storage_blobs do |t|
t.string :key, null: false
t.string :filename, null: false
t.string :key, null: false
t.string :filename, null: false
t.string :content_type
t.text :metadata
t.bigint :byte_size, null: false
t.string :checksum, null: false
t.datetime :created_at, null: false
t.string :service_name, null: false
t.bigint :byte_size, null: false
t.string :checksum, null: false
t.datetime :created_at, null: false
t.index [ :key ], unique: true
end

@ -36,6 +36,7 @@
t.string "filename", null: false
t.string "content_type"
t.text "metadata"
t.string "service_name", null: false
t.bigint "byte_size", null: false
t.string "checksum", null: false
t.datetime "created_at", null: false

@ -2,13 +2,14 @@
class CreateActiveStorageTables < ActiveRecord::Migration[5.2]
def change
create_table :active_storage_blobs do |t|
t.string :key, null: false
t.string :filename, null: false
t.string :key, null: false
t.string :filename, null: false
t.string :content_type
t.text :metadata
t.bigint :byte_size, null: false
t.string :checksum, null: false
t.datetime :created_at, null: false
t.string :service_name, null: false
t.bigint :byte_size, null: false
t.string :checksum, null: false
t.datetime :created_at, null: false
t.index [ :key ], unique: true
end

@ -37,6 +37,7 @@
t.string "filename", null: false
t.string "content_type"
t.text "metadata"
t.string "service_name", null: false
t.bigint "byte_size", null: false
t.string "checksum", null: false
t.datetime "created_at", null: false

@ -1,3 +1,17 @@
* Allow storage services to be configured per attachment
```ruby
class User < ActiveRecord::Base
has_one_attached :avatar, service: :s3
end
class Gallery < ActiveRecord::Base
has_many_attached :photos, service: :s3
end
```
*Dmitry Tsepelev*
* You can optionally provide a custom blob key when attaching a new file:
```ruby

@ -38,10 +38,24 @@ class ActiveStorage::Blob < ActiveRecord::Base
scope :unattached, -> { left_joins(:attachments).where(ActiveStorage::Attachment.table_name => { blob_id: nil }) }
after_initialize do
self.service_name ||= Rails.configuration.active_storage.service
end
before_destroy(prepend: true) do
raise ActiveRecord::InvalidForeignKey if attachments.exists?
end
validates :service_name, presence: true
validate do
if service_name_changed? && service_name
ActiveStorage::ServiceRegistry.fetch(service_name) do
errors.add(:service_name, :invalid)
end
end
end
class << self
# You can use the signed ID of a blob to refer to it on the client side without fear of tampering.
# This is particularly helpful for direct uploads where the client-side needs to refer to the blob
@ -52,22 +66,22 @@ def find_signed(id, record: nil)
find ActiveStorage.verifier.verify(id, purpose: :blob_id)
end
def build_after_upload(io:, filename:, content_type: nil, metadata: nil, identify: true, record: nil) #:nodoc:
new(filename: filename, content_type: content_type, metadata: metadata).tap do |blob|
def build_after_upload(io:, filename:, content_type: nil, metadata: nil, service_name: nil, identify: true, record: nil) #:nodoc:
new(filename: filename, content_type: content_type, metadata: metadata, service_name: service_name).tap do |blob|
blob.upload(io, identify: identify)
end
end
deprecate :build_after_upload
def build_after_unfurling(key: nil, io:, filename:, content_type: nil, metadata: nil, identify: true, record: nil) #:nodoc:
new(key: key, filename: filename, content_type: content_type, metadata: metadata).tap do |blob|
def build_after_unfurling(key: nil, io:, filename:, content_type: nil, metadata: nil, service_name: nil, identify: true, record: nil) #:nodoc:
new(key: key, filename: filename, content_type: content_type, metadata: metadata, service_name: service_name).tap do |blob|
blob.unfurl(io, identify: identify)
end
end
def create_after_unfurling!(key: nil, io:, filename:, content_type: nil, metadata: nil, identify: true, record: nil) #:nodoc:
build_after_unfurling(key: key, io: io, filename: filename, content_type: content_type, metadata: metadata, identify: identify).tap(&:save!)
def create_after_unfurling!(key: nil, io:, filename:, content_type: nil, metadata: nil, service_name: nil, identify: true, record: nil) #:nodoc:
build_after_unfurling(key: key, io: io, filename: filename, content_type: content_type, metadata: metadata, service_name: service_name, identify: identify).tap(&:save!)
end
# Creates a new blob instance and then uploads the contents of
@ -75,8 +89,8 @@ def create_after_unfurling!(key: nil, io:, filename:, content_type: nil, metadat
# be saved before the upload begins to prevent the upload clobbering another due to key collisions.
# When providing a content type, pass <tt>identify: false</tt> to bypass
# automatic content type inference.
def create_and_upload!(key: nil, io:, filename:, content_type: nil, metadata: nil, identify: true, record: nil)
create_after_unfurling!(key: key, io: io, filename: filename, content_type: content_type, metadata: metadata, identify: identify).tap do |blob|
def create_and_upload!(key: nil, io:, filename:, content_type: nil, metadata: nil, service_name: nil, identify: true, record: nil)
create_after_unfurling!(key: key, io: io, filename: filename, content_type: content_type, metadata: metadata, service_name: service_name, identify: identify).tap do |blob|
blob.upload_without_unfurling(io)
end
end
@ -89,8 +103,8 @@ def create_and_upload!(key: nil, io:, filename:, content_type: nil, metadata: ni
# in order to produce the signed URL for uploading. This signed URL points to the key generated by the blob.
# Once the form using the direct upload is submitted, the blob can be associated with the right record using
# the signed ID.
def create_before_direct_upload!(key: nil, filename:, byte_size:, checksum:, content_type: nil, metadata: nil, record: nil)
create! key: key, filename: filename, byte_size: byte_size, checksum: checksum, content_type: content_type, metadata: metadata
def create_before_direct_upload!(key: nil, filename:, byte_size:, checksum:, content_type: nil, metadata: nil, service_name: nil, record: nil)
create! key: key, filename: filename, byte_size: byte_size, checksum: checksum, content_type: content_type, metadata: metadata, service_name: service_name
end
# To prevent problems with case-insensitive filesystems, especially in combination
@ -248,6 +262,11 @@ def purge_later
ActiveStorage::PurgeJob.perform_later(self)
end
# Returns an instance of service, which can be configured globally or per attachment
def service
ActiveStorage::ServiceRegistry.fetch(service_name)
end
private
def compute_checksum_in_chunks(io)
Digest::MD5.new.tap do |checksum|

@ -1,13 +1,14 @@
class CreateActiveStorageTables < ActiveRecord::Migration[5.2]
def change
create_table :active_storage_blobs do |t|
t.string :key, null: false
t.string :filename, null: false
t.string :key, null: false
t.string :filename, null: false
t.string :content_type
t.text :metadata
t.bigint :byte_size, null: false
t.string :checksum, null: false
t.datetime :created_at, null: false
t.string :service_name, null: false
t.bigint :byte_size, null: false
t.string :checksum, null: false
t.datetime :created_at, null: false
t.index [ :key ], unique: true
end

@ -0,0 +1,13 @@
class AddServiceNameToActiveStorageBlobs < ActiveRecord::Migration[6.0]
def up
unless column_exists?(:active_storage_blobs, :service_name)
add_column :active_storage_blobs, :service_name, :string
if configured_service = Rails.configuration.active_storage.service
ActiveStorage::Blob.unscoped.update_all(service_name: configured_service)
end
change_column :active_storage_blobs, :service_name, :string, null: false
end
end
end

@ -38,6 +38,7 @@ module ActiveStorage
autoload :Attached
autoload :Service
autoload :ServiceRegistry
autoload :Previewer
autoload :Analyzer

@ -58,14 +58,20 @@ def find_or_build_blob
filename: attachable.original_filename,
content_type: attachable.content_type,
record: record,
service_name: attachment_service_name
)
when Hash
ActiveStorage::Blob.build_after_unfurling(**attachable.merge(record: record))
args = attachable.reverse_merge(record: record, service_name: attachment_service_name)
ActiveStorage::Blob.build_after_unfurling(**args)
when String
ActiveStorage::Blob.find_signed(attachable, record: record)
else
raise ArgumentError, "Could not find or build blob: expected attachable, got #{attachable.inspect}"
end
end
def attachment_service_name
record.attachment_reflections[name].options[:service_name]
end
end
end

@ -32,7 +32,17 @@ module Attached::Model
#
# If the +:dependent+ option isn't set, the attachment will be purged
# (i.e. destroyed) whenever the record is destroyed.
def has_one_attached(name, dependent: :purge_later)
#
# If you need the attachment to use a service which differs from the globally configured one,
# pass the +:service+ option. For instance:
#
# class User < ActiveRecord::Base
# has_one_attached :avatar, service: :s3
# end
#
def has_one_attached(name, dependent: :purge_later, service: nil)
validate_service_configuration(name, service)
generated_association_methods.class_eval <<-CODE, __FILE__, __LINE__ + 1
def #{name}
@active_storage_attached_#{name} ||= ActiveStorage::Attached::One.new("#{name}", self)
@ -57,11 +67,14 @@ def #{name}=(attachable)
after_commit(on: %i[ create update ]) { attachment_changes.delete(name.to_s).try(:upload) }
ActiveRecord::Reflection.add_attachment_reflection(
self,
reflection = ActiveRecord::Reflection.create(
:has_one_attached,
name,
ActiveRecord::Reflection.create(:has_one_attached, name, nil, { dependent: dependent }, self)
nil,
{ dependent: dependent, service_name: service },
self
)
ActiveRecord::Reflection.add_attachment_reflection(self, name, reflection)
end
# Specifies the relation between multiple attachments and the model.
@ -88,7 +101,17 @@ def #{name}=(attachable)
#
# If the +:dependent+ option isn't set, all the attachments will be purged
# (i.e. destroyed) whenever the record is destroyed.
def has_many_attached(name, dependent: :purge_later)
#
# If you need the attachment to use a service which differs from the globally configured one,
# pass the +:service+ option. For instance:
#
# class Gallery < ActiveRecord::Base
# has_many_attached :photos, service: :s3
# end
#
def has_many_attached(name, dependent: :purge_later, service: nil)
validate_service_configuration(name, service)
generated_association_methods.class_eval <<-CODE, __FILE__, __LINE__ + 1
def #{name}
@active_storage_attached_#{name} ||= ActiveStorage::Attached::Many.new("#{name}", self)
@ -130,12 +153,24 @@ def purge_later
after_commit(on: %i[ create update ]) { attachment_changes.delete(name.to_s).try(:upload) }
ActiveRecord::Reflection.add_attachment_reflection(
self,
reflection = ActiveRecord::Reflection.create(
:has_many_attached,
name,
ActiveRecord::Reflection.create(:has_many_attached, name, nil, { dependent: dependent }, self)
nil,
{ dependent: dependent, service_name: service },
self
)
ActiveRecord::Reflection.add_attachment_reflection(self, name, reflection)
end
private
def validate_service_configuration(association_name, service)
return unless service
ServiceRegistry.fetch(service) do
raise ArgumentError, "Cannot configure service :#{service} for #{name}##{association_name}"
end
end
end
def attachment_changes #:nodoc:

@ -102,26 +102,9 @@ class Engine < Rails::Engine # :nodoc:
initializer "active_storage.services" do
ActiveSupport.on_load(:active_storage_blob) do
if config_choice = Rails.configuration.active_storage.service
configs = Rails.configuration.active_storage.service_configurations ||= begin
config_file = Pathname.new(Rails.root.join("config/storage.yml"))
raise("Couldn't find Active Storage configuration in #{config_file}") unless config_file.exist?
require "yaml"
require "erb"
YAML.load(ERB.new(config_file.read).result) || {}
rescue Psych::SyntaxError => e
raise "YAML syntax error occurred while parsing #{config_file}. " \
"Please note that YAML must be consistently indented using spaces. Tabs are not allowed. " \
"Error: #{e.message}"
ActiveStorage::Blob.service = ActiveStorage::ServiceRegistry.fetch(config_choice) do
raise ArgumentError, "Cannot load `Rails.application.config.active_storage.service`:\n#{config_choice}"
end
ActiveStorage::Blob.service =
begin
ActiveStorage::Service.configure config_choice, configs
rescue => e
raise e, "Cannot load `Rails.config.active_storage.service`:\n#{e.message}", e.backtrace
end
end
end
end

@ -0,0 +1,34 @@
# frozen_string_literal: true
module ActiveStorage
class ServiceRegistry #:nodoc:
class << self
def fetch(service_name, &block)
services.fetch(service_name.to_s, &block)
end
private
def services
@services ||= configs.keys.each_with_object({}) do |service_name, hash|
hash[service_name] = ActiveStorage::Service.configure(service_name, configs)
end
end
def configs
Rails.configuration.active_storage.service_configurations ||= begin
config_file = Pathname.new(Rails.root.join("config/storage.yml"))
raise("Couldn't find Active Storage configuration in #{config_file}") unless config_file.exist?
require "yaml"
require "erb"
YAML.load(ERB.new(config_file.read).result) || {}
rescue Psych::SyntaxError => e
raise "YAML syntax error occurred while parsing #{config_file}. " \
"Please note that YAML must be consistently indented using spaces. Tabs are not allowed. " \
"Error: #{e.message}"
end
end
end
end
end

@ -578,6 +578,36 @@ def highlights
end
end
test "attaching a new blob from a Hash with a custom service" do
with_service("mirror") do
@user.highlights.attach io: StringIO.new("STUFF"), filename: "town.jpg", content_type: "image/jpg"
@user.vlogs.attach io: StringIO.new("STUFF"), filename: "town.jpg", content_type: "image/jpg"
assert_instance_of ActiveStorage::Service::MirrorService, @user.highlights.first.service
assert_instance_of ActiveStorage::Service::DiskService, @user.vlogs.first.service
end
end
test "attaching a new blob from an uploaded file with a custom_ ervice" do
with_service("mirror") do
@user.highlights.attach fixture_file_upload("racecar.jpg")
@user.vlogs.attach fixture_file_upload("racecar.jpg")
assert_instance_of ActiveStorage::Service::MirrorService, @user.highlights.first.service
assert_instance_of ActiveStorage::Service::DiskService, @user.vlogs.first.service
end
end
test "raises error when misconfigured service is passed" do
error = assert_raises ArgumentError do
User.class_eval do
has_many_attached :featured_photos, service: :unknown
end
end
assert_match(/Cannot configure service :unknown for User#featured_photos/, error.message)
end
private
def append_on_assign
ActiveStorage.replace_on_assign_to_many, previous = false, ActiveStorage.replace_on_assign_to_many

@ -43,7 +43,7 @@ class ActiveStorage::OneAttachedTest < ActiveSupport::TestCase
test "attaching a new blob from a Hash to an existing record passes record" do
hash = { io: StringIO.new("STUFF"), filename: "town.jpg", content_type: "image/jpg" }
blob = ActiveStorage::Blob.build_after_unfurling(**hash)
arguments = [hash.merge(record: @user)]
arguments = [hash.merge(record: @user, service_name: nil)]
assert_called_with(ActiveStorage::Blob, :build_after_unfurling, arguments, returns: blob) do
@user.avatar.attach hash
end
@ -59,7 +59,7 @@ class ActiveStorage::OneAttachedTest < ActiveSupport::TestCase
def upload.open
@io ||= StringIO.new("")
end
arguments = { io: upload.open, filename: upload.original_filename, content_type: upload.content_type, record: @user }
arguments = { io: upload.open, filename: upload.original_filename, content_type: upload.content_type, record: @user, service_name: nil }
blob = ActiveStorage::Blob.build_after_unfurling(**arguments)
assert_called_with(ActiveStorage::Blob, :build_after_unfurling, [arguments], returns: blob) do
@user.avatar.attach upload
@ -544,4 +544,34 @@ def avatar
User.remove_method :avatar
end
end
test "attaching a new blob from a Hash with a custom service" do
with_service("mirror") do
@user.avatar.attach io: StringIO.new("STUFF"), filename: "town.jpg", content_type: "image/jpg"
@user.cover_photo.attach io: StringIO.new("STUFF"), filename: "town.jpg", content_type: "image/jpg"
assert_instance_of ActiveStorage::Service::MirrorService, @user.avatar.service
assert_instance_of ActiveStorage::Service::DiskService, @user.cover_photo.service
end
end
test "attaching a new blob from an uploaded file with a custom_service" do
with_service("mirror") do
@user.avatar.attach fixture_file_upload("racecar.jpg")
@user.cover_photo.attach fixture_file_upload("racecar.jpg")
assert_instance_of ActiveStorage::Service::MirrorService, @user.avatar.service
assert_instance_of ActiveStorage::Service::DiskService, @user.cover_photo.service
end
end
test "raises error when misconfigured service is passed" do
error = assert_raises ArgumentError do
User.class_eval do
has_one_attached :featured_photo, service: :unknown
end
end
assert_match(/Cannot configure service :unknown for User#featured_photo/, error.message)
end
end

@ -26,28 +26,15 @@ class ActiveStorage::AttachmentTest < ActiveSupport::TestCase
end
test "mirroring a directly-uploaded blob after attaching it" do
previous_service, ActiveStorage::Blob.service = ActiveStorage::Blob.service, build_mirror_service
with_service("mirror") do
blob = directly_upload_file_blob
assert_not ActiveStorage::Blob.service.mirrors.second.exist?(blob.key)
blob = directly_upload_file_blob
assert_not ActiveStorage::Blob.service.mirrors.second.exist?(blob.key)
perform_enqueued_jobs do
@user.highlights.attach(blob)
end
perform_enqueued_jobs do
@user.highlights.attach(blob)
assert ActiveStorage::Blob.service.mirrors.second.exist?(blob.key)
end
assert ActiveStorage::Blob.service.mirrors.second.exist?(blob.key)
ensure
ActiveStorage::Blob.service = previous_service
end
private
def build_mirror_service
ActiveStorage::Service::MirrorService.new \
primary: build_disk_service("primary"),
mirrors: 3.times.collect { |i| build_disk_service("mirror_#{i + 1}") }
end
def build_disk_service(purpose)
ActiveStorage::Service::DiskService.new(root: Dir.mktmpdir("active_storage_tests_#{purpose}"))
end
end

@ -245,6 +245,21 @@ class ActiveStorage::BlobTest < ActiveSupport::TestCase
end
end
test "uses service from blob when provided" do
with_service("mirror") do
blob = create_blob(filename: "funky.jpg", service_name: :local)
assert_instance_of ActiveStorage::Service::DiskService, blob.service
end
end
test "invalidates record when provided service_name is invalid" do
blob = create_blob(filename: "funky.jpg")
blob.update(service_name: :unknown)
assert_not blob.valid?
assert_equal ["is invalid"], blob.errors[:service_name]
end
private
def expected_url_for(blob, disposition: :attachment, filename: nil, content_type: nil)
filename ||= blob.filename

@ -9,6 +9,9 @@ class ActiveStorage::ReflectionTest < ActiveSupport::TestCase
assert_equal :avatar, reflection.name
assert_equal :has_one_attached, reflection.macro
assert_equal :purge_later, reflection.options[:dependent]
reflection = User.reflect_on_attachment(:cover_photo)
assert_equal :local, reflection.options[:service_name]
end
test "reflection on a singular attachment with the same name as an attachment on another model" do
@ -22,6 +25,9 @@ class ActiveStorage::ReflectionTest < ActiveSupport::TestCase
assert_equal :highlights, reflection.name
assert_equal :has_many_attached, reflection.macro
assert_equal :purge_later, reflection.options[:dependent]
reflection = User.reflect_on_attachment(:vlogs)
assert_equal :local, reflection.options[:service_name]
end
test "reflecting on all attachments" do

@ -34,7 +34,16 @@
end
require "tmpdir"
ActiveStorage::Blob.service = ActiveStorage::Service::DiskService.new(root: Dir.mktmpdir("active_storage_tests"))
Rails.configuration.active_storage.service_configurations = {
"local" => { "service" => "Disk", "root" => Dir.mktmpdir("active_storage_tests") },
"disk_mirror_1" => { "service" => "Disk", "root" => Dir.mktmpdir("active_storage_tests_1") },
"disk_mirror_2" => { "service" => "Disk", "root" => Dir.mktmpdir("active_storage_tests_2") },
"disk_mirror_3" => { "service" => "Disk", "root" => Dir.mktmpdir("active_storage_tests_3") },
"mirror" => { "service" => "Mirror", "primary" => "local", "mirrors" => ["disk_mirror_1", "disk_mirror_2", "disk_mirror_3"] }
}
Rails.configuration.active_storage.service = "local"
ActiveStorage.logger = ActiveSupport::Logger.new(nil)
ActiveStorage.verifier = ActiveSupport::MessageVerifier.new("Testing")
@ -51,8 +60,8 @@ class ActiveSupport::TestCase
end
private
def create_blob(key: nil, data: "Hello world!", filename: "hello.txt", content_type: "text/plain", identify: true, record: nil)
ActiveStorage::Blob.create_and_upload! key: key, io: StringIO.new(data), filename: filename, content_type: content_type, identify: identify, record: record
def create_blob(key: nil, data: "Hello world!", filename: "hello.txt", content_type: "text/plain", identify: true, service_name: nil, record: nil)
ActiveStorage::Blob.create_and_upload! key: key, io: StringIO.new(data), filename: filename, content_type: content_type, identify: identify, service_name: service_name, record: record
end
def create_file_blob(key: nil, filename: "racecar.jpg", content_type: "image/jpeg", metadata: nil, record: nil)
@ -89,6 +98,18 @@ def extract_metadata_from(blob)
def fixture_file_upload(filename)
Rack::Test::UploadedFile.new file_fixture(filename).to_s
end
def with_service(service_name)
service = ActiveStorage::ServiceRegistry.fetch(service_name)
ActiveStorage::Blob.service, previous_service = service, ActiveStorage::Blob.service
Rails.configuration.active_storage.service, previous_service_name = service_name, Rails.configuration.active_storage.service
yield
ensure
ActiveStorage::Blob.service = previous_service
Rails.configuration.active_storage.service = previous_service_name
end
end
require "global_id"
@ -99,10 +120,10 @@ class User < ActiveRecord::Base
validates :name, presence: true
has_one_attached :avatar
has_one_attached :cover_photo, dependent: false
has_one_attached :cover_photo, dependent: false, service: :local
has_many_attached :highlights
has_many_attached :vlogs, dependent: false
has_many_attached :vlogs, dependent: false, service: :local
end
class Group < ActiveRecord::Base

@ -287,6 +287,15 @@ Call `avatar.attached?` to determine whether a particular user has an avatar:
user.avatar.attached?
```
In some cases you might want to override a default service for a specific attachment.
You can configure specific services per attachment using the `service` option:
```ruby
class User < ApplicationRecord
has_one_attached :avatar, service: :s3
end
```
### `has_many_attached`
The `has_many_attached` macro sets up a one-to-many relationship between records
@ -329,6 +338,14 @@ Call `images.attached?` to determine whether a particular message has any images
@message.images.attached?
```
Overriding the default service is done done the same way as `has_one_attached`, by using the `service` option:
```ruby
class Message < ApplicationRecord
has_many_attached :images, service: :s3
end
```
### Attaching File/IO Objects
Sometimes you need to attach a file that doesnt arrive via an HTTP request.