From 50e322962b6b3d591be2ff89a58f9b22d788f9fe Mon Sep 17 00:00:00 2001 From: Donal McBreen Date: Fri, 16 Feb 2024 17:07:25 +0000 Subject: [PATCH] Add MessagePackMessageSerializer for binary data (#51102) --- activerecord/CHANGELOG.md | 9 +++ .../fixtures/journal_mode_test.sqlite3 | 0 .../encryption/encrypted_attribute_type.rb | 6 +- .../message_pack_message_serializer.rb | 72 +++++++++++++++++++ ...ble_record_message_pack_serialized_test.rb | 39 ++++++++++ .../message_pack_message_serializer_test.rb | 70 ++++++++++++++++++ 6 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 activerecord/fixtures/journal_mode_test.sqlite3 create mode 100644 activerecord/lib/active_record/encryption/message_pack_message_serializer.rb create mode 100644 activerecord/test/cases/encryption/encryptable_record_message_pack_serialized_test.rb create mode 100644 activerecord/test/cases/encryption/message_pack_message_serializer_test.rb diff --git a/activerecord/CHANGELOG.md b/activerecord/CHANGELOG.md index f4efdf4aae..f820b41099 100644 --- a/activerecord/CHANGELOG.md +++ b/activerecord/CHANGELOG.md @@ -1,3 +1,12 @@ +* Add ActiveRecord::Encryption::MessagePackMessageSerializer + + Serialize data to the MessagePack format, for efficient storage in binary columns. + + The binary encoding requires around 30% less space than the base64 encoding + used by the default serializer. + + *Donal McBreen* + * Add support for encrypting binary columns Ensure encryption and decryption pass `Type::Binary::Data` around for binary data. diff --git a/activerecord/fixtures/journal_mode_test.sqlite3 b/activerecord/fixtures/journal_mode_test.sqlite3 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/activerecord/lib/active_record/encryption/encrypted_attribute_type.rb b/activerecord/lib/active_record/encryption/encrypted_attribute_type.rb index 8da9ac31df..8829962b34 100644 --- a/activerecord/lib/active_record/encryption/encrypted_attribute_type.rb +++ b/activerecord/lib/active_record/encryption/encrypted_attribute_type.rb @@ -135,7 +135,11 @@ def serialize_with_current(value) def encrypt_as_text(value) with_context do - encryptor.encrypt(value, **encryption_options) + encryptor.encrypt(value, **encryption_options).tap do |encrypted| + if !cast_type.binary? && encrypted.encoding == Encoding::BINARY + raise Errors::Encoding, "Binary encoded data can only be stored in binary columns" + end + end end end diff --git a/activerecord/lib/active_record/encryption/message_pack_message_serializer.rb b/activerecord/lib/active_record/encryption/message_pack_message_serializer.rb new file mode 100644 index 0000000000..90fad90944 --- /dev/null +++ b/activerecord/lib/active_record/encryption/message_pack_message_serializer.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require "active_support/message_pack" + +module ActiveRecord + module Encryption + # A message serializer that serializes +Messages+ with MessagePack. + # + # The message is converted to a hash with this structure: + # + # { + # p: , + # h: { + # header1: value1, + # header2: value2, + # ... + # } + # } + # + # Then it is converted to the MessagePack format. + class MessagePackMessageSerializer + def dump(message) + raise Errors::ForbiddenClass unless message.is_a?(Message) + ActiveSupport::MessagePack.dump(message_to_hash(message)) + end + + def load(serialized_content) + data = ActiveSupport::MessagePack.load(serialized_content) + hash_to_message(data, 1) + rescue RuntimeError + raise Errors::Decryption + end + + private + def message_to_hash(message) + { + "p" => message.payload, + "h" => headers_to_hash(message.headers) + } + end + + def headers_to_hash(headers) + headers.transform_values do |value| + value.is_a?(Message) ? message_to_hash(value) : value + end + end + + def hash_to_message(data, level) + validate_message_data_format(data, level) + Message.new(payload: data["p"], headers: parse_properties(data["h"], level)) + end + + def validate_message_data_format(data, level) + if level > 2 + raise Errors::Decryption, "More than one level of hash nesting in headers is not supported" + end + + unless data.is_a?(Hash) && data.has_key?("p") + raise Errors::Decryption, "Invalid data format: hash without payload" + end + end + + def parse_properties(headers, level) + Properties.new.tap do |properties| + headers&.each do |key, value| + properties[key] = value.is_a?(Hash) ? hash_to_message(value, level + 1) : value + end + end + end + end + end +end diff --git a/activerecord/test/cases/encryption/encryptable_record_message_pack_serialized_test.rb b/activerecord/test/cases/encryption/encryptable_record_message_pack_serialized_test.rb new file mode 100644 index 0000000000..d84c8191b0 --- /dev/null +++ b/activerecord/test/cases/encryption/encryptable_record_message_pack_serialized_test.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require "cases/encryption/helper" +require "models/author_encrypted" +require "models/book_encrypted" +require "active_record/encryption/message_pack_message_serializer" + +class ActiveRecord::Encryption::EncryptableRecordTest < ActiveRecord::EncryptionTestCase + fixtures :encrypted_books + + test "binary data can be serialized with message pack" do + all_bytes = (0..255).map(&:chr).join + assert_equal all_bytes, EncryptedBookWithBinaryMessagePackSerialized.create!(logo: all_bytes).logo + end + + test "binary data can be encrypted uncompressed and serialized with message pack" do + low_bytes = (0..127).map(&:chr).join + high_bytes = (128..255).map(&:chr).join + assert_equal low_bytes, EncryptedBookWithBinaryMessagePackSerialized.create!(logo: low_bytes).logo + assert_equal high_bytes, EncryptedBookWithBinaryMessagePackSerialized.create!(logo: high_bytes).logo + end + + test "text columns cannot be serialized with message pack" do + assert_raises(ActiveRecord::Encryption::Errors::Encoding) do + message_pack_serialized_text_class = Class.new(ActiveRecord::Base) do + self.table_name = "encrypted_books" + + encrypts :name, message_serializer: ActiveRecord::Encryption::MessagePackMessageSerializer.new + end + message_pack_serialized_text_class.create(name: "Dune") + end + end + + class EncryptedBookWithBinaryMessagePackSerialized < ActiveRecord::Base + self.table_name = "encrypted_books" + + encrypts :logo, message_serializer: ActiveRecord::Encryption::MessagePackMessageSerializer.new + end +end diff --git a/activerecord/test/cases/encryption/message_pack_message_serializer_test.rb b/activerecord/test/cases/encryption/message_pack_message_serializer_test.rb new file mode 100644 index 0000000000..e82e9667c9 --- /dev/null +++ b/activerecord/test/cases/encryption/message_pack_message_serializer_test.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require "cases/encryption/helper" +require "base64" +require "active_record/encryption/message_pack_message_serializer" + +class ActiveRecord::Encryption::MessagePackMessageSerializerTest < ActiveRecord::EncryptionTestCase + setup do + @serializer = ActiveRecord::Encryption::MessagePackMessageSerializer.new + end + + test "serializes messages" do + message = build_message + deserialized_message = serialize_and_deserialize(message) + assert_equal message, deserialized_message + end + + test "serializes messages with nested messages in their headers" do + message = build_message + message.headers[:other_message] = ActiveRecord::Encryption::Message.new(payload: "some other secret payload", headers: { some_header: "some other value" }) + + deserialized_message = serialize_and_deserialize(message) + assert_equal message, deserialized_message + end + + test "detects random data and raises a decryption error" do + assert_raises ActiveRecord::Encryption::Errors::Decryption do + @serializer.load "hey there" + end + end + + test "detects random JSON hashes and raises a decryption error" do + assert_raises ActiveRecord::Encryption::Errors::Decryption do + @serializer.load JSON.dump({ some: "other data" }) + end + end + + test "raises a TypeError when trying to deserialize other data types" do + assert_raises TypeError do + @serializer.load(:it_can_only_deserialize_strings) + end + end + + test "raises ForbiddenClass when trying to serialize other data types" do + assert_raises ActiveRecord::Encryption::Errors::ForbiddenClass do + @serializer.dump("it can only serialize messages!") + end + end + + test "raises Decryption when trying to parse message with more than one nested message" do + message = build_message + message.headers[:other_message] = ActiveRecord::Encryption::Message.new(payload: "some other secret payload", headers: { some_header: "some other value" }) + message.headers[:other_message].headers[:yet_another_message] = ActiveRecord::Encryption::Message.new(payload: "yet some other secret payload", headers: { some_header: "yet some other value" }) + + assert_raises ActiveRecord::Encryption::Errors::Decryption do + serialize_and_deserialize(message) + end + end + + private + def build_message + payload = "some payload" + headers = { key_1: "1" } + ActiveRecord::Encryption::Message.new(payload: payload, headers: headers) + end + + def serialize_and_deserialize(message, with: @serializer) + @serializer.load @serializer.dump(message) + end +end