Only set the association target if there is no foreign key mismatch

fixes #40018

Prior to this commit, the following test would fail. `book`'s author was
actually being returned as `author_b`. (The same would happen for
`unscope`.)

    def test
      author_a = Author.create!
      author_b = Author.create!

      Book.create(author: author_a)

      book = author_b.books.or(author_a.books).first

      assert_equal book.author, author_a
    end

Co-authored-by: Adam Hess <adamhess1991@gmail.com>

activerecord/lib/active_record/associations/association.rb

@@ -131,8 +131,12 @@ def remove_inverse_instance(record)
       end
 
       def inversed_from(record)
-        self.target = record
-        @inversed = !!record
+        if inversable?(record)
+          self.target = record
+          @inversed = true
+        else
+          @inversed = false
+        end
       end
       alias :inversed_from_queries :inversed_from
 
@@ -325,6 +329,19 @@ def skip_statement_cache?(scope)
         def enqueue_destroy_association(options)
           owner.class.destroy_association_async_job&.perform_later(**options)
         end
+
+        def inversable?(record)
+          record &&
+            ((!record.persisted? || !owner.persisted?) || matches_foreign_key?(record))
+        end
+
+        def matches_foreign_key?(record)
+          if foreign_key_for?(record)
+            record.read_attribute(reflection.foreign_key) == owner.id
+          else
+            owner.read_attribute(reflection.foreign_key) == record.id
+          end
+        end
     end
   end
 end

activerecord/test/cases/associations/inverse_associations_test.rb

@@ -658,6 +658,24 @@ def test_with_has_many_inversing_does_not_trigger_association_callbacks_on_set_w
     end
   end
 
+  def test_unscope_does_not_set_inverse_when_incorrect
+    interest = interests(:trainspotting)
+    human = interest.human
+    created_human = Human.create(name: "wrong human")
+    found_interest = created_human.interests.or(human.interests).detect { |this_interest| interest.id == this_interest.id }
+
+    assert_equal human, found_interest.human
+  end
+
+  def test_or_does_not_set_inverse_when_incorrect
+    interest = interests(:trainspotting)
+    human = interest.human
+    created_human = Human.create(name: "wrong human")
+    found_interest = created_human.interests.unscope(:where).detect { |this_interest| interest.id == this_interest.id }
+
+    assert_equal human, found_interest.human
+  end
+
   def test_child_instance_should_be_shared_with_replaced_via_accessor_parent
     face = Face.first
     human = Human.new(name: "Charles")