diff --git a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
index cfd2655f35b..a133c9d4782 100644
--- a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
+++ b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
@@ -8,7 +8,7 @@
 use rustc_hir::def_id::LocalDefId;
 use rustc_index::bit_set::BitSet;
 use rustc_middle::mir::visit::{NonMutatingUseContext, PlaceContext, Visitor};
-use rustc_middle::mir::{Body, Local, Location, Operand, Terminator, TerminatorKind, RETURN_PLACE};
+use rustc_middle::mir::{Body, Location, Operand, Place, Terminator, TerminatorKind, RETURN_PLACE};
 use rustc_middle::ty::{self, DeducedParamAttrs, Ty, TyCtxt};
 use rustc_session::config::OptLevel;
 
@@ -29,20 +29,31 @@ fn new(arg_count: usize) -> Self {
 }
 
 impl<'tcx> Visitor<'tcx> for DeduceReadOnly {
-    fn visit_local(&mut self, local: Local, context: PlaceContext, _location: Location) {
+    fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, _location: Location) {
         // We're only interested in arguments.
-        if local == RETURN_PLACE || local.index() > self.mutable_args.domain_size() {
+        if place.local == RETURN_PLACE || place.local.index() > self.mutable_args.domain_size() {
             return;
         }
 
-        match context {
+        let mark_as_mutable = match context {
             PlaceContext::MutatingUse(..) => {
                 // This is a mutation, so mark it as such.
-                self.mutable_args.insert(local.index() - 1);
+                true
+            }
+            PlaceContext::NonMutatingUse(NonMutatingUseContext::AddressOf) => {
+                // Whether mutating though a `&raw const` is allowed is still undecided, so we
+                // disable any sketchy `readonly` optimizations for now.
+                // But we only need to do this if the pointer would point into the argument.
+                !place.is_indirect()
             }
             PlaceContext::NonMutatingUse(..) | PlaceContext::NonUse(..) => {
                 // Not mutating, so it's fine.
+                false
             }
+        };
+
+        if mark_as_mutable {
+            self.mutable_args.insert(place.local.index() - 1);
         }
     }
 
diff --git a/tests/codegen/addr-of-mutate.rs b/tests/codegen/addr-of-mutate.rs
new file mode 100644
index 00000000000..bea1aad2352
--- /dev/null
+++ b/tests/codegen/addr-of-mutate.rs
@@ -0,0 +1,34 @@
+// compile-flags: -C opt-level=3 -C no-prepopulate-passes
+// min-llvm-version: 15.0 (for opaque pointers)
+
+#![crate_type = "lib"]
+
+// Test for the absence of `readonly` on the argument when it is mutated via `&raw const`.
+// See <https://github.com/rust-lang/rust/issues/111502>.
+
+// CHECK: i8 @foo(ptr noalias nocapture noundef dereferenceable(128) %x)
+#[no_mangle]
+pub fn foo(x: [u8; 128]) -> u8 {
+    let ptr = core::ptr::addr_of!(x).cast_mut();
+    unsafe {
+        (*ptr)[0] = 1;
+    }
+    x[0]
+}
+
+// CHECK: i1 @second(ptr noalias nocapture noundef dereferenceable({{[0-9]+}}) %a_ptr_and_b)
+#[no_mangle]
+pub unsafe fn second(a_ptr_and_b: (*mut (i32, bool), (i64, bool))) -> bool {
+    let b_bool_ptr = core::ptr::addr_of!(a_ptr_and_b.1.1).cast_mut();
+    (*b_bool_ptr) = true;
+    a_ptr_and_b.1.1
+}
+
+// If going through a deref (and there are no other mutating accesses), then `readonly` is fine.
+// CHECK: i1 @third(ptr noalias nocapture noundef readonly dereferenceable({{[0-9]+}}) %a_ptr_and_b)
+#[no_mangle]
+pub unsafe fn third(a_ptr_and_b: (*mut (i32, bool), (i64, bool))) -> bool {
+    let b_bool_ptr = core::ptr::addr_of!((*a_ptr_and_b.0).1).cast_mut();
+    (*b_bool_ptr) = true;
+    a_ptr_and_b.1.1
+}