diff --git a/compiler/rustc_resolve/src/build_reduced_graph.rs b/compiler/rustc_resolve/src/build_reduced_graph.rs
index 307625bcfb1..d57dabdd78d 100644
--- a/compiler/rustc_resolve/src/build_reduced_graph.rs
+++ b/compiler/rustc_resolve/src/build_reduced_graph.rs
@@ -374,7 +374,6 @@ fn add_import(
             root_span,
             root_id,
             vis,
-            used: Default::default(),
         });
 
         self.r.indeterminate_imports.push(import);
@@ -890,8 +889,10 @@ fn build_reduced_graph_for_extern_crate(
             span: item.span,
             module_path: Vec::new(),
             vis,
-            used: Cell::new(used.then_some(Used::Other)),
         });
+        if used {
+            self.r.import_use_map.insert(import, Used::Other);
+        }
         self.r.potentially_unused_imports.push(import);
         let imported_binding = self.r.import(binding, import);
         if parent == self.r.graph_root {
@@ -1091,7 +1092,6 @@ fn process_macro_use_imports(&mut self, item: &Item, module: Module<'a>) -> bool
                 span,
                 module_path: Vec::new(),
                 vis: ty::Visibility::Restricted(CRATE_DEF_ID),
-                used: Default::default(),
             })
         };
 
@@ -1256,8 +1256,8 @@ fn define_macro(&mut self, item: &ast::Item) -> MacroRulesScopeRef<'a> {
                     span,
                     module_path: Vec::new(),
                     vis,
-                    used: Cell::new(Some(Used::Other)),
                 });
+                self.r.import_use_map.insert(import, Used::Other);
                 let import_binding = self.r.import(binding, import);
                 self.r.define(self.r.graph_root, ident, MacroNS, import_binding);
             } else {
diff --git a/compiler/rustc_resolve/src/check_unused.rs b/compiler/rustc_resolve/src/check_unused.rs
index 75b8ecebdd9..1cee876b80f 100644
--- a/compiler/rustc_resolve/src/check_unused.rs
+++ b/compiler/rustc_resolve/src/check_unused.rs
@@ -381,9 +381,9 @@ pub(crate) fn check_unused(&mut self, krate: &ast::Crate) {
 
         for import in self.potentially_unused_imports.iter() {
             match import.kind {
-                _ if import.used.get().is_some()
-                    || import.vis.is_public()
-                    || import.span.is_dummy() =>
+                _ if import.vis.is_public()
+                    || import.span.is_dummy()
+                    || self.import_use_map.contains_key(import) =>
                 {
                     if let ImportKind::MacroUse { .. } = import.kind {
                         if !import.span.is_dummy() {
diff --git a/compiler/rustc_resolve/src/imports.rs b/compiler/rustc_resolve/src/imports.rs
index 4a891d12281..42171edf757 100644
--- a/compiler/rustc_resolve/src/imports.rs
+++ b/compiler/rustc_resolve/src/imports.rs
@@ -176,7 +176,6 @@ pub(crate) struct ImportData<'a> {
     /// The resolution of `module_path`.
     pub imported_module: Cell<Option<ModuleOrUniformRoot<'a>>>,
     pub vis: ty::Visibility,
-    pub used: Cell<Option<Used>>,
 }
 
 /// All imports are unique and allocated on a same arena,
@@ -484,7 +483,7 @@ fn import_dummy_binding(&mut self, import: Import<'a>, is_indeterminate: bool) {
             });
             self.record_use(target, dummy_binding, Used::Other);
         } else if import.imported_module.get().is_none() {
-            import.used.set(Some(Used::Other));
+            self.import_use_map.insert(import, Used::Other);
             if let Some(id) = import.id() {
                 self.used_imports.insert(id);
             }
@@ -1347,7 +1346,7 @@ pub(crate) fn check_for_redundant_imports(&mut self, import: Import<'a>) -> bool
         // module defined by a block).
         // Skip if the import is public or was used through non scope-based resolution,
         // e.g. through a module-relative path.
-        if import.used.get() == Some(Used::Other)
+        if self.import_use_map.get(&import) == Some(&Used::Other)
             || self.effective_visibilities.is_exported(self.local_def_id(id))
         {
             return false;
diff --git a/compiler/rustc_resolve/src/lib.rs b/compiler/rustc_resolve/src/lib.rs
index 89ac839651f..02fdc1ae668 100644
--- a/compiler/rustc_resolve/src/lib.rs
+++ b/compiler/rustc_resolve/src/lib.rs
@@ -1016,6 +1016,8 @@ pub struct Resolver<'a, 'tcx> {
     partial_res_map: NodeMap<PartialRes>,
     /// Resolutions for import nodes, which have multiple resolutions in different namespaces.
     import_res_map: NodeMap<PerNS<Option<Res>>>,
+    /// An import will be inserted into this map if it has been used.
+    import_use_map: FxHashMap<Import<'a>, Used>,
     /// Resolutions for labels (node IDs of their corresponding blocks or loops).
     label_res_map: NodeMap<NodeId>,
     /// Resolutions for lifetimes.
@@ -1422,6 +1424,7 @@ pub fn new(
             pat_span_map: Default::default(),
             partial_res_map: Default::default(),
             import_res_map: Default::default(),
+            import_use_map: Default::default(),
             label_res_map: Default::default(),
             lifetimes_res_map: Default::default(),
             extra_lifetime_params_map: Default::default(),
@@ -1839,7 +1842,7 @@ fn resolution(
     }
 
     /// Test if AmbiguityError ambi is any identical to any one inside ambiguity_errors
-    fn matches_previous_ambiguity_error(&mut self, ambi: &AmbiguityError<'_>) -> bool {
+    fn matches_previous_ambiguity_error(&self, ambi: &AmbiguityError<'_>) -> bool {
         for ambiguity_error in &self.ambiguity_errors {
             // if the span location and ident as well as its span are the same
             if ambiguity_error.kind == ambi.kind
@@ -1900,10 +1903,9 @@ fn record_use_inner(
                     }
                 }
             }
-            let old_used = import.used.get();
-            let new_used = Some(used);
-            if new_used > old_used {
-                import.used.set(new_used);
+            let old_used = self.import_use_map.entry(import).or_insert(used);
+            if *old_used < used {
+                *old_used = used;
             }
             if let Some(id) = import.id() {
                 self.used_imports.insert(id);