failed attempt at using rust-analyzer to map macro expansion to original lines in a project. always gets no items in macro expansion

broken-ra.rs

[dependencies]
# HIR analysis - use consistent versions
ra_ap_hir = "0.0.289"
ra_ap_base_db = "0.0.289"
ra_ap_hir_def = "0.0.289"
ra_ap_hir_expand = "0.0.289"
ra_ap_ide_db = "0.0.289"
ra_ap_load-cargo = "0.0.289"
ra_ap_paths = "0.0.289"
ra_ap_project_model = "0.0.289"
ra_ap_syntax = "0.0.289"
ra_ap_vfs = "0.0.289"
rustc-hash = "2.1"

use anyhow::{Context, Result};
use ra_ap_base_db::{FileId, SourceDatabase};
use ra_ap_hir::{HirFileId, Semantics};
use ra_ap_ide_db::RootDatabase;
use ra_ap_load_cargo::{load_workspace, LoadCargoConfig, ProcMacroServerChoice};
use ra_ap_paths::{AbsPathBuf, Utf8PathBuf};
use ra_ap_project_model::{CargoConfig, ProjectManifest};
use ra_ap_syntax::{ast, AstNode, SyntaxNode, TextRange};
use ra_ap_syntax::ast::{HasName, HasVisibility, MacroItems, HasModuleItem};
use ra_ap_hir_expand::ExpandTo;
use ra_ap_vfs::Vfs;
use std::path::Path;

pub struct HirAnalyzer {
    db: RootDatabase,
    vfs: Vfs,
    project_root: AbsPathBuf,
}

impl HirAnalyzer {
    pub fn new(project_path: &Path) -> Result<Self> {
        let utf8_path = Utf8PathBuf::from_path_buf(project_path.to_path_buf())
            .map_err(|_| anyhow::anyhow!("Project path is not valid UTF-8"))?;
        let manifest_path = AbsPathBuf::assert(utf8_path);
        let manifest = ProjectManifest::discover_single(&manifest_path)
            .context("Failed to discover project manifest")?;
				let manifests = ProjectManifest::discover(&manifest_path).unwrap();
				eprintln!("{manifest_path} {manifests:?}");

        let cargo_config = CargoConfig {
            all_targets: true, // Enable all targets like CodeQL does
            ..CargoConfig::default()
        };
        let load_config = LoadCargoConfig {
            load_out_dirs_from_check: true,
            with_proc_macro_server: ProcMacroServerChoice::Sysroot,
            prefill_caches: true, // Enable cache prefilling
        };

        if let Some((db, vfs)) = Self::load_workspace(&manifest, &cargo_config, &load_config) {
            Ok(Self { db, vfs, project_root: manifest_path })
        } else {
            anyhow::bail!("Failed to load workspace")
        }
    }

    /// Load workspace using ra_ap_load_cargo
    fn load_workspace(
        project: &ProjectManifest,
        config: &CargoConfig,
        load_config: &LoadCargoConfig,
    ) -> Option<(RootDatabase, Vfs)> {
        let workspace = ra_ap_project_model::ProjectWorkspace::load(
            project.clone(),
            config,
            &|msg| eprintln!("Loading: {}", msg),
        ).ok()?;

        let extra_env: rustc_hash::FxHashMap<String, Option<String>> = rustc_hash::FxHashMap::default();
        let (db, vfs, _proc_macro_client) = load_workspace(
            workspace,
            &extra_env,
            load_config,
        ).ok()?;

        Some((db, vfs))
    }

    /// Analyze macro expansions in the project, filtering to only workspace files
    pub fn analyze_macro_expansions(&self) -> Result<Vec<MacroExpansionInfo>> {
        let semantics = Semantics::new(&self.db);
        let mut expansions = Vec::new();

        // Get all source files from VFS, but filter to only workspace files (not dependencies)
        for (file_id, vfs_file) in self.vfs.iter() {
            // Skip files that are from dependencies (libraries)
            // let source_root_id = self.db.file_source_root(file_id);
            // let source_root_input = self.db.source_root(source_root_id.source_root_id(&self.db));
						// let source_root = source_root_input.source_root(&self.db);

            // Only include files that are within our project directory
            if let Some(file_path) = vfs_file.as_path() {
                if !file_path.as_str().starts_with(self.project_root.as_str()) {
                    continue;
                }

                if !file_path.as_str().ends_with(".rs") {
                    continue;
                }

                eprintln!("Analyzing workspace file: {}", file_path.as_str());

							let hir_file_id = HirFileId::from(ra_ap_base_db::EditionedFileId::current_edition(&self.db, file_id));
							self.analyze_hir_file(&semantics, hir_file_id, &mut expansions)?;
                eprintln!("Analyzed workspace file: {}", file_path.as_str());
            }
        }

        Ok(expansions)
    }

    fn analyze_hir_file(
        &self,
        semantics: &Semantics<'_, RootDatabase>,
        hir_file_id: HirFileId,
        expansions: &mut Vec<MacroExpansionInfo>,
    ) -> Result<()> {
        // Get the syntax tree for this HIR file
        // We need to extract the EditionedFileId from HirFileId for parsing
        let root = if let Some(file_id) = hir_file_id.file_id() {
            let parse = semantics.parse(file_id);
            parse.syntax().clone()
        } else {
            // This is a macro file, skip it since we'll find macro calls in source files
            return Ok(());
        };

        // Find macro calls in this file
        self.find_macro_calls(&root, semantics, hir_file_id, expansions)?;

        Ok(())
    }

    fn find_macro_calls(
        &self,
        node: &SyntaxNode,
        semantics: &Semantics<'_, RootDatabase>,
        current_file: HirFileId,
        expansions: &mut Vec<MacroExpansionInfo>,
    ) -> Result<()> {
				use ra_ap_syntax::SyntaxKind::*;
				match node.kind() {
					MACRO_CALL => {

					if let Some(macro_call) = ast::MacroCall::cast(node.clone()) {
						// node.tok
							if let Some(expansion_info) = self.analyze_macro_call(&macro_call, semantics, current_file)? {
									expansions.push(expansion_info);
							}
					} else {
						unreachable!();
					}
					},
								MODULE | SOURCE_FILE => {
									// Recursively search child nodes
									for child in node.children() {
											self.find_macro_calls(&child, semantics, current_file, expansions)?;
									}
								}
								unk => {
									println!("Seen: {unk:?}");
								},
						}

        Ok(())
    }

    fn analyze_macro_call(
        &self,
        macro_call: &ast::MacroCall,
        semantics: &Semantics<'_, RootDatabase>,
        current_file: HirFileId,
    ) -> Result<Option<MacroExpansionInfo>> {
        // Get the macro name
        let macro_name = macro_call
            .path()
            .and_then(|path| path.segment())
            .and_then(|seg| seg.name_ref())
            .map(|name| name.text().to_string());

        // Try to expand the macro call directly, following CodeQL pattern
        if let Some(expanded) = semantics.expand_macro_call(macro_call) {
            eprintln!("Macro expanded successfully");

            // Determine what type of expansion this should be
            let expand_to = ExpandTo::from_call_site(macro_call);
            eprintln!("Expected expansion type: {:?}", expand_to);
            eprintln!("Actual expansion kind: {:?}", expanded.kind());

            let expanded_items = match expand_to {
                ExpandTo::Items => {
                    if let Some(macro_items) = MacroItems::cast(expanded.value) {
                        eprintln!("Successfully cast to MacroItems");
                        self.collect_items_from_macro_items(&macro_items)?
                    } else {
                        eprintln!("Failed to cast expanded result to MacroItems");
                        Vec::new()
                    }
                }
                _ => {
                    eprintln!("⚠Macro expands to {:?}, not items - skipping", expand_to);
                    Vec::new()
                }
            };

            if expanded_items.is_empty() {
                eprintln!("No items found in expansion");
                return Ok(None);
            }

            let range = macro_call.syntax().text_range();

            Ok(Some(MacroExpansionInfo {
                macro_name,
                call_range: range,
                expanded_items,
            }))
        } else {
            eprintln!("Failed to expand {macro_name:?}");
            Ok(None)
        }
    }


    fn collect_items_from_macro_items(&self, macro_items: &MacroItems) -> Result<Vec<String>> {
        let mut items = Vec::new();

        eprintln!("Collecting items from MacroItems");
        eprintln!("MacroItems text: '{}'", macro_items.syntax().text());
        eprintln!("MacroItems children count: {}", macro_items.syntax().children().count());

        let items_iter: Vec<_> = macro_items.items().collect();
        eprintln!("items() returned {} items", items_iter.len());

        for item in items_iter {
            let description = self.describe_item(&item);
            eprintln!("Found item via items(): {}", description);
            items.push(description);
        }

        eprintln!("Manual syntax traversal:");
        for (i, child) in macro_items.syntax().children().enumerate() {
            eprintln!("  Child {}: {:?} - '{}'", i, child.kind(), child.text());
            if let Some(item) = ast::Item::cast(child) {
                let description = self.describe_item(&item);
                eprintln!("Found item via syntax: {}", description);
                // Don't add to items to avoid duplicates - just for debugging
            }
        }

        eprintln!("Total items found: {}", items.len());
        Ok(items)
    }

    fn collect_expanded_items_from_node(&self, node: &SyntaxNode) -> Result<Vec<String>> {
        let mut items = Vec::new();

        eprintln!("Collecting items from node: {:?}", node.kind());
        eprintln!("Node text: '{}'", node.text());
        eprintln!("Node children: {:?}", node.children());
				// let mi = MacroItems::cast(node.clone()).unwrap();
				// mi.syntax()


        match node.kind() {
            ra_ap_syntax::SyntaxKind::MACRO_ITEMS => {
                for child in node.children() {
									eprintln!("Child type {:?}", child.kind());
                    if let Some(item) = ast::Item::cast(child) {
                        let description = self.describe_item(&item);
                        eprintln!("Found item: {}", description);
                        items.push(description);
                    }
                }
            }
            _ => {
                if let Some(item) = ast::Item::cast(node.clone()) {
                    let description = self.describe_item(&item);
                    eprintln!("Found item: {}", description);
                    items.push(description);
                } else {
                    for child in node.children() {
                        if let Some(item) = ast::Item::cast(child) {
                            let description = self.describe_item(&item);
                            eprintln!("Found item: {}", &description);
                            items.push(description);
                        }
                    }
                }
            }
        }

        eprintln!("Total items found: {}", items.len());
        Ok(items)
    }

    fn describe_item(&self, item: &ast::Item) -> String {
        match item {
            ast::Item::Fn(func) => {
                let name = func.name().map_or_else(|| "_".to_string(), |n| n.text().to_string());
                let vis = if func.visibility().is_some() { "pub " } else { "" };
                format!("{}fn {}", vis, name)
            }
            ast::Item::Struct(struct_) => {
                let name = struct_.name().map_or_else(|| "_".to_string(), |n| n.text().to_string());
                let vis = if struct_.visibility().is_some() { "pub " } else { "" };
                format!("{}struct {}", vis, name)
            }
            ast::Item::Enum(enum_) => {
                let name = enum_.name().map_or_else(|| "_".to_string(), |n| n.text().to_string());
                let vis = if enum_.visibility().is_some() { "pub " } else { "" };
                format!("{}enum {}", vis, name)
            }
            ast::Item::TypeAlias(alias) => {
                let name = alias.name().map_or_else(|| "_".to_string(), |n| n.text().to_string());
                let vis = if alias.visibility().is_some() { "pub " } else { "" };
                format!("{}type {}", vis, name)
            }
            ast::Item::Impl(impl_) => {
                if let Some(trait_) = impl_.trait_() {
                    let trait_name = trait_.to_string();
                    let target = impl_.self_ty().map_or_else(|| "_".to_string(), |ty| ty.to_string());
                    format!("impl {} for {}", trait_name, target)
                } else {
                    let target = impl_.self_ty().map_or_else(|| "_".to_string(), |ty| ty.to_string());
                    format!("impl {}", target)
                }
            }
            ast::Item::Trait(trait_) => {
                let name = trait_.name().map_or_else(|| "_".to_string(), |n| n.text().to_string());
                let vis = if trait_.visibility().is_some() { "pub " } else { "" };
                format!("{}trait {}", vis, name)
            }
            ast::Item::Const(const_) => {
                let name = const_.name().map_or_else(|| "_".to_string(), |n| n.text().to_string());
                let vis = if const_.visibility().is_some() { "pub " } else { "" };
                format!("{}const {}", vis, name)
            }
            ast::Item::Static(static_) => {
                let name = static_.name().map_or_else(|| "_".to_string(), |n| n.text().to_string());
                let vis = if static_.visibility().is_some() { "pub " } else { "" };
                format!("{}static {}", vis, name)
            }
            _ => "unknown item".to_string(),
        }
    }

    /// Get file path for a FileId
    pub fn file_path(&self, file_id: FileId) -> Option<&Path> {
        self.vfs.file_path(file_id).as_path().map(|p| p.as_ref())
    }
}

#[derive(Debug)]
pub struct MacroExpansionInfo {
    /// Name of the macro that was called
    pub macro_name: Option<String>,
    /// Text range of the macro call in the original source
    pub call_range: TextRange,
    /// Items that were generated by this macro expansion
    pub expanded_items: Vec<String>,
}