PubMed Scraping with Elixir Flow

pubmed_scraper.exs

#!/usr/bin/env elixir

Mix.install([
  {:httpoison, "~> 2.2"},
  {:floki, "~> 0.36"},
  {:jason, "~> 1.4"},
  {:flow, "~> 1.2"}
])

defmodule PubMedScraper do
  def search_pubmed(query, max_results \\ 5) do
    search_url = "https://pubmed.ncbi.nlm.nih.gov/?term=#{URI.encode(query)}"
    IO.puts("Searching PubMed for: #{query}")

    case HTTPoison.get(search_url) do
      {:ok, %HTTPoison.Response{status_code: 200, body: body}} ->
        IO.puts("Response status code: 200")
        {:ok, document} = Floki.parse_document(body)

        document
        |> Floki.find("article")
        |> Enum.take(max_results)
        |> Flow.from_enumerable()
        |> Flow.partition(max_demand: 5)
        |> Flow.map(&extract_article_info/1)
        |> Flow.map(&fetch_article_details/1)
        |> Enum.to_list()

      {:ok, %HTTPoison.Response{status_code: status_code}} ->
        IO.puts("Response status code: #{status_code}")
        []

      {:error, %HTTPoison.Error{reason: reason}} ->
        IO.puts("Error: #{reason}")
        []
    end
  end

  defp extract_article_info(article) do
    title = article |> Floki.find("a.docsum-title") |> Floki.text() |> String.trim()
    link = article |> Floki.find("a.docsum-title") |> Floki.attribute("href") |> List.first()
    full_link = "https://pubmed.ncbi.nlm.nih.gov#{link}"
    authors = article |> Floki.find("span.docsum-authors") |> Floki.text() |> String.trim()

    %{
      title: title,
      link: full_link,
      authors: authors
    }
  end

  defp fetch_article_details(article) do
    IO.puts("Fetching article from: #{article.link}")
    case HTTPoison.get(article.link, [], recv_timeout: 15000) do
      {:ok, %HTTPoison.Response{status_code: 200, body: article_body}} ->
        {:ok, article_document} = Floki.parse_document(article_body)
        abstract = extract_abstract(article_document)
        doi = extract_doi(article_document)

        Map.merge(article, %{abstract: abstract, doi: doi})

      _ ->
        Map.merge(article, %{abstract: "Failed to fetch abstract", doi: "Failed to fetch DOI"})
    end
  end

  defp extract_abstract(document) do
    abstract = document
               |> Floki.find("#abstract")
               |> Floki.find(".abstract-content")
               |> Floki.raw_html()
               |> process_abstract_html()

    case abstract do
      "" -> "No abstract available"
      abstract -> abstract
    end
  end

  defp process_abstract_html(html) do
    html
    |> String.replace(~r/<sup>(.*?)<\/sup>/, "^\\1")
    |> String.replace(~r/<sub>(.*?)<\/sub>/, "_\\1")
    |> String.replace(~r/<p.*?>(.*?)<\/p>/s, "\\1\n\n")
    |> String.replace(~r/<[^>]*>/, "")
    |> String.split("\n")
    |> Enum.map(&String.trim/1)
    |> Enum.join("\n")
    |> String.replace(~r/\n{3,}/,  "\n\n")
    |> String.trim()
  end

  defp extract_doi(document) do
    document
    |> Floki.find(".citation-doi")
    |> Floki.text()
    |> String.trim()
    |> case do
      "" -> "No DOI available"
      doi -> doi
    end
  end
end

# Get the query from command-line arguments
case System.argv() do
  [query | _] ->
    results = PubMedScraper.search_pubmed(query)
    IO.puts(Jason.encode!(results, pretty: true))

  [] ->
    IO.puts("Error: Please provide a search query as a command-line argument.")
    System.halt(1)
end

zlogs.txt

Searching PubMed for: Does arterial stiffness mediate or suppress the associations of blood pressure with cardiac structure and function in adolescents?
Response status code: 200
Fetching article from: https://pubmed.ncbi.nlm.nih.gov/36930657/
Fetching article from: https://pubmed.ncbi.nlm.nih.gov/34785587/
Fetching article from: https://pubmed.ncbi.nlm.nih.gov/28954852/
Fetching article from: https://pubmed.ncbi.nlm.nih.gov/33766485/
Fetching article from: https://pubmed.ncbi.nlm.nih.gov/31995135/
[
  {
    "abstract": "There is limited understanding of the role of arterial stiffness in cardiovascular disease risk in the pediatric population, lagging behind strong evidence in the adult population. Arterial stiffness progression among adolescents with hypertension has been considered hypertension-mediated vascular damage. However, emerging pediatric reports suggest that arterial stiffness may precede increased blood pressure and hypertension, whereas increased blood pressure from childhood has been associated with signs of cardiac damage in mid-adulthood. Thus, this study used a third variable analytical approach to examine whether arterial stiffness mediates or suppresses the effects of increasing blood pressure on cardiac structure and function in the Avon Longitudinal Study of Parents and Children (ALSPAC) cohort of 1,778 adolescents. After an adjustment for cardiometabolic and lifestyle factors, arterial stiffness measured as carotid-femoral pulse wave velocity partly suppressed the association of higher systolic blood pressure with higher left ventricular mass (standardized regression coefficient, β = -0.012; P = 0.017; suppression effect = 4%), partly mediated the associations of higher systolic and diastolic blood pressure with higher relative ventricular wall thickness, and partly suppressed the association of higher diastolic blood pressure with lower left ventricular diastolic function (β = -0.021; P = 0.003; suppression effect = 14.5%). In conclusion, increasing arterial stiffness could attenuate some of the adverse effects of increased blood pressure on cardiac structure and function in adolescents possibly by modifying the Windkessel effects.NEW & NOTEWORTHY The present study demonstrates that the associations of blood pressure with cardiac function and structure in adolescents may be mediated or suppressed by arterial stiffness depending on the blood pressure phenotype: systolic or diastolic. Arterial stiffness may be considered as an intermediate pathway to attenuate the effect of increased blood pressure on altered cardiac structure and function in youth.",
    "authors": "Agbaje AO.",
    "doi": "doi: 10.1152/ajpheart.00094.2023.\n      \n        doi: 10.1152/ajpheart.00094.2023.",
    "link": "https://pubmed.ncbi.nlm.nih.gov/36930657/",
    "title": "Does arterial stiffness mediate or suppress the associations of blood pressure with cardiac structure and function in adolescents?"
  },
  {
    "abstract": "The cardiometabolic syndrome (CMS) and obesity are typically characterized by a state of metabolic insulin resistance. As global and US rates of obesity increase there is an acceleration of the incidence and prevalence of insulin resistance along with associated cardiovascular disease (CVD). Under physiological conditions insulin regulates glucose homeostasis by enhancing glucose disposal in insulin sensitive tissues while also regulating delivery of nutrients through its vasodilation actions on small feed arteries. Specifically, insulin-mediated production of nitric oxide (NO) from the vascular endothelium leads to increased blood flow enhancing disposal of glucose. Typically, insulin resistance is considered as a decrease in sensitivity or responsiveness to the metabolic actions of insulin including insulin-mediated glucose disposal. However, a decreased sensitivity to the normal vascular actions of insulin, especially diminished nitric oxide production, plays an additional important role in the development of CVD in states of insulin resistance. One mechanism by which insulin resistance and attendant hyperinsulinemia promote CVD is via increases in vascular stiffness. Although obesity and insulin resistance are known to be associated with substantial increases in the prevalence of vascular fibrosis and stiffness the mechanisms and mediators that underlie vascular stiffening in insulin resistant states are complex and have only recently begun to be addressed. Current evidence supports the role of increased plasma levels of aldosterone and insulin and attendant reductions in bioavailable NO in the pathogenesis of impaired vascular relaxation and vascular stiffness in the CMS and obesity. Aldosterone and insulin both increase the activity of serum and glucocorticoid kinase 1 (SGK-1) which in turn is a major regulator of vascular and renal sodium (Na^+) channel activity.The importance of SGK-1 in the pathogenesis of the CMS is highlighted by observations that gain of function mutations in SGK-1 in humans promotes hypertension, insulin resistance and obesity. In endothelial cells, an increase in Na^+ flux contributes to remodeling of the cytoskeleton, reduced NO bioavailability and vascular stiffening. Thus, endothelial SGK-1 may represent a point of convergence for insulin and aldosterone signaling in arterial stiffness associated with obesity and the CMS. This review examines our contemporary understanding of the link between insulin resistance and increased vascular stiffness with emphasis placed on a role for enhanced SGK-1 signaling as a key node in this pathological process.",
    "authors": "Hill MA, Yang Y, Zhang L, Sun Z, Jia G, Parrish AR, Sowers JR.Hill MA, et al.",
    "doi": "doi: 10.1016/j.metabol.2021.154766.\n      \n        doi: 10.1016/j.metabol.2021.154766.",
    "link": "https://pubmed.ncbi.nlm.nih.gov/33766485/",
    "title": "Insulin resistance, cardiovascular stiffening and cardiovascular disease."
  },
  {
    "abstract": "Vitamin K_2 serves an important role in cardiovascular health through regulation of calcium homeostasis. Its effects on the cardiovascular system are mediated through activation of the anti-calcific protein known as matrix Gla protein. In its inactive form, this protein is associated with various markers of cardiovascular disease including increased arterial stiffness, vascular and valvular calcification, insulin resistance and heart failure indices which ultimately increase cardiovascular mortality. Supplementation of vitamin K_2 has been strongly associated with improved cardiovascular outcomes through its modification of systemic calcification and arterial stiffness. Although its direct effects on delaying the progression of vascular and valvular calcification is currently the subject of multiple randomised clinical trials, prior reports suggest potential improved survival among cardiac patients with vitamin K_2 supplementation. Strengthened by its affordability and Food and Drug Adminstration (FDA)-proven safety, vitamin K_2 supplementation is a viable and promising option to improve cardiovascular outcomes.",
    "authors": "Hariri E, Kassis N, Iskandar JP, Schurgers LJ, Saad A, Abdelfattah O, Bansal A, Isogai T, Harb SC, Kapadia S.Hariri E, et al.",
    "doi": "doi: 10.1136/openhrt-2021-001715.\n      \n        doi: 10.1136/openhrt-2021-001715.",
    "link": "https://pubmed.ncbi.nlm.nih.gov/34785587/",
    "title": "Vitamin K2-a neglected player in cardiovascular health: a narrative review."
  },
  {
    "abstract": "The cushioning function of large arteries encompasses distension during systole and recoil during diastole which transforms pulsatile flow into a steady flow in the microcirculation. Arterial stiffness, the inverse of distensibility, has been implicated in various etiologies of chronic common and monogenic cardiovascular diseases and is a major cause of morbidity and mortality globally. The first components that contribute to arterial stiffening are extracellular matrix (ECM) proteins that support the mechanical load, while the second important components are vascular smooth muscle cells (VSMCs), which not only regulate actomyosin interactions for contraction but mediate also mechanotransduction in cell-ECM homeostasis. Eventually, VSMC plasticity and signaling in both conductance and resistance arteries are highly relevant to the physiology of normal and early vascular aging. This review summarizes current concepts of central pressure and tensile pulsatile circumferential stress as key mechanical determinants of arterial wall remodeling, cell-ECM interactions depending mainly on the architecture of cytoskeletal proteins and focal adhesion, the large/small arteries cross-talk that gives rise to target organ damage, and inflammatory pathways leading to calcification or atherosclerosis. We further speculate on the contribution of cellular stiffness along the arterial tree to vascular wall stiffness. In addition, this review provides the latest advances in the identification of gene variants affecting arterial stiffening. Now that important hemodynamic and molecular mechanisms of arterial stiffness have been elucidated, and the complex interplay between ECM, cells, and sensors identified, further research should study their potential to halt or to reverse the development of arterial stiffness.",
    "authors": "Lacolley P, Regnault V, Segers P, Laurent S.Lacolley P, et al.",
    "doi": "doi: 10.1152/physrev.00003.2017.\n      \n        doi: 10.1152/physrev.00003.2017.",
    "link": "https://pubmed.ncbi.nlm.nih.gov/28954852/",
    "title": "Vascular Smooth Muscle Cells and Arterial Stiffening: Relevance in Development, Aging, and Disease."
  },
  {
    "abstract": "Importance:\n\nPrevious research has linked a history of depression with arterial stiffness (AS) during midlife.\n\nObjective:\n\nTo assess the association of depression with elevated midlife AS and to investigate the extent to which this association is mediated via metabolic syndrome (MetS).\n\nDesign, settings, and participants:\n\nThis population-based retrospective cohort study analyzed data collected between March 2006 and December 2010 from 124 445 participants aged 40 to 69 years from the UK Biobank. Participants without data on AS at baseline (n = 332 780) or who reported a previous diagnosis of cardiovascular disease (n = 45 374) were not eligible. Data analysis was performed from May to August 2019.\n\nExposures:\n\nLifetime history of depression was assessed via verbal interview and linked hospital-based clinical depression diagnosis. Metabolic syndrome was defined as the presence of 3 or more of hypertension, dyslipidemia, hyperglycemia, hypertriglyceridemia, and unhealthy waist circumference.\n\nMain outcomes and measures:\n\nPeripherally assessed AS index (ASI) using digital photoplethysmography.\n\nResults:\n\nOf 124 445 included participants with ASI assessed, 71 799 (57.7%) were women, and the mean (SD) age was 56 (8) years. A total of 10 304 participants (8.3%) reported a history of depression. Study findings indicated a significant direct association between depression and ASI levels (β = 0.25; 95% CI, 0.17-0.32). A significant indirect association was also observed between depression and ASI levels (β = 0.10; 95% CI, 0.07-0.13), indicating that 29% of the association of depression with ASI was mediated by MetS. The proportion of mediation increased to 37% when C-reactive protein was added to the MetS criteria (direct association: β = 0.21; 95% CI, 0.15-0.28; indirect association: β = 0.13; 95% CI, 0.10-0.17). Concerning components of MetS, the strongest indirect association was for waist circumference, accounting for 25% of the association between depression and ASI levels (direct association: β = 0.26; 95% CI, 0.18-0.34; indirect association: β = 0.09; 95% CI, 0.06-0.11). Among men, hypertriglyceridemia accounted for 19% of the association between depression and ASI (direct association: β = 0.22; 95% CI, 0.05-0.40; indirect association: β = 0.05; 95% CI, 0.02-0.08).\n\nConclusions and relevance:\n\nOne-third of the association of depression with elevated ASI levels during midlife may be accounted for by combined MetS and inflammatory processes. Unhealthy waist circumference and hypertriglyceridemia emerged as the most important potential targets for preventive interventions within women and men, respectively.",
    "authors": "Dregan A, Rayner L, Davis KAS, Bakolis I, Arias de la Torre J, Das-Munshi J, Hatch SL, Stewart R, Hotopf M.Dregan A, et al.",
    "doi": "doi: 10.1001/jamapsychiatry.2019.4712.\n      \n        doi: 10.1001/jamapsychiatry.2019.4712.",
    "link": "https://pubmed.ncbi.nlm.nih.gov/31995135/",
    "title": "Associations Between Depression, ArterialStiffness, and Metabolic Syndrome Among Adults in the UK Biobank Population Study: A Mediation Analysis."
  }
]