Scrape links off TV Tropes to find common tropes in a set of shows

_tv-tropes-scraper.rs

use scraper::{Html, Selector};

use std::collections::HashMap;

fn main() {
    let source_sites: &[&[&str]] = &[
        &[
            "https://tvtropes.org/pmwiki/pmwiki.php/BreakingBad/TropesAToB",
            "https://tvtropes.org/pmwiki/pmwiki.php/BreakingBad/TropesCToD",
            "https://tvtropes.org/pmwiki/pmwiki.php/BreakingBad/TropesEToL",
            "https://tvtropes.org/pmwiki/pmwiki.php/BreakingBad/TropesMToR",
            "https://tvtropes.org/pmwiki/pmwiki.php/BreakingBad/TropesSToZ",
        ],
        &["https://tvtropes.org/pmwiki/pmwiki.php/Manga/DeathNote"],
        &["https://tvtropes.org/pmwiki/pmwiki.php/Series/HouseOfCardsUS"],
        &["https://tvtropes.org/pmwiki/pmwiki.php/Series/Impulse2018"],
        &["https://tvtropes.org/pmwiki/pmwiki.php/Film/InsideMan"],
        &["https://tvtropes.org/pmwiki/pmwiki.php/Series/IZombie"],
        &["https://tvtropes.org/pmwiki/pmwiki.php/Series/LaCasaDePapel"],
        &["https://tvtropes.org/pmwiki/pmwiki.php/Series/OrphanBlack"],
        &["https://tvtropes.org/pmwiki/pmwiki.php/Literature/TheExpanse"],
    ];

    let mut occurs = HashMap::new();
    for (show_index, &show) in source_sites.iter().enumerate() {
        for &page in show {
            let response = reqwest::blocking::get(page).expect("Failed to get page content.");
            let response_text = response.text().expect("Response doesn't have text.");
            let html_document = Html::parse_document(&response_text);

            let selector = Selector::parse("a").unwrap();
            for element in html_document.select(&selector) {
                if let Some(href) = element.value().attr("href") {
                    *occurs.entry(String::from(href)).or_insert(0u64) |= 1 << show_index;
                }
            }
        }
    }

    let mut trope_set_with_occurence_count: Vec<(u32, String)> = occurs
        .into_iter()
        .filter_map(|(mut href, set)| {
            let instances = set.count_ones();
            let is_unrelated = instances == source_sites.len() as u32;
            let is_trope = href.starts_with("/pmwiki/pmwiki.php/Main/");

            if instances > 1 && !is_unrelated && is_trope {
                href.drain(0..24);
                Some((instances, href))
            } else {
                None
            }
        }).collect();
    trope_set_with_occurence_count.sort_by_key(|(instances, href)| (u32::MAX - instances, href.clone()));

    for (instances, href) in trope_set_with_occurence_count.iter() {
        println!("{}; {}", instances, href);
    }
}

out.txt

7; AssholeVictim
7; Foreshadowing
7; ShoutOut
6; ChekhovsGun
5; AscendedExtra
5; BerserkButton
5; BlackComedy
5; BookEnds
5; BoomHeadshot
5; CallBack
5; DoesThisRemindYouOfAnything
5; DownerEnding
5; EnemyMine
5; EstablishingCharacterMoment
5; EvenEvilHasStandards
5; KansasCityShuffle
5; KarmaHoudini
5; ManipulativeBastard
5; MeaningfulName
5; MercyKill
5; OhCrap
5; PetTheDog
5; SarcasticConfession
5; SeriesOfThe2010s
5; TakeThat
5; TooDumbToLive
5; TookALevelInBadass
5; UpToEleven
5; VillainProtagonist
5; WhatTheHellHero
4; AbusiveParents
4; AffablyEvil
4; AlasPoorVillain
4; AnyoneCanDie
4; AxCrazy
4; BatmanGambit
4; BigBad
4; BittersweetEnding
4; BlatantLies
4; ButtMonkey
4; CelebrityParadox
4; CharacterDevelopment
4; ContrivedCoincidence
4; DramaticIrony
4; DrivenToSuicide
4; EvenEvilHasLovedOnes
4; HeelFaceTurn
4; HeelRealization
4; HeroicBSOD
4; HeroicSacrifice
4; HoistByHisOwnPetard
4; Hypocrite
4; InUniverse
4; JumpingOffTheSlipperySlope
4; MoodWhiplash
4; MoralityPet
4; NiceGuy
4; NiceJobBreakingItHero
4; PapaWolf
4; PlayingWithATrope
4; PoliceAreUseless
4; PoorCommunicationKills
4; RealLife
4; RedHerring
4; SpannerInTheWorks
4; StealthPun
4; TheDragon
4; TheReasonYouSuckSpeech
4; TheSociopath
4; ThoseTwoGuys
4; WhamEpisode
4; WhamLine
4; WhamShot
4; XanatosSpeedChess
3; ALighterShadeOfBlack
3; AbortedArc
3; ActionGirl
3; AdultFear
3; AndIMustScream
3; AntiHero
3; ArcWords
3; ArmorPiercingQuestion
3; ArtisticLicenseLaw
3; BadassBoast
3; BecomingTheMask
3; BilingualBonus
3; BloodFromTheMouth
3; BreakTheCutie
3; BreakingTheFourthWall
3; CatchPhrase
3; ChekhovsBoomerang
3; ChekhovsGunman
3; ChekhovsSkill
3; ColorCodedForYourConvenience
3; ConsummateLiar
3; CorruptCorporateExecutive
3; CountryMatters
3; CurbStompBattle
3; DemotedToExtra
3; DiscussedTrope
3; DisposingOfABody
3; DoubleMeaningTitle
3; EarlyBirdCameo
3; EarlyInstallmentWeirdness
3; EurekaMoment
3; EvilVersusEvil
3; ExactWords
3; Expy
3; EyeScream
3; Fanservice
3; ForWantOfANail
3; ForegoneConclusion
3; ForeignRemake
3; FourTemperamentEnsemble
3; FreezeFrameBonus
3; FriendlyEnemy
3; GambitRoulette
3; GenreSavvy
3; GoneHorriblyRight
3; GoryDiscretionShot
3; GrandFinale
3; HaveYouToldAnyoneElse
3; HeroAntagonist
3; HiddenDepths
3; HollywoodLaw
3; HopeSpot
3; IgnoredEpiphany
3; IronicEcho
3; Irony
3; JerkJock
3; JustForPun
3; KickTheDog
3; LampshadeHanging
3; LeaningOnTheFourthWall
3; LoadsAndLoadsOfCharacters
3; LoveTriangle
3; MagnificentBastard
3; MaleGaze
3; MamaBear
3; ManChild
3; MaybeMagicMaybeMundane
3; MegaCorp
3; MoralEventHorizon
3; MyGodWhatHaveIDone
3; MythologyGag
3; NiceJobFixingItVillain
3; NightmareFuel
3; NoodleIncident
3; ObfuscatingStupidity
3; ObliviousGuiltSlinging
3; OnceMoreWithClarity
3; OohMeAccentsSlipping
3; OvertRendezvous
3; PacManFever
3; PlayedWith
3; PlotArmor
3; PragmaticAdaptation
3; PragmaticVillainy
3; PrecisionFStrike
3; ProductPlacement
3; PunchClockVillain
3; PutOnABus
3; RealityEnsues
3; RealityIsUnrealistic
3; RefugeInAudacity
3; RuleOfSymbolism
3; SerialKiller
3; SexyDiscretionShot
3; ShownTheirWork
3; SmugSnake
3; SpinOff
3; StagingAnIntervention
3; StepfordSmiler
3; SympatheticInspectorAntagonist
3; TakingYouWithMe
3; TheBadGuyWins
3; TheDogBitesBack
3; TheGhost
3; TheMole
3; ThereIsNoKillLikeOverkill
3; TimeSkip
3; UndyingLoyalty
3; UnspokenPlanGuarantee
3; VillainousBreakdown
3; WhatHappenedToTheMouse
3; WordOfGod
3; WrongGenreSavvy
3; XanatosGambit
3; YouHaveFailedMe
3; YouHaveOutlivedYourUsefulness
3; YourDaysAreNumbered
2; ADayInTheLimelight
2; AMillionIsAStatistic
2; AccidentalMurder
2; ActorAllusion
2; AdaptationExpansion
2; AllLoveIsUnrequited
2; AllThereInTheManual
2; AmbitionIsEvil
2; AmericanSeries
2; AmoralAttorney
2; AnAesop
2; AnalogyBackfire
2; AndStarring
2; AnimalMotifs
2; AntiVillain
2; ArchEnemy
2; ArtisticLicenseChemistry
2; ArtisticLicenseMedicine
2; ArtisticLicensePhysics
2; AsHimself
2; AttemptedRape
2; AvertedTrope
2; AwesomeMcCoolname
2; AwesomenessByAnalysis
2; AwfulTruth
2; BackFromTheDead
2; BadBoss
2; BaitAndSwitch
2; Beat
2; BeautyIsNeverTarnished
2; BewareTheNiceOnes
2; BigDamnHeroes
2; BigFancyHouse
2; BitchInSheepsClothing
2; BlackAndGrayMorality
2; Blackmail
2; BlueAndOrangeMorality
2; BodyHorror
2; BottomlessMagazines
2; BreakHisHeartToSaveHim
2; BreakTheHaughty
2; BullyHunter
2; BunnyEarsLawyer
2; BurnerPhones
2; BusCrash
2; CallingTheOldManOut
2; CameBackWrong
2; CaperRationalization
2; CaptainObvious
2; CassandraTruth
2; CastingGag
2; Catchphrase
2; CentralTheme
2; CharactersDroppingLikeFlies
2; CigaretteOfAnxiety
2; CliffHanger
2; Cliffhanger
2; ClosetGeek
2; Cloudcuckoolander
2; ColdOpen
2; CompoundTitle
2; ConcealmentEqualsCover
2; ContinuityNod
2; ControlFreak
2; CoolOldGuy
2; CorporateConspiracy
2; CosmicPlaything
2; CrapsackWorld
2; CrazyPrepared
2; CrossReferencedTitles
2; CutLexLuthorACheck
2; DamselInDistress
2; DarkerAndEdgier
2; DeadpanSnarker
2; DeathByAdaptation
2; DeathFakedForYou
2; DeathTrap
2; DeliberateValuesDissonance
2; DescentIntoAddiction
2; DespairEventHorizon
2; DeusExMachina
2; Deuteragonist
2; DiabolusExMachina
2; DiegeticSoundtrackUsage
2; DirtyCop
2; DiscOneFinalBoss
2; DisproportionateRetribution
2; DoomedMoralVictor
2; DroppedABridgeOnHim
2; DrowningMySorrows
2; DrunkWithPower
2; DuctTapeForEverything
2; DueToTheDead
2; DyingMomentOfAwesome
2; EnhanceButton
2; EpicFail
2; EvenBadMenLoveTheirMamas
2; EvilCounterpart
2; EvilIsPetty
2; ExploitedTrope
2; FaceDeathWithDignity
2; FaceFramedInShadow
2; FakingTheDead
2; FanDisservice
2; FateWorseThanDeath
2; FauxAffablyEvil
2; FemmeFatale
2; FireForgedFriends
2; FlippingTheBird
2; FoodPorn
2; ForScience
2; FrameUp
2; FreudianExcuse
2; FreudianTrio
2; FridgeBrilliance
2; FriendToAllChildren
2; FromBadToWorse
2; FromNobodyToNightmare
2; FunnyBackgroundEvent
2; GambitPileup
2; GilliganCut
2; GivenNameReveal
2; Gorn
2; GreaterScopeVillain
2; GreenEyedMonster
2; GreyAndGrayMorality
2; GuileHero
2; HandWave
2; Handwave
2; HappierHomeMovie
2; HeKnowsTooMuch
2; HeWhoFightsMonsters
2; HealingFactor
2; HeelFaceDoorSlam
2; HeterosexualLifePartners
2; HitmanWithAHeart
2; HollywoodHacking
2; HollywoodHealing
2; HoneyTrap
2; HowsYourBritishAccent
2; HumiliationConga
2; IAmTheNoun
2; ICannotSelfTerminate
2; IKnowYouKnowIKnow
2; IdiosyncraticEpisodeNaming
2; IdiotBall
2; ImagineSpot
2; InNameOnly
2; IncrediblyLamePun
2; IncurableCoughOfDeath
2; IndyPloy
2; InnerMonologue
2; InsideJob
2; InsistentTerminology
2; InspectorJavert
2; InvokedTrope
2; ItAmusedMe
2; ItHasBeenAnHonor
2; ItsAlwaysSpring
2; JackBauerInterrogationTechnique
2; JerkWithAHeartOfGold
2; JerkWithAHeartOfJerk
2; JustifiedTrope
2; KarmicDeath
2; KickTheSonOfABitch
2; KillEmAll
2; KilledMidSentence
2; KilledOffscreen
2; Lampshade
2; Lampshaded
2; LaserGuidedKarma
2; LastNameBasis
2; LaughingMad
2; Leitmotif
2; LenoDevice
2; LiteralMetaphor
2; LiteraryAllusionTitle
2; LonelyAtTheTop
2; LoopholeAbuse
2; LoveInterest
2; LoveMakesYouDumb
2; LoveMakesYouEvil
2; MacGyvering
2; MachiavelliWasWrong
2; MadeOfIron
2; MedicalRapeAndImpregnate
2; MemeticMutation
2; MexicanStandoff
2; MissingMom
2; MohsScaleOfSciFiHardness
2; MomentKiller
2; MoneyIsNotPower
2; Mooks
2; MundaneMadeAwesome
2; MurderByInaction
2; MyBelovedSmother
2; NakedApron
2; NeatFreak
2; NeverHurtAnInnocent
2; NewscasterCameo
2; NextSundayAD
2; NiceToTheWaiter
2; NoCelebritiesWereHarmed
2; NoNameGiven
2; NoPreggerSex
2; NoSenseOfPersonalSpace
2; NobleBigotWithABadge
2; NotDistractedByTheSexy
2; NotQuiteDead
2; NotSoDifferent
2; NotSoDifferentRemark
2; NotTheFallThatKillsYou
2; ObviouslyEvil
2; OffingTheOffspring
2; OffstageVillainy
2; OminousLatinChanting
2; OmnidisciplinaryScientist
2; OneSteveLimit
2; OnlyAFleshWound
2; OnlySaneMan
2; OurZombiesAreDifferent
2; OutGambitted
2; OutOfCharacterMoment
2; OutOfFocus
2; OutsideContextProblem
2; PaperThinDisguise
2; ParentalSubstitute
2; PayEvilUntoEvil
2; Pilot
2; PlayedForDrama
2; PlayedForLaughs
2; PlayingSick
2; PlayingWithSyringes
2; PlotHole
2; PoisonousFriend
2; PornStash
2; PosthumousCharacter
2; PowerTrio
2; PowersInTheFirstEpisode
2; PreMortemOneLiner
2; PromotedToOpeningTitles
2; ProperlyParanoid
2; ProtagonistJourneyToVillain
2; PunctuatedForEmphasis
2; PunnyName
2; RecklessGunUsage
2; RedEyesTakeWarning
2; RedOniBlueOni
2; RedemptionEqualsDeath
2; RefusalOfTheCall
2; RemovingTheHeadOrDestroyingTheBrain
2; ReplacementGoldfish
2; RevealingCoverup
2; ReverseWhodunnit
2; RoaringRampageOfRevenge
2; RootingForTheEmpire
2; RunningGag
2; SWATTeam
2; SanitySlippage
2; SarcasmMode
2; ScareChord
2; ScienceFiction
2; ScienceHero
2; SecretTestOfCharacter
2; SelectiveObliviousness
2; SelfServingMemory
2; SeriousBusiness
2; ShootTheDog
2; SickbedSlaying
2; SignificantAnagram
2; SlidingScaleOfIdealismVersusCynicism
2; SoundtrackDissonance
2; SpiritualSuccessor
2; SplashOfColor
2; Squick
2; StalkerWithACrush
2; StepfordSuburbia
2; StrangeMindsThinkAlike
2; SubvertedTrope
2; SuicideByCop
2; SuperStrength
2; SurprisinglyRealisticOutcome
2; Symbolism
2; TeethClenchedTeamwork
2; TemptingFate
2; ThanatosGambit
2; TheAtoner
2; TheBusCameBack
2; TheCameo
2; TheCaper
2; TheChessmaster
2; TheCoronerDothProtestTooMuch
2; TheDarkSideWillMakeYouForget
2; TheDreaded
2; TheDyingWalk
2; TheHeroDies
2; TheInformant
2; TheLancer
2; TheLeader
2; ThePiratesWhoDontDoAnything
2; TheQuietOne
2; TheReveal
2; TheScapegoat
2; TheUnreveal
2; TheVoiceless
2; TheWoobie
2; ThereAreNoTherapists
2; Thriller
2; TitleDrop
2; TomatoInTheMirror
2; TookALevelInJerkass
2; TrademarkFavoriteFood
2; Tragedy
2; TraumaCongaLine
2; Tykebomb
2; Ubermensch
2; Understatement
2; UnholyMatrimony
2; UnwittingPawn
2; UrbanFantasy
2; VertigoEffect
2; VillainWithGoodPublicity
2; VillainousFriendship
2; VisionaryVillain
2; VisualPun
2; VitriolicBestBuds
2; VomitIndiscretionShot
2; WasItAllALie
2; WellIntentionedExtremist
2; WildCard
2; WoobieDestroyerOfWorlds
2; WorthyOpponent
2; WouldHurtAChild
2; WouldntHurtAChild
2; WoundedGazelleGambit
2; YankTheDogsChain
2; YouAreBetterThanYouThinkYouAre
2; YouAreTooLate
2; YouKillItYouBoughtIt
2; ZombieApocalypse