devxpy · August 22, 2023 16:54
diff --git a/gistfile1.py b/gistfile1.py
 from llama_index import (
    VectorStoreIndex,
    SimpleWebPageReader,
    set_global_service_context,
    ServiceContext,
 )
 from llama_index.callbacks import CallbackManager, TokenCountingHandler
 from llama_index.chat_engine.types import ChatMode


 token_counter = TokenCountingHandler()
 callback_manager = CallbackManager([token_counter])
 service_context = ServiceContext.from_defaults(callback_manager=callback_manager)
 # set the global default!
 set_global_service_context(service_context)

 # Load documents and build index
 documents = SimpleWebPageReader().load_data(
    urls=["https://memory-beta.fandom.com/wiki/Jean-Luc_Picard"]
 )
 index = VectorStoreIndex.from_documents(documents)

 chat_engine = index.as_chat_engine(chat_mode=ChatMode.CONTEXT)

 chat_engine.reset()
 message = "When was he granted a leave?"
 response = chat_engine.chat(message)
 print(">> Query:", token_counter.embedding_token_counts[0].prompt)
 print(">> Prompt:", token_counter.llm_token_counts[0].prompt)
 print(">> Response:", response)
diff --git a/out.txt b/out.txt
 >> Query: <!DOCTYPE html>
 <html class="client-nojs" lang="en" dir="ltr">
 <head>
 <meta charset="UTF-8"/>
 <title>Jean-Luc Picard | Memory Beta, non-canon Star Trek Wiki | Fandom</title>
 <script>document.documentElement.className="client-js";RLCONF={"wgBreakFrames":false,"wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat":"dmy","wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgRequestId":"dc86010c6b4e52606a661d746e02c2ff","wgCSPNonce":false,"wgCanonicalNamespace":"","wgCanonicalSpecialPageName":false,"wgNamespaceNumber":0,"wgPageName":"Jean-Luc_Picard","wgTitle":"Jean-Luc Picard","wgCurRevisionId":853962,"wgRevisionId":853962,"wgArticleId":128925,"wgIsArticle":true,"wgIsRedirect":false,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":["Articles with spoilers","Articles with major spoilers","Memory Beta articles sourced from episodes and movies","Memory Beta pages needing citation","Memory Beta articles sourced from comics","Memory Beta articles sourced from novels","Memory Beta articles sourced from video games",
 "Memory Beta articles sourced from games","Memory Beta articles sourced from short stories","Memory Beta articles sourced from eBooks","Memory Beta articles sourced from RPGs","Memory Beta articles sourced from novelizations","Articles with continuity notations","Memory Beta articles sourced from websites","Memory Beta continuity database","Memory Beta incomplete articles","Memory Beta articles sourced from novellas","Memory Beta articles sourced from comic adaptations","Memory Beta good articles","Humans","Humans (24th century)","Starfleet personnel","Starfleet personnel (24th century)","Starfleet helmsmen and flight controllers","Starfleet cadets","Starfleet ensigns","Starfleet lieutenants","Starfleet commanders","Starfleet captains","USS Stargazer personnel","USS Portia personnel","USS Enterprise-D personnel","USS Enterprise-E personnel","USS Verity personnel","USS Enterprise-F personnel","Federation ambassadors","Athletes","Scientists","Time travellers","2305 births"],
 "wgPageContentLanguage":"en","wgPageContentModel":"wikitext","wgRelevantPageName":"Jean-Luc_Picard","wgRelevantArticleId":128925,"wgIsProbablyEditable":false,"wgRelevantPageIsProbablyEditable":false,"wgRestrictionEdit":[],"wgRestrictionMove":[],"wgNoExternals":false,"wgArticleInterlangList":[],"wikiaPageType":"article","isDarkTheme":false,"wgVisualEditor":{"pageLanguageCode":"en","pageLanguageDir":"ltr","pageVariantFallbacks":"en"},"wgMFDisplayWikibaseDescriptions":{"search":false,"nearby":false,"watchlist":false,"tagline":false},"egMapsScriptPath":"/extensions-ucp/mw139/Maps/","egMapsDebugJS":false,"egMapsAvailableServices":["leaflet","googlemaps3"],"egMapsLeafletLayersApiKeys":{"MapBox":"","MapQuestOpen":"","Thunderforest":"","GeoportailFrance":""},"wgIsTestModeEnabled":false,"wgEnableLightboxExt":true,"wgDisableCMSNotifications":false,"wgEditSubmitButtonLabelPublish":false,"mwAuthBaseUrl":"https://auth.fandom.com","personalizedRecommendationsMessage":"Recommended for you",
 "wgPerformanceMonitoringSamplingFactor":10,"wgPerformanceMonitoringEndpointUrl":"https://beacon.wikia-services.
 >> Prompt: system: Context information is below.
 --------------------
 src="data:image/gif;base64,R0lGODlhAQABAIABAAAAAP///yH5BAEAAAEALAAAAAABAAEAQAICTAEAOw%3D%3D" decoding="async" loading="lazy" width="110" height="32" data-image-name="2350s-2360s command capt.png" data-image-key="2350s-2360s_command_capt.png" data-relevant="0" data-src="https://static.wikia.nocookie.net/startrek/images/9/99/2350s-2360s_command_capt.png/revision/latest/scale-to-width-down/110?cb=20090503195619" class="lazyload"></a>
 </td></tr>
 <tr>
 <td colspan="2">Extended leave of absence</td>
 <td><a href="/wiki/2355" title="2355">2355</a>-<a href="/wiki/2359" title="2359">2359</a>
 </td></tr>
 <tr>
 <td><a href="/wiki/USS_Portia" title="USS Portia">USS <i>Portia</i></a></td>
 <td><a href="/wiki/Commanding_officer" title="Commanding officer">commanding officer</a></td>
 <td><a href="/wiki/2360" title="2360">2360</a>
 </td></tr>
 <tr>
 <td><a href="/wiki/Starfleet_Tactical" title="Starfleet Tactical">Starfleet Tactical</a></td>
 <td>Head of <span class="new" title="Long-range Threat Assessment and Response Division (page does not exist)" data-uncrawlable-url="L3dpa2kvTG9uZy1yYW5nZV9UaHJlYXRfQXNzZXNzbWVudF9hbmRfUmVzcG9uc2VfRGl2aXNpb24/YWN0aW9uPWVkaXQmcmVkbGluaz0x">Long-range Threat Assessment and Response Division</span></td>
 <td><a href="/wiki/2360" title="2360">2360</a>-<a href="/wiki/2363" title="2363">2363</a>
 </td></tr>
 <tr>
 <td><a href="/wiki/Starfleet_Headquarters" title="Starfleet Headquarters">Starfleet Headquarters</a></td>
 <td>Starfleet liaison to the <a href="/wiki/Federation_President" class="mw-redirect" title="Federation President">Federation President</a></td>
 <td><a href="/wiki/2363" title="2363">2363</a>-<a href="/wiki/2364" title="2364">2364</a>
 </td></tr>
 <tr>
 <td rowspan="3"><a href="/wiki/USS_Enterprise_(NCC-1701-D)" title="USS Enterprise (NCC-1701-D)">USS <i>Enterprise</i>-D</a></td>
 <td rowspan="9"><a href="/wiki/Commanding_officer" title="Commanding officer">commanding officer</a></td>
 <td rowspan="2"><a href="/wiki/2364" title="2364">2364</a>-<a href="/wiki/2366" title="2366">2366</a>
 </td></tr>
 <tr>
 <td rowspan="6"><a href="https://static.wikia.nocookie.net/startrek/images/c/c1/Red_Capt_2364.png/revision/latest?cb=20070413095934" class="image" title="Collar rank."><img alt="Collar rank."

 He decided to visit his former <a href="/wiki/Second_officer" title="Second officer">second officer</a> <a href="/wiki/Elizabeth_Wu" title="Elizabeth Wu">Elizabeth Wu</a> on the <span class="new" title="Hydra IV (page does not exist)" data-uncrawlable-url="L3dpa2kvSHlkcmFfSVY/YWN0aW9uPWVkaXQmcmVkbGluaz0x">Hydra IV</span> colony.Wu had left the <i>Stargazer</i> in <a href="/wiki/2336" title="2336">2336</a>, but Picard had kept in touch with her over the years, and desired the perspective of someone who had been in Starfleet, but left it behind.</p><p>Picard's craft crash-landed on an uncharted and unexplored planet.There, he encountered a group of <span class="new" title="Rhitori (page does not exist)" data-uncrawlable-url="L3dpa2kvUmhpdG9yaT9hY3Rpb249ZWRpdCZyZWRsaW5rPTE=">Rhitori</span>, whose ancestors owed their freedom to the <i>Stargazer</i> responding to their distress call over 20 years prior.The incident made Picard realize all the good that he had done, and could continue to do, as a starship captain.Picard decided to remain in Starfleet.(<i><a href="/wiki/STA" class="mw-redirect" title="STA">STA</a></i>  <a href="/wiki/Short_story" title="Short story">short story</a>: "<a href="/wiki/Darkness" title="Darkness">Darkness</a>")
 </p><p>Soon after, Picard began guest lecturing at Starfleet Academy, as well as transferring between numerous assignments.(<i><a href="/wiki/TNG" class="mw-redirect" title="TNG">TNG</a></i>   <a href="/wiki/Novel" title="Novel">novel</a>: <i><a href="/wiki/Doors_Into_Chaos" title="Doors Into Chaos">Doors Into Chaos</a></i>)
 </p><p>Picard also enrolled as a Ph.D. student in archaeology at the <a href="/wiki/University_of_Alpha_Centauri" title="University of Alpha Centauri">University of Alpha Centauri</a> in 2355.Three years into this program, he was teaching an introductory class in archaeology when <a href="/wiki/Guinan" title="Guinan">Guinan</a> paid him a visit, and questioned whether he really wanted to avoid going back to commanding a starship.</p><p>During his time at the <a href="/wiki/University_of_Alpha_Centauri" title="University of Alpha Centauri">University of Alpha Centauri</a> Picard assembled a group of archeologists who wanted discovered some survivors of the ancient race of <a href="/wiki/Manraloth" title="Manraloth">Manraloth</a>.Picard fell in love with the <a href="/wiki/Giriaenn" title="Giriaenn">first survivor</a> they were able to rescue from a time stasis field.Not remembering her past, she accepted the name "Ariel".Picard and Ariel spent many months together putting together a team of Starfleet specialists who would eventually search for other Manraloth.Knowing that he could achieve their goal only with the help of Starfleet, Picard rejoined the fleet and was given command of the <a href="/wiki/Miranda_class" title="Miranda class"><i>Miranda</i>-class</a> <a href="/wiki/USS_Portia" title="USS Portia">USS <i>Portia</i></a>.For this mission he recruited <a href="/wiki/Lieutenant" title="Lieutenant">Lieutenant</a> <a href="/wiki/Data" title="Data">Data</a> as his <a href="/wiki/Science_officer" title="Science officer">science officer</a>.</p><p>After they found other survivors, Picard realized that Ariel (who in the meanwhile regained her memory, and remembered her true name: Giriaenn) had tricked him by using his feelings for her.
 --------------------

 user: When was he granted a leave?
 >> Response: Captain Picard was granted an extended leave of absence from 2355 to 2359.
	from llama_index import (
	VectorStoreIndex,
	SimpleWebPageReader,
	set_global_service_context,
	ServiceContext,
	)
	from llama_index.callbacks import CallbackManager, TokenCountingHandler
	from llama_index.chat_engine.types import ChatMode


	token_counter = TokenCountingHandler()
	callback_manager = CallbackManager([token_counter])
	service_context = ServiceContext.from_defaults(callback_manager=callback_manager)
	# set the global default!
	set_global_service_context(service_context)

	# Load documents and build index
	documents = SimpleWebPageReader().load_data(
	urls=["https://memory-beta.fandom.com/wiki/Jean-Luc_Picard"]
	)
	index = VectorStoreIndex.from_documents(documents)

	chat_engine = index.as_chat_engine(chat_mode=ChatMode.CONTEXT)

	chat_engine.reset()
	message = "When was he granted a leave?"
	response = chat_engine.chat(message)
	print(">> Query:", token_counter.embedding_token_counts[0].prompt)
	print(">> Prompt:", token_counter.llm_token_counts[0].prompt)
	print(">> Response:", response)
	>> Query: <!DOCTYPE html>
	<html class="client-nojs" lang="en" dir="ltr">
	<head>
	<meta charset="UTF-8"/>
	<title>Jean-Luc Picard \| Memory Beta, non-canon Star Trek Wiki \| Fandom</title>
	<script>document.documentElement.className="client-js";RLCONF={"wgBreakFrames":false,"wgSeparatorTransformTable":["",""],"wgDigitTransformTable":["",""],"wgDefaultDateFormat":"dmy","wgMonthNames":["","January","February","March","April","May","June","July","August","September","October","November","December"],"wgRequestId":"dc86010c6b4e52606a661d746e02c2ff","wgCSPNonce":false,"wgCanonicalNamespace":"","wgCanonicalSpecialPageName":false,"wgNamespaceNumber":0,"wgPageName":"Jean-Luc_Picard","wgTitle":"Jean-Luc Picard","wgCurRevisionId":853962,"wgRevisionId":853962,"wgArticleId":128925,"wgIsArticle":true,"wgIsRedirect":false,"wgAction":"view","wgUserName":null,"wgUserGroups":["*"],"wgCategories":["Articles with spoilers","Articles with major spoilers","Memory Beta articles sourced from episodes and movies","Memory Beta pages needing citation","Memory Beta articles sourced from comics","Memory Beta articles sourced from novels","Memory Beta articles sourced from video games",
	"Memory Beta articles sourced from games","Memory Beta articles sourced from short stories","Memory Beta articles sourced from eBooks","Memory Beta articles sourced from RPGs","Memory Beta articles sourced from novelizations","Articles with continuity notations","Memory Beta articles sourced from websites","Memory Beta continuity database","Memory Beta incomplete articles","Memory Beta articles sourced from novellas","Memory Beta articles sourced from comic adaptations","Memory Beta good articles","Humans","Humans (24th century)","Starfleet personnel","Starfleet personnel (24th century)","Starfleet helmsmen and flight controllers","Starfleet cadets","Starfleet ensigns","Starfleet lieutenants","Starfleet commanders","Starfleet captains","USS Stargazer personnel","USS Portia personnel","USS Enterprise-D personnel","USS Enterprise-E personnel","USS Verity personnel","USS Enterprise-F personnel","Federation ambassadors","Athletes","Scientists","Time travellers","2305 births"],
	"wgPageContentLanguage":"en","wgPageContentModel":"wikitext","wgRelevantPageName":"Jean-Luc_Picard","wgRelevantArticleId":128925,"wgIsProbablyEditable":false,"wgRelevantPageIsProbablyEditable":false,"wgRestrictionEdit":[],"wgRestrictionMove":[],"wgNoExternals":false,"wgArticleInterlangList":[],"wikiaPageType":"article","isDarkTheme":false,"wgVisualEditor":{"pageLanguageCode":"en","pageLanguageDir":"ltr","pageVariantFallbacks":"en"},"wgMFDisplayWikibaseDescriptions":{"search":false,"nearby":false,"watchlist":false,"tagline":false},"egMapsScriptPath":"/extensions-ucp/mw139/Maps/","egMapsDebugJS":false,"egMapsAvailableServices":["leaflet","googlemaps3"],"egMapsLeafletLayersApiKeys":{"MapBox":"","MapQuestOpen":"","Thunderforest":"","GeoportailFrance":""},"wgIsTestModeEnabled":false,"wgEnableLightboxExt":true,"wgDisableCMSNotifications":false,"wgEditSubmitButtonLabelPublish":false,"mwAuthBaseUrl":"https://auth.fandom.com","personalizedRecommendationsMessage":"Recommended for you",
	"wgPerformanceMonitoringSamplingFactor":10,"wgPerformanceMonitoringEndpointUrl":"https://beacon.wikia-services.
	>> Prompt: system: Context information is below.
	--------------------
	src="data:image/gif;base64,R0lGODlhAQABAIABAAAAAP///yH5BAEAAAEALAAAAAABAAEAQAICTAEAOw%3D%3D" decoding="async" loading="lazy" width="110" height="32" data-image-name="2350s-2360s command capt.png" data-image-key="2350s-2360s_command_capt.png" data-relevant="0" data-src="https://static.wikia.nocookie.net/startrek/images/9/99/2350s-2360s_command_capt.png/revision/latest/scale-to-width-down/110?cb=20090503195619" class="lazyload"></a>
	</td></tr>
	<tr>
	<td colspan="2">Extended leave of absence</td>
	<td><a href="/wiki/2355" title="2355">2355</a>-<a href="/wiki/2359" title="2359">2359</a>
	</td></tr>
	<tr>
	<td><a href="/wiki/USS_Portia" title="USS Portia">USS <i>Portia</i></a></td>
	<td><a href="/wiki/Commanding_officer" title="Commanding officer">commanding officer</a></td>
	<td><a href="/wiki/2360" title="2360">2360</a>
	</td></tr>
	<tr>
	<td><a href="/wiki/Starfleet_Tactical" title="Starfleet Tactical">Starfleet Tactical</a></td>
	<td>Head of <span class="new" title="Long-range Threat Assessment and Response Division (page does not exist)" data-uncrawlable-url="L3dpa2kvTG9uZy1yYW5nZV9UaHJlYXRfQXNzZXNzbWVudF9hbmRfUmVzcG9uc2VfRGl2aXNpb24/YWN0aW9uPWVkaXQmcmVkbGluaz0x">Long-range Threat Assessment and Response Division</span></td>
	<td><a href="/wiki/2360" title="2360">2360</a>-<a href="/wiki/2363" title="2363">2363</a>
	</td></tr>
	<tr>
	<td><a href="/wiki/Starfleet_Headquarters" title="Starfleet Headquarters">Starfleet Headquarters</a></td>
	<td>Starfleet liaison to the <a href="/wiki/Federation_President" class="mw-redirect" title="Federation President">Federation President</a></td>
	<td><a href="/wiki/2363" title="2363">2363</a>-<a href="/wiki/2364" title="2364">2364</a>
	</td></tr>
	<tr>
	<td rowspan="3"><a href="/wiki/USS_Enterprise_(NCC-1701-D)" title="USS Enterprise (NCC-1701-D)">USS <i>Enterprise</i>-D</a></td>
	<td rowspan="9"><a href="/wiki/Commanding_officer" title="Commanding officer">commanding officer</a></td>
	<td rowspan="2"><a href="/wiki/2364" title="2364">2364</a>-<a href="/wiki/2366" title="2366">2366</a>
	</td></tr>
	<tr>
	<td rowspan="6"><a href="https://static.wikia.nocookie.net/startrek/images/c/c1/Red_Capt_2364.png/revision/latest?cb=20070413095934" class="image" title="Collar rank."><img alt="Collar rank."

	He decided to visit his former <a href="/wiki/Second_officer" title="Second officer">second officer</a> <a href="/wiki/Elizabeth_Wu" title="Elizabeth Wu">Elizabeth Wu</a> on the <span class="new" title="Hydra IV (page does not exist)" data-uncrawlable-url="L3dpa2kvSHlkcmFfSVY/YWN0aW9uPWVkaXQmcmVkbGluaz0x">Hydra IV</span> colony.Wu had left the <i>Stargazer</i> in <a href="/wiki/2336" title="2336">2336</a>, but Picard had kept in touch with her over the years, and desired the perspective of someone who had been in Starfleet, but left it behind.</p><p>Picard's craft crash-landed on an uncharted and unexplored planet.There, he encountered a group of <span class="new" title="Rhitori (page does not exist)" data-uncrawlable-url="L3dpa2kvUmhpdG9yaT9hY3Rpb249ZWRpdCZyZWRsaW5rPTE=">Rhitori</span>, whose ancestors owed their freedom to the <i>Stargazer</i> responding to their distress call over 20 years prior.The incident made Picard realize all the good that he had done, and could continue to do, as a starship captain.Picard decided to remain in Starfleet.(<i><a href="/wiki/STA" class="mw-redirect" title="STA">STA</a></i> <a href="/wiki/Short_story" title="Short story">short story</a>: "<a href="/wiki/Darkness" title="Darkness">Darkness</a>")
	</p><p>Soon after, Picard began guest lecturing at Starfleet Academy, as well as transferring between numerous assignments.(<i><a href="/wiki/TNG" class="mw-redirect" title="TNG">TNG</a></i> <a href="/wiki/Novel" title="Novel">novel</a>: <i><a href="/wiki/Doors_Into_Chaos" title="Doors Into Chaos">Doors Into Chaos</a></i>)
	</p><p>Picard also enrolled as a Ph.D. student in archaeology at the <a href="/wiki/University_of_Alpha_Centauri" title="University of Alpha Centauri">University of Alpha Centauri</a> in 2355.Three years into this program, he was teaching an introductory class in archaeology when <a href="/wiki/Guinan" title="Guinan">Guinan</a> paid him a visit, and questioned whether he really wanted to avoid going back to commanding a starship.</p><p>During his time at the <a href="/wiki/University_of_Alpha_Centauri" title="University of Alpha Centauri">University of Alpha Centauri</a> Picard assembled a group of archeologists who wanted discovered some survivors of the ancient race of <a href="/wiki/Manraloth" title="Manraloth">Manraloth</a>.Picard fell in love with the <a href="/wiki/Giriaenn" title="Giriaenn">first survivor</a> they were able to rescue from a time stasis field.Not remembering her past, she accepted the name "Ariel".Picard and Ariel spent many months together putting together a team of Starfleet specialists who would eventually search for other Manraloth.Knowing that he could achieve their goal only with the help of Starfleet, Picard rejoined the fleet and was given command of the <a href="/wiki/Miranda_class" title="Miranda class"><i>Miranda</i>-class</a> <a href="/wiki/USS_Portia" title="USS Portia">USS <i>Portia</i></a>.For this mission he recruited <a href="/wiki/Lieutenant" title="Lieutenant">Lieutenant</a> <a href="/wiki/Data" title="Data">Data</a> as his <a href="/wiki/Science_officer" title="Science officer">science officer</a>.</p><p>After they found other survivors, Picard realized that Ariel (who in the meanwhile regained her memory, and remembered her true name: Giriaenn) had tricked him by using his feelings for her.
	--------------------

	user: When was he granted a leave?
	>> Response: Captain Picard was granted an extended leave of absence from 2355 to 2359.