Created
March 7, 2020 12:19
-
-
Save lanfon72/7284f83552eb871220804f8ee850fe0e to your computer and use it in GitHub Desktop.
integrate youtube_dl with asyncio.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import asyncio | |
from datetime import datetime | |
from functools import partial | |
from concurrent.futures import ProcessPoolExecutor | |
from youtube_dl import YoutubeDL as YDL | |
PPE = ProcessPoolExecutor() | |
def extract_info(url, quiet=False): | |
# use `quiet=True` to avoid noisy | |
return YDL(dict(quiet=quiet)).extract_info(url, download=False) | |
async def flatten_urls(*urls, loop=None): | |
print(f"trying to flat {len(urls)} URLs.") | |
loop = asyncio.get_event_loop() | |
futs = [loop.run_in_executor(PPE, extract_info, url) for url in urls] | |
infos = await asyncio.gather(*futs) | |
urlmatrix = [i.get('entries', [i]) for i in infos] | |
return [u.get('webpage_url', "") for urls in urlmatrix for u in urls] | |
async def progress(*urls, loop=None): | |
loop = loop or asyncio.get_event_loop() | |
print(f"{len(urls)} URLs will be progress.") | |
pure_urls = await flatten_urls(*urls, loop=loop) | |
print(pure_urls) | |
futs = [loop.run_in_executor(None, YDL().download, [url]) for url in pure_urls] | |
return await asyncio.gather(*futs) | |
async def keep_noise(): | |
while True: | |
await asyncio.sleep(1) | |
print(f"Time is: {datetime.now()}", flush=True) | |
def main(): | |
u1 = ["https://youtu.be/-JNeBKlG0cI", | |
"https://youtu.be/mVEItYOsXjM", | |
"https://youtu.be/0JoMqP5UwQ8"] | |
u2 = ["https://youtu.be/m3gLNa-fx_w"] | |
u3 = ["https://www.youtube.com/watch?v=sm4BID47dTI&list=PLFWyAXHl5a6ffjKkNGu27BEk1US055MzB"] | |
loop = asyncio.get_event_loop() | |
coros = [progress(*u1), progress(*u2), progress(*u3), keep_noise()] | |
loop.run_until_complete(asyncio.gather(*coros)) | |
if __name__ == '__main__': | |
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
3 URLs will be progress. | |
trying to flat 3 URLs. | |
1 URLs will be progress. | |
trying to flat 1 URLs. | |
1 URLs will be progress. | |
trying to flat 1 URLs. | |
[youtube] -JNeBKlG0cI: Downloading webpage | |
[youtube] 0JoMqP5UwQ8: Downloading webpage | |
[youtube] mVEItYOsXjM: Downloading webpage | |
Time is: 2020-03-07 20:17:22.442724 | |
[youtube] -JNeBKlG0cI: Downloading video info webpage | |
[youtube] mVEItYOsXjM: Downloading video info webpage | |
[youtube] 0JoMqP5UwQ8: Downloading video info webpage | |
[youtube] m3gLNa-fx_w: Downloading webpage | |
Time is: 2020-03-07 20:17:23.442782 | |
[youtube] m3gLNa-fx_w: Downloading video info webpage | |
[youtube:playlist] Downloading playlist PLFWyAXHl5a6ffjKkNGu27BEk1US055MzB - add --no-playlist to just download video sm4BID47dTI | |
[youtube:playlist] PLFWyAXHl5a6ffjKkNGu27BEk1US055MzB: Downloading webpage | |
['https://www.youtube.com/watch?v=-JNeBKlG0cI', 'https://www.youtube.com/watch?v=mVEItYOsXjM', 'https://www.youtube.com/watch?v=0JoMqP5UwQ8'] | |
[download] Downloading playlist: 【誰來晚餐 番外篇】 | |
[youtube:playlist] playlist 【誰來晚餐 番外篇】: Downloading 7 videos | |
[download] Downloading video 1 of 7 | |
[youtube] sm4BID47dTI: Downloading webpage | |
Time is: 2020-03-07 20:17:24.459840 | |
[youtube] 0JoMqP5UwQ8: Downloading webpage | |
[youtube] -JNeBKlG0cI: Downloading webpage | |
[youtube] mVEItYOsXjM: Downloading webpage | |
[youtube] sm4BID47dTI: Downloading video info webpage | |
['https://www.youtube.com/watch?v=m3gLNa-fx_w'] | |
[youtube] m3gLNa-fx_w: Downloading webpage | |
[youtube] mVEItYOsXjM: Downloading video info webpage | |
[youtube] 0JoMqP5UwQ8: Downloading video info webpage | |
[youtube] -JNeBKlG0cI: Downloading video info webpage | |
[youtube] m3gLNa-fx_w: Downloading video info webpage | |
Time is: 2020-03-07 20:17:25.465897 | |
[download] Downloading video 2 of 7 | |
[youtube] r5WApOc0WLI: Downloading webpage | |
[youtube] r5WApOc0WLI: Downloading video info webpage | |
[download] Destination: 台灣觀眾如何被媒體出賣?「媒體洗腦」完全破解!紅色滲透是啥?【記者真心話】Vol.2|懶人包-mVEItYOsXjM.mp4 | |
[download] 0.8% of 32.14MiB at 4.88MiB/s ETA 00:06 [download] Destination: 你也被媒體帶風向了嗎?ft.回覆酸民留言與頭版質疑【 | |
記者真心話】Vol.3|懶人包|媒體識讀-0JoMqP5UwQ8.mp4 | |
[download] 0.1% of 95.98MiB at 3.76MiB/s ETA 00:25 [download] Destination: 台灣媒體是怎麼爛掉的?七分鐘看懂媒體亂象五大關鍵!【記者真 | |
心話】Vol.1|懶人包--JNeBKlG0cI.mp4 | |
[download] 0.0% of 93.69MiB at 4.88MiB/s ETA 00:19 Time is: 2020-03-07 20:17:26.465955 | |
[download] 0.1% of 93.69MiB at 1.02MiB/s ETA 01:32 [download] Destination: 假新聞出沒! 破解 #媒體與牠們的產地 ft.劣質媒體黑 | |
名單【記者真心話】Vol.4|懶人包|媒體識讀-m3gLNa-fx_w.mp4 | |
[download] 1.1% of 93.69MiB at 1.22MiB/s ETA 01:16 Time is: 2020-03-07 20:17:27.466012 | |
[download] Downloading video 3 of 7 | |
[youtube] W5oa03IOkzE: Downloading webpage | |
[download] 33.4% of 23.95MiB at 4.72MiB/s ETA 00:03 [youtube] W5oa03IOkzE: Downloading video info webpage | |
Time is: 2020-03-07 20:17:28.466069 | |
[download] 37.3% of 32.14MiB at 3.95MiB/s ETA 00:05 Time is: 2020-03-07 20:17:29.466126 | |
[download] 10.2% of 95.98MiB at 2.96MiB/s ETA 00:29 [download] Downloading video 4 of 7 | |
[youtube] RxALj5QWsjo: Downloading webpage | |
[download] 8.5% of 93.69MiB at 2.00MiB/s ETA 00:42 Time is: 2020-03-07 20:17:30.466183 | |
[download] 47.0% of 32.14MiB at 3.59MiB/s ETA 00:04 [youtube] RxALj5QWsjo: Downloading video info webpage | |
[download] 100% of 23.95MiB in 00:04 | |
[download] 11.4% of 93.69MiB at 2.19MiB/s ETA 00:37 Time is: 2020-03-07 20:17:31.465241 | |
[download] 15.8% of 95.98MiB at 2.99MiB/s ETA 00:26 [download] Downloading video 5 of 7 | |
[youtube] aG2iEAs78V0: Downloading webpage | |
[download] 19.3% of 95.98MiB at 3.19MiB/s ETA 00:24 [download] 64.2% of 32.14MiB at 3.51MiB/s ETA 00:03 [youtube] aG2iEAs78V0: Downloading video info webpage | |
Time is: 2020-03-07 20:17:32.465298 | |
[download] 23.1% of 93.69MiB at 3.12MiB/s ETA 00:23 Time is: 2020-03-07 20:17:33.465355 | |
[download] Downloading video 6 of 7 | |
[youtube] qO9J-WwUiEc: Downloading webpage | |
[download] 87.8% of 32.14MiB at 3.58MiB/s ETA 00:01 [youtube] qO9J-WwUiEc: Downloading video info webpage | |
Time is: 2020-03-07 20:17:34.465412 | |
[download] 100% of 32.14MiB in 00:09 | |
Time is: 2020-03-07 20:17:35.456469 | |
[download] Downloading video 7 of 7 | |
[youtube] YvVthtbNtEE: Downloading webpage | |
[download] 36.0% of 95.98MiB at 3.72MiB/s ETA 00:16 [youtube] YvVthtbNtEE: Downloading video info webpage | |
[download] 40.2% of 95.98MiB at 3.90MiB/s ETA 00:14 Time is: 2020-03-07 20:17:36.456526 | |
[download] 44.3% of 95.98MiB at 4.06MiB/s ETA 00:13 [download] Finished downloading playlist: 【誰來晚餐 番外篇】 | |
['https://www.youtube.com/watch?v=sm4BID47dTI', 'https://www.youtube.com/watch?v=r5WApOc0WLI', 'https://www.youtube.com/watch?v=W5oa03IOkzE', 'https://www.youtube.com/watch?v=RxALj5QWsjo', 'https://www.youtube.com/watch?v=aG2iEAs78V0', 'https://www.youtube.com/watch?v=qO9J-WwUiEc', 'https://www.youtube.com/watch?v=YvVthtbNtEE'] | |
[youtube] sm4BID47dTI: Downloading webpage | |
[youtube] r5WApOc0WLI: Downloading webpage | |
[download] 48.8% of 93.69MiB at 4.22MiB/s ETA 00:11 [youtube] W5oa03IOkzE: Downloading webpage | |
[youtube] RxALj5QWsjo: Downloading webpage | |
[youtube] aG2iEAs78V0: Downloading webpage | |
[youtube] qO9J-WwUiEc: Downloading webpage | |
Time is: 2020-03-07 20:17:37.462584 | |
[download] 48.5% of 95.98MiB at 4.19MiB/s ETA 00:11 [youtube] r5WApOc0WLI: Downloading video info webpage | |
[youtube] sm4BID47dTI: Downloading video info webpage | |
[youtube] W5oa03IOkzE: Downloading video info webpage | |
[download] 53.0% of 93.69MiB at 4.32MiB/s ETA 00:10 [youtube] RxALj5QWsjo: Downloading video info webpage | |
[youtube] qO9J-WwUiEc: Downloading video info webpage | |
[youtube] aG2iEAs78V0: Downloading video info webpage | |
[download] 52.7% of 95.98MiB at 4.32MiB/s ETA 00:10 Time is: 2020-03-07 20:17:38.462641 | |
[download] 61.6% of 93.69MiB at 4.50MiB/s ETA 00:08 Time is: 2020-03-07 20:17:39.462698 | |
[download] 65.8% of 93.69MiB at 4.59MiB/s ETA 00:06 [download] Destination: 【 誰來晚餐_番外篇 Ep1 】香港到底怎麼了?民主當飯吃 | |
,吃飽繼續說!-r5WApOc0WLI.mp4 | |
[download] 0.4% of 16.27MiB at 1.10MiB/s ETA 00:14 [download] Destination: 【 誰來晚餐_番外篇 Ep 4 】投你所好,好好投-aG2iEAs7 | |
8V0.mp4 | |
[download] 0.1% of 17.17MiB at 499.97KiB/s ETA 00:35 [download] Destination: 【 誰來晚餐_番外篇 Ep2 】浪浪別哭!毛小孩怎麼找幸福 | |
?-W5oa03IOkzE.mp4 | |
[download] 0.2% of 33.42MiB at 588.75KiB/s ETA 00:58[download] Destination: 【 誰來晚餐_番外篇 Ep3 】買房好還是租房好-RxALj5QWsjo.mp4 | |
[download] 0.7% of 33.42MiB at 885.37KiB/s ETA 00:38[download] Destination: 【 誰來晚餐_番外篇 Ep5 】台北街頭有話要說-qO9J-WwUiEc.mp4 | |
[download] 0.3% of 9.36MiB at 815.74KiB/s ETA 00:11Time is: 2020-03-07 20:17:40.462755 | |
[download] 5.3% of 9.36MiB at 1.80MiB/s ETA 00:04 [download] Destination: 【 誰來晚餐_番外篇 Ep0 】 吃飽繼續說-sm4BID47dTI.mp4 | |
[download] 24.6% of 16.27MiB at 2.72MiB/s ETA 00:04 Time is: 2020-03-07 20:17:41.462812ETA 00:06 | |
[download] 40.8% of 16.27MiB at 2.70MiB/s ETA 00:03 Time is: 2020-03-07 20:17:42.462870 | |
[download] 57.2% of 16.27MiB at 2.72MiB/s ETA 00:02 Time is: 2020-03-07 20:17:43.462927 | |
[download] 93.3% of 9.36MiB at 2.22MiB/s ETA 00:00 Time is: 2020-03-07 20:17:44.462984 | |
[download] 100% of 9.36MiB in 00:05 | |
[youtube] YvVthtbNtEE: Downloading webpage | |
[download] 100% of 6.34MiB in 00:06 | |
[download] 49.0% of 17.17MiB at 1.59MiB/s ETA 00:05 [youtube] YvVthtbNtEE: Downloading video info webpage | |
Time is: 2020-03-07 20:17:45.465041 | |
[download] 100% of 16.27MiB in 00:07 | |
Time is: 2020-03-07 20:17:46.467099 | |
[download] 80.7% of 93.69MiB at 3.61MiB/s ETA 00:05 Time is: 2020-03-07 20:17:47.467156 | |
[download] Destination: 【 誰來晚餐_番外篇 Ep6 】回家過年快樂嗎?-YvVthtbNtEE.mp4 | |
[download] 100% of 17.17MiB in 00:08 | |
[download] 23.1% of 8.66MiB at 3.48MiB/s ETA 00:01 Time is: 2020-03-07 20:17:48.478214 | |
[download] 46.0% of 33.42MiB at 1.68MiB/s ETA 00:10 Time is: 2020-03-07 20:17:49.478271 | |
[download] 100% of 8.66MiB in 00:02 | |
[download] 53.0% of 33.42MiB at 1.77MiB/s ETA 00:08 Time is: 2020-03-07 20:17:50.489329 | |
[download] 93.0% of 95.98MiB at 3.56MiB/s ETA 00:01 Time is: 2020-03-07 20:17:51.489386 | |
[download] 97.1% of 95.98MiB at 3.60MiB/s ETA 00:00 Time is: 2020-03-07 20:17:52.489443 | |
[download] 100% of 34.30MiB in 00:13 | |
[download] 100% of 95.98MiB in 00:26 | |
[download] 90.7% of 33.42MiB at 2.26MiB/s ETA 00:01 Time is: 2020-03-07 20:17:53.489500 | |
[download] 100% of 33.42MiB in 00:14 | |
[download] 100% of 93.69MiB in 00:27 | |
Time is: 2020-03-07 20:17:54.501558 | |
Time is: 2020-03-07 20:17:55.501615 | |
Time is: 2020-03-07 20:17:56.501673 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Great code, can you add progress hooks?