-
-
Save nestukh/4da25f3aa2360d770490ee434bff3dde to your computer and use it in GitHub Desktop.
#!/bin/bash | |
pypcks="python3-pip python3 python3-all-dev python3-dev libffi-dev libssl-dev librtmp-dev python-dev python3 python3-doc python3-tk python3-setuptools tix xvfb python-bluez python-gobject python-dbus python cython python-doc python-tk python-numpy python-scipy python-qt4 python3-pyqt5 python3-pyqt5.q* python3-qtpy python-pyqt5.q* python-lxml fontconfig python-demjson qt5-default libqt5webkit5-dev build-essential libudev-dev python-lxml libxml2-dev libxslt-dev libpq-dev python-pyside python-distlib python-pip python-setuptools" # python-examples python3-examples python-vte | |
allgoodpcks="ca-certificates virtualenv autotools-dev cdbs git expect libnss3-tools util-linux xvfb curl bridge-utils chromium-browser chromium-chromedriver firefox-esr" | |
sudo apt-get install --reinstall -y $pypcks $allgoodpcks | |
if [[ ! -f /usr/lib/chromium-browser/chromedriver ]]; then | |
sudo ln -s /usr/bin/chromedriver /usr/lib/chromium-browser/chromedriver | |
fi | |
sudo pip install --upgrade pip | |
# python 2to3 transitional setup | |
sudo apt-mark hold python-pip | |
sudo rm -frR ~/.cache/pip | |
sudo rm -frR /root/.cache/pip | |
if [[ -f /usr/bin/pip3 ]]; then | |
sudo update-alternatives --install /usr/bin/pip pip "/usr/bin/pip3" 900 | |
else | |
sudo pip3 install --upgrade pip ## prereformat | |
sudo ln -s /usr/local/bin/pip3 /usr/bin/pip3 | |
sudo update-alternatives --install /usr/bin/pip pip "/usr/bin/pip3" 900 | |
fi | |
sudo pip install --upgrade pip | |
python3 -m virtualenv $HOME/personal_python_env --python=$(ls -1 /usr/bin/python3* | grep -P "\d$" | tail -n1) | |
source $HOME/personal_python_env/bin/activate | |
pip install --upgrade pip | |
pip install --no-cache-dir requests lxml beautifulsoup4 ftfy blink1 python-librtmp pyOpenSSL pathlib certifi python-crontab pexpect python-magic pyquery regex xvfbwrapper selenium pyvirtualdisplay python-librtmp xvfbwrapper youtube_dl selenium pyvirtualdisplay | |
export DISPLAY=:1 | |
sudo touch /root/.Xauthority | |
XAUTHORITY='/root/.Xauthority' sudo Xvfb :1 -screen 0 1280x960x16 & | |
sudo apt-get install --reinstall -y python3-xlib python-xlib scrot python3-tk python3-dev python-tk python-dev | |
pip install --no-cache-dir Xlib | |
pip install --no-cache-dir Pillow RPi.GPIO spidev pyautogui # pillow | |
sudo kill "$(ps -auxf | grep -P "sudo Xvfb .1 -screen 0 1280x960x16" | grep -v grep | sed "s/\\\\_.*$//g;s/[ \t]*$//g;s/^root[ \t]*//g;s/ .*$//g")" | |
unset DISPLAY | |
ln -s "/usr/lib/python3/dist-packages/PyQt5" "$HOME/personal_python_env/lib/python3."*"/site-packages/" | |
#ln -s "/usr/lib/python2.7/dist-packages/PyQt5" "$HOME/personal_python_env/lib/python2.7/site-packages/" | |
for FILE in "/usr/lib/python3/dist-packages/sip"*; do ln -s "$FILE" "$HOME/personal_python_env/lib/python3."*"/site-packages/"; done | |
deactivate | |
## https://firefox-source-docs.mozilla.org/testing/geckodriver/Support.html | |
## https://firefox-source-docs.mozilla.org/testing/geckodriver/ARM.html | |
sudo apt install -y --reinstall gcc-arm-linux-gnueabihf libc6-armhf-cross libc6-dev-armhf-cross | |
######################################## | |
######################################## | |
########## START OF GECKODRIVER REINSTALL | |
# To disable user crontab | |
if [[ "$(crontab -l| cut -c1-3 | head -n1)" != "#@@" ]]; then | |
crontab -l | awk '{print "#@@ "$0}' | crontab | |
fi | |
if [[ "$(cat "$HOME/.gitconfig" | grep -P "helper.*=.*store")" == "" ]]; then | |
echo -e "\n[credential]\n helper = store\n" | tee -a "$HOME/.gitconfig" > /dev/null | |
fi | |
if [[ -f "$HOME/.cargo/env" ]]; then | |
source $HOME/.cargo/env | |
expect -c "spawn $HOME/.cargo/bin/rustup self uninstall; expect -re \"^.*Continue. .y.N..*\"; send -- \"y\r\"; expect eof; exit" # rustup self uninstall | |
fi | |
rm -fRr gecko-dev ############################## | |
rm -fRr "$HOME/.cargo" ##################### thanks to rust bugs it's easier to level everything and install env again... | |
rm -fRr "$HOME/.rustup" ######################## | |
git clone --depth=1 https://github.com/mozilla/gecko-dev | |
if [[ ! -d "$HOME/.cargo/bin" ]]; then | |
curl https://sh.rustup.rs -sSf | bash -s -- -v -y | |
fi | |
declare -a bindirs=("$HOME/.cargo/bin") | |
if [[ "$(cat $HOME/.bashrc | grep -P "^PATH=" )" == "" ]]; then | |
echo -e "PATH=\$PATH\n" >> $HOME/.bashrc | |
fi | |
for binfolder in "${bindirs[@]}" | |
do | |
if [[ "$(cat $HOME/.bashrc | grep -P "^PATH=" | grep "${binfolder}:")" == "" ]]; then | |
sed -i -e "s/^PATH=\"\(.*\)\"/PATH=\"$(echo "$binfolder" | sed "s/\//\\\\\//g"):\1\"/g" $HOME/.bashrc # place with no spaces in it | |
export PATH="$PATH:$binfolder" | |
fi | |
done | |
source $HOME/.cargo/env | |
source $HOME/.bashrc | |
rustup target install armv7-unknown-linux-gnueabihf | |
rustup update # https://github.com/rust-lang/rustup | |
echo -e "[target.armv7-unknown-linux-gnueabihf] | |
linker = \"arm-linux-gnueabihf-gcc\"" > "$HOME/gecko-dev/testing/geckodriver/.cargo/config" | |
cd "$HOME/gecko-dev/testing/geckodriver" | |
exitstatus=99 | |
while [[ "$exitstatus" != "0" ]]; do | |
cargo clean | |
CARGO_NET_GIT_FETCH_WITH_CLI=true cargo build --verbose --release --target armv7-unknown-linux-gnueabihf; exitstatus=$? | |
sleep 2 | |
done | |
cd | |
#To enable user crontab | |
if [[ "$(crontab -l| cut -c1-3 | head -n1)" == "#@@" ]]; then | |
crontab -l | cut -c 5- | crontab | |
fi | |
### | |
# test binary file with this command | |
$HOME/gecko-dev/target/armv7-unknown-linux-gnueabihf/release/geckodriver --version | |
########## END OF GECKODRIVER REINSTALL | |
######################################## | |
######################################## | |
exit 0 | |
###### open python with | |
source $HOME/personal_python_env/bin/activate | |
xvfb-run -a python -u -B ## xvfb for headless firefox-esr | |
#################################################### | |
########################## | |
#!/usr/bin/python | |
# -*- coding: utf-8 -*- | |
import os | |
import sys | |
import io | |
import getpass | |
import six | |
import requests | |
import ssl | |
import certifi | |
import ftfy | |
import locale | |
import datetime | |
import platform | |
from selenium import webdriver | |
from selenium.webdriver.common.action_chains import ActionChains | |
from selenium.webdriver.common.by import By | |
from selenium.webdriver.firefox.options import Options | |
from selenium.webdriver.common.keys import Keys | |
from selenium.webdriver.support.ui import Select | |
from selenium.webdriver.support.ui import WebDriverWait | |
from selenium.webdriver.support import expected_conditions as EC | |
from selenium.common.exceptions import TimeoutException | |
from selenium.webdriver.firefox.firefox_binary import FirefoxBinary | |
from pyvirtualdisplay import Display | |
// start browser.. | |
print("waiting Selenium + Firefox to load..") | |
display = Display(visible=0, size=(800, 600)) | |
display.start() | |
mime_types = "application/gpx+tcx,application/octet-stream,application/x-pdf,application/acrobat,applications/vnd.pdf,application/pdf,text/pdf,text/x-pdf,application/vnd.cups-pdf,application/vnd.adobe.xfdf,application/vnd.fdf,application/vnd.adobe.xdp+xml" | |
fp = webdriver.FirefoxProfile() | |
geckopath='/home/'+getpass.getuser()+'/gecko-dev/target/armv7-unknown-linux-gnueabihf/release/geckodriver' | |
tempdownloaddir='/home/'+getpass.getuser()+'/download' | |
fp.set_preference("webdriver.gecko.driver", geckopath) | |
fp.set_preference("browser.cache.disk.enable", False) | |
#fp.set_preference("browser.cache.memory.enable", False) | |
fp.set_preference("browser.cache.offline.enable", False) | |
fp.set_preference("network.http.use-cache", False) | |
fp.set_preference("plugin.scan.Acrobat", "99.0") # | |
fp.set_preference("plugin.scan.plid.all", False) # | |
fp.set_preference("browser.helperApps.alwaysAsk.force", False) # | |
fp.set_preference("browser.download.folderList", 2) | |
fp.set_preference("browser.download.manager.showWhenStarting", False) | |
fp.set_preference("browser.download.dir", tempdownloaddir) | |
fp.set_preference("browser.helperApps.neverAsk.saveToDisk", mime_types) | |
fp.set_preference("browser.download.manager.alertOnEXEOpen",False) | |
fp.set_preference("plugin.disable_full_page_plugin_for_types", mime_types) | |
fp.set_preference("pdfjs.disabled", True) | |
fp.set_preference('browser.download.manager.showAlertOnComplete', False) | |
fp.set_preference('browser.download.useDownloadDir', True) | |
fp.set_preference('browser.download.defaultFolder', tempdownloaddir) | |
fp.set_preference('browser.download.lastDir', tempdownloaddir) | |
#fp.set_preference('plugin.state.java', 2) | |
fp.accept_untrusted_certs = True | |
#fp.set_preference("security.default_personal_cert", "Select Automatically") | |
fp.set_preference("datareporting.healthreport.uploadEnabled", False) | |
#fp.set_preference("general.useragent.override", "Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0") | |
fp.set_preference("general.useragent.override", "Mozilla/5.0 (iPad; CPU OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53") | |
fp.set_preference("javascript.enabled", True) | |
fp.set_preference("configplugins.click_to_play", True) | |
fp.set_preference("http.response.timeout", 9999) | |
fp.set_preference("dom.max_script_run_time", 9999) | |
fp.set_preference("permissions.default.image", 2) ## Disable images | |
# fp.set_preference('permissions.default.stylesheet', 2) ## Disable CSS | |
#fp.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',flashplugin) | |
fp.set_preference("media.volume_scale", "0.0") | |
fp.set_preference("app.update.auto", False); | |
fp.set_preference("app.update.lastUpdateTime.addon-background-update-timer", 1280826385); | |
fp.set_preference("app.update.lastUpdateTime.background-update-timer", 1280826385); | |
fp.set_preference("app.update.lastUpdateTime.blocklist-background-update-timer", 1280826385); | |
fp.set_preference("app.update.lastUpdateTime.microsummary-generator-update-timer", 1280502030); | |
fp.set_preference("app.update.lastUpdateTime.places-maintenance-timer", 1280826385); | |
fp.set_preference("app.update.lastUpdateTime.search-engine-update-timer", 1280826385); | |
fp.set_preference("browser.EULA.3.accepted", True); | |
fp.set_preference("browser.EULA.override", True); | |
fp.set_preference("browser.allowpopups", False); | |
fp.set_preference("browser.bookmarks.restore_default_bookmarks", False); | |
fp.set_preference("browser.history_expire_days.mirror", 180); | |
#fp.set_preference("browser.link.open_external", 2); | |
#fp.set_preference("browser.migration.version", 1); | |
#fp.set_preference("browser.places.smartBookmarksVersion", 1); | |
#fp.set_preference("browser.preferences.advanced.selectedTabIndex", 2); | |
#fp.set_preference("browser.privatebrowsing.autostart", True); | |
#fp.set_preference("browser.rights.3.shown", True); | |
#fp.set_preference("browser.safebrowsing.enabled", False); | |
fp.set_preference("browser.search.update", False); | |
#fp.set_preference("browser.sessionstore.resume_session_once", True); | |
fp.set_preference("browser.startup.homepage", "about:blank"); | |
fp.set_preference("browser.startup.homepage_override.mstone", "rv:1.9.1.11"); | |
fp.set_preference("browser.startup.page", 0); | |
fp.set_preference("browser.tabs.warnOnClose", False); | |
fp.set_preference("browser.tabs.warnOnOpen", False); | |
fp.set_preference("browser.urlbar.autocomplete.enabled", False); | |
fp.set_preference("dom.disable_open_during_load", False); | |
fp.set_preference("dom.max_chrome_script_run_time", 1800); | |
fp.set_preference("dom.max_script_run_time", 1800); | |
#fp.set_preference("extensions.lastAppVersion", "3.5.11"); | |
fp.set_preference("extensions.update.enabled", False); | |
fp.set_preference("extensions.update.notifyUser", False); | |
fp.set_preference("idle.lastDailyNotification", 1280826384); | |
fp.set_preference("intl.charsetmenu.browser.cache", "UTF-8, ISO-8859-1"); | |
fp.set_preference("network.cookie.prefsMigrated", True); | |
fp.set_preference("network.dns.disableIPv6", True); | |
fp.set_preference("network.http.phishy-userpass-length", 255); | |
#fp.set_preference("pref.browser.homepage.disable_button.bookmark_page", False); | |
#fp.set_preference("pref.browser.homepage.disable_button.current_page", False); | |
#fp.set_preference("pref.browser.homepage.disable_button.restore_default", False); | |
#fp.set_preference("privacy.sanitize.migrateFx3Prefs", True); | |
#fp.set_preference("privacy.sanitize.timeSpan", 0); | |
fp.set_preference("security.warn_entering_weak", False); | |
fp.set_preference("security.warn_entering_weak.show_once", False); | |
fp.set_preference("security.warn_viewing_mixed", False); | |
fp.set_preference("security.warn_viewing_mixed.show_once", False); | |
fp.set_preference("signon.rememberSignons", False); | |
#fp.set_preference("spellchecker.dictionary", "en_US"); | |
fp.set_preference("startup.homepage_welcome_url", ""); | |
#fp.set_preference("urlclassifier.keyupdatetime.https://sb-ssl.google.com/safebrowsing/newkey", 1287053252); | |
#fp.set_preference("xpinstall.whitelist.required", False); | |
capabilities = webdriver.DesiredCapabilities().FIREFOX | |
capabilities["marionette"] = True | |
ffoptions = Options() | |
#options.add_argument("--headless") | |
#options.add_argument("start-maximized") | |
#options.add_argument("--disable-gpu") | |
#options.add_argument("--disable-extensions") | |
ffbinaryx='/usr/bin/firefox-esr' | |
print('using Firefox in path '+ffbinaryx) | |
driver = webdriver.Firefox(firefox_binary=FirefoxBinary(ffbinaryx),firefox_profile=fp,options=ffoptions,executable_path=geckopath,service_log_path=os.devnull,timeout=720,capabilities=capabilities) | |
firefoxpid = driver.service.process.pid | |
print("Firefox PID = "+str(firefoxpid)+"\n") | |
driver.maximize_window() | |
// do stuff | |
driver.get('https://google.com') # goes to google.com | |
datetime.datetime.today().strftime('%Y-%m-%d %H:%M:%S');driver.save_screenshot('/home/'+getpass.getuser()+datetime.datetime.today().strftime('%Y-%m-%d %H:%M:%S')+'.png') # screenshot | |
soup = BeautifulSoup(driver.page_source.encode('utf-8'), 'html.parser') # pass stuff to BeautifulSoup | |
driver.find_element_by_xpath("//div[contains(@class,'flux capacitor')]/.//li/span[contains(.,"+os.RTLD_LAZY+")]").find_element_by_xpath("..").click() # clicks somewhere | |
dir(driver) # lists commands and values | |
// quit everything | |
driver.quit() | |
displaypid=display.pid | |
display.stop() | |
// for good measure (it depends on the firefox version) | |
os.kill(firefoxpid, signal.SIGKILL) | |
os.kill(displaypid, signal.SIGKILL) | |
exit() | |
########################## | |
#################################################### | |
#################### | |
########## | |
// start browser.. | |
print("waiting Selenium + Chromium to load..") | |
options = webdriver.ChromeOptions() | |
#options.binary_location='/usr/bin/chromium-browser' | |
#options.add_argument("--incognito") | |
options.add_argument("user-data-dir="+'/home/'+getpass.getuser()+'/chromiumprofiledir') | |
options.add_argument("--start-maximized") | |
options.add_argument("--disable-notifications") | |
options.add_argument("--headless") | |
options.add_argument("--window-size=1920,1080") | |
#options.add_argument("download.default_directory=/home/"+getpass.getuser()+"/chromiumdownloaddir") | |
prefs = { | |
"download.default_directory" : "/home/"+getpass.getuser()+"/chromiumdownloaddir", | |
"download.directory_upgrade": True, | |
"download.prom pt_for_download": False, | |
"disable-popup-blocking": True, | |
"safebrowsing.enabled": False, | |
"safebrowsing.disable_download_protection": True, | |
} | |
options.add_experimental_option("prefs", prefs) | |
chromedriverpath="/usr/lib/chromium-browser/chromedriver" | |
driver = webdriver.Chrome(executable_path=chromedriverpath, chrome_options=options) | |
driver.command_executor._commands["send_command"] = ("POST", '/session/$sessionId/chromium/send_command') | |
params = {'cmd': 'Page.setDownloadBehavior', 'params': {'behavior': 'allow', 'downloadPath': chromiumdownloaddir}} | |
driver.execute("send_command", params) | |
// do stuff... | |
########## | |
#################### | |
### close python env with | |
deactivate |
This is amazing thanks geckodriver worked perfectly.
Should i be creating the folder personal_python_env and what is the use for this?
The python3 -m virtualenv $HOME/personal_python_env [...]
command creates the folder for you. In this folder a Python Virtual Enviroment will be located, specifically for automating firefox interactions.
A VirtualENV for Python is like a small containter for a subset of locally installed python packages. In this way:
- you can install the same packages across various system using similar commands (gnu/linux, windows, macos)
- you can expand the list of globally installed python packages, which is in
/usr/local/lib/python*
- there are less conficts (e.g. some python package Y requires a very specific version of another package X, while a different package Z absolutely requires a newer version of X, this is a classic situation).
You can install python package globally viasudo apt-get
(old stable releases) or viasudo pip
(always newer stable releases). But if you hit an unfixable conflict upgrading global python packages, you're in bad luck. A virtualenv in a personal folder can be fixed with ease, and you only use thepip
mechanism. Two cooks in the kitchen are too many for baking the same cake. - you can update python packages to the latest stable version, without superuser privileges (e.g. useful for youtube-dl, as websites change their internal code very frequently).
Thanks very helpful
new firefox-esr 78.7
does not work with selenium at current date,
but firefox-esr 78.6
does.
here the version of firefox-esr that works (it's a missing debian package in the raspbian archives):
firefox-esr_78.6.1esr-1~deb10u1+rpi1_armhf.deb (rename it after downloading)
if you have it on your raspberry, it's in /var/cache/apt/archives/
.
After installing it with sudo dpkg -i firefox-esr*.deb
, you have to disable upgrading with sudo apt-mark hold firefox-esr
@nestukh
If this works I'ma lose my s^^^.... awesome regardless. People like you doing the Lord's work. I'll be back...
Hi there, thank you for your work. Is this somehow specific to 32bit versions of raspberry pi OS? If so, is there a tweak I can make to the script to make it work for the 64bit version? Again thanks for the great effort
no it is not specific to 32bit raspberry pi os, but the armhf.deb above is, it's essential for working with selenium3. But they have updated selelium to version 4 in the meantime, and you don't need that anymore.
On raspi 3b+ with 64bit os you can use only a couple of v91+ firefox-esr instances, otherwise it hangs/reboot, I've tried.
I'll update this after getting a raspberry pi 5 (with risk-v architecture). It will be out this year, pi day (3.14) has passed, so I believe you could have one in your hands by 2*pi day (6.28) like it was with raspi 4. I'll wait for the typical second hardware revision in November.
And what about using the latest release of firefox-esr? It seems to me that the script downloads and builds the latest version of geckodriver, which should work with the latest version of firefox, right?
yes with the latest release of firefox-esr, but it's the combination of firefox-esr + geckodriver + selenium that makes the magic work. Anyway lastest FF is more cpu hungry and uses multiple cpus, even containing within a single core with cpu affinity and other tricks, it hangs, using 100% of it on a raspi3b.
True, however, I have noticed that the script does not download firefox-esr; notice that the last element of allgoodpcks
is firefox-esrt
with an extra 't'.
fixed. my bad
updated geckodriver install instructions for anyone who needs them