Last active
June 17, 2020 17:21
-
-
Save ruebot/b4d565c2a0cdc5e72678366f6d616388 to your computer and use it in GitHub Desktop.
AUT Spark 3.0.0 Testing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| /home/nruest/bin/spark-3.0.0-bin-hadoop2.7/bin/spark-submit --master local\[2\] --driver-memory 4g --conf spark.driver.maxResultSize=0 --class io.archivesunleashed.app.CommandLineAppRunner /home/nruest/Projects/au/aut/target/aut-0.80.1-SNAPSHOT-fatjar.jar --extractor AudioInformationExtractor --input /home/nruest/Projects/au/sample-data/geocities --output /home/nruest/Projects/au/sample-data/3.0.0-testing/audio/csv | |
| /home/nruest/bin/spark-3.0.0-bin-hadoop2.7/bin/spark-submit --master local\[2\] --driver-memory 4g --conf spark.driver.maxResultSize=0 --class io.archivesunleashed.app.CommandLineAppRunner /home/nruest/Projects/au/aut/target/aut-0.80.1-SNAPSHOT-fatjar.jar --extractor DomainFrequencyExtractor --input /home/nruest/Projects/au/sample-data/geocities --output /home/nruest/Projects/au/sample-data/3.0.0-testing/domains/csv | |
| /home/nruest/bin/spark-3.0.0-bin-hadoop2.7/bin/spark-submit --master local\[2\] --driver-memory 4g --conf spark.driver.maxResultSize=0 --class io.archivesunleashed.app.CommandLineAppRunner /home/nruest/Projects/au/aut/target/aut-0.80.1-SNAPSHOT-fatjar.jar --extractor DomainGraphExtractor --input /home/nruest/Projects/au/sample-data/geocities --output /home/nruest/Projects/au/sample-data/3.0.0-testing/domaingraph/csv | |
| /home/nruest/bin/spark-3.0.0-bin-hadoop2.7/bin/spark-submit --master local\[2\] --driver-memory 4g --conf spark.driver.maxResultSize=0 --class io.archivesunleashed.app.CommandLineAppRunner /home/nruest/Projects/au/aut/target/aut-0.80.1-SNAPSHOT-fatjar.jar --extractor ImageGraph --input /home/nruest/Projects/au/sample-data/geocities --output /home/nruest/Projects/au/sample-data/3.0.0-testing/ImageGraph/csv | |
| /home/nruest/bin/spark-3.0.0-bin-hadoop2.7/bin/spark-submit --master local\[2\] --driver-memory 4g --conf spark.driver.maxResultSize=0 --class io.archivesunleashed.app.CommandLineAppRunner /home/nruest/Projects/au/aut/target/aut-0.80.1-SNAPSHOT-fatjar.jar --extractor ImageInformationExtractor --input /home/nruest/Projects/au/sample-data/geocities --output /home/nruest/Projects/au/sample-data/3.0.0-testing/ImageInformationExtractor/csv | |
| /home/nruest/bin/spark-3.0.0-bin-hadoop2.7/bin/spark-submit --master local\[2\] --driver-memory 4g --conf spark.driver.maxResultSize=0 --class io.archivesunleashed.app.CommandLineAppRunner /home/nruest/Projects/au/aut/target/aut-0.80.1-SNAPSHOT-fatjar.jar --extractor PDFInformationExtractor --input /home/nruest/Projects/au/sample-data/geocities --output /home/nruest/Projects/au/sample-data/3.0.0-testing/PDFInformationExtractor/csv | |
| /home/nruest/bin/spark-3.0.0-bin-hadoop2.7/bin/spark-submit --master local\[2\] --driver-memory 4g --conf spark.driver.maxResultSize=0 --class io.archivesunleashed.app.CommandLineAppRunner /home/nruest/Projects/au/aut/target/aut-0.80.1-SNAPSHOT-fatjar.jar --extractor PlainTextExtractor --input /home/nruest/Projects/au/sample-data/geocities --output /home/nruest/Projects/au/sample-data/3.0.0-testing/PlainTextExtractor/csv | |
| /home/nruest/bin/spark-3.0.0-bin-hadoop2.7/bin/spark-submit --master local\[2\] --driver-memory 4g --conf spark.driver.maxResultSize=0 --class io.archivesunleashed.app.CommandLineAppRunner /home/nruest/Projects/au/aut/target/aut-0.80.1-SNAPSHOT-fatjar.jar --extractor PresentationProgramInformationExtractor --input /home/nruest/Projects/au/sample-data/geocities --output /home/nruest/Projects/au/sample-data/3.0.0-testing/PresentationProgramInformationExtractor/csv | |
| /home/nruest/bin/spark-3.0.0-bin-hadoop2.7/bin/spark-submit --master local\[2\] --driver-memory 4g --conf spark.driver.maxResultSize=0 --class io.archivesunleashed.app.CommandLineAppRunner /home/nruest/Projects/au/aut/target/aut-0.80.1-SNAPSHOT-fatjar.jar --extractor SpreadsheetInformationExtractor --input /home/nruest/Projects/au/sample-data/geocities --output /home/nruest/Projects/au/sample-data/3.0.0-testing/SpreadsheetInformationExtractor/csv | |
| /home/nruest/bin/spark-3.0.0-bin-hadoop2.7/bin/spark-submit --master local\[2\] --driver-memory 4g --conf spark.driver.maxResultSize=0 --class io.archivesunleashed.app.CommandLineAppRunner /home/nruest/Projects/au/aut/target/aut-0.80.1-SNAPSHOT-fatjar.jar --extractor VideoInformationExtractor --input /home/nruest/Projects/au/sample-data/geocities --output /home/nruest/Projects/au/sample-data/3.0.0-testing/VideoInformationExtractor/csv | |
| /home/nruest/bin/spark-3.0.0-bin-hadoop2.7/bin/spark-submit --master local\[2\] --driver-memory 4g --conf spark.driver.maxResultSize=0 --class io.archivesunleashed.app.CommandLineAppRunner /home/nruest/Projects/au/aut/target/aut-0.80.1-SNAPSHOT-fatjar.jar --extractor WebGraphExtractor --input /home/nruest/Projects/au/sample-data/geocities --output /home/nruest/Projects/au/sample-data/3.0.0-testing/WebGraphExtractor/csv | |
| /home/nruest/bin/spark-3.0.0-bin-hadoop2.7/bin/spark-submit --master local\[2\] --driver-memory 4g --conf spark.driver.maxResultSize=0 --class io.archivesunleashed.app.CommandLineAppRunner /home/nruest/Projects/au/aut/target/aut-0.80.1-SNAPSHOT-fatjar.jar --extractor WebPagesExtractor --input /home/nruest/Projects/au/sample-data/geocities --output /home/nruest/Projects/au/sample-data/3.0.0-testing/WebPagesExtractor/csv | |
| /home/nruest/bin/spark-3.0.0-bin-hadoop2.7/bin/spark-submit --master local\[2\] --driver-memory 4g --conf spark.driver.maxResultSize=0 --class io.archivesunleashed.app.CommandLineAppRunner /home/nruest/Projects/au/aut/target/aut-0.80.1-SNAPSHOT-fatjar.jar --extractor WordProcessorInformationExtractor --input /home/nruest/Projects/au/sample-data/geocities --output /home/nruest/Projects/au/sample-data/3.0.0-testing/WordProcessorInformationExtractor/csv |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| export PYSPARK_DRIVER_PYTHON=jupyter; export PYSPARK_DRIVER_PYTHON_OPTS=notebook; ~/bin/spark-3.0.0-bin-hadoop2.7/bin/pyspark --py-files /home/nruest/Projects/au/aut/target/aut.zip --jars /home/nruest/Projects/au/aut/target/aut-0.80.1-SNAPSHOT-fatjar.jar --driver-memory 6g |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment