Perl script to localise OpenTripPlanner's provided html clients so that they operate without network access by detecting and fetching all external javascript/css dependencies. This works OK for the GraphQL explorer (e.g. http://localhost:8080/graphiql) but the query interface (e.g. http://localhost:8080) still needs the network to fetch OpenStre…

otp-devel-localise-index.pl

#!/usr/bin/env perl

##############################################################################
#
# By: Andreas Hadjiprocopis (c) 2024 [ andreashad2 at gmail dot com ]
# Date: 13-Sep-2024
# Free to modify and/or distribute with attribution
#
# Script to localise OpenTripPlanner's client user-interface
# so that it can operate without network access.
# It works OK for the GraphQL explorer (http://localhost:8080/graphiql)
# but the query client (http://localhost:8080) still needs to
# fetch OpenStreetMap tiles, unfortunately this can not be fixed at the moment.
#
##############################################################################

use strict;
use warnings;

use LWP::UserAgent;
use HTML5::DOM;
use HTTP::Cookies;
use URI;
use FindBin;
use File::Copy;
use File::Util;
use Getopt::Long;

use Data::Roundtrip qw/perl2dump no-unicode-escape-permanently/;

# if FORCE is 0 then it does not fetch JS,CSS files if already exist on the local path specified
my $FORCE = 0; # << default

# the dir relative to the index file to save the JS,CSS files that may be fetched locally
my $DEPSDIR = 'deps'; # << default

# if RESTORE is 1 then it will restore all index.html content back to original
# (from backups we made earlier) and exit.
# note: the fetched js/css files will remain in local disk
my $RESTORE = 0; # << default

# the dir to the distribution of OpenTripPlanner
# (e.g. where it was git clone'd from the repository)
my $DISTDIR; # << no default, this is required

if( ! Getopt::Long::GetOptions(
	'force' => sub { $FORCE = 1 },
	'depsdir=s' => \$DEPSDIR,
	'restore' => sub { $RESTORE = 1 },
	'distdir=s' => \$DISTDIR,
	'help|?|h' => sub { print STDOUT usage($0); exit(0) },
) ){ print STDERR usage($0) . "\nSomething wrong with command-line parameters...\n"; exit(1); }

if( ! defined $DISTDIR ){ die usage($0)."\n\nerror, you need to specify the distribution's dir (e.g. where it was git clone'd from the repository). Use --distdir <adir> to specify it." }

my $updir = File::Spec->catdir($FindBin::Bin, '..', '3rdparty');
my @INDEXFILES = map { 
	[
	  File::Spec->catfile($DISTDIR, 'src', 'client'),
	  $_
	]
     }	(
		'index.html', # this means: <DISTDIR>/src/client/index.html
		File::Spec->catfile('graphiql', 'index.html') # for <DISTDIR>/src/client/graphiql/index.html
		# add here more paths ...
	)
;

#####################################
# nothing to change below
#####################################

$DEPSDIR =~ s#${File::Util::SL}##g;

my $lwp = LWP::UserAgent->new;
$lwp->cookie_jar(HTTP::Cookies->new());

for my $anitem (@INDEXFILES){
	my ($basedir, $indexrest) = @$anitem;
	if( ! defined $indexrest ){ print perl2dump($anitem)."$0, line ".__LINE__." : error, above item from INDEXFILES does not contain required elements, something seriously wrong.\n"; exit(1) }

	my ($volume, $prepend, $index_filename, $index_filedir, $tmp);
	($volume, $prepend, $index_filename) = File::Spec->splitpath($indexrest);

	my $index_filepath = File::Spec->catfile($basedir, $indexrest);
	($volume, $index_filedir, $tmp) = File::Spec->splitpath($index_filepath);
	my $index_filepath_original = $index_filepath.'.original';

	if( $RESTORE == 1 ){
		print "$0 : resetting the contents of '$index_filepath' into its original state (from '$index_filepath_original') ...\n";
		copy($index_filepath_original, $index_filepath);
		print "$0 : done, nothing else to do.\n";
		next
	}

	# if index.html.original does not exist then backup
	# the index.html into this
	# if it does then copy the original into index.html
	# so as not to accumulate changes
	if( -f $index_filepath_original ){
		print "$0 : resetting the contents of '$index_filepath' into its original state (from '$index_filepath_original') ...\n";
		copy($index_filepath_original, $index_filepath);
	} else {
		print "$0 : backing up '$index_filepath_original' into '$index_filepath' ...\n";
		copy($index_filepath, $index_filepath_original);
	}

	print "$0 : processing index file '$index_filepath' ...\n";

	my ($FH, $content);
	if( ! open($FH, '<:encoding(UTF-8)', $index_filepath) ){ print "$0, line ".__LINE__." : file '$index_filepath' : failed to open file '$index_filepath' for reading, $!\n"; exit(1); }
	{ local $/ = undef; $content = <$FH> } close $FH;
	my $depsdir = File::Spec->catdir($index_filedir, $DEPSDIR);
	if( ! -d $depsdir ){ mkdir $depsdir; if( ! -d $depsdir ){ print "$0, line ".__LINE__." : file '$index_filepath' : failed to make deps dir '$depsdir' or it exists but it is not a dir.\n"; exit(1); } }
	my $parser = HTML5::DOM->new;
	my $tree = $parser->parse($content);

	# handle the script tags
	$tree->find('script')->each(sub {
		my ($node, $index) = @_;
		my $src = $node->attr('src');
		if( ! defined $src ){
			#print "begin node:\n".$node."\nend node.\n$0 : file '$index_filepath' : warning, src is undefined for above node, perhaps an inline script, skipping it ...\n";
			return
		}
		return unless $src =~ m#^(?://|https?://)#i;
		my $url = $src;
		if( $url !~ /^http/i ){ $url = 'https:'.$url }
		my $uriobj = URI->new($url);
		if( ! defined $uriobj ){ print "$0, line ".__LINE__." : file '$index_filepath' : url '$url' does not validate.\n"; exit(1); }
		$url = $uriobj->as_string;
		my $filepath = $uriobj->path;
		my ($volume,$directories,$localfilename) = File::Spec->splitpath( $filepath );
		my $localfilepath = File::Spec->catfile($depsdir, $localfilename);
		my $srcjsname = File::Spec->catfile('/', $prepend, $DEPSDIR, $localfilename);
		if( ! fetch_url_and_save($url, $localfilepath, $FORCE) ){ print "$0, line ".__LINE__." : file '$index_filepath' : error, call to fetch_url_and_save() has failed for url '$url' and local file '$localfilepath'.\n"; exit(1); }
		if( $filepath=~ /\.min\.js$/ ){
			# fetch the .map of this shit
			my $localfilename_map = $localfilename.'.map';
			my $localfilepath_map = $localfilepath.'.map';
			my $url_map = $url.'.map';
			print "$0, line ".__LINE__." : file '$index_filepath' : fetching also the map file ($localfilename_map) for this JS file ($filepath) using url '$url_map' ...\n";
			if( ! fetch_url_and_save($url_map, $localfilepath_map, $FORCE) ){ print "$0, line ".__LINE__." : file '$index_filepath' : error, call to fetch_url_and_save() has failed for url '$url' and local file '$localfilepath'.\n"; exit(1); }
		}

		my $newnode = $tree->createElement('script');
		$newnode->attr('src', $srcjsname);
		$node->replace($newnode);
	});

	# handle the link tags (css) e.g. <link rel="stylesheet" href="js/lib/jquery-ui/css/smoothness/jquery-ui-1.9.1.custom.css">
	$tree->find('link')->each(sub {
		my ($node, $index) = @_;
		my $rel = $node->attr('rel');
		if( ! defined($rel)
		 || (defined($rel) && ($rel ne 'stylesheet')) ){ print "$0, line ".__LINE__." : file '$index_filepath' : 'link' tag does not have a 'rel' attribute or it is not a 'stylesheet', skipping it ...\n"; return }
		my $src = $node->attr('href');
		if( ! defined $src ){
			#print "begin node:\n".$node."\nend node.\n$0 : file '$index_filepath' : warning, src is undefined for above node, perhaps an inline script, skipping it ...\n";
			return
		}
		return unless $src =~ m#^(?://|https?://)#i;
		my $url = $src;
		if( $url !~ /^http/i ){ $url = 'https:'.$url }
		my $uriobj = URI->new($url);
		if( ! defined $uriobj ){ print "$0, line ".__LINE__." : file '$index_filepath' : url '$url' does not validate.\n"; exit(1); }
		$url = $uriobj->as_string;
		my $filepath = $uriobj->path;
		my ($volume,$directories,$localfilename) = File::Spec->splitpath( $filepath );
		my $localfilepath = File::Spec->catfile($depsdir, $localfilename);
		my $srcjsname = File::Spec->catfile('/', $prepend, $DEPSDIR, $localfilename);
		if( ! fetch_url_and_save($url, $localfilepath, $FORCE) ){ print "$0, line ".__LINE__." : file '$index_filepath' : error, call to fetch_url_and_save() has failed for url '$url' and local file '$localfilepath'.\n"; exit(1); }

		my $newnode = $node->clone(1); # deep clone (1)
		$newnode->attr('href', $srcjsname);
		$node->replace($newnode);
	});

	# and save the html, replace the index.html
	# the original has been backed up to index.html.original
	if( ! open($FH, '>:encoding(UTF-8)', $index_filepath) ){ print "$0, line ".__LINE__." : file '$index_filepath' : failed to open index file '$index_filepath' for writing, $!\n"; exit(1); }
	print $FH $tree->html."\n";
	close $FH;
	print "$0 : done, wrote new index file '$index_filepath'.\n";

}

print "$0 : done.\n";

sub fetch_url_and_save {
	my ($url, $localfilepath, $force) = @_;
	if( (defined($force) && $force)
	 || (! -f $localfilepath)
	){
		print "$0 , line ".__LINE__." : fetching '$url' ...\n";
		my $response = $lwp->get($url);
		if( ! defined $response ){ print "$0, line ".__LINE__." : failed to fetch '$url', got undef.\n"; return 0; }
		if( ! $response->is_success ){ print "$0, line ".__LINE__." : failed to fetch '$url', got this error: ".$response->status_line."\n"; return 0 }
		my $FH2;
		if( ! open($FH2, '>:encoding(UTF-8)', $localfilepath) ){ print "$0, line ".__LINE__." : failed to open file '$localfilepath' for writing, $!\n"; return 0; }
		print $FH2 $response->decoded_content;
		close $FH2;
		print "$0 , line ".__LINE__." : fetched and saved '$url'.\n";
	}
	return 1; # success
}

sub usage {
	my ($appname) = @_;
	return "Usage : $appname <options>\n\nwhere options are:\n\n"
	. " --distdir DISTDIR : specify the distributions base dir. E.g. the location where it was git clone'd.\n"
	. "[--force] : fetch all files again even if they already exist locally in their designated location. However, the 'index.html' files will be modified none-the-less. The current default is ".($FORCE==0?'not':'')." to force re-fetching the dependencies.\n"
	. "[--depsdir DEPSDIR] : specify the name of the 'dependencies' dir. This is a directory relative to the input html file's which is created and where the JS/CSS dependencies, downloaded via this script, are saved into. The current default is '$DEPSDIR'.\n"
	. "[--restore] : restore all mentioned 'index.html' files back to their original content and exit. All the fetched dependencies and saved under the '$DEPSDIR' dirs will remain there and you need to erase them manually else they will be bundled into the packaged jar files when you recompile.\n"
	. "\n"
	. "This script will localise OpenTripPlanner so that external dependencies (e.g. javascript and css files) are read from local copies. It reads 'index.html' files, e.g. in src/client/index.html and src/client/graphiql/index.html), finds their external dependencies by searching <script> and <link> tags, fetches those dependencies, saves them locally within the OpenTripPlanner distribution dir under '<DISTDIR>/src/client/<DEPSDIR>' and modifies said index.html files to load those dependencies from said local location.\n\n"
	. "On success, you need to recompile the OTP sources, for example, with 'mvn package -DskipTests'\n\n"
	. "The fetched dependencies will be bundled in the jar files. You may want to run this script (using --force) and compile often in order to catch any changes made by the OTP developers to javascript/css files of the client UI.\n\n"
	. "Caveat: the OpenStreetMap tiles, because of the OTP design, can not be localised yet. So you will need still to be connected to the internet for the plan enquiries (e.g. http://localhost:8080). But you can be disconnected from the internet for exploring the GraphQL interface (e.g. http://localhost:8080/graphiql).\n\n"
	. "Minor glitch: the GraphQL explorer shows tabs, spaces and newlines with 'funny' characters (A with hats etc.), perhaps a unicode encoding problem. Solutions welcome.\n\n"
	. "Note: It has been tested on Linux for OTP v2.5.0 and v.2.6.0. It has not been tested on Windows and never will be.\n\n"
	. 'Script by Andreas Hadjiprocopis ([email protected]) (c) 2024, freely distributed with attribution. Use at your own risk.'
	. "\n"
	;
}