Skip to content

Instantly share code, notes, and snippets.

@dipu-bd
Last active May 2, 2020 23:52
Show Gist options
  • Save dipu-bd/faf7b14d0ffe72a6baca12dd4f4d9bff to your computer and use it in GitHub Desktop.
Save dipu-bd/faf7b14d0ffe72a6baca12dd4f4d9bff to your computer and use it in GitHub Desktop.
This is a direct port of python's urllib.parse.urlparse into dart. Additionally, it splits the netloc (authority) component into user, password, host and port components.
class URL {
final String original;
String _fragment = '';
String _query = '';
String _scheme = '';
String _user = '';
String _password = '';
String _host = '';
String _port = '';
String _path = '';
String _params = '';
String _hostname = '';
String _userinfo = '';
String _authority = '';
String _baseurl = '';
String _absolute = '';
/// ```
/// hostname
/// ┌───────┴─────┐
/// userinfo? host port?
/// ┌───┴───┐ ┌────┴────┐ ┌┴┐
/// ftp://user:[email protected]:123/pa/th/?q=ue&r=y#efg
/// └┬┘ └──────────┬────────────┘└──┬──┘ └───┬──┘ └┬┘
/// scheme authority? path query? fragment?
/// └─────────────┬───────────────┘
/// baseurl?
/// ```
URL.parse(String url) : original = url ?? '' {
_splitURL(original);
}
String get fragment => _fragment;
String get query => _query;
String get scheme => _scheme;
String get user => _user;
String get password => _password;
String get host => _host;
String get port => _port;
String get path => _path;
String get params => _params;
String get hostname => _hostname;
String get baseurl => _baseurl;
String get absolute => _absolute;
@override
String toString() =>
"scheme=$scheme, authority=$_authority{userinfo=$_userinfo($user:$password), " +
"hostname=$hostname($host:$port)}, params=$params, path=$path, query=$query, " +
"fragment=$fragment";
void _splitURL(String url) {
Iterable<int> temp, rest;
Iterable<int> chars = url.runes;
// remove invalid characters
chars = chars.where((c) => c > 32);
// parse scheme
temp = chars.takeWhile((c) => c != 58); // `:` = 58
if (temp.length < chars.length) {
rest = chars.skip(temp.length + 1);
bool valid = temp.skip(1).every((c) =>
(c >= 65 && c <= 90) || // `A` = 65 `Z` = 90
(c >= 97 && c <= 122) || // `a` = 97 `z` = 122
c == 43 || // `+` = 43
c == 45 || // `-` = 45
c == 46 // `.` = 46
) &&
(rest.isEmpty ||
!rest.every((c) => c >= 48 && c <= 57) // `0` = 48 `9` = 57
);
if (valid) {
_scheme = String.fromCharCodes(temp).toLowerCase();
chars = rest;
}
}
// parse authority (netloc)
if ('//' == String.fromCharCodes(chars.take(2))) {
rest = chars.skip(2);
temp = rest.takeWhile(
// `#` = 35, `/` = 47, `?` = 63
(c) => c != 35 && c != 47 && c != 63);
_authority = String.fromCharCodes(temp);
chars = rest.skip(temp.length);
_splitAuthority();
}
// parse fragments
temp = chars.takeWhile((c) => c != 35); // `#` = 35
if (temp.length < chars.length) {
rest = chars.skip(temp.length + 1);
_fragment = String.fromCharCodes(rest);
chars = temp;
}
// parse query
temp = chars.takeWhile((c) => c != 63); // `?` = 63
if (temp.length < chars.length) {
rest = chars.skip(temp.length + 1);
_query = String.fromCharCodes(rest);
chars = temp;
}
// parse path
_path = String.fromCharCodes(chars);
// Build baseurl
_baseurl = (_scheme.isEmpty ? '' : _scheme + ':') +
(_authority.isEmpty ? '' : '//' + _authority);
// Build absolute url
_absolute = _baseurl +
(_baseurl.isEmpty || _path.isEmpty || _path[0] == '/' ? '' : '/') +
_path +
(_query.isEmpty ? '' : '?' + _query) +
(_fragment.isEmpty ? '' : '#' + _fragment);
}
_splitAuthority() {
Iterable<int> temp, rest;
Iterable<int> chars = _authority.runes;
// parse userinfo
temp = chars.takeWhile((c) => c != 64); // `@` = 64
if (temp.length < chars.length) {
_userinfo = String.fromCharCodes(temp);
chars = chars.skip(temp.length + 1);
_splitUserInfo();
}
// parse params
temp = chars.takeWhile((c) => c != 59); // `;` = 59
if (temp.length < chars.length) {
rest = chars.skip(temp.length + 1);
_params = String.fromCharCodes(rest);
chars = temp;
}
// parse hostname
_hostname = String.fromCharCodes(chars);
// parse port
int i = chars.toList().lastIndexOf(58); // `:` = 58
if (i >= 0 && i < chars.length) {
rest = chars.skip(i + 1);
bool valid = (rest.isEmpty || rest.every(
// `0` = 48, `9` = 57
(c) => c >= 48 && c <= 57));
if (valid) {
_port = String.fromCharCodes(rest);
chars = chars.take(i);
}
}
// parse host
_host = String.fromCharCodes(chars);
}
_splitUserInfo() {
Iterable<int> temp, rest;
Iterable<int> chars = _userinfo.runes;
// parse password
temp = chars.takeWhile((c) => c != 58); // `:` = 58
if (temp.length < chars.length) {
rest = chars.skip(temp.length + 1);
_password = String.fromCharCodes(rest);
chars = temp;
}
// parse user
_user = String.fromCharCodes(chars);
}
}
/* ------------------------------------------------------------------------- */
/* TESTING */
/* ------------------------------------------------------------------------- */
import 'dart:math';
void main() {
const scheme_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
const default_chars =
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ012456789+-.%,=&";
final rand = Random();
final randomString = (int len, [chars = default_chars]) =>
List<String>.generate(len, (i) => chars[rand.nextInt(chars.length)])
.join('');
URL url;
expect(actual, original, {reason: ''}) {
if (actual != original) {
print(' Expected: $original');
print(' Found: $actual');
print(' Reason: $reason');
print('');
}
}
print("test URL.parse using randomly generated uri");
print('--------------------------------------------------');
// <scheme>://<user>:<pass>@<host>:<port>;<params>/<path>?<query>#<fragment>
for (int i = 0; i < 1000; ++i) {
String scheme = '',
user = '',
passwd = '',
host = '',
port = '',
params = '',
path = '',
query = '',
fragment = '',
userinfo = '',
hostname = '',
authority = '',
baseurl = '',
absolute = '';
scheme = rand.nextBool()
? randomString(1 + rand.nextInt(3), scheme_chars).toLowerCase()
: '';
user = rand.nextBool() ? randomString(3 + rand.nextInt(10)) : '';
passwd = rand.nextBool() ? randomString(1 + rand.nextInt(10)) : '';
host = rand.nextBool() ? randomString(2 + rand.nextInt(20)) : '';
port = rand.nextBool() ? '${2 + rand.nextInt(50000)}' : '';
params = rand.nextBool() ? randomString(1 + rand.nextInt(10)) : '';
query = rand.nextBool() ? randomString(5 + rand.nextInt(20)) : '';
fragment = rand.nextBool() ? randomString(2 + rand.nextInt(10)) : '';
path = List<String>.generate(
rand.nextInt(5),
(i) => randomString(1 + rand.nextInt(10)),
).join('/');
if (path.isNotEmpty) {
path = '/' + path;
}
userinfo = user + (passwd.isEmpty ? '' : ':') + passwd;
hostname = host + (port.isEmpty ? '' : ':') + port;
authority = userinfo + (userinfo.isEmpty ? '' : '@') + hostname;
baseurl = (authority.isEmpty ? '' : '//') + authority;
if (authority.isNotEmpty) {
baseurl += (params.isEmpty ? '' : ';') + params;
} else {
params = '';
}
baseurl = scheme + (scheme.isEmpty ? '' : ':') + baseurl;
absolute = baseurl +
path +
(query.isEmpty ? '' : '?') +
query +
(fragment.isEmpty ? '' : '#') +
fragment;
url = URL.parse(absolute);
expect(url.scheme, scheme, reason: '$i [scheme] $absolute | $url');
expect(url.user, user, reason: '$i [user] $absolute | $userinfo');
expect(url.password, passwd, reason: '$i [passwd] $absolute | $url');
expect(url.host, host, reason: '$i [host] $absolute | $url');
expect(url.port, port, reason: '$i [port] $absolute | $url');
expect(url.params, params, reason: '$i [params] $absolute | $url');
expect(url.path, path, reason: '$i [path] $absolute | $url');
expect(url.query, query, reason: '$i [query] $absolute | $url');
expect(url.fragment, fragment, reason: '$i [fragment] $absolute | $url');
expect(url.hostname, hostname, reason: '$i [hostname] $absolute | $url');
expect(url.baseurl, baseurl, reason: '$i [baseurl] $absolute | $url');
expect(url.absolute, absolute, reason: '$i [absolute] $absolute | $url');
}
print('DONE.\n');
print("foo://example.com:8042/over/there?name=ferret#nose");
print('--------------------------------------------------');
url = URL.parse("foo://example.com:8042/over/there?name=ferret#nose");
expect(url.scheme, 'foo', reason: '[scheme] $url');
expect(url.user, '', reason: '[user] $url');
expect(url.password, '', reason: '[password] $url');
expect(url.host, 'example.com', reason: '[host] $url');
expect(url.port, '8042', reason: '[port] $url');
expect(url.path, '/over/there', reason: '[path] $url');
expect(url.query, 'name=ferret', reason: '[query] $url');
expect(url.fragment, 'nose', reason: '[fragment] $url');
print('DONE.\n');
print("ftp://ftp.is.co.za/rfc/rfc1808.txt");
print('--------------------------------------------------');
url = URL.parse("ftp://ftp.is.co.za/rfc/rfc1808.txt");
expect(url.scheme, 'ftp', reason: '[scheme] $url');
expect(url.user, '', reason: '[user] $url');
expect(url.password, '', reason: '[password] $url');
expect(url.host, 'ftp.is.co.za', reason: '[host] $url');
expect(url.port, '', reason: '[port] $url');
expect(url.path, '/rfc/rfc1808.txt', reason: '[path] $url');
expect(url.query, '', reason: '[query] $url');
expect(url.fragment, '', reason: '[fragment] $url');
print('DONE.\n');
print("http://www.ietf.org/rfc/rfc2396.txt#header1");
print('--------------------------------------------------');
url = URL.parse("http://www.ietf.org/rfc/rfc2396.txt#header1");
expect(url.scheme, 'http', reason: '[scheme] $url');
expect(url.user, '', reason: '[user] $url');
expect(url.password, '', reason: '[password] $url');
expect(url.host, 'www.ietf.org', reason: '[host] $url');
expect(url.port, '', reason: '[port] $url');
expect(url.path, '/rfc/rfc2396.txt', reason: '[path] $url');
expect(url.query, '', reason: '[query] $url');
expect(url.fragment, 'header1', reason: '[fragment] $url');
print('DONE.\n');
print("ldap://[2001:db8::7]/c=GB?objectClass=one&objectClass=two");
print('--------------------------------------------------');
url = URL.parse("ldap://[2001:db8::7]/c=GB?objectClass=one&objectClass=two");
expect(url.scheme, 'ldap', reason: '[scheme] $url');
expect(url.user, '', reason: '[user] $url');
expect(url.password, '', reason: '[password] $url');
expect(url.host, '[2001:db8::7]', reason: '[host] $url');
expect(url.port, '', reason: '[port] $url');
expect(url.path, '/c=GB', reason: '[path] $url');
expect(url.query, 'objectClass=one&objectClass=two', reason: '[query] $url');
expect(url.fragment, '', reason: '[fragment] $url');
print('DONE.\n');
print("https://bob:[email protected]/place");
print('--------------------------------------------------');
url = URL.parse("https://bob:[email protected]/place");
expect(url.scheme, 'https', reason: '[scheme] $url');
expect(url.user, 'bob', reason: '[user] $url');
expect(url.password, 'pass', reason: '[password] $url');
expect(url.host, 'example.com', reason: '[host] $url');
expect(url.port, '', reason: '[port] $url');
expect(url.path, '/place', reason: '[path] $url');
expect(url.query, '', reason: '[query] $url');
expect(url.fragment, '', reason: '[fragment] $url');
print('DONE.\n');
print("http://example.com/?a=1&b=2+2&c=3&c=4&d=%65%6e%63%6F%64%65%64");
print('--------------------------------------------------');
url = URL
.parse("http://example.com/?a=1&b=2+2&c=3&c=4&d=%65%6e%63%6F%64%65%64");
expect(url.scheme, 'http', reason: '[scheme] $url');
expect(url.user, '', reason: '[user] $url');
expect(url.password, '', reason: '[password] $url');
expect(url.host, 'example.com', reason: '[host] $url');
expect(url.port, '', reason: '[port] $url');
expect(url.path, '/', reason: '[path] $url');
expect(url.query, 'a=1&b=2+2&c=3&c=4&d=%65%6e%63%6F%64%65%64',
reason: '[query] $url');
expect(url.fragment, '', reason: '[fragment] $url');
print('DONE.\n');
print("bob:[email protected]/place");
print('--------------------------------------------------');
url = URL.parse("bob:[email protected]/place");
expect(url.scheme, 'bob', reason: '[scheme] $url');
expect(url.user, '', reason: '[user] $url');
expect(url.password, '', reason: '[password] $url');
expect(url.host, '', reason: '[host] $url');
expect(url.port, '', reason: '[port] $url');
expect(url.path, '[email protected]/place', reason: '[path] $url');
expect(url.query, '', reason: '[query] $url');
expect(url.fragment, '', reason: '[fragment] $url');
print('DONE.\n');
print("example.com/place");
print('--------------------------------------------------');
url = URL.parse("example.com/place");
expect(url.scheme, '', reason: '[scheme] $url');
expect(url.user, '', reason: '[user] $url');
expect(url.password, '', reason: '[password] $url');
expect(url.host, '', reason: '[host] $url');
expect(url.port, '', reason: '[port] $url');
expect(url.path, 'example.com/place', reason: '[path] $url');
expect(url.query, '', reason: '[query] $url');
expect(url.fragment, '', reason: '[fragment] $url');
print('DONE.\n');
print("mailto:[email protected]");
print('--------------------------------------------------');
url = URL.parse("mailto:[email protected]");
expect(url.scheme, 'mailto', reason: '[scheme] $url');
expect(url.user, '', reason: '[user] $url');
expect(url.password, '', reason: '[password] $url');
expect(url.host, '', reason: '[host] $url');
expect(url.port, '', reason: '[port] $url');
expect(url.path, '[email protected]', reason: '[path] $url');
expect(url.query, '', reason: '[query] $url');
expect(url.fragment, '', reason: '[fragment] $url');
print('DONE.\n');
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment