Serverpod - Google Cloud Storage Interim

file_uploader.dart

import 'dart:convert';
import 'dart:typed_data';

import 'package:http/http.dart' as http;

/// The file uploader uploads files to Serverpod's cloud storage. On the server
/// you can setup a custom storage service, such as S3 or Google Cloud. To
/// directly upload a file, you first need to retrieve an upload description
/// from your server. After the file is uploaded, make sure to notify the server
/// by calling the verifyDirectFileUpload on the current Session object.
class GoogleCloudStorageUploader {
  late final _UploadDescription _uploadDescription;
  bool _attemptedUpload = false;

  /// Creates a new FileUploader from an [uploadDescription] created by the
  /// server.
  GoogleCloudStorageUploader(String uploadDescription) {
    _uploadDescription = _UploadDescription(uploadDescription);
  }

  /// Uploads a file contained by a [ByteData] object, returns true if
  /// successful.
  Future<bool> uploadByteData(ByteData byteData) async {
    var stream = http.ByteStream.fromBytes(byteData.buffer.asUint8List());
    return upload(stream, byteData.lengthInBytes);
  }

  /// Uploads a file from a [Stream], returns true if successful.
  Future<bool> upload(Stream<List<int>> stream, int length) async {
    if (_attemptedUpload) {
      throw Exception(
          'Data has already been uploaded using this FileUploader.');
    }
    _attemptedUpload = true;

    if (_uploadDescription.type == _UploadType.binary) {
      try {
        var result = switch (_uploadDescription.httpMethod) {
          'PUT' => await http.put(
              _uploadDescription.url,
              body: await _readStreamData(stream),
              headers: _uploadDescription.headers,
            ),
          _ => await http.post(
              _uploadDescription.url,
              body: await _readStreamData(stream),
              headers: _uploadDescription.headers,
            ),
        };

        if (result.statusCode == 200) {
          print('statusCode: ${result.statusCode}, body: ${result.body}');
        }

        return result.statusCode == 200;
      } catch (e) {
        return false;
      }
    } else if (_uploadDescription.type == _UploadType.multipart) {
      // final stream = http.ByteStream(Stream.castFrom(file.openRead()));
      // final length = await file.length();

      // final stream = http.ByteStream.fromBytes(data.buffer.asUint8List());
      // final length = await data.lengthInBytes;

      var request = http.MultipartRequest('POST', _uploadDescription.url);
      var multipartFile = http.MultipartFile(
          _uploadDescription.field!, stream, length,
          filename: _uploadDescription.fileName);

      request.files.add(multipartFile);
      for (var key in _uploadDescription.requestFields.keys) {
        request.fields[key] = _uploadDescription.requestFields[key]!;
      }

      try {
        var result = await request.send();
        // var body = await _readBody(result.stream);
        // print('body: $body');
        return result.statusCode == 204;
      } catch (e) {
        return false;
      }
    }
    throw UnimplementedError('Unknown upload type');
  }

  Future<List<int>> _readStreamData(Stream<List<int>> stream) async {
    // TODO: Find more efficient solution?
    var data = <int>[];
    await for (var segment in stream) {
      data += segment;
    }
    return data;
  }
}

enum _UploadType {
  binary,
  multipart,
}

class _UploadDescription {
  late _UploadType type;
  late Uri url;
  String? field;
  String? fileName;
  String? httpMethod = 'POST';
  Map<String, String> headers = {};
  Map<String, String> requestFields = {};

  _UploadDescription(String description) {
    var data = jsonDecode(description);
    if (data['type'] == 'binary') {
      type = _UploadType.binary;
    } else if (data['type'] == 'multipart') {
      type = _UploadType.multipart;
    } else {
      throw const FormatException('Missing type, can be binary or multipart');
    }

    httpMethod = data['httpMethod'];
    headers = (data['headers'] as Map).cast<String, String>();
    headers.remove('host');
    url = Uri.parse(data['url']);

    if (type == _UploadType.multipart) {
      field = data['field'];
      fileName = data['file-name'];
      requestFields = (data['request-fields'] as Map).cast<String, String>();
    }
  }
}

google_cloud_storage_native.dart

// in mypod_server project, used to configure google cloud storage access

import 'dart:collection';
import 'dart:convert';
import 'dart:io';
import 'dart:typed_data';

import 'package:crypto/crypto.dart';
import 'package:hex/hex.dart';
import 'package:gcloud/storage.dart';
// import 'package:googleapis/storage/v1.dart' as gcpStorage;
import 'package:googleapis_auth/auth_io.dart' as auth;
// ignore: implementation_imports
import 'package:googleapis_auth/src/crypto/pem.dart';
// ignore: implementation_imports
import 'package:googleapis_auth/src/crypto/rsa_sign.dart';
import 'package:intl/intl.dart';
import 'package:serverpod/serverpod.dart';

// This is a service credentials with storage admin permissions 
// (in IAM > service accounts, create a new key for the same service account you were using HMAC keys for)
const _googleClientSecretPath = 'config/google_storage_service_account.json';

/// Concrete implementation of Google Cloud Storage, using native GCP APIs,
/// for use with Serverpod.
class GoogleCloudStorageNative extends CloudStorage {
  final String bucket;
  final bool public;

  late final String publicHost;
  late final Storage storage;

  late final Map<String, dynamic> _serviceAccount;

  /// Creates a new [GoogleCloudStorageNative] object.
  static Future<GoogleCloudStorageNative> create({
    required Serverpod serverpod,
    required String storageId,
    required bool public,
    required String bucket,
    String? publicHost,
  }) async {
    final instance = GoogleCloudStorageNative._(
      serverpod: serverpod,
      storageId: storageId,
      public: public,
      bucket: bucket,
    );

    await instance._initStorage();

    return instance;
  }

  // Initializes the Google Cloud Storage client.
  Future<void> _initStorage() async {
    final jsonCredentials = await File(_googleClientSecretPath).readAsString();

    var json = jsonDecode(jsonCredentials);
    final project = json['project_id'] as String;

    final credentials =
        auth.ServiceAccountCredentials.fromJson(jsonCredentials);

    final client =
        await auth.clientViaServiceAccount(credentials, Storage.SCOPES);

    storage = Storage(client, project);

    final serviceAccount = await File(_googleClientSecretPath).readAsString();
    _serviceAccount = jsonDecode(serviceAccount);
  }

  // Private constructor
  GoogleCloudStorageNative._({
    required Serverpod serverpod,
    required String storageId,
    required this.public,
    required this.bucket,
    String? publicHost,
  }) : super(storageId) {
    this.publicHost = publicHost ?? 'storage.googleapis.com/$bucket';
  }

  @override
  Future<void> storeFile({
    required Session session,
    required String path,
    required ByteData byteData,
    DateTime? expiration,
    bool verified = true,
  }) async {
    await storage
        .bucket(bucket)
        .writeBytes(path, byteData.buffer.asUint8List());
  }

  @override
  Future<ByteData?> retrieveFile({
    required Session session,
    required String path,
  }) async {
    var byteLists = await storage.bucket(bucket).read(path).toList();
    var totLength =
        byteLists.fold<int>(0, (prev, element) => prev + element.length);
    var bytes = Uint8List(totLength);
    var offset = 0;
    for (var byteList in byteLists) {
      bytes.setRange(offset, offset + byteList.length, byteList);
      offset += byteList.length;
    }
    return ByteData.view(bytes.buffer);
  }

  @override
  Future<Uri?> getPublicUrl({
    required Session session,
    required String path,
  }) async {
    if (!public) return null;

    if (await fileExists(session: session, path: path)) {
      return Uri.parse('https://$publicHost/$path');
    }

    return null;
  }

  @override
  Future<bool> fileExists({
    required Session session,
    required String path,
  }) async {
    try {
      await storage.bucket(bucket).info(path);
      return true;
    } catch (e) {
      return false;
    }
  }

  @override
  Future<void> deleteFile({
    required Session session,
    required String path,
  }) async {
    await storage.bucket(bucket).delete(path);
  }

    // Creates a V4 signed url to allow direct API calls to cloud storage.
  String? _createSignedUrl({
    required Session session,
    required String path,
    String? subresource,
    int expiration = 604800,
    String httpMethod = 'GET',
    Map<String, String>? queryParameters,
    Map<String, String>? headers,
  }) {
    if (expiration > 604800) {
      session.log(
          'Expiration Time can\'t be longer than 604800 seconds (7 days).',
          level: LogLevel.error);
      return null;
    }

    final escapedObjectName = Uri.encodeComponent(path);
    final canonicalUri = "/$escapedObjectName";

    final dateTimeNow = DateTime.now().toUtc();
    final requestTimestamp =
        DateFormat("yyyyMMdd'T'HHmmss'Z'").format(dateTimeNow);
    final datestamp = DateFormat('yyyyMMdd').format(dateTimeNow);

    final clientEmail = _serviceAccount['client_email'];
    final credentialScope = '$datestamp/auto/storage/goog4_request';
    final credential = "$clientEmail/$credentialScope";

    headers ??= {};
    final host = '$bucket.storage.googleapis.com';

    headers['host'] = host;

    SplayTreeMap splayTreeMap = SplayTreeMap.from(headers);
    headers = Map.from(splayTreeMap);

    final canonicalHeaders = headers.entries
        .map((entry) =>
            '${entry.key.toLowerCase()}:${entry.value.trim().toLowerCase()}\n')
        .join();
    final signedHeaders = headers.keys.map((k) => k.toLowerCase()).join(';');

    queryParameters ??= {};
    queryParameters.addAll({
      'X-Goog-Algorithm': 'GOOG4-RSA-SHA256',
      'X-Goog-Credential': credential,
      'X-Goog-Date': requestTimestamp,
      'X-Goog-Expires': expiration.toString(),
      'X-Goog-SignedHeaders': signedHeaders,
    });
    if (subresource != null) {
      queryParameters[subresource] = '';
    }

    final canonicalQueryString = queryParameters.entries
        .map((entry) =>
            '${Uri.encodeComponent(entry.key)}=${Uri.encodeComponent(entry.value)}')
        .join('&');

    final canonicalRequest = [
      httpMethod,
      canonicalUri,
      canonicalQueryString,
      canonicalHeaders,
      signedHeaders,
      'UNSIGNED-PAYLOAD',
    ].join("\n");

    final canonicalRequestHash =
        sha256.convert(utf8.encode(canonicalRequest)).toString();

    final stringToSign = [
      'GOOG4-RSA-SHA256',
      requestTimestamp,
      credentialScope,
      canonicalRequestHash,
    ].join('\n');

    var pem = _serviceAccount['private_key'];
    var rsaKey = keyFromString(pem!);
    var signer = RS256Signer(rsaKey);

    List<int> stringToSignList = utf8.encode(stringToSign);
    var signedRequestBytes = signer.sign(stringToSignList);
    var signature = HEX.encode(signedRequestBytes);

    final schemeAndHost = 'https://$host';
    return '$schemeAndHost$canonicalUri?$canonicalQueryString&x-goog-signature=$signature';
  }

  @override
  Future<String?> createDirectFileUploadDescription({
    required Session session,
    required String path,
    Duration expirationDuration = const Duration(minutes: 10),
  }) async {
    if (await fileExists(session: session, path: path)) return null;

    final headers = {
      'content-type': 'application/octet-stream',
      'accept': '*/*',
      if (public) 'x-goog-acl': 'public-read',
    };

    final url = _createSignedUrl(
      session: session,
      path: path,
      httpMethod: 'PUT',
      expiration: expirationDuration.inSeconds,
      headers: headers,
    );

    var uploadDescriptionData = {
      'url': url,
      'type': 'binary',
      'httpMethod': 'PUT',
      'headers': headers,
    };

    return jsonEncode(uploadDescriptionData);
  }

  @override
  Future<bool> verifyDirectFileUpload({
    required Session session,
    required String path,
  }) async {
    return fileExists(session: session, path: path);
  }
}

server.dart

  // in mypod_server project, in the main server.dart to configure the different storage buckets

  pod.addCloudStorage(await GoogleCloudStorageNative.create(
    serverpod: pod,
    storageId: 'public',
    public: true,
    bucket: publicBucket,
  ));

  pod.addCloudStorage(await GoogleCloudStorageNative.create(
    serverpod: pod,
    storageId: 'private',
    public: false,
    bucket: privateBucket,
  ));