Amavisd Courier Patch for Amavisd Version 2.10.1 and Courier Version 0.73.2

amavisd-new-courier.patch

--- amavisd.ori	2014-12-21 19:49:24.757834955 +0200
+++ amavisd	2014-12-21 20:58:38.581469101 +0200
@@ -107,7 +107,7 @@
 #  Amavis::Lookup::LDAPattr (the rest)
 #  Amavis::In::AMPDP
 #  Amavis::In::SMTP
-#( Amavis::In::Courier )
+#  Amavis::In::Courier
 #  Amavis::Out::SMTP::Protocol
 #  Amavis::Out::SMTP::Session
 #  Amavis::Out::SMTP
@@ -229,7 +229,7 @@
   }
   fetch_modules('REQUIRED BASIC MODULES', 1, qw(
     Exporter POSIX Fcntl Socket Errno Carp Time::HiRes
-    IO::Handle IO::File IO::Socket IO::Socket::UNIX
+    IO::Handle IO::File IO::Socket IO::Socket::UNIX IO::Select
     IO::Stringy Digest::MD5 Unix::Syslog File::Basename
     Compress::Zlib MIME::Base64 MIME::QuotedPrint MIME::Words
     MIME::Head MIME::Body MIME::Entity MIME::Parser MIME::Decoder
@@ -12671,12 +12671,26 @@
 }
 
 ### Net::Server hook
+### This hook takes place immediately after the "->run()" method is called.
+### This hook allows for setting up the object before any built in configuration
+### takes place.  This allows for custom configurability.
+sub configure_hook {
+  my($self) = @_;
+  if ($courierfilter_shutdown) {
+    # Duplicate the courierfilter pipe to another fd since STDIN is closed if we
+    # daemonize
+    $self->{courierfilter_pipe} = IO::File->new('<&STDIN')
+      or die "Can't duplicate courierfilter shutdown pipe: $!";
+  }
+}
+
+### Net::Server hook
 ### Occurs in the parent (master) process after (possibly) opening a log file,
 ### creating pid file, reopening STDIN/STDOUT to /dev/null and daemonizing;
 ### but before binding to sockets
 #
 sub post_configure_hook {
-# umask(0007);  # affect protection of Unix sockets created by Net::Server
+  umask(0007);  # affect protection of Unix sockets created by Net::Server
 }
 
 sub set_sockets_access() {
@@ -12695,11 +12709,20 @@
 
 ### Net::Server hook
 ### Occurs in the parent (master) process after binding to sockets,
-### but before chrooting and dropping privileges
-#
+### but before chrooting and dropping privileges. At this point
+### the process will still be running as the user who started the server.
 sub post_bind_hook {
+  my ($self) = @_;
   umask(0027);  # restore our preferred umask
   set_sockets_access()  if defined $warm_restart && !$warm_restart;
+  if (c('protocol') eq 'COURIER') {
+    # Allow courier to write to the socket
+    chmod(0660, $unix_socketname);
+  }
+  if ($self->{courierfilter_pipe}) {
+    # Watch for courierfilter telling us to shut down
+    $self->{server}->{select}->add($self->{courierfilter_pipe});
+  }
 }
 
 ### Net::Server hook
@@ -12755,6 +12778,17 @@
     # elsif (-d _ && !-w _){ die "QUARANTINEDIR directory $name not writable"}
     }
     $spamcontrol_obj->init_pre_fork  if $spamcontrol_obj;
+    if ($courierfilter_shutdown) {
+      # Tell courierfilter we have finished initialisation by closing fd 3
+      # But make sure it's a pipe (and not the courierfilter shutdown pipe)
+      # first: if we have been started using filterctl (i.e. not when
+      # courierfilter itself starts) then there is no initial pipe on fd 3 so
+      # it could be assigned to another file
+      open(my $fh3, '<&3');
+      if (-p $fh3 && $self->{courierfilter_pipe}->fileno() != 3) {
+        POSIX::close(3);
+      }
+    }
     my(@modules_extra) = grep(!exists $modules_basic{$_}, keys %INC);
     if (@modules_extra) {
       do_log(1, "extra modules loaded after daemonizing/chrooting: %s",
@@ -13229,7 +13263,9 @@
       $ampdp_in_obj = Amavis::In::AMPDP->new  if !$ampdp_in_obj;
       $ampdp_in_obj->process_policy_request($sock, $conn, \&check_mail, 0);
     } elsif ($suggested_protocol eq 'COURIER') {
-      die "unavailable support for protocol: $suggested_protocol";
+      # courierfilter client
+      $courier_in_obj = Amavis::In::Courier->new  if !$courier_in_obj;
+      $courier_in_obj->process_courier_request($sock, $conn, \&check_mail);
     } elsif ($suggested_protocol eq 'QMQPqq') {
       die "unavailable support for protocol: $suggested_protocol";
     } elsif ($suggested_protocol eq 'TCP-LOOKUP') { #postfix maps, experimental
@@ -13337,6 +13373,26 @@
   DB::finish_profile() if $profiling;
 }
 
+### Net::Server hook
+### Called in child process when any filehandle Net::Server is selecting on
+### becomes readable
+### Used to detect EOF on the courierfilter pipe
+sub can_read_hook {
+  my($self, $fh) = @_;
+  if ($self->{courierfilter_pipe}
+      && fileno($fh) == $self->{courierfilter_pipe}->fileno())
+  {
+    do_log(0, "Instructed by courierfilter to shutdown");
+    kill('TERM', $self->{server}->{ppid});
+    # Wait for the parent to kill us
+    sleep(1);
+    # Still here?  Close down ourselves
+    $self->child_finish_hook;
+    exit;
+  }
+  return undef;
+}
+
 ### Child is about to be terminated
 ### user customizable Net::Server hook
 #
@@ -14174,7 +14230,7 @@
             defined $msginfo->mime_entity ? '' : ', MIME not decoded',
             !defined $mime_err || $mime_err eq '' ? ''
                                                   : ", MIME error: $mime_err");
-          link($msginfo->mail_text_fn, $newpart)
+          symlink($msginfo->mail_text_fn, $newpart)
             or die sprintf("Can't create hard link %s to %s: %s",
                            $newpart, $msginfo->mail_text_fn, $!);
           $newpart_obj->type_short('MAIL');  # case sensitive
@@ -18595,6 +18651,11 @@
 undef $Amavis::Conf::log_short_templ;
 undef $Amavis::Conf::log_verbose_templ;
 
+# courierfilter shutdown needs can_read_hook, added in Net::Server 0.90
+if ($courierfilter_shutdown && Net::Server->VERSION < 0.90) {
+  die "courierfilter shutdown needs Net::Server 0.90 or better";
+}
+
 if (defined $desired_user && $daemon_user ne '') {
   local($1);
   # compare the config file settings to current UID
@@ -19249,6 +19310,8 @@
     port => \@listen_sockets,  # listen on these sockets (Unix, inet, inet6)
     host => $bind_to[0],  # default bind, redundant, merged to @listen_sockets
     listen => $listen_queue_size, # undef for a default
+    # need to set multi_port for can_read_hook
+    multi_port => $courierfilter_shutdown ? 1 : undef,
     max_servers => $max_servers,  # number of pre-forked children
     !defined($min_servers) ? ()
     : ( min_servers       => $min_servers,
@@ -22857,7 +22920,420 @@
 use warnings FATAL => qw(utf8 void);
 no warnings 'uninitialized';
 
-BEGIN { die "Code not available for module Amavis::In::Courier" }
+BEGIN {
+  require Exporter;
+  use vars qw(@ISA @EXPORT @EXPORT_OK %EXPORT_TAGS $VERSION);
+  $VERSION = '2.102';
+  @ISA = qw(Exporter);
+  import Amavis::Conf qw(:platform :confvars ca c);
+  import Amavis::Util qw(do_log am_id untaint debug_oneshot snmp_counters_init
+  			 switch_to_my_time switch_to_client_time
+                         read_text orcpt_encode xtext_decode);
+  import Amavis::Lookup qw(lookup);
+  import Amavis::Lookup::IP qw(lookup_ip_acl);
+  import Amavis::rfc2821_2822_Tools qw(quote_rfc2821_local qquote_rfc2821_local
+				       unquote_rfc2821_local);
+  import Amavis::Timing qw(section_time);
+  import Amavis::TempDir;
+  import Amavis::In::Message;
+}
+
+use IO::File;
+
+# Amavis::In::Courier->new()
+# Creates a new Amavis::In::Courier object
+sub new() {
+  my($class) = @_;
+  my $tempdir = Amavis::TempDir->new;
+  bless { tempdir => $tempdir }, $class;
+}
+
+# courier_in_obj->process_courier_request(socket, conn, check_mail)
+# Processes a request from Courier to check a single message
+#   socket: the socket to communicate with courierfilter
+#   conn: Amavis::In::Connection object
+#   check_mail: reference to the MTA-independent function called to check the
+#               message
+sub process_courier_request($$$) {
+  my($self, $socket, $conn, $check_mail) = @_;
+  
+  # Save the policy bank so that it can be restored at the end
+  my %baseline_policy_bank = %current_policy_bank;
+  
+  eval {
+    $self->init_request($conn);
+    $self->read_courierfilter_socket($socket);
+    $self->open_mail_text();
+    $self->change_policy_bank();
+    $self->call_check_mail($self->{msginfo},$check_mail);
+    $self->process_result();
+    1;
+  } or do {  # An exception occurred
+    my($eval_stat) = $@ ne '' ? $@ : "errno=$!"; chomp $eval_stat;
+    my $msg = "Error in processing: $eval_stat";
+    do_log(-2, "TROUBLE in process_courier_request: 451 4.5.0 %s", $msg);
+    # Close the mail text file
+    $self->{msginfo}->mail_text->close()  if ($self->{msginfo}->mail_text);
+    $self->{msginfo}->mail_text(undef);
+    # Send a temporary failure to Courier
+    $self->{smtp_resp} = "451 4.5.0 $msg";
+  };
+  
+  # Send the SMTP reponse back to Courier (done outside the eval to ensure that
+  # it always happens exactly once, whether or not there is an exception)
+  do_log(3, "Mail checking ended: %s", $self->{smtp_resp});
+  send($socket, "$self->{smtp_resp}\n", 0);
+  
+  # Record time
+  section_time('send response');
+  do_log(2, Amavis::Timing::report());
+  
+  # Restore the policy bank
+  %current_policy_bank = %baseline_policy_bank;
+  
+  # Clean up object
+  $self->{per_recip_data} = undef;
+  $self->{control_files} = undef;
+}
+
+# courier_in_obj->init_request( )
+# Begins processing for a single request: initialises global variables and
+# creates msginfo object
+#   conn: Amavis::In::Connection object
+sub init_request($) {
+  my($self, $conn) = @_;
+  
+  # Set up globals
+  am_id("$$-$Amavis::child_invocation_count");
+  Amavis::Timing::init();
+  snmp_counters_init();
+  
+  # Create msginfo object
+  $self->{msginfo} = Amavis::In::Message->new;
+  $self->{msginfo}->rx_time(time);
+  $self->{msginfo}->log_id(am_id());
+  $self->{msginfo}->conn_obj($conn);
+
+  $conn->appl_proto('courierfilter');
+}
+
+# courier_in_obj->read_courierfilter_socket(socket)
+# Reads the courierfilter socket, which specifies the path to the mail text and
+# the control files, storing the path to the mail text in the msginfo object
+# Also reads the control files and stores their data in msginfo
+#  socket: The courierfilter socket
+sub read_courierfilter_socket($) {
+  my($self, $socket) = @_;
+  
+  # Just make sure
+  local $/ = "\n";
+  
+  # Read the path to the mail text
+  switch_to_client_time("start receiving message text path");
+  my $text_path = $socket->getline;
+  switch_to_my_time("received message text path");
+  $text_path || die "Can't read message text path: $!";
+  chomp($text_path);
+  $text_path = untaint($text_path)  if ($text_path =~ m{^[A-Za-z0-9/._=+-]+\z});
+  $self->{msginfo}->mail_text_fn($text_path);
+  
+  # Read control files
+  $self->{control_files} = [];
+  my $path;
+  switch_to_client_time("start receving control file paths");
+  for ($! = 0; defined($path = $socket->getline); $! = 0) {
+    chomp($path);
+    # courierfilter indicates end of control files by sending a blank line
+    $path || last;
+    
+    switch_to_my_time("received control file path");
+    $path = untaint($path)  if ($path =~ m{^[A-Za-z0-9/._=+-]+\z});
+    push(@{ $self->{control_files} }, $path);
+    $self->read_control_file($path);
+    switch_to_client_time("receiving control file paths");
+  }
+  switch_to_my_time("finished receiving control file paths");
+  
+  # Check we did actually get a control file
+  @{ $self->{control_files} } || die "No control files specified";
+  # Record the recipients in msginfo
+  $self->{msginfo}->per_recip_data($self->{per_recip_data});
+  
+  # Record time
+  section_time('read control');
+}
+
+# courier_in_obj->read_control_file(path)
+# Reads a single Courier control file, adding its recipients to
+# $self->{per_recip_data} and storing other information in msginfo.
+# $self->{per_recip_data} is an array of Amavis::In::Message::PerRecip objects.
+# (Note that this method will overwrite any previous settings for sender, etc,
+# but if there are multiple control files they should contain the same
+# information.)
+#   path: the path to the control file
+sub read_control_file($) {
+  my($self, $path) = @_;
+  
+  do_log(3, "Reading Courier control file %s", $path);
+  
+  # Read the file
+  my $control_data = read_text($path);
+  my ($rcpt_idx, $recip) = (0, undef);
+  foreach (split(/\n/, $control_data)) {
+    # Parse a line of the control file
+    # Sender
+    do_log(4, "Courier control file line: %s", $_);
+    if (/^s ( .*? (?:  \[  (?: \\. | [^\]\\] )*  \]
+                       |  [^@"<>\[\]\\\s] )*
+            ) \z/xs) {
+      my $sender_quoted = $1;
+      my $sender_unquoted = unquote_rfc2821_local($sender_quoted);
+      $self->{msginfo}->sender_smtp('<'.$sender_quoted.'>');
+      $self->{msginfo}->sender($sender_unquoted);
+    }
+    
+    # Recipient
+    if (/^r ( .*? (?:  \[  (?: \\. | [^\]\\] )*  \]
+                       |  [^@"<>\[\]\\\s] )*
+            ) \z/xs) {
+      $recip = Amavis::In::Message::PerRecip->new;
+      my $addr_quoted = $1;
+      my $addr_unquoted = unquote_rfc2821_local($addr_quoted);
+      $recip->recip_addr_smtp('<'.$addr_quoted.'>');
+      $recip->recip_addr($addr_unquoted);
+      $recip->courier_control_file($path);
+      $recip->courier_recip_index($rcpt_idx);
+      $recip->recip_destiny(D_PASS); # Default destiny
+      push(@{ $self->{per_recip_data} }, $recip);
+      $rcpt_idx++;
+    }
+    
+    # Original Recipient (RFC 3461)
+    if (/^R ( [!-~]+ ) \z/xs) { $recip->dsn_orcpt($1) }
+    # RFC 3461 NOTIFY value
+    if (/^N ( [FSDN]+ ) \z/xs) {
+      my %notify_values = ( F => 'FAILURE', S => 'SUCCESS', D => 'DELAY', N => 'NEVER' );
+      $recip->dsn_notify([ map { $notify_values{$_} } split(m//, $1) ]);
+    }
+    
+    # DSN RET parameter (RFC 3461)
+    if (/^t F \z/xs) { $self->{msginfo}->dsn_ret('FULL') }
+    if (/^t H \z/xs) { $self->{msginfo}->dsn_ret('HDRS') }
+    # Envid (RFC 3461)
+    if (/^e ( [!-~]+ ) \z/xs) { $self->{msginfo}->dsn_envid($1) }
+    
+    # Authenticated submitter (RFC 2554)
+    if (/^i ( [!-~]+ ) \z/xs) { $self->{msginfo}->auth_submitter(xtext_decode($1)) }
+    
+    # Received-From-MTA
+    if (/^f .*? ;\s*  (  [A-Za-z0-9\.-]+  |  \[ [0-9A-Fa-f\.:]+ \]  )  \s*
+            \(  ( [A-Za-z0-9\.-]* )  \s* \[  ( [0-9A-Fa-f\.:]+ )  \] \)
+            \z/xs) {
+      $self->{msginfo}->client_helo($1);
+      $self->{msginfo}->client_name($2);
+      $self->{msginfo}->client_addr($3);
+    }
+    
+    # Courier queue ID
+    if (/^M ( [0-9A-Fa-f]+ \. [0-9A-Fa-f]+ \. [0-9A-Fa-f]+ )
+            \z/xs) {
+      $self->{msginfo}->queue_id($1);
+    }
+  }
+}
+
+# courier_in_obj->open_mail_text( )
+# Opens the mail text file, whose path has been read into msginfo->mail_text_fn
+# The file handle is stored in msginfo->mail_text
+sub open_mail_text() {
+  my($self) = @_;
+  
+  # Open the file
+  my $fh = IO::File->new($self->{msginfo}->mail_text_fn, 'r');
+  $fh || die "Can't open ", $self->{msginfo}->mail_text_fn, ": $!";
+  
+  # Store file handle
+  $self->{msginfo}->mail_text($fh);
+  
+  # Record time
+  section_time('open text');
+}
+
+# courier_in_obj->change_policy_bank( )
+# Loads a new policy bank if necessary
+# Also enables debug_oneshot if necessary
+sub change_policy_bank() {
+  my($self) = @_;
+  my $cl_ip = $self->{msginfo}->client_addr;
+  my $sender = $self->{msginfo}->sender;
+  
+  # Enable debug_oneshot if set for this sender
+  debug_oneshot(1)  if lookup(0, $sender, @{ ca('debug_sender_maps') });
+  
+  # Load MYNETS policy bank if client IP is local
+  my $cl_ip_mynets = ($cl_ip eq '' ? undef
+  		      : lookup_ip_acl($cl_ip, @{ ca('mynetworks_maps') }));
+  if (($cl_ip_mynets?1:0) > (c('originating')?1:0)) {
+    $current_policy_bank{'originating'} = $cl_ip_mynets;
+  }
+  if ($cl_ip_mynets && defined($policy_bank{'MYNETS'})) {
+    Amavis::load_policy_bank('MYNETS');
+  }
+  
+  # Load MYUSERS policy bank if sender is local
+  if ($sender ne '' && defined($policy_bank{'MYUSERS'})
+      && lookup(0, $sender, @{ ca('local_domains_maps') }))
+  {
+    Amavis::load_policy_bank('MYUSERS');
+  }
+}
+
+# courier_in_obj->call_check_mail(conn, check_mail)
+# Calls the check_mail function to check a message - the properties of msginfo
+# must already be set
+# Also handles the tempdir and closes the mail_text file afterwards
+# Saves the STMP response returned by check_mail in $self->{smtp_resp}
+#   conn: Amavis::In::Connection object
+#   check_mail: reference to the function to call
+sub call_check_mail($$) {
+  my($self, $conn, $check_mail) = @_;
+  
+  # Initialise variables
+  Amavis::check_mail_begin_task();
+  
+  # Prepare temporary directory
+  $self->{tempdir}->prepare_dir();
+  $self->{msginfo}->mail_tempdir($self->{tempdir}->path);
+  
+  # Courier is responsible for relaying the message, and so for success DSNs
+  $self->{msginfo}->dsn_passed_on(c('forward_method') eq '' ? 1 : 0);
+  
+  # Log the message
+  do_log(1, 'Courier %s %s: %s -> %s%s',
+            $self->{msginfo}->queue_id, $self->{tempdir}->path,
+            $self->{msginfo}->sender_smtp,
+            join(',', map { $_->recip_addr_smtp }
+		          @{ $self->{msginfo}->per_recip_data }),
+            join('',
+                 !$self->{msginfo}->auth_submitter ||
+                      $self->{msginfo}->auth_submitter eq '<>' ? ():
+                          ' AUTH='.$self->{msginfo}->auth_submitter,
+                 !$self->{msginfo}->dsn_ret   ? () :
+                          ' RET='.$self->{msginfo}->dsn_ret,
+                 !$self->{msginfo}->dsn_envid ? () :
+                          ' ENVID='.xtext_decode($self->{msginfo}->dsn_envid),
+            ));
+  
+  # The temporary directory is about to become non-empty
+  $self->{tempdir}->empty(0);
+  # Do the work
+  my ($smtp_resp, $exit_code, $preserve_evidence)
+    = $check_mail->($conn, $self->{msginfo}, 0);
+  # Preserve evidence if necessary
+  $preserve_evidence && $self->{tempdir}->preserve(1);
+  
+  # Clean the temporary directory
+  $self->{tempdir}->clean();
+  
+  # Close the mail text file
+  $self->{msginfo}->mail_text->close() || die "Can't close temp file: $!";
+  $self->{msginfo}->mail_text(undef);
+  
+  # Save the SMTP response
+  $self->{smtp_resp} = $smtp_resp;
+}
+
+# courier_in_obj->process_result( )
+# Processes the result of mail scanning - recipient addition/deletion
+# Before calling this, the SMTP response must be stored in $self->{smtp_resp}
+# and may be altered
+# This does not send the SMTP response back to Courier
+sub process_result() {
+  my($self) = @_;
+  
+  if ($self->{smtp_resp} =~ /^25/) {
+    foreach my $r (@{ $self->{msginfo}->per_recip_data }) {
+      my ($addr, $newaddr) = ($r->recip_addr, $r->recip_final_addr);
+
+      if ($r->recip_done) {
+        # Deleted recipient
+        $self->delete_recipient($r)  if defined $addr;
+
+      } elsif (!defined($r->courier_control_file)) {
+        # Newly added recipient
+        $self->add_recipient($newaddr, '', $r->dsn_notify);
+
+      } elsif ($newaddr ne $addr) {
+        # Recipient with address changed
+        $r->recip_smtp_response("251 2.1.5 Amavisd replaced recip with <$newaddr>");
+        $self->delete_recipient($r)  if defined $addr;
+
+        my $orcpt = $r->dsn_orcpt || orcpt_encode($r->recip_addr_smtp);
+        $self->add_recipient($newaddr, $orcpt, $r->dsn_notify);
+      }
+    }
+  }
+  
+  # Record time
+  section_time('process result');
+}
+
+# delete_recipient(recip)
+# Deletes a recipient by marking them as successfully delivered in the control
+# file.  If the same recipient appears more than once in the control files,
+# every instance will be marked as done.
+#   recip: Amavis::In::Message::PerRecip object
+sub delete_recipient($) {
+  my($self, $recip) = @_;
+  
+  do_log(1, "Amavis::In::Courier: Deleting recipient <%s>: %s",
+            $recip->recip_addr, $recip->recip_smtp_response);
+  
+  my $filename = $recip->courier_control_file;
+  my $control_file = IO::File->new($filename, 'a');
+  # Not sure why we do the seek when the file is already opened for append,
+  # but courier-pythonfilter does it so it's probably a good idea
+  seek($control_file, 0, 2);
+  # Courier may still append to the control file after calling the filter,
+  # so write a long blank line first to ensure that its additional records
+  # only overwrite the blank line
+  $control_file->print(" " x 254, "\n");
+  
+  # Tell Courier the message is delivered
+  $control_file->printf("I%d R %s\n", $recip->courier_recip_index,
+                        $recip->recip_smtp_response);
+  $control_file->printf("S%d %d\n", $recip->courier_recip_index, time);
+
+  $control_file->close or die "Error closing control file $filename: $!";
+}
+
+# add_recipient(recip)
+# Adds a recipient to the last control file for the message.
+#   address: recipient address to add
+#   orig_recip: RFC 3461 original recipient (if any)
+#   notify: reference to array containing values of the DSN NOTIFY value
+sub add_recipient($$$) {
+  my ($self, $address, $orig_recip, $notify) = @_;
+  
+  do_log(1, "Amavis::In::Courier: Adding recipient <%s>", $address);
+  
+  # Convert $notify array into character string
+  my $notify_str = join('', map { substr($_, 0, 1) } @$notify);
+  
+  # Open the last control file
+  my $filename = $self->{control_files}->[-1];
+  my $control_file = IO::File->new($filename, 'a');
+  # Take care with the control file: see comments in delete_recipient
+  seek($control_file, 0, 2);
+  $control_file->print(" " x 254, "\n");
+  
+  # Add recipient to control file
+  $control_file->print("r$address\n");
+  $control_file->print("R$orig_recip\n");
+  $control_file->print("N$notify_str\n");
+
+  $control_file->close or die "Error closing control file $filename: $!";
+}
 
 1;