uw_medicine_unmoderate.py

#!/usr/bin/python

"""Set moderator settings for UW Medicine HR Lists

This script takes a predefined csv file of mail lists and 
a seperate config file of important people and adjusts
settings accordingy

Usage: %(PROGRAM)s [options]

Options:
    -h / --help
        Print this message and exit.

    -l listname
    --listname=listname
        Process only the given list, otherwise do all lists.
"""

import sys
import time
import getopt
import os
import csv
import re
import pprint
import httplib
import json
from collections import defaultdict

sys.path
sys.path.append('/usr/lusers/mailman')

#from dateutil.parser import parse
import paths

# mm_cfg must be imported before the other modules, due to the side-effect of
# it hacking sys.paths to include site-packages.  Without this, running this
# script from cron with python -S will fail.
from Mailman import mm_cfg
from Mailman import Utils
from Mailman import MailList
from Mailman import Pending
from Mailman import MemberAdaptor
from Mailman import Errors
from Mailman.Bouncer import _BounceInfo
from Mailman.Logging.Syslog import syslog
from Mailman.i18n import _
from Mailman.Errors import MMUnknownListError

# Work around known problems with some RedHat cron daemons
import signal
signal.signal(signal.SIGCHLD, signal.SIG_DFL)


GWS_HOST  = 'groups.uw.edu'
GWS_PORT  = '443'
GWS_BASE = '/group_sws/v3'

USE_PROD = 1

EXTRAS = '?synchronized=true'

if USE_PROD != 1:
  GWS_HOST  = 'eval.groups.uw.edu'
  GWS_PORT  = '443'
  GWS_BASE = '/group_sws/v3'

GWS_KEY_FILE  = '/usr/lusers/mailman/uw_medicine/certs/uwmlists.key'
GWS_CERT_FILE = '/usr/lusers/mailman/uw_medicine/certs/uwmlists.cert'


# set if wanting to not update anything
#DEVMODE = True
DEVMODE = False

uwm_lists_file = "uwmc_check2"
uwm_lists_perms_file = "uw_medicine_lists_permissions_v6.csv"

email_regex = re.compile(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)")
email_sub = re.compile(r"(@uw.edu)")
list_regex = re.compile(r"(^(uwm|wwami_faculty|alliedhealth)_[a-z_-]+$)", re.I)
all_lists_regex = re.compile(r"(^all lists$)", re.I)
pp=pprint.PrettyPrinter( indent = 2 )

PROGRAM = sys.argv[0]

LISTPERMS = {}
ALLPERMS = []

_gws_connection = None
GWS_ADMINS = [ {"type": "group", "id": "u_uwmlists_group-admins"} ]
GWS_HEAD = { "Content-type": "application/json", "Accept": "application/json" }
GWS_UPDATE_HEAD = { "Content-type": "application/json", "Accept": "application/json", "If-Match": "*" }

def is_grouplist(listname):
  enrolled = ''.join(["/mm/lists/", listname, "/enrolled"])
  try:
    os.stat(enrolled)
  except:
    return False
  return True

def is_mailman_folder(listname):
  config = ''.join(["/mm/lists/", listname, "/config.pck"])
  try:
    os.stat(config)
  except:
    return False
  return True

def usage(code, msg=''):
  if code:
    fd = sys.stderr
  else:
    fd = sys.stdout
  print >> fd, _(__doc__)
  if msg:
    print >> fd, msg
  sys.exit(code)

def get_unmoderated_members(mlist):
  # Get the lowercased member addresses
  rmembers = mlist.getRegularMemberKeys()
  dmembers = mlist.getDigestMemberKeys()
  unmod_members = []
  mod_members = []

  all = rmembers + dmembers
  all.sort()
  for addr in all:
    if not mlist.getMemberOption(addr, mm_cfg.Moderate):
      unmod_members.append(addr)
    else:
      mod_members.append(addr)
  #print "get_unmoderated_members() has %s" % unmod_members
  return list(unmod_members), list(mod_members)

def unmoderate_user(addr, mlist):
  try:
    if DEVMODE:
      print "Would have called mlist.setMemberOption(%s,mm_cfg.Moderate, False)" % addr
    else:
      mlist.setMemberOption(addr,mm_cfg.Moderate, False)
  except NotAMemberError:
    return False
  return True

def moderate_user(addr, mlist):
  try:
    if DEVMODE:
      print "Would have called mlist.setMemberOption(%s,mm_cfg.Moderate, True)" % addr
    else:
      mlist.setMemberOption(addr,mm_cfg.Moderate, True)
  except NotAMemberError:
    return False
  return True

def read_listnames_file():
  listnames = []
  with open(uwm_lists_file, 'r') as source_list_file:
    reader = csv.reader(source_list_file)
    for row in reader:
      listnames.append(row[0])
  return listnames

#Noting this hhhis highly dependant on the patterns used by 
# UW Medicine Communications in their DraftApprovedSenders_v2.xlsx file
def load_permissions_file():
  with open(uwm_lists_perms_file, 'r') as source_list_file:
    reader = csv.reader(source_list_file)

    # processes each row in the csv file separately
    for row in reader:
      emails = []
      lists = []
      all_lists = False
      if DEVMODE:
          print row

      # process each field on this row
      for field in row:
        field = field.strip()
        field = field.lower()
        # if this is an email address add it to the list of emails found on that row
        if email_regex.match( field ):
          emails.append( email_sub.sub('@uw.edu', field ) )
        # if this looks like uwm_ then add it as a list
        if list_regex.match( field ):
          #print "matched list_regex for "+field
          lists.append( re.sub(r'-',r'_',field) )
        # if we found "All Lists" in a field that row is a full permissions row
        if all_lists_regex.match( field ):
          all_lists = True
      # start updating the list of permissions
      # if row is flagged as "All Lists" then emails go there
      if all_lists == True:
        if DEVMODE:
          print " Matched All-Lists:",str(row)
        for addr in emails:
          try:
            ALLPERMS.index(addr)
          except ValueError:
            ALLPERMS.append(addr)
      # if row doesn'thave "All Lists" then put emsils on the list of lists that "follow"
      else:
        if DEVMODE:
          print " emails: " + ','.join(emails)
          print " lists: " + ','.join(lists)
        for uwm in lists:
          if DEVMODE:
            print " CHECKING LIST %s with addresses: %s" % (uwm, str(emails))
          if uwm in LISTPERMS:
            if DEVMODE:
              print " found %s in LISTPERMS for %s\n   adding %s" % (str(LISTPERMS[uwm]), uwm, str(emails))
            for addr in emails:
              if addr not in LISTPERMS[uwm]:
                LISTPERMS[ uwm ].append(addr)
          # if the uwm dict object doesn't exist yet add the list to it.  This is where I made may error..
          else:
            LISTPERMS[uwm] = list(emails)
  return True

def unmod_list(permissioned, mlist):
    list_real_name = mlist.real_name
    for person in permissioned:
       if mlist.isMember(person):
           if DEVMODE:
               print "would have unset moderation flag for %s on %s" % (person, list_real_name)
           else:
               unmoderate_user(person,mlist)
       #else:
       #    if DEVMODE:
       #        print "%s not a member of list %s" % (person, list_real_name)
    return True

def remod_list(unauthorized, mlist):
    list_real_name = mlist.real_name
    for person in unauthorized:
       if mlist.isMember(person):
           if DEVMODE:
               print "would have set moderation flag for %s on %s" % (person, list_real_name)
           else:
               moderate_user(person,mlist)
       #else:
       #    if DEVMODE:
       #        print "%s not a member of list %s" % (person, list_real_name)
    return True

def intersection(list1, list2):
    return list(set(list1) & set(list2))

#remove list2 items from list1
def filter_set (list1, list2):
    list_to_filter = list1[:]
    items_to_filter = list2[:]
    for item in items_to_filter:
        if item in list_to_filter:
            list_to_filter.remove(item)
    return list(list_to_filter)

def create_reader_group(_gws_connection, listname):
  groupname = 'u_uwmlists_' + re.sub(r'_', r'-',listname)
  readergroupname = 'u_uwmlists_' + re.sub(r'_', r'-',listname) + '_approvedsenders'
  request = GWS_BASE + '/group/'+readergroupname
  METHOD = 'PUT'

  #create group or try to
  DATA = {"data": {
    "id": readergroupname,
    "displayName": "Readers of "+groupname,
    "description": "Individuals who can read group "+groupname +" and send email to "+listname+"@uw.edu",
    "contact": "uwmlists",
    "readers": [
       { "type": "group", "id": "u_uwmlists_group-readers"},
       { "type": "group", "id": "u_uwmlists_approved-senders-all-lists"},
       { "type": "group", "id": readergroupname }
    ],
    "admins": GWS_ADMINS } }
  _gws_connection.request(METHOD, request, json.dumps(DATA), headers=GWS_HEAD)
  response1 = _gws_connection.getresponse()
  code = response1.status
  body = response1.read()
  print "PUT #1 group code status: "+str(code)

  #update parent group with this new group name
  DATAPARENT = { "data": {
    "id": groupname,
    "admins": GWS_ADMINS,
    "readers": [
       { "type": "group", "id": "u_uwmlists_group-readers"},
       { "type": "group", "id": readergroupname }
    ]
  } }
  _gws_connection.request(METHOD, request, json.dumps(DATAPARENT), headers=GWS_UPDATE_HEAD)
  response2 = _gws_connection.getresponse()
  code = response2.status
  body = response2.read()
  #pp.pprint(body)
  print "PUT #2 parent group code status: "+str(code)

  #update reader group so this credential isn't explicitly added
  _gws_connection.request(METHOD, request, json.dumps(DATA), headers=GWS_UPDATE_HEAD)
  response3 = _gws_connection.getresponse()
  code = response3.status
  body = response3.read()
  print "PUT #3 group code status: "+str(code)

def update_reader_group(_gws_connection, listname, specific_approved_senders):
  REQUEST = GWS_BASE + '/group/u_uwmlists_' + re.sub(r'_', r'-',listname) + '_approvedsenders'
  METHOD = 'GET'
  print "Request: " +REQUEST
  _gws_connection.request(METHOD, REQUEST)
  response1 = _gws_connection.getresponse()
  code = response1.status
  body = response1.read()
  print 'GWS member response 1: ' + str(code)

  if(code == 404):
    create_reader_group(_gws_connection, listname)
  #elif DEVMODE:
  #  create_reader_group(_gws_connection, listname)

  METHOD = 'PUT'
  members = []
  if specific_approved_senders:
    for person in specific_approved_senders:
      if re.search('@uw.edu',person):
        netid = person.split('@')[0]
        members.append( { "type": "uwnetid", "id": netid } )
      else:
        continue
  members.append( { "type": "group", "id": "u_uwmlists_approved-senders-all-lists" } )
  DATA = { "data": members }
  _gws_connection.request(METHOD, REQUEST + "/member", json.dumps(DATA), headers=GWS_HEAD)
  response2 = _gws_connection.getresponse()
  code = response2.status
  body = response2.read()

  print 'GWS member response 2: ' + str(code)

def update_all_lists_group(_gws_connection, people_permitted_for_all):
  REQUEST = GWS_BASE + '/group/u_uwmlists_approved-senders-all-lists'
  METHOD = 'PUT'
  members = []
  if people_permitted_for_all:
    for person in people_permitted_for_all:
      if re.search('@uw.edu',person):
        netid = person.split('@')[0]
        members.append( { "type": "uwnetid", "id": netid } )
  else:
    print "Permissions list for u_uwmlists_approved-senders-all-lists is empty"
    return False
  DATA = { "data": members }
  _gws_connection.request(METHOD, REQUEST + "/member", json.dumps(DATA), headers=GWS_HEAD)
  response2 = _gws_connection.getresponse()
  code = response2.status
  body = response2.read()

  print 'GWS all list response: ' + str(code) +'\n'


def main():
  #process arguments
  try:
    opts, args = getopt.getopt(
      sys.argv[1:], 'hl:omubaf',
      ['listname=', 'help'])
  except getopt.error, msg:
    usage(1, msg)
  try:
    _gws_connection = httplib.HTTPSConnection(GWS_HOST, GWS_PORT, GWS_KEY_FILE, GWS_CERT_FILE)
  except:
    print "unable to connect to groups service"

  if args:
    usage(1)
  listnames = []
  for opt, arg in opts:
    if opt in ('-h', '--help'):
      usage(0)
    elif opt in ('-l', '--list'):
      listnames.append(arg)
  if not listnames:
   listnames = read_listnames_file()

  #load permissions file
  load_permissions_file()
  if DEVMODE:
    print"\nLIST PERMISSONS"
    pp.pprint( LISTPERMS )
    print "\nALL PERMISSONS"
    print str(ALLPERMS)

  #update all lists group
  update_all_lists_group(_gws_connection, ALLPERMS)

  #begin processing mail lists
  for listname in listnames:
    permissions = []
    unmod_members = []
    unmod_notmembers = []
    if(is_mailman_folder(listname) == False):
      print "%s Not a list on this mailman server" % listname
      continue
    try:
      mlist = MailList.MailList(listname)
    except MMUnknownListError:
      print "Couldn't find list: "+listname
      continue

    #set permissions
    try:
      print "%s list:" % listname
      # set permissions afresh
      permissions = list(ALLPERMS)

      # overlay list specific people
      if listname in LISTPERMS:
        permissions = permissions + LISTPERMS.get(listname)
      #update UW Group if list has people in permissions file
      update_reader_group(_gws_connection, listname, LISTPERMS.get(listname))

      #get list of people who are unmoderated and unmoderated
      unmod_members, mod_members = get_unmoderated_members(mlist)
      if DEVMODE:
        print "unmod members\n" + str(unmod_members)
        #print "mod members\n" + str(mod_members) #some lists are very large

      # get list of people who need to be updated, then set those
      should_be_authorized = intersection(permissions, mod_members)
      if should_be_authorized:
        print "  moderated members who are approved:\n    " + '\n    '.join(should_be_authorized)
        unmod_list(should_be_authorized, mlist)

      # get list of people who have access but shouldn't 
      unauthorized = filter_set(unmod_members, permissions)
      if unauthorized:
        print "  unmoderated members who are not approved:\n    " + '\n    '.join(unauthorized)
        remod_list(unauthorized, mlist)

      # we can set accept these non-members in one call so here it is
      if DEVMODE:
        print "would have set accept_these_nonmembers on %s\n  %s" % (listname, ', '.join(permissions))
      else:
        mlist.accept_these_nonmembers = permissions

    #cleanup step
    finally:
      # if you don't do mlist.Save() the mlist.accept_these_nonmembers 
      #  setting doesn't get saved
      if not DEVMODE:
        mlist.Save()
      mlist.Unlock()
  _gws_connection.close()

if __name__ == '__main__':
    main()