gistfile1.txt

bluecmd@provision-rojter:~$ sudo puppet agent -t
Warning: Facter: Could not process routing table entry: Expected a destination followed by key/value pairs, got 'default via 2a05:2242:920::1 dev ens192 metric 1024 onlink pref medium'
Warning: Downgrading to PSON for future requests
Info: Using configured environment 'production'
Info: Retrieving pluginfacts
Info: Retrieving plugin
Info: Loading facts
Warning: Facter: Could not process routing table entry: Expected a destination followed by key/value pairs, got 'default via 2a05:2242:920::1 dev ens192 metric 1024 onlink pref medium'
Info: Caching catalog for provision-rojter.event.dreamhack.se
Info: Applying configuration version '1540149772'
Notice: /Stage[main]/Provision/File[bin:provisiond]/content:
--- /usr/local/bin/provisiond   2018-09-20 17:20:08.998637942 +0200
+++ /tmp/puppet-file20181021-1407-2ofc6g        2018-10-21 23:17:40.955652545 +0200
@@ -6,7 +6,6 @@

 import json
 import hvac
-import logging
 import netsnmp
 import os
 import pysphere
@@ -16,15 +15,13 @@
 import sqlite3
 import subprocess
 import sys
+import syslog
 import tempfile
 import threading
 import time
 import traceback
 import yaml

-import pyghmi.ipmi.command
-import pyghmi.exceptions
-
 from provision import c7000
 from provision import esxi
 from provision import snmp
@@ -78,16 +75,14 @@
     self.manager = config['domain']
     self.vault = vault
     self.redis = redis
-    self.host = config.get('host', None)
-    self.fqdn = config.get('fqdn', None)
+    self.host = config['host']
+    self.fqdn = config['fqdn']
     self.username = config['username']
     self.password = config['password']
     self.thread = threading.Thread(target=self.run)
     self.thread.daemon = True

   def is_alive(self):
-    if self.host is None:
-      return True
     with open(os.devnull, 'w') as null:
       return subprocess.call(
           ['ping', '-c' , '1', self.host], stdout=null) == 0
@@ -107,13 +102,16 @@
   def run(self):
     while True:
       if not self.is_alive():
-        logging.info('Skipping host %s, it is not alive', self.host)
+        syslog.syslog(syslog.LOG_INFO,
+                      'Skipping host %s, it is not alive' % self.host)
         continue

       try:
         self.execute()
       except:
-        logging.exception('Exception while executing')
+        syslog.syslog(syslog.LOG_ERR,
+                      'Uncaught exception: %s' % traceback.format_exc())
+        print 'Uncaught exception: %s' % traceback.format_exc()
       time.sleep(RUN_INTERVAL)

   def execute(self):
@@ -121,14 +119,15 @@


 class Esxi(Backend):
-  def __init__(self, config, vault, redis):
-    super(Esxi, self).__init__(config, vault, redis)
+  def __init__(self, config):
+    super(Esxi, self).__init__(config)
     self.deploy_vlan = config['deploy-vlan']
     self.esxi_cache = None

   def vcenter_deploy(self, host):
     # vCenter uses its own ISO with deploy appliance, use that instead
-    logging.info('Preparing vCenter installation VM %s', host['name'])
+    syslog.syslog(syslog.LOG_INFO,
+                  'Preparing vCenter installation VM %s' % host['name'])

     ip = host['ipv4']['address']
     prefix = str(host['ipv4']['prefix'])
@@ -143,7 +142,8 @@
       self.server, host['name'], vlan, ip, prefix, gateway, password,
       host['datastore'], domain)

-    logging.info('Starting vCenter installation VM %s', host['name'])
+    syslog.syslog(syslog.LOG_INFO,
+                  'Starting vCenter installation VM %s' % host['name'])
     mount = tempfile.mkdtemp()
     subprocess.check_call([
       '/bin/mount', '-o', 'loop', os.environ['VMWARE_VCENTER_ISO'], mount])
@@ -161,9 +161,11 @@
               host['name'], username='administrator@%s' % domain,
               password=password)
           vm = self.server.get_vm_by_name(host['name'])
-          logging.info('Created new vCenter VM %s', host['name'])
+          syslog.syslog(syslog.LOG_INFO,
+                        'Created new vCenter VM %s' % host['name'])
       except:
-        logging.error('Failed to create vCenter VM %s', host['name'])
+        syslog.syslog(syslog.LOG_ERR,
+                      'Failed to create vCenter VM %s' % host['name'])
       finally:
         subprocess.check_call([ '/bin/umount', mount])
     # Start this in a seperate thread as it will be running for long
@@ -188,12 +190,13 @@
         cluster = esxi.get_or_create_cluster(self.server, datacenter,
             DEFAULT_CLUSTER)
         esxi.add_esxi_to_vcenter(self.server, target, cluster)
-        logging.info('Added ESXi %s to vCenter %s',
-                request['name'], self.host)
+        syslog.syslog(syslog.LOG_INFO,
+                      'Added ESXi %s to vCenter %s' % (
+                        request['name'], self.host))
       elif request['operation'] == 'add-host-to-dvs':
-        esxi.add_host_to_dvs(self.server, esxi.get_server_fqdn(target),
+        esxi.add_host_to_dvs(self.server, esxi.get_server_ip(target),
             datacenter, DEFAULT_DVS, request['interface'])
-        logging.info('Added host %s to DVS', request['name'])
+        syslog.syslog(syslog.LOG_INFO, 'Added host %s to DVS' % request['name'])
       else:
         # Unknown option
         pass
@@ -214,18 +217,20 @@
       # Verify that the VM doesn't exist already
       known_names = [name for (name, _) in self.esxi_cache.values()]
       if host['name'] in known_names:
-        logging.error('Tried to create already existing VM %s', host['name'])
+        syslog.syslog(syslog.LOG_ERR,
+                      'Tried to create already existing VM %s' % host['name'])
       elif host['os'] == 'vcenter':
         self.vcenter_deploy(host)
       else:
-        logging.info('Creating new VM using configuration %s',
-                ', '.join(k + '=' + str(v) for k, v in host.items()))
+        syslog.syslog(syslog.LOG_INFO,
+                  'Creating new VM using configuration %s' % (
+                      ', '.join(k + '=' + str(v) for k, v in host.items())))
         vm = esxi.create_vm(
             self.server, host['name'], self.deploy_vlan, host['datastore'],
             disk_size=host['disk'], num_cpus=host['cpus'],
             memory=host['memory'], os=host['os'])
         esxi.power_on(self.server, vm)
-        logging.info('Created new VM %s', host['name'])
+        syslog.syslog(syslog.LOG_INFO, 'Created new VM %s' % host['name'])
       # Delete request since we're done.
       # If anything above failed, we will retry
       self.redis.delete(key)
@@ -248,17 +253,8 @@
         vm = self.server.get_vm_by_path(path)
         name = vm.properties.name
         uuid = vm.properties.config.uuid.lower()
-        # vmx-12 and forward has problem with endian conv
-        # this reverses in step of 2 the first 3 strings in the uuid
-        # AABBCCDD -> DDCCBBAA
-        luuid = uuid.split('-')
-        l = []
-        for u in luuid[0:3]:
-          l.append(''.join([u[i:i + 2] for i in range(0, len(u), 2)][::-1]))
-        uuid = '{0}-{1}'.format('-'.join(l),'-'.join(luuid[3:]))
-
         self.esxi_cache[path] = (name, uuid)
-        logging.info('Found new VM %s', path)
+        syslog.syslog(syslog.LOG_INFO, 'Found new VM %s' % path)
       else:
         name, uuid = self.esxi_cache[path]
       visited_keys.add(path)
@@ -268,7 +264,7 @@

     removed_keys = set(self.esxi_cache.keys()) - visited_keys
     for key in removed_keys:
-      logging.info('Forgot VM %s', key)
+      syslog.syslog(syslog.LOG_INFO, 'Forgot VM %s' % key)
       del self.esxi_cache[key]

   def provision(self):
@@ -297,18 +293,21 @@

       # If we have no network configuration for the VM, we cannot configure
       if not host['network']:
-        logging.error('VM %s lacking network config', path)
+        syslog.syslog(syslog.LOG_ERR, 'VM %s lacking network config' % path)
         continue
       vlan = host['network']['vlan']

       try:
         esxi.provision_vm(self.server, vm, vlan)
-        logging.info('Provisioned VLAN %d on VM %s', vlan, name)
+        syslog.syslog(syslog.LOG_INFO,
+                      'Provisioned VLAN %d on VM %s' % (vlan, name))
       except esxi.Error, e:
-        logging.error('Failed to provision VM %s: %s', name, e.message)
+        syslog.syslog(syslog.LOG_ERR, 'Failed to provision VM %s: %s' % (
+          name, e.message))

   def setup_vcenter(self):
-    logging.info('Discovered new vCenter %s, setting up', self.host)
+    syslog.syslog(syslog.LOG_INFO, 'Discovered new vCenter %s, setting up' % (
+      self.host))
     datacenter = esxi.get_or_create_datacenter(self.server, DEFAULT_DATACENTER)
     esxi.get_or_create_cluster(self.server, datacenter, DEFAULT_CLUSTER)
     esxi.create_dvswitch(self.server, datacenter, DEFAULT_DVS)
@@ -318,7 +317,7 @@
       name = '%s: %s' % (vlan, network.split('@', 1)[1])
       vlan_map[name] = vlan
     esxi.create_dvs_portgroup(self.server, datacenter, DEFAULT_DVS, vlan_map)
-    logging.info('Setup for vCenter %s done', self.host)
+    syslog.syslog(syslog.LOG_INFO, 'Setup for vCenter %s done' % self.host)

   def execute(self):
     self.server = pysphere.VIServer()
@@ -338,8 +337,8 @@


 class C7000(Backend):
-  def __init__(self, config, vault, redis):
-    super(C7000, self).__init__(config, vault, redis)
+  def __init__(self, config):
+    super(C7000, self).__init__(config)
     self.community = config['community']

   def execute(self):
@@ -382,8 +381,9 @@
         power_type = c7000.MOMENTARY_PRESS
       else:
         # Unknown state, fail
-        logging.error('C7000 bay %d has unknown power-state %s',
-                bay, power_state)
+        syslog.syslog(syslog.LOG_ERR,
+                      'C7000 bay %d has unknown power-state %s' % (
+                          bay, power_state))
         continue
       session = c7000.login(self.host, self.username, self.password)
       c7000.setup_boot_order(session, install['bay'])
@@ -394,42 +394,6 @@
       self.redis.setex('install-' + sn, 3600, json.dumps(install))


-class OCP(Backend):
-  def __init__(self, config, vault, redis):
-    super(OCP, self).__init__(config, vault, redis)
-    self.machines = config['machines']
-
-  def execute(self):
-    bays = {}
-    for name, entry in self.machines.iteritems():
-      # Using mac as serial to be able to re-use deploy-bay
-      mac = entry['mac']
-      bays[name] = {'mac': mac, 'serial': mac, 'ip': entry['ip']}
-    self.redis.setex('bays-' + self.manager, 600, json.dumps(bays))
-
-    # Provision machines
-    for bay_id, bay in bays.iteritems():
-      install = self.redis.get('install-' + bay['mac'])
-      if not install:
-        continue
-
-      install = json.loads(install)
-      if install.get('initialized', True):
-        continue
-
-      try:
-        impi = pyghmi.ipmi.command.Command(
-                bay['ip'], self.username, self.password)
-        impi.set_bootdev('network', uefiboot=True)
-        impi.set_power('boot')
-      except pyghmi.exceptions.IpmiException:
-        logging.exception('OCP failed to IPMI node %s', bay_id)
-        continue
-
-      install['initialized'] = True
-      self.redis.setex('install-' + bay['mac'], 3600, json.dumps(install))
-
-
 def new_vault_client(**kwargs):
   cert = os.environ.get('VAULT_CERT', None)
   key = os.environ.get('VAULT_KEY', None)
@@ -439,7 +403,6 @@


 if __name__ == '__main__':
-  logging.basicConfig(level=logging.INFO)
   # SSL verification hack until pySphere get their stuff together
   # https://www.python.org/dev/peps/pep-0476/#opting-out
   import ssl
@@ -464,7 +427,5 @@
     Esxi(c, vault, redis).start()
   for c in config.get('c7000', None) or []:
     C7000(c, vault, redis).start()
-  for c in config.get('ocp', None) or []:
-    OCP(c, vault, redis).start()
   while True:
     signal.pause()

Info: Computing checksum on file /usr/local/bin/provisiond
Info: /Stage[main]/Provision/File[bin:provisiond]: Filebucketed /usr/local/bin/provisiond to puppet with sum 772ce6dcfb7a605fb6b01646dc044d28
Notice: /Stage[main]/Provision/File[bin:provisiond]/content: content changed '{md5}772ce6dcfb7a605fb6b01646dc044d28' to '{md5}8eb3c019f0283ed2f4feb0dfc4943916'
Info: /Stage[main]/Provision/File[bin:provisiond]: Scheduling refresh of Supervisor::Restart[provisiond]
Notice: /Stage[main]/Provision/File[/usr/local/lib/python2.7/dist-packages/provision/esxi.py]/content:
--- /usr/local/lib/python2.7/dist-packages/provision/esxi.py    2018-09-20 17:20:09.222641654 +0200
+++ /tmp/puppet-file20181021-1407-eoyxd3        2018-10-21 23:17:41.307658400 +0200
@@ -6,14 +6,14 @@
 import collections
 import hashlib
 import json
-import logging
 import pysphere
+import syslog
 from pysphere.resources import VimService_services as VI

 # WARNING(2014-10-18): If you set this higher than 8 the
 # vSphere client will not allow you to modify the VMs
-ESXI_HW_VERSION = 'vmx-13'
-VCENTER_HW_VERSION = 'vmx-13'
+ESXI_HW_VERSION = 'vmx-09'
+VCENTER_HW_VERSION = 'vmx-09'

 # This value can be overridden by setting an 'os' option in ipplan.
 DEFAULT_OS = 'debian'
@@ -29,7 +29,7 @@

 # Map OS -> Hardware configurations
 SYSTEM_CONFIGURATION_MAP = {
-  'debian': SysConf(osid='debian10_64Guest', scsi='paravirtual'),
+  'debian': SysConf(osid='debian6_64Guest', scsi='paravirtual'),
   'ubuntu': SysConf(osid='ubuntu64Guest', scsi='paravirtual'),
   'openbsd': SysConf(osid='otherGuest64', scsi='lsi_sas'),
   'coreos': SysConf(osid='otherGuest64', scsi='paravirtual')
@@ -229,16 +229,6 @@
   return prop.config.network.vnic[0].spec.ip.ipAddress


-def get_server_fqdn(server):
-  """Extract server's configured fqdn."""
-  host = next(server.get_hosts().iterkeys())
-  prop = pysphere.VIProperty(server, host)
-  # Prop is a "HostSystem"
-  # http://pubs.vmware.com/vsphere-60/index.jsp#com.vmware.wssdk.apiref.doc/vim.HostSystem.html
-  dnsconfig = prop.config.network.dnsConfig
-  return dnsconfig.hostName + '.' + dnsconfig.domainName
-
-
 def provision_vm(server, vm, vlan):

   # Set VMs first NIC to the correct label
@@ -280,8 +270,8 @@
   for d in target_config.Datastore:
     if not d.Datastore.Accessible:
       continue
-    logging.info('Considering datastore %s with %s free',
-            d.Datastore.Name, d.Datastore.FreeSpace)
+    syslog.syslog(syslog.LOG_INFO, 'Considering datastore %s with %s free' % (
+        d.Datastore.Name, d.Datastore.FreeSpace))
     datastores[d.Datastore.Name] = (
             d.Datastore.Datastore, d.Datastore.FreeSpace)

@@ -289,11 +279,13 @@
     if datastore not in datastores:
       raise DatastoreNotFoundError('Datastore %s does not appear to exist' %
                datastore)
-    logging.info('Selected datastore %s (user provided)', datastore)
+    syslog.syslog(syslog.LOG_INFO, 'Selected datastore %s (user provided)' % (
+        datastore))
   else:
     # Use the datastore with most free space
     datastore = max(datastores.iteritems(), key=lambda x: int(x[1][1]))[0]
-    logging.info('Selected datastore %s (max free)', datastore)
+    syslog.syslog(syslog.LOG_INFO, 'Selected datastore %s (max free)' % (
+        datastore))

   ds, _ = datastores[datastore]
   return '[%s]' % datastore if brackets else datastore, ds
@@ -493,7 +485,7 @@
   request.AsConnected = True
   spec = request.new_spec()
   spec.Force = True
-  spec.HostName = get_server_fqdn(esxi)
+  spec.HostName = get_server_ip(esxi)
   spec.UserName = esxi._VIServer__user
   spec.Password = esxi._VIServer__password
   spec.SslThumbprint = get_ssl_thumbprint(esxi)
@@ -590,8 +582,6 @@

   specs = []
   for name, vlan_id in name_vlan_map.iteritems():
-    if vlan_id is None:
-      continue
     spec = request.new_spec()
     spec.Name = name
     spec.Type = 'earlyBinding'

Info: Computing checksum on file /usr/local/lib/python2.7/dist-packages/provision/esxi.py
Info: /Stage[main]/Provision/File[/usr/local/lib/python2.7/dist-packages/provision/esxi.py]: Filebucketed /usr/local/lib/python2.7/dist-packages/provision/esxi.py to puppet with sum 833b9ebefc874da02d4c7a04644c0ba3
Notice: /Stage[main]/Provision/File[/usr/local/lib/python2.7/dist-packages/provision/esxi.py]/content: content changed '{md5}833b9ebefc874da02d4c7a04644c0ba3' to '{md5}ddc1892e5d1ff8c1ec09deeff84fa158'
Info: lib:provisiond: Scheduling refresh of Supervisor::Restart[provisiond]
Info: Supervisor::Restart[provisiond]: Scheduling refresh of Exec[supervisorctl_restart_provisiond]
Notice: /Stage[main]/Provision/Supervisor::Register[provisiond]/Supervisor::Restart[provisiond]/Exec[supervisorctl_restart_provisiond]: Triggered 'refresh' from 1 event
Notice: Applied catalog in 63.57 seconds