puppet agent run failing after regenerating all certificates on monolithic master PE 2016.2

gistfile1.txt

root@master:~ # puppet agent -t
Warning: Unable to fetch my node definition, but the agent run will continue:
Warning: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [unable to get local issuer certif
icate for /CN=master.puppetlabs.vm]
Info: Retrieving pluginfacts
Error: /File[/opt/puppetlabs/puppet/cache/facts.d]: Failed to generate additional resources using 'eval_generate': SSL_connect returned=1 errno=0 state=error: certificate verify failed: [unable to get local issuer certificate fo
r /CN=master.puppetlabs.vm]
Error: /File[/opt/puppetlabs/puppet/cache/facts.d]: Could not evaluate: Could not retrieve file metadata for puppe
t:///pluginfacts: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [unable to get local issu
er certificate for /CN=master.puppetlabs.vm]
Info: Retrieving plugin
Error: /File[/opt/puppetlabs/puppet/cache/lib]: Failed to generate additional resources using 'eval_generate': SSL
_connect returned=1 errno=0 state=error: certificate verify failed: [unable to get local issuer certificate for /C
N=master.puppetlabs.vm]
Error: /File[/opt/puppetlabs/puppet/cache/lib]: Could not evaluate: Could not retrieve file metadata for puppet://
/plugins: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [unable to get local issuer certi
ficate for /CN=master.puppetlabs.vm]
Info: Loading facts
Error: Could not retrieve catalog from remote server: SSL_connect returned=1 errno=0 state=error: certificate veri
fy failed: [unable to get local issuer certificate for /CN=master.puppetlabs.vm]
Warning: Not using cache on failed catalog
Error: Could not retrieve catalog; skipping run
Error: Could not send report: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [unable to ge
t local issuer certificate for /CN=master.puppetlabs.vm]

tmux output

root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 # verify_classroom.sh
Version: 4.1

Verifying Puppet Labs Training classroom setup:

Checking hostname                                                                                     [  OK  ]
Checking local hostname resolution                                                                    [  OK  ]
Checking core count for classroom Master                                                              [  OK  ]
Checking available memory for classroom Master                                                        [  OK  ]
Verifying that the default password has been changed                                                  [  OK  ]
Attempting to synchronize time...                                                                     [ FAIL ]
  > Network time server unavailable. You should run class in offline mode
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 #
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 #
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 # ps aufx | grep ntp
root     19768  0.0  0.0 112644   956 pts/0    S+   11:07   0:00          \_ grep --color=auto ntp
ntp      18626  0.0  0.0  33648  2160 ?        Ss   10:56   0:00 /usr/sbin/ntpd -u ntp:ntp -g
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 # date
Tue Jun 28 11:07:56 UTC 2016
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 #
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 #
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 # vi /etc/ntp
ntp/      ntp.conf
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 # vi /etc/ntp
ntp/      ntp.conf
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 # vi /etc/ntp.conf
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 # host ntp.det.nsw.edu.au
ntp.det.nsw.edu.au has address 153.107.101.53
ntp.det.nsw.edu.au has address 153.107.37.53
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 # ps aufx | grep ntp^C
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 # puppet agent -t
Info: Using configured environment 'production'
Info: Retrieving pluginfacts
Info: Retrieving plugin
Info: Loading facts
Error: Could not retrieve catalog from remote server: Error 400 on SERVER: Evaluation Error: Error while evaluating a Resource Statement, Evaluation Error: Error while evaluating a Function Call, "ntp.det.nsw.edu.au" is not an Array.  It looks to be a String at /etc/puppetlabs/code/modules/classroom/manifests/init.pp:35:3 on node master.puppetlabs.vm
Warning: Not using cache on failed catalog
Error: Could not retrieve catalog; skipping run
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 # puppet agent -t
Notice: Run of Puppet configuration client already in progress; skipping  (/opt/puppetlabs/puppet/cache/state/agent_catalog_run.lock exists)
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 # puppet agent -t
Warning: Unable to fetch my node definition, but the agent run will continue:
Warning: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [CRL is not yet valid for /CN=Puppet Enterprise CA]
Info: Retrieving pluginfacts
Error: /File[/opt/puppetlabs/puppet/cache/facts.d]: Failed to generate additional resources using 'eval_generate': SSL_connect returned=1 errno=0 state=error: certificate verify failed: [CRL is not yet valid for /CN=Puppet Enterprise CA]
Error: /File[/opt/puppetlabs/puppet/cache/facts.d]: Could not evaluate: Could not retrieve file metadata for puppet:///pluginfacts: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [CRL is not yet valid for /CN=Puppet Enterprise CA]
Info: Retrieving plugin
Error: /File[/opt/puppetlabs/puppet/cache/lib]: Failed to generate additional resources using 'eval_generate': SSL_connect returned=1 errno=0 state=error: certificate verify failed: [CRL is not yet valid for /CN=Puppet Enterprise CA]
Error: /File[/opt/puppetlabs/puppet/cache/lib]: Could not evaluate: Could not retrieve file metadata for puppet:///plugins: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [CRL is not yet valid for /CN=Puppet Enterprise CA]
Info: Loading facts
Error: Could not retrieve catalog from remote server: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [CRL is not yet valid for /CN=Puppet Enterprise CA]
Warning: Not using cache on failed catalog
Error: Could not retrieve catalog; skipping run
Error: Could not send report: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [CRL is not yet valid for /CN=Puppet Enterprise CA]
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 #
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 #
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 # puppet agent -t
Warning: Unable to fetch my node definition, but the agent run will continue:
Warning: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [CRL is not yet valid for /CN=Puppet Enterprise CA]
Info: Retrieving pluginfacts
Error: /File[/opt/puppetlabs/puppet/cache/facts.d]: Failed to generate additional resources using 'eval_generate': SSL_connect returned=1 errno=0 state=error: certificate verify failed: [CRL is not yet valid for /CN=Puppet Enterprise CA]
Error: /File[/opt/puppetlabs/puppet/cache/facts.d]: Could not evaluate: Could not retrieve file metadata for puppet:///pluginfacts: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [CRL is not yet valid for /CN=Puppet Enterprise CA]
Info: Retrieving plugin
Error: /File[/opt/puppetlabs/puppet/cache/lib]: Failed to generate additional resources using 'eval_generate': SSL_connect returned=1 errno=0 state=error: certificate verify failed: [CRL is not yet valid for /CN=Puppet Enterprise CA]
Error: /File[/opt/puppetlabs/puppet/cache/lib]: Could not evaluate: Could not retrieve file metadata for puppet:///plugins: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [CRL is not yet valid for /CN=Puppet Enterprise CA]
Info: Loading facts
Error: Could not retrieve catalog from remote server: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [CRL is not yet valid for /CN=Puppet Enterprise CA]
Warning: Not using cache on failed catalog
Error: Could not retrieve catalog; skipping run
Error: Could not send report: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [CRL is not yet valid for /CN=Puppet Enterprise CA]
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 # date
Tue Jun 28 01:11:59 UTC 2016
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 #
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 #
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 # sudo puppet resource service puppetserver ensure=stopped
sudo: puppet: command not found
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 # puppet resource service puppetserver ensure=stopped
service { 'puppetserver':
  ensure => 'stopped',
}
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 #
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 #
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 #
root@master:~/puppet-enterprise-2016.2.0-el-7-x86_64 # cd /etc/puppetlabs/puppet
root@master:/etc/puppetlabs/puppet # ls
auth.conf      classifier.yaml  hiera.yaml   puppetdb.conf  ssl
autosign.conf  fileserver.conf  puppet.conf  routes.yaml
root@master:/etc/puppetlabs/puppet # ls -l ssl
total 28
drwxr-xr-x 5 pe-puppet pe-puppet 4096 Jun 28  2016 ca
drwxr-xr-x 2 pe-puppet pe-puppet 4096 Jun 28  2016 certificate_requests
drwxr-xr-x 2 pe-puppet pe-puppet 4096 Jun 28  2016 certs
-rw-r--r-- 1 pe-puppet pe-puppet  954 Jun 28  2016 crl.pem
drwxr-x--- 2 pe-puppet pe-puppet 4096 Apr  8 17:15 private
drwxr-x--- 2 pe-puppet pe-puppet 4096 Jun 28  2016 private_keys
drwxr-xr-x 2 pe-puppet pe-puppet 4096 Jun 28  2016 public_keys
root@master:/etc/puppetlabs/puppet # mv ssl /tmp
root@master:/etc/puppetlabs/puppet # puppet cert list -a
Notice: Signed certificate request for ca
root@master:/etc/puppetlabs/puppet # puppet master --no-daemonize --verbose
Info: Caching certificate for ca
Info: Creating a new SSL key for master.puppetlabs.vm
Info: csr_attributes file loading from /etc/puppetlabs/puppet/csr_attributes.yaml
Info: Creating a new SSL certificate request for master.puppetlabs.vm
Info: Certificate Request fingerprint (SHA256): 28:BE:46:A8:E0:6F:EA:EB:3F:6E:5E:D6:A1:45:A6:27:4F:E7:1C:01:0B:74:D6:3E:1F:F7:A6:9C:10:02:8F:14
Notice: master.puppetlabs.vm has a waiting certificate request
Info: authstore: defaulting to no access for master.puppetlabs.vm
Notice: Signed certificate request for master.puppetlabs.vm
Notice: Removing file Puppet::SSL::CertificateRequest master.puppetlabs.vm at '/etc/puppetlabs/puppet/ssl/ca/requests/master.puppetlabs.vm.pem'
Notice: Removing file Puppet::SSL::CertificateRequest master.puppetlabs.vm at '/etc/puppetlabs/puppet/ssl/certificate_requests/master.puppetlabs.vm.pem'
Warning: The WEBrick Puppet master server is deprecated and will be removed in a future release. Please use Puppet Server instead. See http://links.puppetlabs.com/deprecate-rack-webrick-servers for more information.
   (at /opt/puppetlabs/puppet/lib/ruby/vendor_ruby/puppet/application/master.rb:210:in `main')
Notice: Starting Puppet master version 4.5.2
Error: Could not run: Address already in use - bind(2) for 0.0.0.0:8140
root@master:/etc/puppetlabs/puppet # systemctl status puppetserver
● puppetserver.service
   Loaded: not-found (Reason: No such file or directory)
   Active: inactive (dead)
root@master:/etc/puppetlabs/puppet # systemctl status pe-puppetserver
● pe-puppetserver.service - pe-puppetserver Service
   Loaded: loaded (/usr/lib/systemd/system/pe-puppetserver.service; enabled; vendor preset: disabled)
   Active: active (running) since Tue 2016-06-28 10:12:36 UTC; 8h left
 Main PID: 13033 (java)
   CGroup: /system.slice/pe-puppetserver.service
           └─13033 /opt/puppetlabs/server/bin/java -Xms2048m -Xmx2048m -XX:OnOutOfMemoryError=kill -9 %p -Djava...

Jun 28 10:11:55 master.puppetlabs.vm systemd[1]: Starting pe-puppetserver Service...
Jun 28 10:12:36 master.puppetlabs.vm systemd[1]: Started pe-puppetserver Service.
root@master:/etc/puppetlabs/puppet # systemctl stop pe-puppetserver
root@master:/etc/puppetlabs/puppet #
root@master:/etc/puppetlabs/puppet #
root@master:/etc/puppetlabs/puppet #
root@master:/etc/puppetlabs/puppet # systemctl status pe-puppetserver
● pe-puppetserver.service - pe-puppetserver Service
   Loaded: loaded (/usr/lib/systemd/system/pe-puppetserver.service; enabled; vendor preset: disabled)
   Active: inactive (dead) since Tue 2016-06-28 01:16:01 UTC; 2s ago
 Main PID: 13033 (code=exited, status=143)

Jun 28 10:11:55 master.puppetlabs.vm systemd[1]: Starting pe-puppetserver Service...
Jun 28 10:12:36 master.puppetlabs.vm systemd[1]: Started pe-puppetserver Service.
Jun 28 01:16:01 master.puppetlabs.vm systemd[1]: Stopping pe-puppetserver Service...
Jun 28 01:16:01 master.puppetlabs.vm systemd[1]: Stopped pe-puppetserver Service.
root@master:/etc/puppetlabs/puppet # puppet master --no-daemonize --verbose
Warning: The WEBrick Puppet master server is deprecated and will be removed in a future release. Please use Puppet Server instead. See http://links.puppetlabs.com/deprecate-rack-webrick-servers for more information.
   (at /opt/puppetlabs/puppet/lib/ruby/vendor_ruby/puppet/application/master.rb:210:in `main')
Notice: Starting Puppet master version 4.5.2
^CNotice: Caught INT; exiting
root@master:/etc/puppetlabs/puppet # systemctl start pe-puppetserver
Job for pe-puppetserver.service failed because the control process exited with error code. See "systemctl status pe-puppetserver.service" and "journalctl -xe" for details.
root@master:/etc/puppetlabs/puppet # systemctl status pe-puppetserver.service
● pe-puppetserver.service - pe-puppetserver Service
   Loaded: loaded (/usr/lib/systemd/system/pe-puppetserver.service; enabled; vendor preset: disabled)
   Active: activating (start-post) since Tue 2016-06-28 01:18:35 UTC; 4s ago
  Process: 22113 ExecStartPre=/usr/bin/install --group=pe-puppet --owner=pe-puppet -d /var/run/puppetlabs/puppetserver (code=exited, status=0/SUCCESS)
 Main PID: 22115 (java);         : 22116 (bash)
   CGroup: /system.slice/pe-puppetserver.service
           ├─22115 /opt/puppetlabs/server/bin/java -Xms2048m -Xmx2048m -XX:OnOutOfMemoryError=kill -9 %p -Djava...
           └─control
             ├─22116 /bin/bash /opt/puppetlabs/server/apps/puppetserver/ezbake-functions.sh wait_for_app
             └─22153 sleep 1

Jun 28 01:18:35 master.puppetlabs.vm systemd[1]: Starting pe-puppetserver Service...
root@master:/etc/puppetlabs/puppet #
root@master:/etc/puppetlabs/puppet #
root@master:/etc/puppetlabs/puppet # puppet resource service puppet ensure=stopped
Notice: /Service[puppet]/ensure: ensure changed 'running' to 'stopped'
service { 'puppet':
  ensure => 'stopped',
}
root@master:/etc/puppetlabs/puppet # puppet resource service
service { 'NetworkManager-dispatcher.service':
  ensure => 'stopped',
  enable => 'true',
}
service { 'NetworkManager-wait-online.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'NetworkManager.service':
  ensure => 'running',
  enable => 'true',
}
service { 'abrt-ccpp.service':
  ensure => 'running',
  enable => 'true',
}
service { 'abrt-oops.service':
  ensure => 'running',
  enable => 'true',
}
service { 'abrt-pstoreoops.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'abrt-vmcore.service':
  ensure => 'stopped',
  enable => 'true',
}
service { 'abrt-xorg.service':
  ensure => 'stopped',
  enable => 'true',
}
service { 'abrtd.service':
  ensure => 'running',
  enable => 'true',
}
service { 'arp-ethers.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'atd.service':
  ensure => 'running',
  enable => 'true',
}
service { 'auditd.service':
  ensure => 'running',
  enable => 'true',
}
service { '[email protected]':
  ensure => 'stopped',
  enable => 'false',
}
service { 'blk-availability.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'chrony-wait.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'chronyd.service':
  ensure => 'stopped',
  enable => 'true',
}
service { 'console-getty.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'console-shell.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'cpupower.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'crond.service':
  ensure => 'running',
  enable => 'true',
}
service { 'dbus-org.freedesktop.NetworkManager.service':
  ensure => 'stopped',
  enable => 'true',
}
service { 'dbus-org.freedesktop.nm-dispatcher.service':
  ensure => 'stopped',
  enable => 'true',
}
service { 'debug-shell.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'dm-event.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'dmraid-activation.service':
  ensure => 'stopped',
  enable => 'true',
}
service { 'dnsmasq.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'ebtables.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'firewalld.service':
  ensure => 'stopped',
  enable => 'false',
}
service { '[email protected]':
  ensure => 'stopped',
  enable => 'true',
}
service { 'gssproxy.service':
  ensure => 'running',
  enable => 'false',
}
service { 'iprdump.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'iprinit.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'iprupdate.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'irqbalance.service':
  ensure => 'running',
  enable => 'true',
}
service { 'kdump.service':
  ensure => 'running',
  enable => 'true',
}
service { 'kpatch.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'libstoragemgmt.service':
  ensure => 'running',
  enable => 'true',
}
service { 'lvm2-lvmetad.service':
  ensure => 'running',
  enable => 'false',
}
service { 'lvm2-lvmpolld.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'lvm2-monitor.service':
  ensure => 'running',
  enable => 'true',
}
service { 'mcollective.service':
  ensure => 'running',
  enable => 'true',
}
service { 'mdmonitor.service':
  ensure => 'stopped',
  enable => 'true',
}
service { 'microcode.service':
  ensure => 'stopped',
  enable => 'true',
}
service { 'netconsole':
  ensure => 'stopped',
  enable => 'false',
}
service { 'network':
  ensure => 'running',
  enable => 'true',
}
service { 'nfs-blkmap.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'nfs-rquotad.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'nfs-server.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'nfs.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'ntpd.service':
  ensure => 'running',
  enable => 'true',
}
service { 'ntpdate.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'pe-activemq.service':
  ensure => 'running',
  enable => 'true',
}
service { 'pe-console-services.service':
  ensure => 'running',
  enable => 'true',
}
service { 'pe-nginx.service':
  ensure => 'running',
  enable => 'true',
}
service { 'pe-orchestration-services.service':
  ensure => 'running',
  enable => 'true',
}
service { 'pe-postgresql.service':
  ensure => 'running',
  enable => 'true',
}
service { 'pe-puppetdb.service':
  ensure => 'running',
  enable => 'true',
}
service { 'pe-puppetserver.service':
  ensure => 'stopped',
  enable => 'true',
}
service { 'plymouth-halt.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'plymouth-kexec.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'plymouth-poweroff.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'plymouth-quit-wait.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'plymouth-quit.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'plymouth-read-write.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'plymouth-reboot.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'plymouth-start.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'postfix.service':
  ensure => 'running',
  enable => 'true',
}
service { 'psacct.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'puppet.service':
  ensure => 'stopped',
  enable => 'true',
}
service { 'pxp-agent.service':
  ensure => 'running',
  enable => 'true',
}
service { 'rdisc.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'rdma.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'rhel-dmesg.service':
  ensure => 'running',
  enable => 'false',
}
service { 'rhel-domainname.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'rngd.service':
  ensure => 'running',
  enable => 'true',
}
service { 'rpc-rquotad.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'rsyncd.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'rsyslog.service':
  ensure => 'running',
  enable => 'true',
}
service { '[email protected]':
  ensure => 'stopped',
  enable => 'false',
}
service { 'smartd.service':
  ensure => 'running',
  enable => 'true',
}
service { 'sshd.service':
  ensure => 'running',
  enable => 'true',
}
service { 'svnserve.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'sysstat.service':
  ensure => 'running',
  enable => 'true',
}
service { 'systemd-bootchart.service':
  ensure => 'stopped',
  enable => 'false',
}
service { '[email protected]':
  ensure => 'stopped',
  enable => 'false',
}
service { 'systemd-readahead-collect.service':
  ensure => 'stopped',
  enable => 'true',
}
service { 'systemd-readahead-drop.service':
  ensure => 'stopped',
  enable => 'true',
}
service { 'systemd-readahead-replay.service':
  ensure => 'stopped',
  enable => 'true',
}
service { 'tcsd.service':
  ensure => 'stopped',
  enable => 'false',
}
service { 'tuned.service':
  ensure => 'running',
  enable => 'true',
}
service { 'wpa_supplicant.service':
  ensure => 'running',
  enable => 'false',
}
root@master:/etc/puppetlabs/puppet #
root@master:/etc/puppetlabs/puppet #
root@master:/etc/puppetlabs/puppet # puppet resource service puppet ensure=stopped
service { 'puppet':
  ensure => 'stopped',
}
root@master:/etc/puppetlabs/puppet # puppet resource service pe-puppetserver ensure=stopped
service { 'pe-puppetserver':
  ensure => 'stopped',
}
root@master:/etc/puppetlabs/puppet # puppet resource service pe-activemq ensure=stopped
Notice: /Service[pe-activemq]/ensure: ensure changed 'running' to 'stopped'
service { 'pe-activemq':
  ensure => 'stopped',
}
root@master:/etc/puppetlabs/puppet # puppet resource service mcollective ensure=stopped
Notice: /Service[mcollective]/ensure: ensure changed 'running' to 'stopped'
service { 'mcollective':
  ensure => 'stopped',
}
root@master:/etc/puppetlabs/puppet # puppet resource service pe-puppetdb ensure=stopped
Notice: /Service[pe-puppetdb]/ensure: ensure changed 'running' to 'stopped'
service { 'pe-puppetdb':
  ensure => 'stopped',
}
root@master:/etc/puppetlabs/puppet # puppet resource service pe-postgresql ensure=stopped
Notice: /Service[pe-postgresql]/ensure: ensure changed 'running' to 'stopped'
service { 'pe-postgresql':
  ensure => 'stopped',
}
root@master:/etc/puppetlabs/puppet # puppet resource service pe-console-services ensure=stopped
Notice: /Service[pe-console-services]/ensure: ensure changed 'running' to 'stopped'
service { 'pe-console-services':
  ensure => 'stopped',
}
root@master:/etc/puppetlabs/puppet # puppet resource service pe-nginx ensure=stopped
Notice: /Service[pe-nginx]/ensure: ensure changed 'running' to 'stopped'
service { 'pe-nginx':
  ensure => 'stopped',
}
root@master:/etc/puppetlabs/puppet # puppet resource service pe-orchestration-services ensure=stopped
^Z
[1]+  Stopped                 puppet resource service pe-orchestration-services ensure=stopped
root@master:/etc/puppetlabs/puppet # bg
[1]+ puppet resource service pe-orchestration-services ensure=stopped &
root@master:/etc/puppetlabs/puppet # top
top - 01:24:01 up  2:21,  1 user,  load average: 2.56, 2.16, 1.12
Tasks: 122 total,   1 running, 121 sleeping,   0 stopped,   0 zombie
%Cpu(s): 25.2 us,  0.2 sy,  0.0 ni, 74.5 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem :  8011232 total,  3327040 free,  1441700 used,  3242492 buff/cache
KiB Swap:  4194300 total,  4194300 free,        0 used.  6284232 avail Mem

  PID USER      PR  NI    VIRT    RES    SHR S  %CPU %MEM     TIME+ COMMAND
25786 pe-pupp+  20   0 5140164 881432  16052 S 100.3 11.0   0:39.38 java
  666 root      20   0    4372    596    500 S   0.7  0.0   0:10.82 rngd
25787 root      20   0  113116   1508   1296 S   0.3  0.0   0:00.01 bash
    1 root      20   0  188908   4024   2412 S   0.0  0.1   0:03.62 systemd
    2 root      20   0       0      0      0 S   0.0  0.0   0:00.00 kthreadd
    3 root      20   0       0      0      0 S   0.0  0.0   0:00.01 ksoftirqd/0
    5 root       0 -20       0      0      0 S   0.0  0.0   0:00.00 kworker/0:0H
    7 root      rt   0       0      0      0 S   0.0  0.0   0:00.36 migration/0
    8 root      20   0       0      0      0 S   0.0  0.0   0:00.00 rcu_bh
    9 root      20   0       0      0      0 S   0.0  0.0   0:00.00 rcuob/0
   10 root      20   0       0      0      0 S   0.0  0.0   0:00.00 rcuob/1
   11 root      20   0       0      0      0 S   0.0  0.0   0:00.00 rcuob/2
   12 root      20   0       0      0      0 S   0.0  0.0   0:00.00 rcuob/3
root@master:/etc/puppetlabs/puppet # ps aufx | less
[1]+  Done                    puppet resource service pe-orchestration-services ensure=stopped
root@master:/etc/puppetlabs/puppet #
root@master:/etc/puppetlabs/puppet #
root@master:/etc/puppetlabs/puppet # puppet resource service pxp-agent ensure=stopped
Notice: /Service[pxp-agent]/ensure: ensure changed 'running' to 'stopped'
service { 'pxp-agent':
  ensure => 'stopped',
}
root@master:/etc/puppetlabs/puppet # puppet resource service pe-puppetserver
service { 'pe-puppetserver':
  ensure => 'stopped',
  enable => 'true',
}
root@master:/etc/puppetlabs/puppet # ^Cppet resource service pe-puppetserver
root@master:/etc/puppetlabs/puppet # ps awwux | grep java
pe-pupp+ 26349  349  9.1 4683224 734812 ?      Ssl  01:24   0:17 /opt/puppetlabs/server/bin/java -Xms2048m -Xmx2048m -XX:OnOutOfMemoryError=kill -9 %p -Djava.security.egd=/dev/urandom -cp /opt/puppetlabs/server/apps/puppetserver/puppet-server-release.jar clojure.main -m puppetlabs.trapperkeeper.main --config /etc/puppetlabs/puppetserver/conf.d -b /etc/puppetlabs/puppetserver/bootstrap.cfg
root     26393  0.0  0.0 112644   956 pts/0    R+   01:24   0:00 grep --color=auto java
root@master:/etc/puppetlabs/puppet #
root@master:/etc/puppetlabs/puppet #
root@master:/etc/puppetlabs/puppet # kill 26349
root@master:/etc/puppetlabs/puppet # ps awwux | grep java
pe-pupp+ 26349  266 13.4 5300336 1080188 ?     Ssl  01:24   1:01 /opt/puppetlabs/server/bin/java -Xms2048m -Xmx2048m -XX:OnOutOfMemoryError=kill -9 %p -Djava.security.egd=/dev/urandom -cp /opt/puppetlabs/server/apps/puppetserver/puppet-server-release.jar clojure.main -m puppetlabs.trapperkeeper.main --config /etc/puppetlabs/puppetserver/conf.d -b /etc/puppetlabs/puppetserver/bootstrap.cfg
root     26513  0.0  0.0 112644   956 pts/0    S+   01:25   0:00 grep --color=auto java
root@master:/etc/puppetlabs/puppet #
root@master:/etc/puppetlabs/puppet # kill 26349
root@master:/etc/puppetlabs/puppet # kill -9 26349
-bash: kill: (26349) - No such process
root@master:/etc/puppetlabs/puppet # ps awwux | grep java
pe-pupp+ 26594  273  9.4 4693636 758164 ?      Ssl  01:25   0:19 /opt/puppetlabs/server/bin/java -Xms2048m -Xmx2048m -XX:OnOutOfMemoryError=kill -9 %p -Djava.security.egd=/dev/urandom -cp /opt/puppetlabs/server/apps/puppetserver/puppet-server-release.jar clojure.main -m puppetlabs.trapperkeeper.main --config /etc/puppetlabs/puppetserver/conf.d -b /etc/puppetlabs/puppetserver/bootstrap.cfg
root     26642  0.0  0.0 112644   956 pts/0    R+   01:25   0:00 grep --color=auto java
root@master:/etc/puppetlabs/puppet #
root@master:/etc/puppetlabs/puppet #
root@master:/etc/puppetlabs/puppet #
root@master:/etc/puppetlabs/puppet # puppet resource service | less
root@master:/etc/puppetlabs/puppet # puppet resource service | less
root@master:/etc/puppetlabs/puppet #
root@master:/etc/puppetlabs/puppet #
root@master:/etc/puppetlabs/puppet #
root@master:/etc/puppetlabs/puppet # reboot
Connection to 10.0.99.20 closed by remote host.
Connection to 10.0.99.20 closed.
~ $ ssh [email protected]
[email protected]'s password:
Last login: Tue Jun 28 09:09:15 2016 from 10.0.99.2
root@master:~ #
root@master:~ #
root@master:~ #
root@master:~ # puppet resource service pe-puppetserver
service { 'pe-puppetserver':
  ensure => 'stopped',
  enable => 'true',
}
root@master:~ # puppet resource service puppet ensure=stopped
Notice: /Service[puppet]/ensure: ensure changed 'running' to 'stopped'
service { 'puppet':
  ensure => 'stopped',
}
root@master:~ #  puppet resource service pe-puppetserver ensure=stopped
service { 'pe-puppetserver':
  ensure => 'stopped',
}
root@master:~ #  puppet resource service pe-activemq ensure=stopped
service { 'pe-activemq':
  ensure => 'stopped',
}
root@master:~ #  puppet resource service mcollective ensure=stopped
Notice: /Service[mcollective]/ensure: ensure changed 'running' to 'stopped'
service { 'mcollective':
  ensure => 'stopped',
}
root@master:~ #  puppet resource service pe-puppetdb ensure=stopped
service { 'pe-puppetdb':
  ensure => 'stopped',
}
root@master:~ #  puppet resource service pe-postgresql ensure=stopped
Notice: /Service[pe-postgresql]/ensure: ensure changed 'running' to 'stopped'
service { 'pe-postgresql':
  ensure => 'stopped',
}
root@master:~ #  puppet resource service pe-console-services ensure=stopped
service { 'pe-console-services':
  ensure => 'stopped',
}
root@master:~ #  puppet resource service pe-nginx ensure=stopped
service { 'pe-nginx':
  ensure => 'stopped',
}
root@master:~ #  puppet resource service pe-orchestration-services ensure=stopped
service { 'pe-orchestration-services':
  ensure => 'stopped',
}
root@master:~ #  puppet resource service pxp-agent ensure=stopped
Notice: /Service[pxp-agent]/ensure: ensure changed 'running' to 'stopped'
service { 'pxp-agent':
  ensure => 'stopped',
}
root@master:~ # ps awwux | grep java
pe-pupp+   904 76.9  5.6 3931712 449912 ?      Ssl  01:26   1:04 /opt/puppetlabs/server/bin/java -Xmx256m -Xms256m -XX:OnOutOfMemoryError=kill -9 %p -Djava.security.egd=/dev/urandom -cp /opt/puppetlabs/server/apps/puppetdb/puppetdb.jar clojure.main -m puppetlabs.puppetdb.main --config /etc/puppetlabs/puppetdb/conf.d -b /etc/puppetlabs/puppetdb/bootstrap.cfg
pe-pupp+   919  101 14.6 5887956 1175212 ?     Ssl  01:26   1:25 /opt/puppetlabs/server/bin/java -Xms2048m -Xmx2048m -XX:OnOutOfMemoryError=kill -9 %p -Djava.security.egd=/dev/urandom -cp /opt/puppetlabs/server/apps/puppetserver/puppet-server-release.jar clojure.main -m puppetlabs.trapperkeeper.main --config /etc/puppetlabs/puppetserver/conf.d -b /etc/puppetlabs/puppetserver/bootstrap.cfg
pe-orch+  1447 97.8  3.6 2932648 291488 ?      Ssl  01:26   1:21 /opt/puppetlabs/server/bin/java -Xmx192m -Xms192m -XX:OnOutOfMemoryError=kill -9 %p -Djava.security.egd=/dev/urandom -cp /opt/puppetlabs/server/apps/orchestration-services/orchestration-services-release.jar clojure.main -m puppetlabs.trapperkeeper.main --config /etc/puppetlabs/orchestration-services/conf.d -b /etc/puppetlabs/orchestration-services/bootstrap.cfg
pe-acti+  3822  9.5  2.1 3368144 172656 ?      Sl   01:27   0:02 /opt/puppetlabs/server/apps/java/lib/jvm/java/jre/bin/java -Xms512m -Xmx512m -Dorg.apache.activemq.UseDedicatedTaskRunner=false -Djava.util.logging.config.file=logging.properties -Dhawtio.realm=activemq -Dhawtio.role=admins -Dhawtio.rolePrincipalClasses=org.apache.activemq.jaas.GroupPrincipal -Djava.security.auth.login.config=/etc/puppetlabs/activemq/login.config -Dcom.sun.management.jmxremote -Djava.awt.headless=true -Djava.io.tmpdir=/opt/puppetlabs/server/apps/activemq/tmp -Dactivemq.classpath=/etc/puppetlabs/activemq; -Dactivemq.home=/opt/puppetlabs/server/apps/activemq -Dactivemq.base=/opt/puppetlabs/server/apps/activemq -Dactivemq.conf=/etc/puppetlabs/activemq -Dactivemq.data=/opt/puppetlabs/server/data/activemq -jar /opt/puppetlabs/server/apps/activemq/bin/activemq.jar start
pe-cons+  4292  273  3.7 2796304 297228 ?      Ssl  01:28   0:27 /opt/puppetlabs/server/bin/java -Xmx256m -Xms256m -XX:OnOutOfMemoryError=kill -9 %p -Djava.security.egd=/dev/urandom -cp /opt/puppetlabs/server/apps/console-services/console-services-release.jar clojure.main -m puppetlabs.trapperkeeper.main --config /etc/puppetlabs/console-services/conf.d -b /etc/puppetlabs/console-services/bootstrap.cfg
root      4438  0.0  0.0 112644   956 pts/0    R+   01:28   0:00 grep --color=auto java
root@master:~ #
root@master:~ #
root@master:~ #
root@master:~ # ps awwux | grep java
pe-pupp+   904 58.9  5.6 3931712 450340 ?      Ssl  01:26   1:04 /opt/puppetlabs/server/bin/java -Xmx256m -Xms256m -XX:OnOutOfMemoryError=kill -9 %p -Djava.security.egd=/dev/urandom -cp /opt/puppetlabs/server/apps/puppetdb/puppetdb.jar clojure.main -m puppetlabs.puppetdb.main --config /etc/puppetlabs/puppetdb/conf.d -b /etc/puppetlabs/puppetdb/bootstrap.cfg
pe-pupp+   919 77.8 14.6 5887956 1175320 ?     Ssl  01:26   1:25 /opt/puppetlabs/server/bin/java -Xms2048m -Xmx2048m -XX:OnOutOfMemoryError=kill -9 %p -Djava.security.egd=/dev/urandom -cp /opt/puppetlabs/server/apps/puppetserver/puppet-server-release.jar clojure.main -m puppetlabs.trapperkeeper.main --config /etc/puppetlabs/puppetserver/conf.d -b /etc/puppetlabs/puppetserver/bootstrap.cfg
pe-orch+  1447 98.4  3.6 2932648 291488 ?      Ssl  01:26   1:47 /opt/puppetlabs/server/bin/java -Xmx192m -Xms192m -XX:OnOutOfMemoryError=kill -9 %p -Djava.security.egd=/dev/urandom -cp /opt/puppetlabs/server/apps/orchestration-services/orchestration-services-release.jar clojure.main -m puppetlabs.trapperkeeper.main --config /etc/puppetlabs/orchestration-services/conf.d -b /etc/puppetlabs/orchestration-services/bootstrap.cfg
pe-acti+  3822  5.1  2.2 3368144 178884 ?      Sl   01:27   0:02 /opt/puppetlabs/server/apps/java/lib/jvm/java/jre/bin/java -Xms512m -Xmx512m -Dorg.apache.activemq.UseDedicatedTaskRunner=false -Djava.util.logging.config.file=logging.properties -Dhawtio.realm=activemq -Dhawtio.role=admins -Dhawtio.rolePrincipalClasses=org.apache.activemq.jaas.GroupPrincipal -Djava.security.auth.login.config=/etc/puppetlabs/activemq/login.config -Dcom.sun.management.jmxremote -Djava.awt.headless=true -Djava.io.tmpdir=/opt/puppetlabs/server/apps/activemq/tmp -Dactivemq.classpath=/etc/puppetlabs/activemq; -Dactivemq.home=/opt/puppetlabs/server/apps/activemq -Dactivemq.base=/opt/puppetlabs/server/apps/activemq -Dactivemq.conf=/etc/puppetlabs/activemq -Dactivemq.data=/opt/puppetlabs/server/data/activemq -jar /opt/puppetlabs/server/apps/activemq/bin/activemq.jar start
pe-cons+  4292  137  5.1 3124040 412380 ?      Ssl  01:28   0:49 /opt/puppetlabs/server/bin/java -Xmx256m -Xms256m -XX:OnOutOfMemoryError=kill -9 %p -Djava.security.egd=/dev/urandom -cp /opt/puppetlabs/server/apps/console-services/console-services-release.jar clojure.main -m puppetlabs.trapperkeeper.main --config /etc/puppetlabs/console-services/conf.d -b /etc/puppetlabs/console-services/bootstrap.cfg
root      4652  0.0  0.0 112644   956 pts/0    R+   01:28   0:00 grep --color=auto java
root@master:~ # ^C
root@master:~ #
root@master:~ # ls -l /etc/puppetlabs/puppet/ssl/
total 28
drwxr-xr-x 5 pe-puppet pe-puppet 4096 Jun 28 01:14 ca
drwxr-xr-x 2 pe-puppet pe-puppet 4096 Jun 28 01:15 certificate_requests
drwxr-xr-x 2 pe-puppet pe-puppet 4096 Jun 28 01:27 certs
-rw-r--r-- 1 pe-puppet pe-puppet  954 Jun 28 01:27 crl.pem
drwxr-x--- 2 pe-puppet pe-puppet 4096 Jun 28 01:14 private
drwxr-x--- 2 pe-puppet pe-puppet 4096 Jun 28 01:27 private_keys
drwxr-xr-x 2 pe-puppet pe-puppet 4096 Jun 28 01:27 public_keys
root@master:~ # date
Tue Jun 28 01:29:08 UTC 2016
root@master:~ # netstat -an ^C
root@master:~ #
root@master:~ #
root@master:~ # ps awwux | grep java
pe-pupp+   904 39.1  5.6 3931712 450384 ?      Ssl  01:26   1:04 /opt/puppetlabs/server/bin/java -Xmx256m -Xms256m -XX:OnOutOfMemoryError=kill -9 %p -Djava.security.egd=/dev/urandom -cp /opt/puppetlabs/server/apps/puppetdb/puppetdb.jar clojure.main -m puppetlabs.puppetdb.main --config /etc/puppetlabs/puppetdb/conf.d -b /etc/puppetlabs/puppetdb/bootstrap.cfg
pe-pupp+   919 51.5 14.6 5887956 1175320 ?     Ssl  01:26   1:25 /opt/puppetlabs/server/bin/java -Xms2048m -Xmx2048m -XX:OnOutOfMemoryError=kill -9 %p -Djava.security.egd=/dev/urandom -cp /opt/puppetlabs/server/apps/puppetserver/puppet-server-release.jar clojure.main -m puppetlabs.trapperkeeper.main --config /etc/puppetlabs/puppetserver/conf.d -b /etc/puppetlabs/puppetserver/bootstrap.cfg
pe-orch+  1447 98.4  3.6 2932648 291488 ?      Ssl  01:26   2:42 /opt/puppetlabs/server/bin/java -Xmx192m -Xms192m -XX:OnOutOfMemoryError=kill -9 %p -Djava.security.egd=/dev/urandom -cp /opt/puppetlabs/server/apps/orchestration-services/orchestration-services-release.jar clojure.main -m puppetlabs.trapperkeeper.main --config /etc/puppetlabs/orchestration-services/conf.d -b /etc/puppetlabs/orchestration-services/bootstrap.cfg
pe-acti+  3822  2.7  2.2 3368144 183308 ?      Sl   01:27   0:03 /opt/puppetlabs/server/apps/java/lib/jvm/java/jre/bin/java -Xms512m -Xmx512m -Dorg.apache.activemq.UseDedicatedTaskRunner=false -Djava.util.logging.config.file=logging.properties -Dhawtio.realm=activemq -Dhawtio.role=admins -Dhawtio.rolePrincipalClasses=org.apache.activemq.jaas.GroupPrincipal -Djava.security.auth.login.config=/etc/puppetlabs/activemq/login.config -Dcom.sun.management.jmxremote -Djava.awt.headless=true -Djava.io.tmpdir=/opt/puppetlabs/server/apps/activemq/tmp -Dactivemq.classpath=/etc/puppetlabs/activemq; -Dactivemq.home=/opt/puppetlabs/server/apps/activemq -Dactivemq.base=/opt/puppetlabs/server/apps/activemq -Dactivemq.conf=/etc/puppetlabs/activemq -Dactivemq.data=/opt/puppetlabs/server/data/activemq -jar /opt/puppetlabs/server/apps/activemq/bin/activemq.jar start
pe-cons+  4774  116  5.0 3124040 407208 ?      Ssl  01:28   0:47 /opt/puppetlabs/server/bin/java -Xmx256m -Xms256m -XX:OnOutOfMemoryError=kill -9 %p -Djava.security.egd=/dev/urandom -cp /opt/puppetlabs/server/apps/console-services/console-services-release.jar clojure.main -m puppetlabs.trapperkeeper.main --config /etc/puppetlabs/console-services/conf.d -b /etc/puppetlabs/console-services/bootstrap.cfg
root      5123  0.0  0.0 112644   956 pts/0    R+   01:29   0:00 grep --color=auto java
root@master:~ # puppet resource service pe-orchestration-services
service { 'pe-orchestration-services':
  ensure => 'stopped',
  enable => 'true',
}
root@master:~ # puppet resource service mcollective
service { 'mcollective':
  ensure => 'stopped',
  enable => 'true',
}
root@master:~ # rm -rf /etc/puppetlabs/puppet/ssl /etc/puppetlabs/puppetdb/ssl /opt/puppetlabs/server/data/console-services/certs/ /opt/puppetlabs/server/data/postgresql/9.4/data/certs/ /etc/puppetlabs/orchestration-services/ssl/
root@master:~ # rm -f /opt/puppetlabs/puppet/cache/client_data/catalog/master.puppetlabs.vm.json
root@master:~ # puppet cert list -a
Notice: Signed certificate request for ca
root@master:~ # puppet cert generate master.puppetlabs.vm --dns_alt_names="master,puppet.puppetlabs.vm"
Notice: master.puppetlabs.vm has a waiting certificate request
Notice: Signed certificate request for master.puppetlabs.vm
Notice: Removing file Puppet::SSL::CertificateRequest master.puppetlabs.vm at '/etc/puppetlabs/puppet/ssl/ca/requests/master.puppetlabs.vm.pem'
Notice: Removing file Puppet::SSL::CertificateRequest master.puppetlabs.vm at '/etc/puppetlabs/puppet/ssl/certificate_requests/master.puppetlabs.vm.pem'
root@master:~ # puppet cert generate pe-internal-classifier
Notice: pe-internal-classifier has a waiting certificate request
Notice: Signed certificate request for pe-internal-classifier
Notice: Removing file Puppet::SSL::CertificateRequest pe-internal-classifier at '/etc/puppetlabs/puppet/ssl/ca/requests/pe-internal-classifier.pem'
Notice: Removing file Puppet::SSL::CertificateRequest pe-internal-classifier at '/etc/puppetlabs/puppet/ssl/certificate_requests/pe-internal-classifier.pem'
root@master:~ #  puppet cert generate pe-internal-dashboard
Notice: pe-internal-dashboard has a waiting certificate request
Notice: Signed certificate request for pe-internal-dashboard
Notice: Removing file Puppet::SSL::CertificateRequest pe-internal-dashboard at '/etc/puppetlabs/puppet/ssl/ca/requests/pe-internal-dashboard.pem'
Notice: Removing file Puppet::SSL::CertificateRequest pe-internal-dashboard at '/etc/puppetlabs/puppet/ssl/certificate_requests/pe-internal-dashboard.pem'
root@master:~ #  puppet cert generate pe-internal-mcollective-servers
Notice: pe-internal-mcollective-servers has a waiting certificate request
Notice: Signed certificate request for pe-internal-mcollective-servers
Notice: Removing file Puppet::SSL::CertificateRequest pe-internal-mcollective-servers at '/etc/puppetlabs/puppet/ssl/ca/requests/pe-internal-mcollective-servers.pem'
Notice: Removing file Puppet::SSL::CertificateRequest pe-internal-mcollective-servers at '/etc/puppetlabs/puppet/ssl/certificate_requests/pe-internal-mcollective-servers.pem'
root@master:~ #  puppet cert generate pe-internal-peadmin-mcollective-client
Notice: pe-internal-peadmin-mcollective-client has a waiting certificate request
Notice: Signed certificate request for pe-internal-peadmin-mcollective-client
Notice: Removing file Puppet::SSL::CertificateRequest pe-internal-peadmin-mcollective-client at '/etc/puppetlabs/puppet/ssl/ca/requests/pe-internal-peadmin-mcollective-client.pem'
Notice: Removing file Puppet::SSL::CertificateRequest pe-internal-peadmin-mcollective-client at '/etc/puppetlabs/puppet/ssl/certificate_requests/pe-internal-peadmin-mcollective-client.pem'
root@master:~ #  puppet cert generate pe-internal-puppet-console-mcollective-client
Notice: pe-internal-puppet-console-mcollective-client has a waiting certificate request
Notice: Signed certificate request for pe-internal-puppet-console-mcollective-client
Notice: Removing file Puppet::SSL::CertificateRequest pe-internal-puppet-console-mcollective-client at '/etc/puppetlabs/puppet/ssl/ca/requests/pe-internal-puppet-console-mcollective-client.pem'
Notice: Removing file Puppet::SSL::CertificateRequest pe-internal-puppet-console-mcollective-client at '/etc/puppetlabs/puppet/ssl/certificate_requests/pe-internal-puppet-console-mcollective-client.pem'
root@master:~ #  puppet cert generate pe-internal-orchestrator
Notice: pe-internal-orchestrator has a waiting certificate request
Notice: Signed certificate request for pe-internal-orchestrator
Notice: Removing file Puppet::SSL::CertificateRequest pe-internal-orchestrator at '/etc/puppetlabs/puppet/ssl/ca/requests/pe-internal-orchestrator.pem'
Notice: Removing file Puppet::SSL::CertificateRequest pe-internal-orchestrator at '/etc/puppetlabs/puppet/ssl/certificate_requests/pe-internal-orchestrator.pem'
root@master:~ # cp /etc/puppetlabs/puppet/ssl/ca/ca_crl.pem /etc/puppetlabs/puppet/ssl/crl.pem
root@master:~ #  chown -R pe-puppet:pe-puppet /etc/puppetlabs/puppet/ssl
root@master:~ # rm -rf /etc/puppetlabs/puppetdb/ssl/*
root@master:~ # vi /tmp/a.sh
root@master:~ # set -o vi
root@master:~ # bash /tmp/a.sh
cp: cannot create regular file ‘/etc/puppetlabs/puppetdb/ssl/master.puppetlabs.vm.cert.pem’: No such file or directory
cp: cannot create regular file ‘/etc/puppetlabs/puppetdb/ssl/master.puppetlabs.vm.public_key.pem’: No such file or directory
cp: cannot create regular file ‘/etc/puppetlabs/puppetdb/ssl/master.puppetlabs.vm.private_key.pem’: No such file or directory
Can't open input file /etc/puppetlabs/puppetdb/ssl/master.puppetlabs.vm.private_key.pem
chown: cannot access ‘/etc/puppetlabs/puppetdb/ssl’: No such file or directory
root@master:~ #
root@master:~ #
root@master:~ # ls -l /etc/puppetlabs/puppet/ssl/certs/
total 32
-rw-r--r-- 1 pe-puppet pe-puppet 1980 Jun 28 01:32 ca.pem
-rw-r--r-- 1 pe-puppet pe-puppet 2053 Jun 28 01:32 master.puppetlabs.vm.pem
-rw-r--r-- 1 pe-puppet pe-puppet 1972 Jun 28 01:33 pe-internal-classifier.pem
-rw-r--r-- 1 pe-puppet pe-puppet 1972 Jun 28 01:33 pe-internal-dashboard.pem
-rw-r--r-- 1 pe-puppet pe-puppet 1984 Jun 28 01:33 pe-internal-mcollective-servers.pem
-rw-r--r-- 1 pe-puppet pe-puppet 1976 Jun 28 01:33 pe-internal-orchestrator.pem
-rw-r--r-- 1 pe-puppet pe-puppet 1992 Jun 28 01:33 pe-internal-peadmin-mcollective-client.pem
-rw-r--r-- 1 pe-puppet pe-puppet 2004 Jun 28 01:33 pe-internal-puppet-console-mcollective-client.pem
root@master:~ # ls ld /etc/puppetlabs/puppetdb/ssl
ls: cannot access ld: No such file or directory
ls: cannot access /etc/puppetlabs/puppetdb/ssl: No such file or directory
root@master:~ # mkdir -p /etc/puppetlabs/puppetdb/ssl
root@master:~ #
root@master:~ #
root@master:~ #
root@master:~ # bash /tmp/a.sh
root@master:~ # ls -l /etc/puppetlabs/puppetdb/ssl
total 16
-rw-r--r-- 1 pe-puppetdb pe-puppetdb 2053 Jun 28 01:37 master.puppetlabs.vm.cert.pem
-rw-r----- 1 pe-puppetdb pe-puppetdb 3243 Jun 28 01:37 master.puppetlabs.vm.private_key.pem
-rw-r--r-- 1 pe-puppetdb pe-puppetdb 2373 Jun 28 01:37 master.puppetlabs.vm.private_key.pk8
-rw-r--r-- 1 pe-puppetdb pe-puppetdb  800 Jun 28 01:37 master.puppetlabs.vm.public_key.pem
root@master:~ # rm -rf /opt/puppetlabs/server/data/postgresql/9.4/data/certs/*
root@master:~ # vi /tmp/b.sh
root@master:~ # bash /tmp/b.sh
cp: cannot create regular file ‘/opt/puppetlabs/server/data/postgresql/9.4/data/certs/master.puppetlabs.vm.cert.pem’: No such file or directory
cp: cannot create regular file ‘/opt/puppetlabs/server/data/postgresql/9.4/data/certs/master.puppetlabs.vm.public_key.pem’: No such file or directory
cp: cannot create regular file ‘/opt/puppetlabs/server/data/postgresql/9.4/data/certs/master.puppetlabs.vm.private_key.pem’: No such file or directory
chmod: cannot access ‘/opt/puppetlabs/server/data/postgresql/9.4/data/certs/*’: No such file or directory
chown: cannot access ‘/opt/puppetlabs/server/data/postgresql/9.4/data/certs/*’: No such file or directory
root@master:~ # ls -ld /opt/puppetlabs/server/data/postgresql/9.4/data/certs
ls: cannot access /opt/puppetlabs/server/data/postgresql/9.4/data/certs: No such file or directory
root@master:~ # ls -ld /opt/puppetlabs/server/data/postgresql/9.4/data/
drwx------ 18 pe-postgres pe-postgres 4096 Jun 28 01:31 /opt/puppetlabs/server/data/postgresql/9.4/data/
root@master:~ # mkdir /opt/puppetlabs/server/data/postgresql/9.4/data/certs
root@master:~ #
root@master:~ #
root@master:~ #
root@master:~ # bash /tmp/b.sh
root@master:~ # rm -rf /etc/puppetlabs/orchestration-services/ssl/*
root@master:~ #
root@master:~ # cp /etc/puppetlabs/puppet/ssl/certs/pe-internal-orchestrator.pem /etc/puppetlabs/orchestration-services/ssl/pe-internal-orchestrator.cert.pem
cp: cannot create regular file ‘/etc/puppetlabs/orchestration-services/ssl/pe-internal-orchestrator.cert.pem’: No such file or directory
root@master:~ # cp /etc/puppetlabs/puppet/ssl/public_keys/pe-internal-orchestrator.pem /etc/puppetlabs/orchestration-services/ssl/pe-internal-orchestrator.public_key.pem
cp: cannot create regular file ‘/etc/puppetlabs/orchestration-services/ssl/pe-internal-orchestrator.public_key.pem’: No such file or directory
root@master:~ # cp /etc/puppetlabs/puppet/ssl/private_keys/pe-internal-orchestrator.pem /etc/puppetlabs/orchestration-services/ssl/pe-internal-orchestrator.private_key.pem
cp: cannot create regular file ‘/etc/puppetlabs/orchestration-services/ssl/pe-internal-orchestrator.private_key.pem’: No such file or directory
root@master:~ # ls -ld /etc/puppetlabs/orchestration-services/ssl
ls: cannot access /etc/puppetlabs/orchestration-services/ssl: No such file or directory
root@master:~ # ls -ld /etc/puppetlabs/orchestration-services
drwxr-x--- 3 pe-orchestration-services pe-orchestration-services 4096 Jun 28 01:31 /etc/puppetlabs/orchestration-services
root@master:~ # ^Cetc/puppetlabs/orchestration-services
root@master:~ # mkdir /etc/puppetlabs/orchestration-services/ssl
root@master:~ # ^C /etc/puppetlabs/puppet/ssl/private_keys/pe-internal-orchestrator.pem /etc/puppetlabs/orchestrat
ion-services/ssl/pe-internal-orchestrator.private_key.pem
root@master:~ # /etc/puppetlabs/orchestration-services/ssl^C
root@master:~ # cp /etc/puppetlabs/puppet/ssl/certs/pe-internal-orchestrator.pem /etc/puppetlabs/orchestration-services/ssl/pe-internal-orchestrator.cert.pem
root@master:~ # cp /etc/puppetlabs/puppet/ssl/public_keys/pe-internal-orchestrator.pem /etc/puppetlabs/orchestration-services/ssl/pe-internal-orchestrator.public_key.pem
root@master:~ # cp /etc/puppetlabs/puppet/ssl/private_keys/pe-internal-orchestrator.pem /etc/puppetlabs/orchestration-services/ssl/pe-internal-orchestrator.private_key.pem
root@master:~ # vi /tmp/c.sh
root@master:~ #
root@master:~ # bash /tmp/c.sh
root@master:~ #
root@master:~ #
root@master:~ # rm -rf /opt/puppetlabs/server/data/console-services/certs/*
root@master:~ # cp /etc/puppetlabs/puppet/ssl/certs/pe-internal-classifier.pem /opt/puppetlabs/server/data/console-services/certs/pe-internal-classifier.cert.pem
cp: cannot create regular file ‘/opt/puppetlabs/server/data/console-services/certs/pe-internal-classifier.cert.pem’: No such file or directory
root@master:~ #  cp /etc/puppetlabs/puppet/ssl/public_keys/pe-internal-classifier.pem /opt/puppetlabs/server/data/console-services/certs/pe-internal-classifier.public_key.pem
cp: cannot create regular file ‘/opt/puppetlabs/server/data/console-services/certs/pe-internal-classifier.public_key.pem’: No such file or directory
root@master:~ #  cp /etc/puppetlabs/puppet/ssl/private_keys/pe-internal-classifier.pem /opt/puppetlabs/server/data/console-services/certs/pe-internal-classifier.private_key.pem
cp: cannot create regular file ‘/opt/puppetlabs/server/data/console-services/certs/pe-internal-classifier.private_key.pem’: No such file or directory
root@master:~ # ls -ld /opt/puppetlabs/server/data/console-services/certs
ls: cannot access /opt/puppetlabs/server/data/console-services/certs: No such file or directory
root@master:~ # ls -ld /opt/puppetlabs/server/data/console-services
drwxrwx--- 2 pe-console-services pe-console-services 4096 Jun 28 01:31 /opt/puppetlabs/server/data/console-services
root@master:~ # mkdir -p /opt/puppetlabs/server/data/console-services/certs
root@master:~ #
root@master:~ #
root@master:~ #
root@master:~ # /opt/puppetlabs/server/data/console-services/certs^C
root@master:~ # cp /etc/puppetlabs/puppet/ssl/certs/pe-internal-classifier.pem /opt/puppetlabs/server/data/console-services/certs/pe-internal-classifier.cert.pem
root@master:~ #  cp /etc/puppetlabs/puppet/ssl/public_keys/pe-internal-classifier.pem /opt/puppetlabs/server/data/console-services/certs/pe-internal-classifier.public_key.pem
root@master:~ #  cp /etc/puppetlabs/puppet/ssl/private_keys/pe-internal-classifier.pem /opt/puppetlabs/server/data/console-services/certs/pe-internal-classifier.private_key.pem
root@master:~ # vi /tmp/d.sh
root@master:~ # ls -ld /opt/puppetlabs/server/data/console-services/certs
drwxr-xr-x 2 root root 4096 Jun 28 01:42 /opt/puppetlabs/server/data/console-services/certs
root@master:~ # ^C -ld /opt/puppetlabs/server/data/console-services/certs
root@master:~ # bash /tmp/d.sh
root@master:~ #
root@master:~ #
root@master:~ #  cp /etc/puppetlabs/puppet/ssl/certs/pe-internal-dashboard.pem /opt/puppetlabs/server/data/console-services/certs/pe-internal-dashboard.cert.pem
root@master:~ #  cp /etc/puppetlabs/puppet/ssl/public_keys/pe-internal-dashboard.pem /opt/puppetlabs/server/data/console-services/certs/pe-internal-dashboard.public_key.pem
root@master:~ #  cp /etc/puppetlabs/puppet/ssl/private_keys/pe-internal-dashboard.pem /opt/puppetlabs/server/data/console-services/certs/pe-internal-dashboard.private_key.pem
root@master:~ # chown -R pe-console-services:pe-console-services /opt/puppetlabs/server/data/console-services/certs
root@master:~ # puppet resource service pe-puppetserver ensure=running
service { 'pe-puppetserver':
  ensure => 'running',
}
root@master:~ #   puppet resource service pe-postgresql ensure=running
Notice: /Service[pe-postgresql]/ensure: ensure changed 'stopped' to 'running'
service { 'pe-postgresql':
  ensure => 'running',
}
root@master:~ #   puppet resource service pe-puppetdb ensure=running
service { 'pe-puppetdb':
  ensure => 'running',
}
root@master:~ #   puppet resource service pe-console-services ensure=running
Notice: /Service[pe-console-services]/ensure: ensure changed 'stopped' to 'running'
service { 'pe-console-services':
  ensure => 'running',
}
root@master:~ #   puppet resource service pe-nginx ensure=running
Notice: /Service[pe-nginx]/ensure: ensure changed 'stopped' to 'running'
service { 'pe-nginx':
  ensure => 'running',
}
root@master:~ #   puppet resource service pe-activemq ensure=running
service { 'pe-activemq':
  ensure => 'running',
}
root@master:~ #   puppet resource service mcollective ensure=running
Notice: /Service[mcollective]/ensure: ensure changed 'stopped' to 'running'
service { 'mcollective':
  ensure => 'running',
}
root@master:~ #   puppet resource service puppet ensure=running
Notice: /Service[puppet]/ensure: ensure changed 'stopped' to 'running'
service { 'puppet':
  ensure => 'running',
}
root@master:~ #   puppet resource service pe-orchestration-services ensure=running
service { 'pe-orchestration-services':
  ensure => 'running',
}
root@master:~ #   puppet resource service pxp-agent ensure=running
Notice: /Service[pxp-agent]/ensure: ensure changed 'stopped' to 'running'
service { 'pxp-agent':
  ensure => 'running',
}
root@master:~ # puppet agent -t
Warning: Unable to fetch my node definition, but the agent run will continue:
Warning: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [unable to get local issuer certificate for /CN=master.puppetlabs.vm]
Info: Retrieving pluginfacts
Error: /File[/opt/puppetlabs/puppet/cache/facts.d]: Failed to generate additional resources using 'eval_generate': SSL_connect returned=1 errno=0 state=error: certificate verify failed: [unable to get local issuer certificate for /CN=master.puppetlabs.vm]
Error: /File[/opt/puppetlabs/puppet/cache/facts.d]: Could not evaluate: Could not retrieve file metadata for puppet:///pluginfacts: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [unable to get local issuer certificate for /CN=master.puppetlabs.vm]
Info: Retrieving plugin
Error: /File[/opt/puppetlabs/puppet/cache/lib]: Failed to generate additional resources using 'eval_generate': SSL_connect returned=1 errno=0 state=error: certificate verify failed: [unable to get local issuer certificate for /CN=master.puppetlabs.vm]
Error: /File[/opt/puppetlabs/puppet/cache/lib]: Could not evaluate: Could not retrieve file metadata for puppet:///plugins: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [unable to get local issuer certificate for /CN=master.puppetlabs.vm]
Info: Loading facts
Error: Could not retrieve catalog from remote server: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [unable to get local issuer certificate for /CN=master.puppetlabs.vm]
Warning: Not using cache on failed catalog
Error: Could not retrieve catalog; skipping run
Error: Could not send report: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [unable to get local issuer certificate for /CN=master.puppetlabs.vm]
root@master:~ # date
Tue Jun 28 01:45:38 UTC 2016
root@master:~ # puppet agent -t
Warning: Unable to fetch my node definition, but the agent run will continue:
Warning: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [unable to get local issuer certificate for /CN=master.puppetlabs.vm]
Info: Retrieving pluginfacts
Error: /File[/opt/puppetlabs/puppet/cache/facts.d]: Failed to generate additional resources using 'eval_generate': SSL_connect returned=1 errno=0 state=error: certificate verify failed: [unable to get local issuer certificate for /CN=master.puppetlabs.vm]
Error: /File[/opt/puppetlabs/puppet/cache/facts.d]: Could not evaluate: Could not retrieve file metadata for puppet:///pluginfacts: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [unable to get local issuer certificate for /CN=master.puppetlabs.vm]
Info: Retrieving plugin
Error: /File[/opt/puppetlabs/puppet/cache/lib]: Failed to generate additional resources using 'eval_generate': SSL_connect returned=1 errno=0 state=error: certificate verify failed: [unable to get local issuer certificate for /CN=master.puppetlabs.vm]
Error: /File[/opt/puppetlabs/puppet/cache/lib]: Could not evaluate: Could not retrieve file metadata for puppet:///plugins: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [unable to get local issuer certificate for /CN=master.puppetlabs.vm]
Info: Loading facts
Error: Could not retrieve catalog from remote server: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [unable to get local issuer certificate for /CN=master.puppetlabs.vm]
Warning: Not using cache on failed catalog
Error: Could not retrieve catalog; skipping run
Error: Could not send report: SSL_connect returned=1 errno=0 state=error: certificate verify failed: [unable to get local issuer certificate for /CN=master.puppetlabs.vm]
root@master:~ #
root@master:~ #
root@master:~ # du -sh /v^C
root@master:~ # cd /var/log
root@master:/var/log # du -sh puppetlabs
11M     puppetlabs
root@master:/var/log # tar czf ~/var_log_puppetlabs-cert-regen-failed-DEC001.tgz puppetlabs
root@master:/var/log #
root@master:/var/log # ^Cr czf ~/var_log_puppetlabs-cert-regen-failed-DEC001.tgz puppetlabs
root@master:/var/log #
root@master:/var/log #