dincosman

[oracle@bltdb02 trace]$ vi /u01/app/oracle/diag/rdbms/clouddg/clouddg2/trace/clouddg2_vktm_7250.trc
	...
	*** 2022-02-22T13:28:12.530124+03:00
	kstmchkdrift (kstmhighrestimecntkeeper:highres): Time jumped forward by (29192208)usec at (238987772848) whereas (1000000) is allowed
	*** 2022-02-22T13:28:52.029681+03:00
	kstmchkdrift (kstmhighrestimecntkeeper:highres): Time jumped forward by (8942061)usec at (239010720349) whereas (1000000) is allowed
	*** 2022-02-22T13:30:35.618002+03:00
	kstmchkdrift (kstmhighrestimecntkeeper:highres): Time jumped forward by (106464068)usec at (239117184417) whereas (1000000) is allowed
	...

dincosman

[oracle@bltdb01 trace]$ vi $ORACLE_BASE/diag/rdbms/clouddg/clouddg1/trace/alert_clouddg1.log
	...
	2022-02-22T13:22:57.223935+03:00
	ARC4 (PID:19605): Archived Log entry 59912 added for T-1.S-40494 ID 0x79b8ac20 LAD:1
	2022-02-22T13:29:06.766088+03:00
	Warning: VKTM detected a forward time drift.
	2022-02-22T13:29:07.895922+03:00
	Time drifts can result in unexpected behavior such as time-outs.
	Please see the VKTM trace file for more details:
	/u01/app/oracle/diag/rdbms/clouddg/clouddg1/trace/clouddg1_vktm_18186.trc

dincosman

 [oracle@bltdb02 ~]$  vi /u01/app/oracle/diag/crs/bltdb02/crs/trace/alert.log
	...
	2022-02-22 13:28:08.409 [GIPCD(6935)]CRS-7510: The 'daemon thread' of Oracle Grid Interprocess communication (GIPC) has been non-responsive for '10220' milliseconds. Additional diagnostics: process name: 'ocssd'.
	2022-02-22 13:28:10.689 [GIPCD(6935)]CRS-7510: The 'daemon thread' of Oracle Grid Interprocess communication (GIPC) has been non-responsive for '10220' milliseconds. Additional diagnostics: process name: 'evmd'.
	2022-02-22 13:28:10.690 [GIPCD(6935)]CRS-7510: The 'daemon thread' of Oracle Grid Interprocess communication (GIPC) has been non-responsive for '10140' milliseconds. Additional diagnostics: process name: 'octssd'.
	2022-02-22 13:28:10.691 [GIPCD(6935)]CRS-7510: The 'daemon thread' of Oracle Grid Interprocess communication (GIPC) has been non-responsive for '10230' milliseconds. Additional diagnostics: process name: 'crsd'.
	2022-02-22 13:28:46.083 [GIPCD(6935)]CRS-7510: The 'daemon thread' of Oracle Grid Int

dincosman

[oracle@bltdb02 ~]$  vi /u01/app/oracle/diag/crs/bltdb02/crs/trace/ocssd_605.trc
	...
	2022-02-22 13:31:21.778 :    CSSD:2703701760: [     INFO] GR+DB_+ASM:10693-22179350,
	2022-02-22 13:31:22.639 :GIPCHTHR:2700547840:  gipchaWorkerWork: workerThread heart beat, time interval since last heartBeat 30010 loopCount 119 sendCount 33 recvCount 87 postCount 33 sendCmplCount 33 recvCmplCount 53
	2022-02-22 13:31:23.586 :    CSSD:2720790272: [     INFO] clssgmpHandleFenceReq: Received fence request from node bltdb01, number 1, for clientID 2:20983:1, request does not kill primary
	2022-02-22 13:31:23.586 :    CSSD:2720790272: [     INFO] clssgmpcSendLocalFenceReq: Sending a fence rquest to node bltdb02, number 2, for clientID 2:20983:1
	2022-02-22 13:31:23.586 :    CSSD:2703701760: [     INFO] clssgmfInitLocalFence: Member level fence for global groupmemberID 14:2:1:33, fence type 1
	2022-02-22 13:31:23.586 :    CSSD:2703701760: [     INFO] clssgmfInitLocalFence: Fencing member (0x7fda4c06b520) with memberID 14:2:1 a

dincosman

 [oracle@bltdb02 ~]$ vi /u01/app/oracle/diag/crs/bltdb02/crs/trace/gipcd_256.trc
	...
	2022-02-22 13:17:52.243 :GIPCLXPT:973010688:  gipcdMonitorWriteOlrString: failed to open OLR key ret 5, root 'SYSTEM.network', ret gipcretFail (1)
	2022-02-22 13:17:53.347 :    GPNP:973010688:  procr_open_key_ext: OLR api procr_open_key_ext failed for key SYSTEM.GPnP.profiles.peer.best
	2022-02-22 13:17:53.347 :    GPNP:973010688:  procr_open_key_ext: OLR current boot level : 7
	2022-02-22 13:17:53.347 :    GPNP:973010688:  procr_open_key_ext: OLR error code    : 5
	2022-02-22 13:17:53.347 :    GPNP:973010688:  procr_open_key_ext: OLR error message : PROCL-5: User does not have permission to perform a local registry operation on this key.
	2022-02-22 13:17:53.347 :    GPNP:973010688:  clsgpnpco_ocr2profile: [at clsgpnpco.c:632] Result: (58) CLSGPNP_OCR_ERR. (:GPNP00701:)Failed to open requested OLR Profile.
	...

dincosman

  [oracle@bltdb02 archive]$ cd /u01/app/oracle/oracle.ahf/data/repository/suptools/bltdb02/oswbb/oracle/archive/oswiostat
  [oracle@bltdb02 archive]$ gunzip bltdb02.kkk.tsk_iostat_22.02.22.1300.dat.gz
  [oracle@bltdb02 archive]$ cd /u01/app/oracle/oracle.ahf/data/repository/suptools/bltdb02/oswbb/oracle/archive/oswpidstat
  [oracle@bltdb02 archive]$ gunzip bltdb02.kkk.tsk_pidstat_22.02.22.1300.dat.gz
  [oracle@bltdb02 archive]$ cd /u01/app/oracle/oracle.ahf/data/repository/suptools/bltdb02/oswbb/oracle/archive/oswpidstatd
  [oracle@bltdb02 archive]$ gunzip bltdb02.kkk.tsk_pidstatd_22.02.22.1300.dat.gz
  [oracle@bltdb02 archive]$ cd /u01/app/oracle/oracle.ahf/data/repository/suptools/bltdb02/oswbb/oracle/archive/oswvmstat
  [oracle@bltdb02 archive]$  gunzip bltdb02.kkk.tsk_vmstat_22.02.22.1300.dat.gz
  [oracle@bltdb02 archive]$ cd /u01/app/oracle/oracle.ahf/data/repository/suptools/bltdb02/oswbb/oracle/archive/oswtop
  [oracle@bltdb02 archive]$  gunzip bltdb02.kkk.tsk_top_22.02.22.1300.dat.gz

dincosman

[oracle@bltdb02 ~]$ ls -ls /dev/mapper
	...
	0 lrwxrwxrwx 1 root root       7 Feb 22 13:35 ol_bltdb02-root -> ../dm-0
	..
[oracle@bltdb02 ~]$ lsblk
	...
	sda                                8:0    0 136.7G  0 disk
	├─sda2                      8:2    0     1G  0 part  /boot
	├─sda3                      8:3    0 135.5G  0 part
	│ ├─ol_bltdb02-swap      252:1    0    16G  0 lvm   [SWAP]

dincosman

[oracle@bltdb02 ~]# export ORACLE_HOME=/u01/app/19/grid
[oracle@bltdb02 ~]# tfactl run oswbb
		--> Enter D for dashboard

dincosman

  # Section 1: System Status					
      ...
      Subsystem       Status
      ------------------------
      CPU             OK
      MEMORY          OK
      I/O             WARNING
      NET             OK
      ...

dincosman

[oracle@bltdb01 trace]$ crsctl stat res ora.cvu -p	
    ...
    RUN_INTERVAL=21600        
    ...

--21600 seconds equal to 6 hours.