#

2019-04-16 16:44:06.949000 +08:00

ORA-04031 heap dump being written to trace file
/oracle/app/oracle/diag/rdbms/cj/cj1/trace/cj1_ora_17506.trc
Errors in file /oracle/app/oracle/diag/rdbms/cj/cj1/trace/cj1_ora_17506.trc
(incident=672884):
ORA-04031: unable to allocate 4096 bytes of shared memory ("java
pool","oracle/aurora/rdbms/DbmsJava","JOXLE^8df486e2",":SGAClass")
Incident details in:
/oracle/app/oracle/diag/rdbms/cj/cj1/incident/incdir_672884/cj1_ora_17506_i672884.t
rc
2019-04-16 16:44:10.319000 +08:00
Errors in file /oracle/app/oracle/diag/rdbms/cj/cj1/trace/cj1_m000_17550.trc
(incident=673052):
ORA-04031: unable to allocate 4096 bytes of shared memory ("java
pool","oracle/aurora/vm/OracleRuntime","JOXLE^bca0300f",":SGAClass")
Incident details in:
/oracle/app/oracle/diag/rdbms/cj/cj1/incident/incdir_673052/cj1_m000_17550_i673052.
trc
2019-04-16 16:44:11.855000 +08:00
Dumping diagnostic data in directory=[cdmp_20190416164411], requested by
(instance=1, osid=17550 (M000)), summary=[incident=673052].
Use ADRCI or Support Workbench to package the incident.
See Note 411.1 at My Oracle Support for error and packaging details.
Errors in file /oracle/app/oracle/diag/rdbms/cj/cj1/trace/cj1_m000_17550.trc:
ORA-04031: unable to allocate 4096 bytes of shared memory ("java
pool","oracle/aurora/vm/OracleRuntime","JOXLE^bca0300f",":SGAClass")
2019-04-16 16:44:14.258000 +08:00

#
/oracle/app/oracle/diag/rdbms/cj/cj1/incident/incdir_672884/cj1_ora_17506_i672884.t
rc

*** 2019-04-16 16:44:07.115

*** SESSION ID:(648.58209) 2019-04-16 16:44:07.115
*** CLIENT ID:() 2019-04-16 16:44:07.115
*** SERVICE NAME:(SYS$USERS) 2019-04-16 16:44:07.115
*** MODULE NAME:(exp@kdcj1 (TNS V1-V3)) 2019-04-16 16:44:07.115
*** ACTION NAME:() 2019-04-16 16:44:07.115

Dump continued from file:

/oracle/app/oracle/diag/rdbms/cj/cj1/trace/cj1_ora_17506.trc
1> ***** Error Stack *****
ORA-04031: unable to allocate 4096 bytes of shared memory ("java
pool","oracle/aurora/rdbms/DbmsJava","JOXLE^8df486e2",":SGAClass")
1< ***** Error Stack *****
1> ***** Dump for incident 672884 (ORA 4031) *****

*** 2019-04-16 16:44:07.116

dbkedDefDump(): Starting incident default dumps (flags=0x2, level=3,
mask=0x0)
2> ***** Current SQL Statement for this session (sql_id=4hjkbgj0815p4) *****
SELECT SYNNAM, DBMS_JAVA.LONGNAME(SYNNAM), DBMS_JAVA.LONGNAME(SYNTAB),
TABOWN, TABNODE, PUBLIC$, SYNOWN, SYNOWNID, TABOWNID, SYNOB
JNO FROM SYS.EXU9PTS WHERE SYNOBJNO IN ( SELECT
SYNOBJNO FROM SYS.EXU9TYPT WHERE TABOBJNO = :1 )
ORDER BY SYNTIME
SQL> @sgastatx ACCESS

-- All allocations:

SUBPOOL BYTES MB
------------------------------ --------------- ---------------
shared pool (1): 4831838208 4608
shared pool (2): 5905580032 5632
shared pool (3): 4831838208 4608
shared pool (4): 3758096384 3584
shared pool (5): 3221225472 3072
shared pool (6): 3758096384 3584
shared pool (7): 4294967296 4096
shared pool (Total): 30601641984 29184

8 rows selected.

-- Allocations matching "ACCESS":

SUBPOOL NAME SUM(BYTES)

MB
------------------------------ -------------------------- ---------------
---------------
shared pool (1): KGH: NO ACCESS 1067057376
1017.63

shared pool (2): KGH: NO ACCESS 1610498272

1535.89

shared pool (3): KGH: NO ACCESS 1610416384

1535.81

shared pool (4): KGH: NO ACCESS 1073643648

1023.91

shared pool (5): KGH: NO ACCESS 536772704

511.91

shared pool (6): KGH: NO ACCESS 536756352

511.89

shared pool (7): KGH: NO ACCESS 532988192

508.3

SUBPOOL NAME SUM(BYTES)

MB
------------------------------ -------------------------- ---------------
---------------
shared pool (1): KGH: NO ACCESS 1067057376
1017.63
SQLA 1049661032
1001.03
free memory 824445472
786.25
KGLH0 627699592
598.62
 kglsim object batch 297524136
283.74
gcs resources 256351264
244.48
kglsim heap 166060800
158.37
KGLHD 88362832
84.27

SQL> show parameter sga

PARAMETER_NAME TYPE VALUE

------------------------------------------------------------ -----------
-----------------------------------------------------------------------------------
-----------------
lock_sga boolean FALSE
pre_page_sga boolean FALSE
sga_max_size big integer 130G
sga_target big integer 128G

AUTO SGA: Errored out rq 0x1fc3c8b1d8 dumped below 0, (nil), 0

SQL> @usid 1350

USERNAME SID AUDSID OSUSER MACHINE

PROGRAM SPID OPID CPID SQL_ID
HASH_VALUE LASTCALL STATUS SADDR PADDR TADDR
LOGON_TIME
----------------------- -------------- ----------- ----------------
------------------ -------------------- -------------- ------
------------------------ ------------- ----------- ---------- --------
---------------- ---------------- ---------------- -----------------
'1350,1' 0 oracle kdcj1
(MMAN) 2636 19 2636
0 11040401 ACTIVE 0000001F22636FC0 0000001F22208510 20181212
14:23:55

SQL> oradebug setorapid 19

Oracle pid: 19, Unix process pid: 2636, image: oracle@kdcj1 (MMAN)
SQL> oradebug short_stack;
ksedsts()+465<-ksdxfstk()+32<-ksdxcb()+1927<-sspuser()+112<-__sighandler()<-
semtimedop()+10<-skgpwwait()+178<-ksliwat()+2022<-kslwaitctx()+163<-kslwait()+141<-
ksarcv()+207<-ksbabs()+330<-ksbrdp()+1045<-opirip()+623<-opidrv()+603<-sou2o()
+103<-opimai_real()+250<-ssthrdmain()+265<-main()+201<-__libc_start_main()+230
SQL> oradebug short_stack;
ksedsts()+465<-ksdxfstk()+32<-ksdxcb()+1927<-sspuser()+112<-__sighandler()<-
semtimedop()+10<-skgpwwait()+178<-ksliwat()+2022<-kslwaitctx()+163<-kslwait()+141<-
ksarcv()+207<-ksbabs()+330<-ksbrdp()+1045<-opirip()+623<-opidrv()+603<-sou2o()
+103<-opimai_real()+250<-ssthrdmain()+265<-main()+201<-__libc_start_main()+230
SQL> oradebug short_stack;
ksedsts()+465<-ksdxfstk()+32<-ksdxcb()+1927<-sspuser()+112<-__sighandler()<-
semtimedop()+10<-skgpwwait()+178<-ksliwat()+2022<-kslwaitctx()+163<-kslwait()+141<-
ksarcv()+207<-ksbabs()+330<-ksbrdp()+1045<-opirip()+623<-opidrv()+603<-sou2o()
+103<-opimai_real()+250<-ssthrdmain()+265<-main()+201<-__libc_start_main()+230
SQL>
9:31:07
anbob 2019/4/19 9:31:07
SQL> @s 1350

SID SQLID_AND_CHILD STATUS STATE EVENT

SEQ# SEC_IN_WAIT BLOCKING_SID P1 P2 P3
P1TRANSL
------- -------------------- -------- -------
---------------------------------------- ---------- ----------- ------------
------------------ ------------------ ------------------
------------------------------------------
1350 0 ACTIVE WAITING rdbms ipc message
46261 1 NO HOLDER timeout= 300 0 0

SQL> @snapper all 5 1 1350

Sampling SID 1350 with interval 5 seconds, taking 1 snapshots...

-- Session Snapper v4.14 BETA - by Tanel Poder ( http://blog.tanelpoder.com ) -

Enjoy the Most Advanced Oracle Troubleshooting Script on the Planet! :)

-----------------------------------------------------------------------------------
-----------------------------------------------------------------------------------
--------------------------------------------
SID @INST, USERNAME , TYPE, STATISTIC
, DELTA, HDELTA/SEC, %TIME, GRAPH , NUM_WAITS, WAITS/SEC,
AVERAGES
-----------------------------------------------------------------------------------
-----------------------------------------------------------------------------------
--------------------------------------------
1350 @1, (MMAN) , STAT, in call idle wait time
, 600, 119.34, , , , ,
~ per execution
1350 @1, (MMAN) , STAT, background timeouts
, 2, .4, , , , ,
~ per execution
1350 @1, (MMAN) , TIME, background elapsed time
, 217, 43.16us, .0%, [ ], , ,
1350 @1, (MMAN) , WAIT, rdbms ipc message
, 4999798, 994.45ms, 99.4%, [WWWWWWWWWW], 2, .4,
2.5s average wait

-- End of Stats snap 1, end=2019-04-19 09:30:29, seconds=5

<No active sessions captured during the sampling period>

-- End of ASH snap 1, end=2019-04-19 09:30:29, seconds=5, samples_taken=48

PL/SQL procedure successfully completed.

10:18:44
anbob 2019/4/19 10:18:44
===============================================================================
HANG ANALYSIS:
instances (db_name.oracle_sid): cj.cj1
oradebug_node_dump_level: 3
os thread scheduling delay history: (sampling every 1.000000 secs)
0.000000 secs at [ 09:25:28 ]
NOTE: scheduling delay has not been sampled for 0.267889 secs 0.000000
secs from [ 09:25:24 - 09:25:29 ], 5 sec avg
0.000000 secs from [ 09:24:29 - 09:25:29 ], 1 min avg
0.000000 secs from [ 09:20:29 - 09:25:29 ], 5 min avg
 vktm time drift history
===============================================================================

Chains most likely to have caused the hang:

[a] Chain 1 Signature: <not in a wait>
Chain 1 Signature Hash: 0x673a0128
[b] Chain 2 Signature: <not in a wait>
Chain 2 Signature Hash: 0x673a0128
[c] Chain 3 Signature: <not in a wait>
Chain 3 Signature Hash: 0x673a0128

SQL> host
oracle@kdcj1:/home/oracle>ps -ef|grep mman
grid 607 1 0 2018 ? 00:10:35 asm_mman_+ASM1
oracle 2636 1 1 2018 ? 2-03:24:35 ora_mman_cj1
oracle 140048 139968 0 10:23 pts/7 00:00:00 grep mman
oracle@kdcj1:/home/oracle>exit
exit

SQL> @uspid 2636

USERNAME SID AUDSID OSUSER MACHINE

PROGRAM SPID SQL_HASH_VALUE LASTCALL STATUS
----------------------- -------------- ----------- ----------------
------------------ -------------------- ------------------------ --------------
---------- --------
'1350,1' 0 oracle kdcj1
oracle@kdcj1 (MMAN) 2636 0 11044816 ACTIVE

SQL> @usid 1350

USERNAME SID AUDSID OSUSER MACHINE

PROGRAM SPID OPID CPID SQL_ID
HASH_VALUE LASTCALL STATUS SADDR PADDR TADDR
LOGON_TIME
----------------------- -------------- ----------- ----------------
------------------ -------------------- -------------- ------
------------------------ --------------- ----------- ---------- --------
---------------- ---------------- ---------------- -----------------
'1350,1' 0 oracle kdcj1
(MMAN) 2636 19 2636
0 11044830 ACTIVE 0000001F22636FC0 0000001F22208510 20181212
14:23:55

SQL> oradebug setorapid 19

Oracle pid: 19, Unix process pid: 2636, image: oracle@kdcj1 (MMAN)

SQL> oradebug dump processstate 10;

Statement processed.

SQL> oradebug event immediate trace name errorstack level 3;

Statement processed.
SQL> oradebug tracefile_name
/oracle/app/oracle/diag/rdbms/cj/cj1/trace/cj1_mman_2636.trc
SQL> exit