script: SI_14610_FM_FCCA.

pl
Log directory: /volume/regressions/results/JUNOS/HEAD/debug/REG_352387/Manhattan
/SI/DCBU_REG/Platform/HA/SI_14610_FM_FCCA.pl.dcbg-rod-chelsea-d_dcbg-tg.6197031.
20130519040653
root@qfabric> show system core-dumps core-file-info /pbdata/export/rdumps/17e0bb
92-6e7f-11e3-8b10-00e081cbbbfe
Repository scope: shared
Repository head: /pbdata/export
Repository name: core
error: No such file "/pbdata/export/rdumps/17e0bb92-6e7f-11e3-8b10-00e081cbbbfe"
root@qfabric> show system core-dumps component 17e0bb92-6e7f-11e3-8b10-00e081cbb
bfe repository core
Repository scope: shared
Repository head: /pbdata/export
Repository name: core
List of core dumps for component 17e0bb92-6e7f-11e3-8b10-00e081cbbbfe
Repository location: /pbdata/export/rdumps/17e0bb92-6e7f-11e3-8b10-00e081cbbbfe
Filename
Date
Size
fcd.core.0.2127.12292013113644.gz
fcd.core.1.7247.12292013113645.gz
Number of core dumps in repository:2

Dec 29 11:36:44 2013

Dec 29 11:36:45 2013

124821
101937

root@qfabric> .../pbdata/export/rdumps/17e0bb92-6e7f-11e3-8b10-00e081cbbbfe/fcd.
core.0.2127.12292013113644.gz
Repository scope: shared
Repository head: /pbdata/export
Repository name: core
'fcd' process terminated
Stack trace:
#0 0x080b3cfd in ?? ()
#0 0x080b3cfd in ?? ()
#1 0x00000051 in ?? ()
#2 0x0000001f in ?? ()
#3 0x0833fef1 in ?? ()
#4 0x085245e8 in ?? ()
root@qfabric> ...core-dumps core-file-info /pbdata/export/rdumps/17e0bb92-6e7f-1
1e3-8b10-00e081cbbbfe/fcd.core.0.2127.12292013113644.gz ?
Possible completions:
<[Enter]>
Execute this command
brief
View details of binary
component
Name of the component (uuid or serial number)
detail
View stack trace with details of binary
|
Pipe through a command
root@qfabric> ...xport/rdumps/17e0bb92-6e7f-11e3-8b10-00e081cbbbfe/fcd.core.0.21
27.12292013113644.gz detail | no-more
Repository scope: shared
Repository head: /pbdata/export
Repository name: core
'fcd' process terminated
Release: 13.2-20131219_rt1_vqfd.0
Build Server: svl-cdbu-pool011.juniper.net
Build Date: 2013-12-19 14:15:34 UTC

Detail:
FCD release 13.2-20131219_rt1_vqfd.0 built by dc-builder on 2013-12-19 14:15:34
UTC
svl-cdbu-pool011.juniper.net:/d/jenkins/20131219040727/sb-junos/obj-i386/dcp-sha
red/usr.sbin/fcd
Stack trace:
#0 0x080b3cfd in ?? ()
#0 0x080b3cfd in ?? ()
#1 0x00000051 in ?? ()
#2 0x0000001f in ?? ()
#3 0x0833fef1 in ?? ()
#4 0x085245e8 in ?? ()

Sun May 19 22:50:09 [pramodpb@eng-shell4: ~/qfx_cores/13.1_cores]ls -l

total 544
-rwxrwxrwx 1 pramodpb ipg 244588 May 19 22:47 eswd.core.0.1734.05192013033447
.gz
-rwxrwxrwx 1 pramodpb ipg 26396 May 19 22:49 eswd.tarball.0.1734.05192013033
448.tgz
Sun May 19 22:50:12 [pramodpb@eng-shell4: ~/qfx_cores/13.1_cores]jdebug eswd.cor
e.0.1734.05192013033447.gz
[File is compressed. This may take a moment...]
File type: executable core shared.
GNU gdb 6.5 juniper_2006a_411
Copyright (C) 2006 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "i386-unknown-freebsd6.2".
Core was generated by `eswd'.
Program terminated with signal 11, Segmentation fault.
#0 0x0000ffff in ?? ()
[New Thread 0x89b7000 (LWP 100092)]
(gdb) bt
#0 0x0000ffff in ?? ()
#1 0x08285faf in _sess_process_packet (sessp=0x8a00850, sp=0x8a2f330, isp=0x8b5
e4e0, transport=0x8a30130, opaque=0x8aa4000, olength=106,
packetptr=0x8b78000 "\001\022", length=88) at ../../../../src/dist/net-snmp/
snmplib/snmp_api.c:5460
#2 0x0828733c in _sess_read (sessp=0x8a00850, fdset=0xbfb7dcec) at ../../../../
src/dist/net-snmp/snmplib/snmp_api.c:6073
#3 0x0828758b in snmp_sess_read2 (sessp=0x8a00850, fdset=0xbfb7dcec) at ../../.
./../src/dist/net-snmp/snmplib/snmp_api.c:6234
#4 0x08287627 in snmp_read2 (fdset=0xbfb7dcec) at ../../../../src/dist/net-snmp
/snmplib/snmp_api.c:5757
#5 0x0821ada9 in jnx_ns_task_io_reader (tp=0x8a2e000) at ../../../../src/junos/
lib/libjtask/snmp/ns_transport_task.c:215
#6 0x0822eda4 in task_process_events (level=0, eventlist=0x89c1000, nevents=1)
at ../../../../src/junos/lib/libjtask/base/task_io.c:3523
#7 0x08227d1d in task_scheduler () at ../../../../src/junos/lib/libjtask/base/t
ask_scheduler.c:441
#8 0x080937fe in main (argc=146109092, argv=0x8b572a4) at ../../../../../../src
/esp-shared/usr.sbin/eswd//core/task_init.c:853
(gdb) quit
################################################################################

################################################################

Archive Log: /pbdata/export/rlogs/P2132-C/log.tarball.0.05062013004938.tgz

Resulting tarfile: /tmp/pblogs.tgz
[root@dg1 P2132-C]# cd /pbdata/export/rlogs/P2132-C
[root@dg1 P2132-C]# ls
log.tarball.0.05062013004938.tgz
scp /pbdata/export/rlogs/P2132-C/log.tarball.0.05062013004938.tgz pramodpb@eng-s
hell1.juniper.net:/volume/labcores/PR/PR-879172

root@qfabric> show system core-dumps

Repository scope: shared
Repository head: /pbdata/export
List of nodes for core repository: /pbdata/export/rdumps/
Node Group
Node Identifier
Num Model
DG-0
DG-1
FM-0
FM-0
DRE-0
NW-NG-0
NW-NG-0
FC-0
FC-1
IC-BBAP0739
IC-BBAP0735
P5606-C
P4782-C
P6079-C
P1442-C
P2132-C
P6445-C
Total usage of

00020003-0004-0005-0006-000700080009
0c24f80d-e678-e111-9969-bf34052a638c
c179fd74-b2c0-11e2-b01d-00e081c1da22
c1e06d52-b2c0-11e2-b428-00e081c1da22
d095cc02-b2c0-11e2-b42b-00e081c1da22
39683ef4-b2c1-11e2-9386-00e081c1da22
424f4ca6-b2c1-11e2-8f82-00e081c1da22
4b4efc7a-b2c1-11e2-8084-00e081c1da22
54566b0a-b2c1-11e2-96f8-00e081c1da22
BBAP0739
BBAP0735
P5606-C
P4782-C
P6079-C
P1442-C
P2132-C
P6445-C
core repository:20M of 70000M (0.0%)

0
0
0
0
0
1
0
0
0
0
0
0
0
0
0
3
0

qfx3100
qfx3100
fx-jvre
fx-jvre
fx-jvre
fx-jvre
fx-jvre
fx-jvre
fx-jvre
qfx3600-i
qfx3600-i
qfx3500
qfx3500
qfx3600-16q
qfx3500
qfx3500
qfx3600-16q

Usage
0M
0M
0M
0M
0M
3M
0M
0M
0M
0M
0M
0M
0M
0M
0M
17M
0M

List of nodes for log repository: /pbdata/export/rlogs/

Node Group
Node Identifier
Num Model

Usage

DG-0
DG-1
FM-0
FM-0
DRE-0
NW-NG-0
NW-NG-0
FC-0
FC-1
IC-BBAP0739
IC-BBAP0735
P5606-C
P4782-C

0M
0M
1M
1M
4M
1M
1M
1M
1M
1M
2M
2M
2M

00020003-0004-0005-0006-000700080009
0c24f80d-e678-e111-9969-bf34052a638c
c179fd74-b2c0-11e2-b01d-00e081c1da22
c1e06d52-b2c0-11e2-b428-00e081c1da22
d095cc02-b2c0-11e2-b42b-00e081c1da22
39683ef4-b2c1-11e2-9386-00e081c1da22
424f4ca6-b2c1-11e2-8f82-00e081c1da22
4b4efc7a-b2c1-11e2-8084-00e081c1da22
54566b0a-b2c1-11e2-96f8-00e081c1da22
BBAP0739
BBAP0735
P5606-C
P4782-C

0
0
1
1
1
2
1
1
1
1
1
1
1

qfx3100
qfx3100
fx-jvre
fx-jvre
fx-jvre
fx-jvre
fx-jvre
fx-jvre
fx-jvre
qfx3600-i
qfx3600-i
qfx3500
qfx3500

P6079-C
P1442-C
P2132-C
P6445-C
Total usage of

P6079-C
P1442-C
P2132-C
P6445-C
log repository:25M of 70000M (0.0%)

1
1
3
1

qfx3600-16q
qfx3500
qfx3500
qfx3600-16q

2M
2M
2M
2M

[root@dg1 tmp]# cd /pbdata/export/rlogs/

[root@dg1 rlogs]# ls
00020003-0004-0005-0006-000700080009 c1e06d52-b2c0-11e2-b428-00e081c1da22
0c24f80d-e678-e111-9969-bf34052a638c d095cc02-b2c0-11e2-b42b-00e081c1da22
39683ef4-b2c1-11e2-9386-00e081c1da22 P1442-C
424f4ca6-b2c1-11e2-8f82-00e081c1da22 P2132-C
4b4efc7a-b2c1-11e2-8084-00e081c1da22 P4782-C
54566b0a-b2c1-11e2-96f8-00e081c1da22 P5606-C
BBAP0735
P6079-C
BBAP0739
P6445-C
c179fd74-b2c0-11e2-b01d-00e081c1da22
[root@dg1 rlogs]# cd P2132-C
[root@dg1 P2132-C]# ls
log.tarball.0.05062013004938.tgz
rpd.tarball.0.1444.05042013225207.tgz
rpdf.tarball.0.1899.05042013224653.tgz
[root@dg1 P2132-C]# ls -l
total 2712
-rw-r--r-- 1 root root 2703909 May 6 00:49 log.tarball.0.05062013004938.tgz
-rw-r--r-- 1 root root 19254 May 4 22:46 rpdf.tarball.0.1899.05042013224653.t
gz
-rw-r--r-- 1 root root 25254 May 4 22:52 rpd.tarball.0.1444.05042013225207.tg
z

[root@dg1 P2132-C]# pwd ; ls -la

/pbdata/export/rdumps/P2132-C
total 17852
drwxr-xr-x 2 root root
3864 May
drwxr-xr-x 19 root root
3864 May
-rwxr-xr-x 1 root root
4 May
-rw-r--r-- 1 root root 9662748 May
z
-rw-r--r-- 1 root root 4319760 May
-rw-r--r-- 1 root root 4222763 May

4
6
6
4

22:57
00:51
04:31
22:57

.
..
.index
pafxpc.core.0.1271.05042013225657.g

4 22:52 rpd.core.0.1444.05042013225201.gz
4 22:46 rpdf.core.0.1899.05042013224648.gz

Mon May 06 04:39:42 [pramodpb@eng-shell1: ~/qfx_cores]jdebug rpdf.core.0.1899.05

042013224648.gz
[File is compressed. This may take a moment...]
File type: executable core shared.
GNU gdb 6.5 juniper_2006a_411
Copyright (C) 2006 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "--host=i386-unknown-freebsd6.2 --target=mips64-junos
".
Core was generated by `rpdf'.

Program terminated with signal 6, Aborted.

#0 0x41e7de88 in kevent () at kevent.S:2
2
kevent.S: No such file or directory.
in kevent.S
Current language: auto; currently asm
(gdb) bt
#0 0x41e7de88 in kevent () at kevent.S:2
#1 0x00664b94 in main (argc=4853, argv=<value optimized out>) at ../../../../..
/src/junos/usr.sbin/rpd/os/task.c:6888
(gdb) quit
Mon May 06 04:40:42 [pramodpb@eng-shell1: ~/qfx_cores]jdebug rpd.core.0.1444.050
42013225201.gz
[File is compressed. This may take a moment...]
File type: executable core shared.
GNU gdb 6.5 juniper_2006a_411
Copyright (C) 2006 Free Software Foundation, Inc.
GDB is free software, covered by the GNU General Public License, and you are
welcome to change it and/or distribute copies of it under certain conditions.
Type "show copying" to see the conditions.
There is absolutely no warranty for GDB. Type "show warranty" for details.
This GDB was configured as "--host=i386-unknown-freebsd6.2 --target=mips64-junos
".
Core was generated by `rpd'.
Program terminated with signal 6, Aborted.
#0 0x41dcde88 in kevent () at kevent.S:2
2
kevent.S: No such file or directory.
in kevent.S
Current language: auto; currently asm
(gdb) bt
#0 0x41dcde88 in kevent () at kevent.S:2
#1 0x005e29b4 in main (argc=5165, argv=<value optimized out>) at ../../../../..
/src/junos/usr.sbin/rpd/os/task.c:6888
(gdb)