Thursday, December 1, 2011

Solaris: metadb repair

$ df -k
Filesystem            kbytes    used   avail capacity  Mounted on
/dev/md/dsk/d100     10086628 8051562 1934200    81%    /
/devices                   0       0       0     0%    /devices
ctfs                       0       0       0     0%    /system/contract
proc                       0       0       0     0%    /proc
mnttab                     0       0       0     0%    /etc/mnttab
swap                 10513824    1072 10512752     1%    /etc/svc/volatile
objfs                      0       0       0     0%    /system/object
/platform/sun4u-us3/lib/libc_psr/libc_psr_hwcap1.so.1
                     10086628 8051562 1934200    81%    /platform/sun4u-us3/lib/libc_psr.so.1
/platform/sun4u-us3/lib/sparcv9/libc_psr/libc_psr_hwcap1.so.1
                     10086628 8051562 1934200    81%    /platform/sun4u-us3/lib/sparcv9/libc_psr.so.1
fd                         0       0       0     0%    /dev/fd
swap                 6146048 4469136 1676912    73%    /tmp
swap                 10512784      32 10512752     1%    /var/run
/dev/md/dsk/d102     10237036 7516386 2618280    75%    /opt
/dev/md/dsk/d104     70569513 50785798 19078020    73%    /opt/app
/dev/md/dsk/d103     1988623   29696 1899269     2%    /var/tmp
$ uname -a
SunOS intdev02 5.10 Generic_125100-10 sun4u sparc SUNW,Sun-Fire-V240


[03:35:01] root@intdev02[1]# grep -i warn /var/adm/messages
Jun 27 21:29:32 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:37 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2 (glm0):
Jun 27 21:30:37 intdev02 glm: [ID 401478 kern.warning] WARNING: ID[SUNWpd.glm.cmd_timeout.6018]
Jun 27 21:30:38 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2 (glm0):
Jun 27 21:30:38 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:38 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:41 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:41 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:46 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:46 intdev02 md_stripe: [ID 641072 kern.warning] WARNING: md: d300: write error on /dev/dsk/c1t1d0s0
Jun 27 21:30:46 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:46 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:46 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:46 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:46 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:46 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:46 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:46 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:51 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:51 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:51 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:51 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:51 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:51 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:51 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:51 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:51 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:56 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:56 intdev02 md_stripe: [ID 641072 kern.warning] WARNING: md: d300: read error on /dev/dsk/c1t1d0s0
Jun 27 21:30:56 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:56 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:56 intdev02 md_stripe: [ID 641072 kern.warning] WARNING: md: d300: write error on /dev/dsk/c1t1d0s0
Jun 27 21:30:56 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:56 intdev02 md_stripe: [ID 641072 kern.warning] WARNING: md: d300: read error on /dev/dsk/c1t1d0s0
Jun 27 21:30:56 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:56 intdev02 md_stripe: [ID 641072 kern.warning] WARNING: md: d300: read error on /dev/dsk/c1t1d0s0
Jun 27 21:30:56 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:56 intdev02 md_stripe: [ID 641072 kern.warning] WARNING: md: d300: write error on /dev/dsk/c1t1d0s0
Jun 27 21:30:56 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:56 intdev02 md_stripe: [ID 641072 kern.warning] WARNING: md: d300: read error on /dev/dsk/c1t1d0s0
Jun 27 21:30:56 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@1,0 (sd0):
Jun 27 21:30:56 intdev02 md_stripe: [ID 641072 kern.warning] WARNING: md: d300: read error on /dev/dsk/c1t1d0s0
Jun 27 21:31:01 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@0,0 (sd3):
Jun 27 21:31:01 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@2,0 (sd1):
Jun 27 21:31:01 intdev02 scsi: [ID 107833 kern.warning] WARNING: /pci@1c,600000/scsi@2/sd@3,0 (sd2):
[03:35:01] root@intdev02[2]# format
Searching for disks...done


AVAILABLE DISK SELECTIONS:
       0. c1t0d0
          /pci@1c,600000/scsi@2/sd@0,0
       1. c1t1d0
          /pci@1c,600000/scsi@2/sd@1,0
       2. c1t2d0
          /pci@1c,600000/scsi@2/sd@2,0
       3. c1t3d0
          /pci@1c,600000/scsi@2/sd@3,0
Specify disk (enter its number): 1
selecting c1t1d0
[disk formatted]
/dev/dsk/c1t1d0s0 is part of SVM volume stripe:d300. Please see metaclear(1M).
/dev/dsk/c1t1d0s1 is part of SVM volume stripe:d301. Please see metaclear(1M).
/dev/dsk/c1t1d0s3 is part of SVM volume stripe:d302. Please see metaclear(1M).
/dev/dsk/c1t1d0s4 is part of SVM volume stripe:d303. Please see metaclear(1M).
/dev/dsk/c1t1d0s5 is part of SVM volume stripe:d302. Please see metaclear(1M).
/dev/dsk/c1t1d0s7 contains an SVM mdb. Please see metadb(1M).


FORMAT MENU:
        disk       - select a disk
        type       - select (define) a disk type
        partition  - select (define) a partition table
        current    - describe the current disk
        format     - format and analyze the disk
        repair     - repair a defective sector
        label      - write label to the disk
        analyze    - surface analysis
        defect     - defect list management
        backup     - search for backup labels
        verify     - read and display labels
        save       - save new disk/partition definitions
        inquiry    - show vendor, product and revision
        volname    - set 8-character volume name
        !     - execute , then return
        quit
format>
[03:35:01] root@intdev02[3]#
[03:35:01] root@intdev02[5]# date
Tue Jul  1 03:39:06 MEST 2008
[03:35:01] root@intdev02[6]# iostat -En | more
c1t1d0           Soft Errors: 0 Hard Errors: 21 Transport Errors: 2
Vendor: SEAGATE  Product: ST373207LSUN72G  Revision: 045A Serial No: 05433302ZN
Size: 73.40GB <73400057856 bytes>
Media Error: 0 Device Not Ready: 19 No Device: 1 Recoverable: 0
Illegal Request: 0 Predictive Failure Analysis: 0
[03:35:01] root@intdev02[9]# metadb -i
        flags           first blk       block count
     a m  p  luo        16              8192            /dev/dsk/c1t0d0s7
     a    p  luo        8208            8192            /dev/dsk/c1t0d0s7
      W   p  l          16              8192            /dev/dsk/c1t1d0s7
     a    p  luo        8208            8192            /dev/dsk/c1t1d0s7
     a    p  luo        16              8192            /dev/dsk/c1t2d0s7
     a    p  luo        8208            8192            /dev/dsk/c1t2d0s7
     a    p  luo        16              8192            /dev/dsk/c1t3d0s7
     a    p  luo        8208            8192            /dev/dsk/c1t3d0s7
 r - replica does not have device relocation information
 o - replica active prior to last mddb configuration change
 u - replica is up to date
 l - locator for this replica was read successfully
 c - replica's location was in /etc/lvm/mddb.cf
 p - replica's location was patched in kernel
 m - replica is master, this is replica selected as input
 W - replica has device write errors
 a - replica is active, commits are occurring to this replica
 M - replica had problem with master blocks
 D - replica had problem with data blocks
 F - replica had format problems
 S - replica is too small to hold current data base
 R - replica had device read errors
[03:35:01] root@intdev02[10]#


     # metadb -a c0t2d0s3 c1t1d0s3

     Example 3: Deleting Two Replicas

     This example shows how to delete two replicas from the  sys-
     tem.   Assume   that   replicas   have   been   set   up  on
     /dev/dsk/c0t2d0s3 and /dev/dsk/c1t1d0s3.

     # metadb -d c0t2d0s3 c1t1d0s3
[03:35:01] root@intdev02[18]# metadb -i
        flags           first blk       block count
     a m  p  luo        16              8192            /dev/dsk/c1t0d0s7
     a    p  luo        8208            8192            /dev/dsk/c1t0d0s7
      W   p  l          16              8192            /dev/dsk/c1t1d0s7
     a    p  luo        8208            8192            /dev/dsk/c1t1d0s7
     a    p  luo        16              8192            /dev/dsk/c1t2d0s7
     a    p  luo        8208            8192            /dev/dsk/c1t2d0s7
     a    p  luo        16              8192            /dev/dsk/c1t3d0s7
     a    p  luo        8208            8192            /dev/dsk/c1t3d0s7
 r - replica does not have device relocation information
 o - replica active prior to last mddb configuration change
 u - replica is up to date
 l - locator for this replica was read successfully
 c - replica's location was in /etc/lvm/mddb.cf
 p - replica's location was patched in kernel
 m - replica is master, this is replica selected as input
 W - replica has device write errors
 a - replica is active, commits are occurring to this replica
 M - replica had problem with master blocks
 D - replica had problem with data blocks
 F - replica had format problems
 S - replica is too small to hold current data base
 R - replica had device read errors
[03:35:01] root@intdev02[19]# metadb -d c1t1d0s7
[03:35:01] root@intdev02[20]# metadb -i
        flags           first blk       block count
     a m  p  luo        16              8192            /dev/dsk/c1t0d0s7
     a    p  luo        8208            8192            /dev/dsk/c1t0d0s7
     a    p  luo        16              8192            /dev/dsk/c1t2d0s7
     a    p  luo        8208            8192            /dev/dsk/c1t2d0s7
     a    p  luo        16              8192            /dev/dsk/c1t3d0s7
     a    p  luo        8208            8192            /dev/dsk/c1t3d0s7
 r - replica does not have device relocation information
 o - replica active prior to last mddb configuration change
 u - replica is up to date
 l - locator for this replica was read successfully
 c - replica's location was in /etc/lvm/mddb.cf
 p - replica's location was patched in kernel
 m - replica is master, this is replica selected as input
 W - replica has device write errors
 a - replica is active, commits are occurring to this replica
 M - replica had problem with master blocks
 D - replica had problem with data blocks
 F - replica had format problems
 S - replica is too small to hold current data base
 R - replica had device read errors
[03:35:01] root@intdev02[21]# metadb -a c1t1d0s7
[03:35:01] root@intdev02[22]# metadb -i
        flags           first blk       block count
     a m  p  luo        16              8192            /dev/dsk/c1t0d0s7
     a    p  luo        8208            8192            /dev/dsk/c1t0d0s7
     a        u         16              8192            /dev/dsk/c1t1d0s7
     a    p  luo        16              8192            /dev/dsk/c1t2d0s7
     a    p  luo        8208            8192            /dev/dsk/c1t2d0s7
     a    p  luo        16              8192            /dev/dsk/c1t3d0s7
     a    p  luo        8208            8192            /dev/dsk/c1t3d0s7
 r - replica does not have device relocation information
 o - replica active prior to last mddb configuration change
 u - replica is up to date
 l - locator for this replica was read successfully
 c - replica's location was in /etc/lvm/mddb.cf
 p - replica's location was patched in kernel
 m - replica is master, this is replica selected as input
 W - replica has device write errors
 a - replica is active, commits are occurring to this replica
 M - replica had problem with master blocks
 D - replica had problem with data blocks
 F - replica had format problems
 S - replica is too small to hold current data base
 R - replica had device read errors
[03:35:01] root@intdev02[23]#