Something went wrong and a compute is complaining some array is corrupt. It was 4x 1T 7.2K rpm disks arrayed together with mdadm to provide /localscratch2tb for heavy IO Gaussian jobs.
This is the process …
# first wipe the disk and partitions for sd[a-d]
[root@n74 ~]# wipefs --all --force /dev/sda1; wipefs --all --force /dev/sda
/dev/sda1: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
/dev/sda: 8 bytes were erased at offset 0x00000200 (gpt): 45 46 49 20 50 41 52 54
/dev/sda: 8 bytes were erased at offset 0xe8e0db5e00 (gpt): 45 46 49 20 50 41 52 54
/dev/sda: 2 bytes were erased at offset 0x000001fe (PMBR): 55 aa
[root@n74 ~]# wipefs --all --force /dev/sdb1; wipefs --all --force /dev/sdb
/dev/sdb1: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
/dev/sdb: 8 bytes were erased at offset 0x00000200 (gpt): 45 46 49 20 50 41 52 54
/dev/sdb: 8 bytes were erased at offset 0xe8e0db5e00 (gpt): 45 46 49 20 50 41 52 54
/dev/sdb: 2 bytes were erased at offset 0x000001fe (PMBR): 55 aa
[root@n74 ~]# wipefs --all --force /dev/sdc1; wipefs --all --force /dev/sdc
/dev/sdc1: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
/dev/sdc: 8 bytes were erased at offset 0x00000200 (gpt): 45 46 49 20 50 41 52 54
/dev/sdc: 8 bytes were erased at offset 0xe8e0db5e00 (gpt): 45 46 49 20 50 41 52 54
/dev/sdc: 2 bytes were erased at offset 0x000001fe (PMBR): 55 aa
[root@n74 ~]# wipefs --all --force /dev/sdd1; wipefs --all --force /dev/sdd
/dev/sdd1: 4 bytes were erased at offset 0x00001000 (linux_raid_member): fc 4e 2b a9
/dev/sdd: 8 bytes were erased at offset 0x00000200 (gpt): 45 46 49 20 50 41 52 54
/dev/sdd: 8 bytes were erased at offset 0xe8e0db5e00 (gpt): 45 46 49 20 50 41 52 54
/dev/sdd: 2 bytes were erased at offset 0x000001fe (PMBR): 55 aa
# make sure the partitions exist or create them
sgdisk -n 1:0:0 /dev/sda
sgdisk -n 1:0:0 /dev/sdb
sgdisk -n 1:0:0 /dev/sdc
sgdisk -n 1:0:0 /dev/sdd
# reboot
[root@n74 ~]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 931.5G 0 disk
└─sda1 8:1 0 931.5G 0 part
└─md127 9:127 0 931.4G 0 raid1
sdb 8:16 0 931.5G 0 disk
└─sdb1 8:17 0 931.5G 0 part
└─md127 9:127 0 931.4G 0 raid1
sdc 8:32 0 931.5G 0 disk
└─sdc1 8:33 0 931.5G 0 part
└─md127 9:127 0 931.4G 0 raid1
sdd 8:48 0 931.5G 0 disk
└─sdd1 8:49 0 931.5G 0 part
└─md127 9:127 0 931.4G 0 raid1
sde 8:64 0 29.5G 0 disk
├─sde1 8:65 0 524M 0 part
├─sde2 8:66 0 525M 0 part
├─sde3 8:67 0 525M 0 part [SWAP]
└─sde4 8:68 0 28G 0 part /
# now we stop and remove md127 device
[root@n74 ~]# mdadm --stop /dev/md127
mdadm: stopped /dev/md127
[root@n74 ~]# mdadm --remove /dev/md127
mdadm: error opening /dev/md127: No such file or directory <-- that's ok
# md127 gone
[root@n74 ~]# lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
sda 8:0 0 931.5G 0 disk
└─sda1 8:1 0 931.5G 0 part
sdb 8:16 0 931.5G 0 disk
└─sdb1 8:17 0 931.5G 0 part
sdc 8:32 0 931.5G 0 disk
└─sdc1 8:33 0 931.5G 0 part
sdd 8:48 0 931.5G 0 disk
└─sdd1 8:49 0 931.5G 0 part
sde 8:64 0 29.5G 0 disk
├─sde1 8:65 0 524M 0 part
├─sde2 8:66 0 525M 0 part
├─sde3 8:67 0 525M 0 part [SWAP]
└─sde4 8:68 0 28G 0 part /
# create the array, raid 10, 4 disks
[root@n74 ~]# mdadm --create --verbose /dev/md127 --level=10 --raid-devices=4 --force /dev/sd[a-d]1
mdadm: layout defaults to n2
mdadm: layout defaults to n2
mdadm: chunk size defaults to 512K
mdadm: size set to 976628736K
mdadm: automatically enabling write-intent bitmap on large array
mdadm: Defaulting to version 1.2 metadata
mdadm: array /dev/md127 started.
# ok, check
[root@n74 ~]# mdadm --detail /dev/md127
/dev/md127:
Version : 1.2
Creation Time : Wed Mar 1 13:21:58 2023
Raid Level : raid10
Array Size : 1953257472 (1862.77 GiB 2000.14 GB)
Used Dev Size : 976628736 (931.39 GiB 1000.07 GB)
Raid Devices : 4
Total Devices : 4
Persistence : Superblock is persistent
Intent Bitmap : Internal
Update Time : Wed Mar 1 13:22:42 2023
State : clean, resyncing
Active Devices : 4
Working Devices : 4
Failed Devices : 0
Spare Devices : 0
Layout : near=2
Chunk Size : 512K
Consistency Policy : bitmap
Resync Status : 0% complete
Name : n74:127 (local to host n74)
UUID : 2ebc7191:8b86aeae:e1b3b05e:b7443fe8
Events : 8
Number Major Minor RaidDevice State
0 8 1 0 active sync set-A /dev/sda1
1 8 17 1 active sync set-B /dev/sdb1
2 8 33 2 active sync set-A /dev/sdc1
3 8 49 3 active sync set-B /dev/sdd1
# make mounting persistent across boots
[root@n74 ~]# rm -f /etc/mdadm/mdadm.conf
[root@n74 ~]# mdadm --detail --scan --verbose | sudo tee -a /etc/mdadm/mdadm.conf
ARRAY /dev/md127 level=raid10 num-devices=4 metadata=1.2 name=n74:127 UUID=2ebc7191:8b86aeae:e1b3b05e:b7443fe8
devices=/dev/sda1,/dev/sdb1,/dev/sdc1,/dev/sdd1
# format
[root@n74 ~]# mkfs.ext4 /dev/md127
mke2fs 1.45.6 (20-Mar-2020)
/dev/md127 contains a xfs file system
Proceed anyway? (y,N) y
Creating filesystem with 488314368 4k blocks and 122085376 inodes
Filesystem UUID: 3da42aee-c22a-45a3-b026-1ebbe4d17809
Superblock backups stored on blocks:
32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208,
4096000, 7962624, 11239424, 20480000, 23887872, 71663616, 78675968,
102400000, 214990848
Allocating group tables: done
Writing inode tables: done
Creating journal (262144 blocks): done
Writing superblocks and filesystem accounting information: done
# edit /etc/fstab and mount
# 2tb
/dev/md127 /localscratch2tb ext4 defaults 0 0
[root@n74 ~]# df -h | egrep -i "used|md127"
Filesystem Size Used Avail Use% Mounted on
/dev/md127 1.8T 77M 1.7T 1% /localscratch2tb
# finish with permissions
[root@n74 ~]# chmod go+rwx /localscratch2tb
[root@n74 ~]# chmod o+t /localscratch2tb
[root@n74 ~]# ls -ld /localscratch2tb/
drwxrwxrwt 3 root root 4096 Mar 1 13:26 /localscratch2tb/
# test
[root@n74 ~]# reboot