Detecting Hidden File System Problems

Nicholas P. Cardo

National Energy Research Scientific Computing Center

Lawrence Berkeley National Laboratory

cardo@nersc.gov

•File system problems can exist•Sometimes there are no errors•How to detect a non-error problem•How to do it quickly

The Dilemma

# grep -i lustre messages | grep -i error | wc –l5836 (6 days)

3

Words of Wisdom

“Strive for perfection in everything. Take the best that exists and make it better. If it does not exist, create it. Accept nothing nearly right or good enough.”

Sir Henry Royceco-founder of Rolls-Royce

4

•llapi_ping ….. ping lustre components•llapi_file_create … create a file w/stripes•llapi_file_get_stripe … read stripe info

Useful API Calls

5

Wanted: Dead or Alive

hc=/proc/fs/lustre/health_check

for node in $mds $ossdo

xx=`ssh $node cat $hc`if [ “$xx” != “healthy” ]then

echo “$node not healthy”fi

done

6

OST Ping

/proc/fs/lustre/osc

readdir

llapi_ping

ping failed

., ..,num_refs DT_DIR

rc=0

Y

Y

Y

Y

N

N

N

N

7

How About

dir = opendir("/proc/fs/lustre/osc");

while((d = readdir(dir)) != NULL) {

if (!strcmp(d->d_name,".")) continue; if (!strcmp(d->d_name,"..")) continue; if (!strcmp(d->d_name,"num_refs")) continue;

if ( d->d_type == DT_DIR ) { if ((llrc=llapi_ping("osc”,d->d_name)) != 0) fprintf(stderr,”problem %s\n”,osc); }}

8

Benchmarking

“benchmark: to study (as a competitor’s product or business practices) in order to improve the performance of one’s own company.”

www.webster.com

“Benchmarking is a process of comparing one’s business processes and performance metrics to industry bests and/or best practices from other industries. Dimensions typically measured are quality, time and cost, improvements from learning mean doing things better, faster, and cheaper.”

www.wikipedia.org

Nick’s Corollary BM1: benchmarks can make anything look good.

9

Create File

rc=0

get failed

stripe cnt

ost match

Y YY

N NN

llapi_file_create

llapi_file_get_stripe

ost mismatchcnt mismatch

10

Create Striped Filerc = llapi_file_create(fname,0,ostnum,1,0);

lum = malloc(LOV_EA_MAX(lum));

/* read back the stripe data */rc = llapi_file_get_stripe(fname,lum);if (!rc) { /* check the stripe count */ if (lum->lmm_stripe_count != 1) { fprintf(stderr,"%s: stripe count mismatch: %s\n", ProgName,fname); } else { /* check the ost number */ oi = lum->lmm_objects[lum->lmm_stripe_offset].l_ost_idx; if (oi != ostnum){ fprintf(stderr,"%s: ost mismatch: %s\n", ProgName,fname); } }}

11

Write Test

size

Start time

End time

MB/swrite

Y

N

12

time(&b_timval); /* start the clock */

for (cnt=0;cnt<fcnt;cnt++) write(ifd,pattern,sizeof(pattern));

time(&e_timval); /* stop the clock */

fsync(ifd); /* flush the cache */

/* * calculate write mega bytes per second */wmbs=(float)fcnt*sizeof(pattern)/(float)(e_timval-b_timval)/1048576;

13

Read Test

size

Start time

End time

MB/sread

Y

N

14

lseek(ifd,0,SEEK_SET); /* rewind */

time(&b_timval); /* start the clock */

for(cnt=0;cnt<fcnt;cnt++) read(ifd,buf,1024);

time(&e_timval); /* stop the clock */

/* * calculate read mega bytes per second */rmbs=(float)fcnt*sizeof(pattern)/(float)(e_timval-b_timval)/1048576.0;

15

Data Check

size

read

Y

N

seek 0

patternY N

Pattern mismatch

16

lseek(ifd,0,SEEK_SET);/* rewind */

/* * repeat the read, but this time check the results * to make sure that what we read matches what we wrote */for(cnt=0;cnt<fcnt;cnt++) { read(ifd,buf,sizeof(buf));

if(strncmp(buf,pattern,sizeof(pattern))) { fprintf(stderr,"%s: read/write mismatch: %s\n", ProgName,fname); rc = 1; break; }}

17

Get OSTs

dp = opendir("/proc/fs/lustre/lov"); while((de=readdir(dp)) != NULL) { /* we only want the requested file system */ if (strncmp(de->d_name,fsp,strlen(fsp))) continue;

sprintf(rc.fsname,"%s\0",fsname);

/* get the number of osts */ sprintf(pname,"/proc/fs/lustre/lov/%s/numobd\0",de->d_name);

ffd = fopen(pname,"r"); fscanf(ffd,"%d",&rc.numobd); fclose(ffd);

break;}

18

In a Nutshell

MPI_Init Create

Write

Verify

Read

Get OSTs

Scale OK

19

MPI_Init (&argc, &argv); MPI_Comm_rank (MPI_COMM_WORLD, &rank);MPI_Comm_size (MPI_COMM_WORLD, &size);/* Parse the command line options */while ((optchr=getopt(argc,argv,"f:s:t:")) != EOF) { switch (optchr) { case 'f': fsname = optarg; break; case 's': fsize = atoi(optarg); break; case 't': tstdir = optarg; break; default : MPI_Abort(MPI_COMM_WORLD,1); }}fs = getfsinfo(fsname);if (size != fs->numobd) { fprintf(stderr,"%s: tasks(%d) != osts(%d)\n", ProgName,size,fs->numobd); MPI_Abort(MPI_COMM_WORLD,1);}/* construct the name of the test file */sprintf(fname,"%s/testfile.%s.%d\0",tstdir,fs->fsname,rank);

rc = StripeFile(fname,rank); /*create the striped file */

if(!rc) rc = RDWR(fname,rank,fsize,fsname);/* read/write test */

if (!rc) unlink (fname); /* delete the test file */MPI_Finalize();

Case #1

0 5 10 15 20 25 30 35 40 45 50130.0

135.0

140.0

145.0

150.0

155.0

160.0

165.0

170.0

175.0

Scratch Write Scratch Read Scratch2 Write Scratch2 Read

21

Case #2

Scratch Write Scratch Read

22

Case #3

write read

23

Case #3b

write read after write after read

Thank You!

24