Charm++: Charm Source Code Documentation

00001 /* -*- Mode: C; c-basic-offset:4 ; -*- */
00002 /*
00003  *   Copyright (C) 1997 University of Chicago.
00004  *   See COPYRIGHT notice in top-level directory.
00005  *
00006  *   Copyright (C) 2007 Oak Ridge National Laboratory
00007  *
00008  *   Copyright (C) 2008 Sun Microsystems, Lustre group
00009  */
00010 
00011 #include "ad_lustre.h"
00012 #include "adio_extern.h"
00013 
00014 void ADIOI_LUSTRE_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
00015 {
00016     char *value;
00017     int flag, stripe_val[3], str_factor = -1, str_unit=0, start_iodev=-1;
00018     struct lov_user_md lum = { 0 };
00019     int err, myrank, fd_sys, perm, amode, old_mask;
00020     int int_val, tmp_val;
00021     static char myname[] = "ADIOI_LUSTRE_SETINFO";
00022 
00023     value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
00024     if ( (fd->info) == MPI_INFO_NULL) {
00025     /* This must be part of the open call. can set striping parameters
00026            if necessary. */
00027     MPI_Info_create(&(fd->info));
00028 
00029     ADIOI_Info_set(fd->info, "direct_read", "false");
00030     ADIOI_Info_set(fd->info, "direct_write", "false");
00031     fd->direct_read = fd->direct_write = 0;
00032         /* initialize lustre hints */
00033     ADIOI_Info_set(fd->info, "romio_lustre_co_ratio", "1");
00034         fd->hints->fs_hints.lustre.co_ratio = 1;
00035     ADIOI_Info_set(fd->info, "romio_lustre_coll_threshold", "0");
00036         fd->hints->fs_hints.lustre.coll_threshold = 0;
00037     ADIOI_Info_set(fd->info, "romio_lustre_ds_in_coll", "enable");
00038         fd->hints->fs_hints.lustre.ds_in_coll = ADIOI_HINT_ENABLE;
00039 
00040     /* has user specified striping or server buffering parameters
00041            and do they have the same value on all processes? */
00042     if (users_info != MPI_INFO_NULL) {
00043             /* striping information */
00044         ADIOI_Info_get(users_info, "striping_unit", MPI_MAX_INFO_VAL,
00045              value, &flag);
00046         if (flag)
00047         str_unit=atoi(value);
00048 
00049         ADIOI_Info_get(users_info, "striping_factor", MPI_MAX_INFO_VAL,
00050              value, &flag);
00051         if (flag)
00052         str_factor=atoi(value);
00053 
00054         ADIOI_Info_get(users_info, "romio_lustre_start_iodevice",
00055                          MPI_MAX_INFO_VAL, value, &flag);
00056         if (flag)
00057         start_iodev=atoi(value);
00058 
00059             /* direct read and write */
00060         ADIOI_Info_get(users_info, "direct_read", MPI_MAX_INFO_VAL,
00061              value, &flag);
00062         if (flag && (!strcmp(value, "true") || !strcmp(value, "TRUE"))) {
00063         ADIOI_Info_set(fd->info, "direct_read", "true");
00064         fd->direct_read = 1;
00065         }
00066         ADIOI_Info_get(users_info, "direct_write", MPI_MAX_INFO_VAL,
00067                  value, &flag);
00068         if (flag && (!strcmp(value, "true") || !strcmp(value, "TRUE"))) {
00069         ADIOI_Info_set(fd->info, "direct_write", "true");
00070         fd->direct_write = 1;
00071         }
00072     }
00073 
00074         /* set striping information with ioctl */
00075     MPI_Comm_rank(fd->comm, &myrank);
00076     if (myrank == 0) {
00077         stripe_val[0] = str_factor;
00078         stripe_val[1] = str_unit;
00079         stripe_val[2] = start_iodev;
00080     }
00081     MPI_Bcast(stripe_val, 3, MPI_INT, 0, fd->comm);
00082 
00083     if (stripe_val[0] != str_factor
00084         || stripe_val[1] != str_unit
00085         || stripe_val[2] != start_iodev) {
00086         FPRINTF(stderr, "ADIOI_LUSTRE_SetInfo: All keys"
00087             "-striping_factor:striping_unit:start_iodevice "
00088             "need to be identical across all processes\n");
00089         MPI_Abort(MPI_COMM_WORLD, 1);
00090     } else if ((str_factor > 0) || (str_unit > 0) || (start_iodev >= 0)) {
00091          /* if user has specified striping info, process 0 tries to set it */
00092         if (!myrank) {
00093         if (fd->perm == ADIO_PERM_NULL) {
00094             old_mask = umask(022);
00095             umask(old_mask);
00096             perm = old_mask ^ 0666;
00097         }
00098         else perm = fd->perm;
00099 
00100         amode = 0;
00101         if (fd->access_mode & ADIO_CREATE)
00102             amode = amode | O_CREAT;
00103         if (fd->access_mode & ADIO_RDONLY)
00104             amode = amode | O_RDONLY;
00105         if (fd->access_mode & ADIO_WRONLY)
00106             amode = amode | O_WRONLY;
00107         if (fd->access_mode & ADIO_RDWR)
00108             amode = amode | O_RDWR;
00109         if (fd->access_mode & ADIO_EXCL)
00110             amode = amode | O_EXCL;
00111 
00112         /* we need to create file so ensure this is set */
00113         amode = amode | O_LOV_DELAY_CREATE | O_CREAT;
00114 
00115         fd_sys = open(fd->filename, amode, perm);
00116         if (fd_sys == -1) {
00117             if (errno != EEXIST)
00118             fprintf(stderr,
00119                 "Failure to open file %s %d %d\n",strerror(errno), amode, perm);
00120         } else {
00121             lum.lmm_magic = LOV_USER_MAGIC;
00122             lum.lmm_pattern = 0;
00123             lum.lmm_stripe_size = str_unit;
00124             lum.lmm_stripe_count = str_factor;
00125             lum.lmm_stripe_offset = start_iodev;
00126 
00127             err = ioctl(fd_sys, LL_IOC_LOV_SETSTRIPE, &lum);
00128             if (err == -1 && errno != EEXIST) {
00129             fprintf(stderr, "Failure to set stripe info %s \n", strerror(errno));
00130             }
00131             close(fd_sys);
00132            }
00133         } /* End of striping parameters validation */
00134     }
00135     MPI_Barrier(fd->comm);
00136     }
00137     /* get other hint */
00138     if (users_info != MPI_INFO_NULL) {
00139         /* CO: IO Clients/OST,
00140          * to keep the load balancing between clients and OSTs */
00141         ADIOI_Info_get(users_info, "romio_lustre_co_ratio", MPI_MAX_INFO_VAL, value,
00142                      &flag);
00143     if (flag && (int_val = atoi(value)) > 0) {
00144             tmp_val = int_val;
00145         MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
00146         if (tmp_val != int_val) {
00147                 MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
00148                                                    "romio_lustre_co_ratio",
00149                                                    error_code);
00150                 ADIOI_Free(value);
00151         return;
00152         }
00153         ADIOI_Info_set(fd->info, "romio_lustre_co_ratio", value);
00154             fd->hints->fs_hints.lustre.co_ratio = atoi(value);
00155     }
00156         /* coll_threshold:
00157          * if the req size is bigger than this, collective IO may not be performed.
00158          */
00159     ADIOI_Info_get(users_info, "romio_lustre_coll_threshold", MPI_MAX_INFO_VAL, value,
00160                      &flag);
00161     if (flag && (int_val = atoi(value)) > 0) {
00162             tmp_val = int_val;
00163         MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
00164         if (tmp_val != int_val) {
00165             MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
00166                                            "romio_lustre_coll_threshold",
00167                                                error_code);
00168                 ADIOI_Free(value);
00169             return;
00170         }
00171         ADIOI_Info_set(fd->info, "romio_lustre_coll_threshold", value);
00172             fd->hints->fs_hints.lustre.coll_threshold = atoi(value);
00173         }
00174         /* ds_in_coll: disable data sieving in collective IO */
00175     ADIOI_Info_get(users_info, "romio_lustre_ds_in_coll", MPI_MAX_INFO_VAL,
00176                  value, &flag);
00177     if (flag && (!strcmp(value, "disable") ||
00178                      !strcmp(value, "DISABLE"))) {
00179             tmp_val = int_val = 2;
00180         MPI_Bcast(&tmp_val, 2, MPI_INT, 0, fd->comm);
00181         if (tmp_val != int_val) {
00182             MPIO_ERR_CREATE_CODE_INFO_NOT_SAME(myname,
00183                                            "romio_lustre_ds_in_coll",
00184                            error_code);
00185                 ADIOI_Free(value);
00186                 return;
00187         }
00188         ADIOI_Info_set(fd->info, "romio_lustre_ds_in_coll", "disable");
00189             fd->hints->fs_hints.lustre.ds_in_coll = ADIOI_HINT_DISABLE;
00190     }
00191     }
00192     /* set the values for collective I/O and data sieving parameters */
00193     ADIOI_GEN_SetInfo(fd, users_info, error_code);
00194 
00195     if (ADIOI_Direct_read) fd->direct_read = 1;
00196     if (ADIOI_Direct_write) fd->direct_write = 1;
00197 
00198     ADIOI_Free(value);
00199 
00200     *error_code = MPI_SUCCESS;
00201 }
libs/ck-libs/ampi/romio/adio/ad_lustre/ad_lustre_hints.c
