Charm++: Charm Source Code Documentation

00001 /* -*- Mode: C; c-basic-offset:4 ; -*- */
00002 /* 
00003  *   $Id$    
00004  *
00005  *   Copyright (C) 1997 University of Chicago. 
00006  *   See COPYRIGHT notice in top-level directory.
00007  */
00008 
00009 #include "ad_nfs.h"
00010 #include "adio_extern.h"
00011 
00012 void ADIOI_NFS_WriteContig(ADIO_File fd, void *buf, int count, 
00013                      MPI_Datatype datatype, int file_ptr_type,
00014                      ADIO_Offset offset, ADIO_Status *status, int *error_code)
00015 {
00016     int err=-1, datatype_size, len;
00017 #ifndef PRINT_ERR_MSG
00018     static char myname[] = "ADIOI_NFS_WRITECONTIG";
00019 #endif
00020 
00021     MPI_Type_size(datatype, &datatype_size);
00022     len = datatype_size * count;
00023 
00024     if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
00025         if (fd->fp_sys_posn != offset)
00026             lseek(fd->fd_sys, offset, SEEK_SET);
00027         ADIOI_WRITE_LOCK(fd, offset, SEEK_SET, len);
00028         err = write(fd->fd_sys, buf, len);
00029         ADIOI_UNLOCK(fd, offset, SEEK_SET, len);
00030         fd->fp_sys_posn = offset + err;
00031         /* individual file pointer not updated */        
00032     }
00033     else { /* write from curr. location of ind. file pointer */
00034         offset = fd->fp_ind;
00035         if (fd->fp_sys_posn != fd->fp_ind)
00036             lseek(fd->fd_sys, fd->fp_ind, SEEK_SET);
00037         ADIOI_WRITE_LOCK(fd, offset, SEEK_SET, len);
00038         err = write(fd->fd_sys, buf, len);
00039         ADIOI_UNLOCK(fd, offset, SEEK_SET, len);
00040         fd->fp_ind += err;
00041         fd->fp_sys_posn = fd->fp_ind;
00042     }
00043 
00044 #ifdef HAVE_STATUS_SET_BYTES
00045     if (err != -1) MPIR_Status_set_bytes(status, datatype, len);
00046 #endif
00047 
00048 #ifdef PRINT_ERR_MSG
00049     *error_code = (err == -1) ? MPI_ERR_UNKNOWN : MPI_SUCCESS;
00050 #else
00051     if (err == -1) {
00052         *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
00053                               myname, "I/O Error", "%s", strerror(errno));
00054         ADIOI_Error(fd, *error_code, myname);
00055     }
00056     else *error_code = MPI_SUCCESS;
00057 #endif
00058 }
00059 
00060 
00061 
00062 
00063 #define ADIOI_BUFFERED_WRITE \
00064 { \
00065     if (req_off >= writebuf_off + writebuf_len) { \
00066         lseek(fd->fd_sys, writebuf_off, SEEK_SET); \
00067         err = write(fd->fd_sys, writebuf, writebuf_len); \
00068         if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
00069         if (err == -1) err_flag = 1; \
00070         writebuf_off = req_off; \
00071         writebuf_len = (int) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));\
00072         if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
00073         lseek(fd->fd_sys, writebuf_off, SEEK_SET); \
00074         err = read(fd->fd_sys, writebuf, writebuf_len); \
00075         if (err == -1) { \
00076             FPRINTF(stderr, "ADIOI_NFS_WriteStrided: ROMIO tries to optimize this access by doing a read-modify-write, but is unable to read the file. Please give the file read permission and open it with MPI_MODE_RDWR.\n"); \
00077             MPI_Abort(MPI_COMM_WORLD, 1); \
00078         } \
00079     } \
00080     write_sz = (int) (ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off)); \
00081     memcpy(writebuf+req_off-writebuf_off, (char *)buf +userbuf_off, write_sz);\
00082     while (write_sz != req_len) { \
00083         lseek(fd->fd_sys, writebuf_off, SEEK_SET); \
00084         err = write(fd->fd_sys, writebuf, writebuf_len); \
00085         if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
00086         if (err == -1) err_flag = 1; \
00087         req_len -= write_sz; \
00088         userbuf_off += write_sz; \
00089         writebuf_off += writebuf_len; \
00090         writebuf_len = (int) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));\
00091         if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
00092         lseek(fd->fd_sys, writebuf_off, SEEK_SET); \
00093         err = read(fd->fd_sys, writebuf, writebuf_len); \
00094         if (err == -1) { \
00095             FPRINTF(stderr, "ADIOI_NFS_WriteStrided: ROMIO tries to optimize this access by doing a read-modify-write, but is unable to read the file. Please give the file read permission and open it with MPI_MODE_RDWR.\n"); \
00096             MPI_Abort(MPI_COMM_WORLD, 1); \
00097         } \
00098         write_sz = ADIOI_MIN(req_len, writebuf_len); \
00099         memcpy(writebuf, (char *)buf + userbuf_off, write_sz);\
00100     } \
00101 }
00102 
00103 
00104 /* this macro is used when filetype is contig and buftype is not contig.
00105    it does not do a read-modify-write and does not lock*/
00106 #define ADIOI_BUFFERED_WRITE_WITHOUT_READ \
00107 { \
00108     if (req_off >= writebuf_off + writebuf_len) { \
00109         lseek(fd->fd_sys, writebuf_off, SEEK_SET); \
00110         if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
00111         err = write(fd->fd_sys, writebuf, writebuf_len); \
00112         if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
00113         if (err == -1) err_flag = 1; \
00114         writebuf_off = req_off; \
00115         writebuf_len = (int) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));\
00116     } \
00117     write_sz = (int) (ADIOI_MIN(req_len, writebuf_off + writebuf_len - req_off)); \
00118     memcpy(writebuf+req_off-writebuf_off, (char *)buf +userbuf_off, write_sz);\
00119     while (write_sz != req_len) { \
00120         lseek(fd->fd_sys, writebuf_off, SEEK_SET); \
00121         if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
00122         err = write(fd->fd_sys, writebuf, writebuf_len); \
00123         if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len); \
00124         if (err == -1) err_flag = 1; \
00125         req_len -= write_sz; \
00126         userbuf_off += write_sz; \
00127         writebuf_off += writebuf_len; \
00128         writebuf_len = (int) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));\
00129         write_sz = ADIOI_MIN(req_len, writebuf_len); \
00130         memcpy(writebuf, (char *)buf + userbuf_off, write_sz);\
00131     } \
00132 }
00133 
00134 
00135 
00136 void ADIOI_NFS_WriteStrided(ADIO_File fd, void *buf, int count,
00137                        MPI_Datatype datatype, int file_ptr_type,
00138                        ADIO_Offset offset, ADIO_Status *status, int
00139                        *error_code)
00140 {
00141 /* offset is in units of etype relative to the filetype. */
00142 
00143     ADIOI_Flatlist_node *flat_buf, *flat_file;
00144     int i, j, k, err=-1, bwr_size, fwr_size=0, st_index=0;
00145     int bufsize, num, size, sum, n_etypes_in_filetype, size_in_filetype;
00146     int n_filetypes, etype_in_filetype;
00147     ADIO_Offset abs_off_in_filetype=0;
00148     int filetype_size, etype_size, buftype_size, req_len;
00149     MPI_Aint filetype_extent, buftype_extent; 
00150     int buf_count, buftype_is_contig, filetype_is_contig;
00151     ADIO_Offset userbuf_off;
00152     ADIO_Offset off, req_off, disp, end_offset=0, writebuf_off, start_off;
00153     char *writebuf, *value;
00154     int flag, st_fwr_size, st_n_filetypes, writebuf_len, write_sz;
00155     int new_bwr_size, new_fwr_size, err_flag=0, info_flag, max_bufsize;
00156 #ifndef PRINT_ERR_MSG
00157     static char myname[] = "ADIOI_NFS_WRITESTRIDED";
00158 #endif
00159 
00160     ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
00161     ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
00162 
00163     MPI_Type_size(fd->filetype, &filetype_size);
00164     if ( ! filetype_size ) {
00165         *error_code = MPI_SUCCESS; 
00166         return;
00167     }
00168 
00169     MPI_Type_extent(fd->filetype, &filetype_extent);
00170     MPI_Type_size(datatype, &buftype_size);
00171     MPI_Type_extent(datatype, &buftype_extent);
00172     etype_size = fd->etype_size;
00173 
00174     bufsize = buftype_size * count;
00175 
00176 /* get max_bufsize from the info object. */
00177 
00178     value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
00179     MPI_Info_get(fd->info, "ind_wr_buffer_size", MPI_MAX_INFO_VAL, value, 
00180                  &info_flag);
00181     max_bufsize = atoi(value);
00182     ADIOI_Free(value);
00183 
00184     if (!buftype_is_contig && filetype_is_contig) {
00185 
00186 /* noncontiguous in memory, contiguous in file. */
00187 
00188         ADIOI_Flatten_datatype(datatype);
00189         flat_buf = ADIOI_Flatlist;
00190         while (flat_buf->type != datatype) flat_buf = flat_buf->next;
00191 
00192         off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind : 
00193                  fd->disp + etype_size * offset;
00194 
00195         start_off = off;
00196         end_offset = off + bufsize - 1;
00197         writebuf_off = off;
00198         writebuf = (char *) ADIOI_Malloc(max_bufsize);
00199         writebuf_len = (int) (ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));
00200 
00201 /* if atomicity is true, lock the region to be accessed */
00202         if (fd->atomicity) 
00203             ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
00204 
00205         for (j=0; j<count; j++) 
00206             for (i=0; i<flat_buf->count; i++) {
00207                 userbuf_off = j*buftype_extent + flat_buf->indices[i];
00208                 req_off = off;
00209                 req_len = flat_buf->blocklens[i];
00210                 ADIOI_BUFFERED_WRITE_WITHOUT_READ
00211                 off += flat_buf->blocklens[i];
00212             }
00213 
00214         /* write the buffer out finally */
00215         lseek(fd->fd_sys, writebuf_off, SEEK_SET); 
00216         if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len);
00217         err = write(fd->fd_sys, writebuf, writebuf_len); 
00218         if (!(fd->atomicity)) ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len);
00219         if (err == -1) err_flag = 1; 
00220 
00221         if (fd->atomicity) 
00222             ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
00223 
00224         ADIOI_Free(writebuf); /* malloced in the buffered_write macro */
00225 
00226         if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
00227 #ifdef PRINT_ERR_MSG
00228         *error_code = (err_flag) ? MPI_ERR_UNKNOWN : MPI_SUCCESS;
00229 #else
00230         if (err_flag) {
00231             *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
00232                               myname, "I/O Error", "%s", strerror(errno));
00233             ADIOI_Error(fd, *error_code, myname);
00234         }
00235         else *error_code = MPI_SUCCESS;
00236 #endif
00237     }
00238 
00239     else {  /* noncontiguous in file */
00240 
00241 /* filetype already flattened in ADIO_Open */
00242         flat_file = ADIOI_Flatlist;
00243         while (flat_file->type != fd->filetype) flat_file = flat_file->next;
00244         disp = fd->disp;
00245 
00246         if (file_ptr_type == ADIO_INDIVIDUAL) {
00247             offset = fd->fp_ind; /* in bytes */
00248             n_filetypes = -1;
00249             flag = 0;
00250             while (!flag) {
00251                 n_filetypes++;
00252                 for (i=0; i<flat_file->count; i++) {
00253                     if (disp + flat_file->indices[i] + 
00254                         (ADIO_Offset) n_filetypes*filetype_extent + flat_file->blocklens[i] 
00255                             >= offset) {
00256                         st_index = i;
00257                         fwr_size = (int) (disp + flat_file->indices[i] + 
00258                                 (ADIO_Offset) n_filetypes*filetype_extent
00259                                  + flat_file->blocklens[i] - offset);
00260                         flag = 1;
00261                         break;
00262                     }
00263                 }
00264             }
00265         }
00266         else {
00267             n_etypes_in_filetype = filetype_size/etype_size;
00268             n_filetypes = (int) (offset / n_etypes_in_filetype);
00269             etype_in_filetype = (int) (offset % n_etypes_in_filetype);
00270             size_in_filetype = etype_in_filetype * etype_size;
00271  
00272             sum = 0;
00273             for (i=0; i<flat_file->count; i++) {
00274                 sum += flat_file->blocklens[i];
00275                 if (sum > size_in_filetype) {
00276                     st_index = i;
00277                     fwr_size = sum - size_in_filetype;
00278                     abs_off_in_filetype = flat_file->indices[i] +
00279                         size_in_filetype - (sum - flat_file->blocklens[i]);
00280                     break;
00281                 }
00282             }
00283 
00284             /* abs. offset in bytes in the file */
00285             offset = disp + (ADIO_Offset) n_filetypes*filetype_extent + abs_off_in_filetype;
00286         }
00287 
00288         start_off = offset;
00289 
00290        /* Calculate end_offset, the last byte-offset that will be accessed.
00291          e.g., if start_offset=0 and 100 bytes to be write, end_offset=99*/
00292 
00293         st_fwr_size = fwr_size;
00294         st_n_filetypes = n_filetypes;
00295         i = 0;
00296         j = st_index;
00297         off = offset;
00298         fwr_size = ADIOI_MIN(st_fwr_size, bufsize);
00299         while (i < bufsize) {
00300             i += fwr_size;
00301             end_offset = off + fwr_size - 1;
00302 
00303             if (j < (flat_file->count - 1)) j++;
00304             else {
00305                 j = 0;
00306                 n_filetypes++;
00307             }
00308 
00309             off = disp + flat_file->indices[j] + (ADIO_Offset) n_filetypes*filetype_extent;
00310             fwr_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i);
00311         }
00312 
00313 /* if atomicity is true, lock the region to be accessed */
00314         if (fd->atomicity) 
00315             ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
00316 
00317         /* initial read for the read-modify-write */
00318         writebuf_off = offset;
00319         writebuf = (char *) ADIOI_Malloc(max_bufsize);
00320         writebuf_len = (int)(ADIOI_MIN(max_bufsize,end_offset-writebuf_off+1));
00321         if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len);
00322         lseek(fd->fd_sys, writebuf_off, SEEK_SET); 
00323         err = read(fd->fd_sys, writebuf, writebuf_len); 
00324         if (err == -1) {
00325             FPRINTF(stderr, "ADIOI_NFS_WriteStrided: ROMIO tries to optimize this access by doing a read-modify-write, but is unable to read the file. Please give the file read permission and open it with MPI_MODE_RDWR.\n"); 
00326             MPI_Abort(MPI_COMM_WORLD, 1); 
00327         } 
00328 
00329         if (buftype_is_contig && !filetype_is_contig) {
00330 
00331 /* contiguous in memory, noncontiguous in file. should be the most
00332    common case. */
00333 
00334             i = 0;
00335             j = st_index;
00336             off = offset;
00337             n_filetypes = st_n_filetypes;
00338             fwr_size = ADIOI_MIN(st_fwr_size, bufsize);
00339             while (i < bufsize) {
00340                 if (fwr_size) { 
00341                     /* TYPE_UB and TYPE_LB can result in 
00342                        fwr_size = 0. save system call in such cases */ 
00343                     /* lseek(fd->fd_sys, off, SEEK_SET);
00344                     err = write(fd->fd_sys, ((char *) buf) + i, fwr_size);*/
00345 
00346                     req_off = off;
00347                     req_len = fwr_size;
00348                     userbuf_off = i;
00349                     ADIOI_BUFFERED_WRITE
00350                 }
00351                 i += fwr_size;
00352 
00353                 if (off + fwr_size < disp + flat_file->indices[j] +
00354                    flat_file->blocklens[j] + (ADIO_Offset) n_filetypes*filetype_extent)
00355                        off += fwr_size;
00356                 /* did not reach end of contiguous block in filetype.
00357                    no more I/O needed. off is incremented by fwr_size. */
00358                 else {
00359                     if (j < (flat_file->count - 1)) j++;
00360                     else {
00361                         j = 0;
00362                         n_filetypes++;
00363                     }
00364                     off = disp + flat_file->indices[j] + 
00365                                         (ADIO_Offset) n_filetypes*filetype_extent;
00366                     fwr_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i);
00367                 }
00368             }
00369         }
00370         else {
00371 /* noncontiguous in memory as well as in file */
00372 
00373             ADIOI_Flatten_datatype(datatype);
00374             flat_buf = ADIOI_Flatlist;
00375             while (flat_buf->type != datatype) flat_buf = flat_buf->next;
00376 
00377             k = num = buf_count = 0;
00378             i = (int) (flat_buf->indices[0]);
00379             j = st_index;
00380             off = offset;
00381             n_filetypes = st_n_filetypes;
00382             fwr_size = st_fwr_size;
00383             bwr_size = flat_buf->blocklens[0];
00384 
00385             while (num < bufsize) {
00386                 size = ADIOI_MIN(fwr_size, bwr_size);
00387                 if (size) {
00388                     /* lseek(fd->fd_sys, off, SEEK_SET);
00389                     err = write(fd->fd_sys, ((char *) buf) + i, size); */
00390 
00391                     req_off = off;
00392                     req_len = size;
00393                     userbuf_off = i;
00394                     ADIOI_BUFFERED_WRITE
00395                 }
00396 
00397                 new_fwr_size = fwr_size;
00398                 new_bwr_size = bwr_size;
00399 
00400                 if (size == fwr_size) {
00401 /* reached end of contiguous block in file */
00402                     if (j < (flat_file->count - 1)) j++;
00403                     else {
00404                         j = 0;
00405                         n_filetypes++;
00406                     }
00407 
00408                     off = disp + flat_file->indices[j] + 
00409                                   (ADIO_Offset) n_filetypes*filetype_extent;
00410 
00411                     new_fwr_size = flat_file->blocklens[j];
00412                     if (size != bwr_size) {
00413                         i += size;
00414                         new_bwr_size -= size;
00415                     }
00416                 }
00417 
00418                 if (size == bwr_size) {
00419 /* reached end of contiguous block in memory */
00420 
00421                     k = (k + 1)%flat_buf->count;
00422                     buf_count++;
00423                     i = (int) (buftype_extent*(buf_count/flat_buf->count) +
00424                         flat_buf->indices[k]); 
00425                     new_bwr_size = flat_buf->blocklens[k];
00426                     if (size != fwr_size) {
00427                         off += size;
00428                         new_fwr_size -= size;
00429                     }
00430                 }
00431                 num += size;
00432                 fwr_size = new_fwr_size;
00433                 bwr_size = new_bwr_size;
00434             }
00435         }
00436 
00437         /* write the buffer out finally */      
00438         lseek(fd->fd_sys, writebuf_off, SEEK_SET); 
00439         if (!(fd->atomicity)) ADIOI_WRITE_LOCK(fd, writebuf_off, SEEK_SET, writebuf_len);
00440         err = write(fd->fd_sys, writebuf, writebuf_len); 
00441 
00442         if (!(fd->atomicity))
00443             ADIOI_UNLOCK(fd, writebuf_off, SEEK_SET, writebuf_len);
00444         else ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
00445 
00446         if (err == -1) err_flag = 1; 
00447 
00448         ADIOI_Free(writebuf); /* malloced in the buffered_write macro */
00449 
00450         if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
00451 #ifdef PRINT_ERR_MSG
00452         *error_code = (err_flag) ? MPI_ERR_UNKNOWN : MPI_SUCCESS;
00453 #else
00454         if (err_flag) {
00455             *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
00456                               myname, "I/O Error", "%s", strerror(errno));
00457             ADIOI_Error(fd, *error_code, myname);
00458         }
00459         else *error_code = MPI_SUCCESS;
00460 #endif
00461     }
00462 
00463     fd->fp_sys_posn = -1;   /* set it to null. */
00464 
00465 #ifdef HAVE_STATUS_SET_BYTES
00466     MPIR_Status_set_bytes(status, datatype, bufsize);
00467 /* This is a temporary way of filling in status. The right way is to 
00468    keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
00469 #endif
00470 
00471     if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
00472 }
libs/ck-libs/ampi/romio/adio/ad_nfs/ad_nfs_write.c
