Charm++: Charm Source Code Documentation

00001 /* -*- Mode: C; c-basic-offset:4 ; -*- */
00002 /* 
00003  *   $Id$    
00004  *
00005  *   Copyright (C) 1997 University of Chicago. 
00006  *   See COPYRIGHT notice in top-level directory.
00007  */
00008 
00009 #include "ad_pvfs.h"
00010 #include "adio_extern.h"
00011 
00012 #ifdef HAVE_PVFS_LISTIO
00013 void ADIOI_PVFS_WriteStridedListIO(ADIO_File fd, void *buf, int count,
00014                        MPI_Datatype datatype, int file_ptr_type,
00015                        ADIO_Offset offset, ADIO_Status *status, int
00016                        *error_code);
00017 #endif
00018 
00019 void ADIOI_PVFS_WriteContig(ADIO_File fd, void *buf, int count, 
00020                      MPI_Datatype datatype, int file_ptr_type,
00021                      ADIO_Offset offset, ADIO_Status *status, int *error_code)
00022 {
00023     int err=-1, datatype_size, len;
00024 #ifndef PRINT_ERR_MSG
00025     static char myname[] = "ADIOI_PVFS_WRITECONTIG";
00026 #endif
00027 
00028     MPI_Type_size(datatype, &datatype_size);
00029     len = datatype_size * count;
00030 
00031     if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
00032         if (fd->fp_sys_posn != offset)
00033             pvfs_lseek64(fd->fd_sys, offset, SEEK_SET);
00034         err = pvfs_write(fd->fd_sys, buf, len);
00035         fd->fp_sys_posn = offset + err;
00036         /* individual file pointer not updated */        
00037     }
00038     else { /* write from curr. location of ind. file pointer */
00039         if (fd->fp_sys_posn != fd->fp_ind)
00040             pvfs_lseek64(fd->fd_sys, fd->fp_ind, SEEK_SET);
00041         err = pvfs_write(fd->fd_sys, buf, len);
00042         fd->fp_ind += err;
00043         fd->fp_sys_posn = fd->fp_ind;
00044     }
00045 
00046 #ifdef HAVE_STATUS_SET_BYTES
00047     if (err != -1) MPIR_Status_set_bytes(status, datatype, err);
00048 #endif
00049 
00050 #ifdef PRINT_ERR_MSG
00051     *error_code = (err == -1) ? MPI_ERR_UNKNOWN : MPI_SUCCESS;
00052 #else
00053     if (err == -1) {
00054         *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
00055                               myname, "I/O Error", "%s", strerror(errno));
00056         ADIOI_Error(fd, *error_code, myname);
00057     }
00058     else *error_code = MPI_SUCCESS;
00059 #endif
00060 }
00061 
00062 
00063 
00064 void ADIOI_PVFS_WriteStrided(ADIO_File fd, void *buf, int count,
00065                        MPI_Datatype datatype, int file_ptr_type,
00066                        ADIO_Offset offset, ADIO_Status *status, int
00067                        *error_code)
00068 {
00069 /* Since PVFS does not support file locking, can't do buffered writes
00070    as on Unix */
00071 
00072 /* offset is in units of etype relative to the filetype. */
00073 
00074     ADIOI_Flatlist_node *flat_buf, *flat_file;
00075     struct iovec *iov;
00076     int i, j, k, err=-1, bwr_size, fwr_size=0, st_index=0;
00077     int bufsize, num, size, sum, n_etypes_in_filetype, size_in_filetype;
00078     int n_filetypes, etype_in_filetype;
00079     ADIO_Offset abs_off_in_filetype=0;
00080     int filetype_size, etype_size, buftype_size;
00081     MPI_Aint filetype_extent, buftype_extent, indx;
00082     int buf_count, buftype_is_contig, filetype_is_contig;
00083     ADIO_Offset off, disp;
00084     int flag, new_bwr_size, new_fwr_size, err_flag=0;
00085 #ifndef PRINT_ERR_MSG
00086     static char myname[] = "ADIOI_PVFS_WRITESTRIDED";
00087 #endif
00088 
00089 #ifdef HAVE_PVFS_LISTIO
00090     if ( fd->hints->fs_hints.pvfs.listio_write == ADIOI_HINT_ENABLE ) {
00091             ADIOI_PVFS_WriteStridedListIO(fd, buf, count, datatype, 
00092                             file_ptr_type, offset, status, error_code);
00093             return;
00094     }
00095 #endif
00096     /* if hint set to DISABLE or AUTOMATIC, don't use listio */
00097 
00098     if (fd->atomicity) {
00099         FPRINTF(stderr, "ROMIO cannot guarantee atomicity of noncontiguous accesses in atomic mode, as PVFS doesn't support file locking. Use nonatomic mode and its associated semantics.\n");
00100         MPI_Abort(MPI_COMM_WORLD, 1);
00101     }
00102 
00103     ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
00104     ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
00105 
00106     MPI_Type_size(fd->filetype, &filetype_size);
00107     if ( ! filetype_size ) {
00108         *error_code = MPI_SUCCESS; 
00109         return;
00110     }
00111 
00112     MPI_Type_extent(fd->filetype, &filetype_extent);
00113     MPI_Type_size(datatype, &buftype_size);
00114     MPI_Type_extent(datatype, &buftype_extent);
00115     etype_size = fd->etype_size;
00116     
00117     bufsize = buftype_size * count;
00118 
00119     if (!buftype_is_contig && filetype_is_contig) {
00120 
00121 /* noncontiguous in memory, contiguous in file. use writev */
00122 
00123         ADIOI_Flatten_datatype(datatype);
00124         flat_buf = ADIOI_Flatlist;
00125         while (flat_buf->type != datatype) flat_buf = flat_buf->next;
00126 
00127 /* There is a limit of 16 on the number of iovecs for readv/writev! */
00128 
00129         iov = (struct iovec *) ADIOI_Malloc(16*sizeof(struct iovec));
00130 
00131         if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
00132             off = fd->disp + etype_size * offset;
00133             pvfs_lseek64(fd->fd_sys, off, SEEK_SET);
00134         }
00135         else off = pvfs_lseek64(fd->fd_sys, fd->fp_ind, SEEK_SET);
00136 
00137         k = 0;
00138         for (j=0; j<count; j++) 
00139             for (i=0; i<flat_buf->count; i++) {
00140                 iov[k].iov_base = ((char *) buf) + j*buftype_extent +
00141                     flat_buf->indices[i]; 
00142                 iov[k].iov_len = flat_buf->blocklens[i];
00143                 /*FPRINTF(stderr, "%d %d\n", iov[k].iov_base, iov[k].iov_len);*/
00144 
00145                 off += flat_buf->blocklens[i];
00146                 k = (k+1)%16;
00147 
00148                 if (!k) {
00149                     err = pvfs_writev(fd->fd_sys, iov, 16);
00150                     if (err == -1) err_flag = 1;
00151                 }
00152             }
00153 
00154         if (k) {
00155             err = pvfs_writev(fd->fd_sys, iov, k);
00156             if (err == -1) err_flag = 1;
00157         }
00158 
00159         if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
00160 
00161         ADIOI_Free(iov);
00162 #ifdef PRINT_ERR_MSG
00163         *error_code = (err_flag) ? MPI_ERR_UNKNOWN : MPI_SUCCESS;
00164 #else
00165         if (err_flag) {
00166             *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
00167                               myname, "I/O Error", "%s", strerror(errno));
00168             ADIOI_Error(fd, *error_code, myname);
00169         }
00170         else *error_code = MPI_SUCCESS;
00171 #endif
00172     }
00173 
00174     else {  /* noncontiguous in file */
00175 
00176 /* split up into several contiguous writes */
00177 
00178 /* find starting location in the file */
00179 
00180 /* filetype already flattened in ADIO_Open */
00181         flat_file = ADIOI_Flatlist;
00182         while (flat_file->type != fd->filetype) flat_file = flat_file->next;
00183         disp = fd->disp;
00184 
00185         if (file_ptr_type == ADIO_INDIVIDUAL) {
00186             offset = fd->fp_ind; /* in bytes */
00187             n_filetypes = -1;
00188             flag = 0;
00189             while (!flag) {
00190                 n_filetypes++;
00191                 for (i=0; i<flat_file->count; i++) {
00192                     if (disp + flat_file->indices[i] + 
00193                         (ADIO_Offset) n_filetypes*filetype_extent + flat_file->blocklens[i] 
00194                             >= offset) {
00195                         st_index = i;
00196                         fwr_size = disp + flat_file->indices[i] + 
00197                                 (ADIO_Offset) n_filetypes*filetype_extent
00198                                  + flat_file->blocklens[i] - offset;
00199                         flag = 1;
00200                         break;
00201                     }
00202                 }
00203             }
00204         }
00205         else {
00206             n_etypes_in_filetype = filetype_size/etype_size;
00207             n_filetypes = (int) (offset / n_etypes_in_filetype);
00208             etype_in_filetype = (int) (offset % n_etypes_in_filetype);
00209             size_in_filetype = etype_in_filetype * etype_size;
00210  
00211             sum = 0;
00212             for (i=0; i<flat_file->count; i++) {
00213                 sum += flat_file->blocklens[i];
00214                 if (sum > size_in_filetype) {
00215                     st_index = i;
00216                     fwr_size = sum - size_in_filetype;
00217                     abs_off_in_filetype = flat_file->indices[i] +
00218                         size_in_filetype - (sum - flat_file->blocklens[i]);
00219                     break;
00220                 }
00221             }
00222 
00223             /* abs. offset in bytes in the file */
00224             offset = disp + (ADIO_Offset) n_filetypes*filetype_extent + abs_off_in_filetype;
00225         }
00226 
00227         if (buftype_is_contig && !filetype_is_contig) {
00228 
00229 /* contiguous in memory, noncontiguous in file. should be the most
00230    common case. */
00231 
00232             i = 0;
00233             j = st_index;
00234             off = offset;
00235             fwr_size = ADIOI_MIN(fwr_size, bufsize);
00236             while (i < bufsize) {
00237                 if (fwr_size) { 
00238                     /* TYPE_UB and TYPE_LB can result in 
00239                        fwr_size = 0. save system call in such cases */ 
00240 #ifdef PROFILE
00241                     MPE_Log_event(11, 0, "start seek");
00242 #endif
00243                     pvfs_lseek64(fd->fd_sys, off, SEEK_SET);
00244 #ifdef PROFILE
00245                     MPE_Log_event(12, 0, "end seek");
00246                     MPE_Log_event(5, 0, "start write");
00247 #endif
00248                     err = pvfs_write(fd->fd_sys, ((char *) buf) + i, fwr_size);
00249 #ifdef PROFILE
00250                     MPE_Log_event(6, 0, "end write");
00251 #endif
00252                     if (err == -1) err_flag = 1;
00253                 }
00254                 i += fwr_size;
00255 
00256                 if (off + fwr_size < disp + flat_file->indices[j] +
00257                    flat_file->blocklens[j] + (ADIO_Offset) n_filetypes*filetype_extent)
00258                        off += fwr_size;
00259                 /* did not reach end of contiguous block in filetype.
00260                    no more I/O needed. off is incremented by fwr_size. */
00261                 else {
00262                     if (j < (flat_file->count - 1)) j++;
00263                     else {
00264                         j = 0;
00265                         n_filetypes++;
00266                     }
00267                     off = disp + flat_file->indices[j] + 
00268                                         (ADIO_Offset) n_filetypes*filetype_extent;
00269                     fwr_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i);
00270                 }
00271             }
00272         }
00273         else {
00274 /* noncontiguous in memory as well as in file */
00275 
00276             ADIOI_Flatten_datatype(datatype);
00277             flat_buf = ADIOI_Flatlist;
00278             while (flat_buf->type != datatype) flat_buf = flat_buf->next;
00279 
00280             k = num = buf_count = 0;
00281             indx = flat_buf->indices[0];
00282             j = st_index;
00283             off = offset;
00284             bwr_size = flat_buf->blocklens[0];
00285 
00286             while (num < bufsize) {
00287                 size = ADIOI_MIN(fwr_size, bwr_size);
00288                 if (size) {
00289 #ifdef PROFILE
00290                     MPE_Log_event(11, 0, "start seek");
00291 #endif
00292                     pvfs_lseek64(fd->fd_sys, off, SEEK_SET);
00293 #ifdef PROFILE
00294                     MPE_Log_event(12, 0, "end seek");
00295                     MPE_Log_event(5, 0, "start write");
00296 #endif
00297                     err = pvfs_write(fd->fd_sys, ((char *) buf) + indx, size);
00298 #ifdef PROFILE
00299                     MPE_Log_event(6, 0, "end write");
00300 #endif
00301                     if (err == -1) err_flag = 1;
00302                 }
00303 
00304                 new_fwr_size = fwr_size;
00305                 new_bwr_size = bwr_size;
00306 
00307                 if (size == fwr_size) {
00308 /* reached end of contiguous block in file */
00309                     if (j < (flat_file->count - 1)) j++;
00310                     else {
00311                         j = 0;
00312                         n_filetypes++;
00313                     }
00314 
00315                     off = disp + flat_file->indices[j] + 
00316                                    (ADIO_Offset) n_filetypes*filetype_extent;
00317 
00318                     new_fwr_size = flat_file->blocklens[j];
00319                     if (size != bwr_size) {
00320                         indx += size;
00321                         new_bwr_size -= size;
00322                     }
00323                 }
00324 
00325                 if (size == bwr_size) {
00326 /* reached end of contiguous block in memory */
00327 
00328                     k = (k + 1)%flat_buf->count;
00329                     buf_count++;
00330                     indx = buftype_extent*(buf_count/flat_buf->count) +
00331                         flat_buf->indices[k]; 
00332                     new_bwr_size = flat_buf->blocklens[k];
00333                     if (size != fwr_size) {
00334                         off += size;
00335                         new_fwr_size -= size;
00336                     }
00337                 }
00338                 num += size;
00339                 fwr_size = new_fwr_size;
00340                 bwr_size = new_bwr_size;
00341             }
00342         }
00343 
00344         if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
00345 #ifdef PRINT_ERR_MSG
00346         *error_code = (err_flag) ? MPI_ERR_UNKNOWN : MPI_SUCCESS;
00347 #else
00348         if (err_flag) {
00349             *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
00350                               myname, "I/O Error", "%s", strerror(errno));
00351             ADIOI_Error(fd, *error_code, myname);
00352         }
00353         else *error_code = MPI_SUCCESS;
00354 #endif
00355     }
00356 
00357     fd->fp_sys_posn = -1;   /* set it to null. */
00358 
00359 #ifdef HAVE_STATUS_SET_BYTES
00360     MPIR_Status_set_bytes(status, datatype, bufsize);
00361 /* This is a temporary way of filling in status. The right way is to 
00362    keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
00363 #endif
00364 
00365     if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
00366 }
00367 
00368 #ifdef HAVE_PVFS_LISTIO
00369 void ADIOI_PVFS_WriteStridedListIO(ADIO_File fd, void *buf, int count,
00370                        MPI_Datatype datatype, int file_ptr_type,
00371                        ADIO_Offset offset, ADIO_Status *status, int
00372                        *error_code) 
00373 {
00374 /* Since PVFS does not support file locking, can't do buffered writes
00375    as on Unix */
00376 
00377 /* offset is in units of etype relative to the filetype. */
00378 
00379     ADIOI_Flatlist_node *flat_buf, *flat_file;
00380     int i, j, k, err=-1, bwr_size, fwr_size=0, st_index=0;
00381     int bufsize, size, sum, n_etypes_in_filetype, size_in_filetype;
00382     int n_filetypes, etype_in_filetype;
00383     ADIO_Offset abs_off_in_filetype=0;
00384     int filetype_size, etype_size, buftype_size;
00385     MPI_Aint filetype_extent, buftype_extent;
00386     int buf_count, buftype_is_contig, filetype_is_contig;
00387     ADIO_Offset userbuf_off;
00388     ADIO_Offset off, disp, start_off;
00389     int flag, st_fwr_size, st_n_filetypes;
00390     int new_bwr_size, new_fwr_size, err_flag=0;
00391 
00392     int mem_list_count, file_list_count;
00393     char ** mem_offsets;
00394     int64_t *file_offsets;
00395     int *mem_lengths;
00396     int32_t *file_lengths;
00397     int total_blks_to_write;
00398 
00399     int max_mem_list, max_file_list;
00400 
00401     int b_blks_wrote;
00402     int f_data_wrote;
00403     int size_wrote=0, n_write_lists, extra_blks;
00404 
00405     int end_bwr_size, end_fwr_size;
00406     int start_k, start_j, new_file_write, new_buffer_write;
00407     int start_mem_offset;
00408 #define MAX_ARRAY_SIZE 1024
00409 
00410 #ifndef PRINT_ERR_MSG
00411     static char myname[] = "ADIOI_PVFS_WRITESTRIDED";
00412 #endif
00413 
00414 /* PFS file pointer modes are not relevant here, because PFS does
00415    not support strided accesses. */
00416 
00417     if ((fd->iomode != M_ASYNC) && (fd->iomode != M_UNIX)) {
00418         FPRINTF(stderr, "ADIOI_PVFS_WriteStrided: only M_ASYNC and M_UNIX iomodes are valid\n");
00419         MPI_Abort(MPI_COMM_WORLD, 1);
00420     }
00421 
00422     if (fd->atomicity) {
00423         FPRINTF(stderr, "ROMIO cannot guarantee atomicity of noncontiguous accesses in atomic mode, as PVFS doesn't support file locking. Use nonatomic mode and its associated semantics.\n");
00424         MPI_Abort(MPI_COMM_WORLD, 1);
00425     }
00426 
00427     ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
00428     ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
00429 
00430     MPI_Type_size(fd->filetype, &filetype_size);
00431     if ( ! filetype_size ) {
00432         *error_code = MPI_SUCCESS; 
00433         return;
00434     }
00435 
00436     MPI_Type_extent(fd->filetype, &filetype_extent);
00437     MPI_Type_size(datatype, &buftype_size);
00438     MPI_Type_extent(datatype, &buftype_extent);
00439     etype_size = fd->etype_size;
00440     
00441     bufsize = buftype_size * count;
00442 
00443     if (!buftype_is_contig && filetype_is_contig) {
00444 
00445 /* noncontiguous in memory, contiguous in file.  */
00446         int64_t file_offsets;
00447         int32_t file_lengths;
00448 
00449         ADIOI_Flatten_datatype(datatype);
00450         flat_buf = ADIOI_Flatlist;
00451         while (flat_buf->type != datatype) flat_buf = flat_buf->next;
00452         
00453         if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
00454             off = fd->disp + etype_size * offset;
00455             pvfs_lseek64(fd->fd_sys, fd->fp_ind, SEEK_SET);
00456         }
00457         else off = pvfs_lseek64(fd->fd_sys, fd->fp_ind, SEEK_SET);
00458 
00459         file_list_count = 1;
00460         file_offsets = off;
00461         file_lengths = 0;
00462         total_blks_to_write = count*flat_buf->count;
00463         b_blks_wrote = 0;
00464 
00465         /* allocate arrays according to max usage */
00466         if (total_blks_to_write > MAX_ARRAY_SIZE)
00467             mem_list_count = MAX_ARRAY_SIZE;
00468         else mem_list_count = total_blks_to_write;
00469         mem_offsets = (char**)ADIOI_Malloc(mem_list_count*sizeof(char*));
00470         mem_lengths = (int*)ADIOI_Malloc(mem_list_count*sizeof(int));
00471 
00472         j = 0;
00473         /* step through each block in memory, filling memory arrays */
00474         while (b_blks_wrote < total_blks_to_write) {
00475             for (i=0; i<flat_buf->count; i++) {
00476                 mem_offsets[b_blks_wrote % MAX_ARRAY_SIZE] = 
00477                     ((char*)buf + j*buftype_extent + flat_buf->indices[i]);
00478                 mem_lengths[b_blks_wrote % MAX_ARRAY_SIZE] = 
00479                     flat_buf->blocklens[i];
00480                 file_lengths += flat_buf->blocklens[i];
00481                 b_blks_wrote++;
00482                 if (!(b_blks_wrote % MAX_ARRAY_SIZE) ||
00483                     (b_blks_wrote == total_blks_to_write)) {
00484 
00485                     /* in the case of the last read list call,
00486                        adjust mem_list_count */
00487                     if (b_blks_wrote == total_blks_to_write) {
00488                         mem_list_count = total_blks_to_write % MAX_ARRAY_SIZE;
00489                         /* in case last read list call fills max arrays */
00490                         if (!mem_list_count) mem_list_count = MAX_ARRAY_SIZE;
00491                     }
00492 
00493                     pvfs_write_list(fd->fd_sys ,mem_list_count, mem_offsets,
00494                                    mem_lengths, file_list_count,
00495                                    &file_offsets, &file_lengths);
00496                   
00497                     /* in the case of the last read list call, leave here */
00498                     if (b_blks_wrote == total_blks_to_write) break;
00499 
00500                     file_offsets += file_lengths;
00501                     file_lengths = 0;
00502                 } 
00503             } /* for (i=0; i<flat_buf->count; i++) */
00504             j++;
00505         } /* while (b_blks_wrote < total_blks_to_write) */
00506         ADIOI_Free(mem_offsets);
00507         ADIOI_Free(mem_lengths);
00508 
00509         if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
00510 
00511 #ifdef PRINT_ERR_MSG
00512         *error_code = (err_flag) ? MPI_ERR_UNKNOWN : MPI_SUCCESS;
00513 #else
00514         if (err_flag) {
00515             *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
00516                               myname, "I/O Error", "%s", strerror(errno));
00517             ADIOI_Error(fd, *error_code, myname);
00518         }
00519         else *error_code = MPI_SUCCESS;
00520 #endif
00521         ADIOI_Delete_flattened(datatype);
00522         return;
00523     } /* if (!buftype_is_contig && filetype_is_contig) */
00524 
00525     /* already know that file is noncontiguous from above */
00526     /* noncontiguous in file */
00527 
00528 /* filetype already flattened in ADIO_Open */
00529     flat_file = ADIOI_Flatlist;
00530     while (flat_file->type != fd->filetype) flat_file = flat_file->next;
00531 
00532     disp = fd->disp;
00533 
00534     /* for each case - ADIO_Individual pointer or explicit, find offset
00535        (file offset in bytes), n_filetypes (how many filetypes into file 
00536        to start), fwr_size (remaining amount of data in present file
00537        block), and st_index (start point in terms of blocks in starting
00538        filetype) */
00539     if (file_ptr_type == ADIO_INDIVIDUAL) {
00540         offset = fd->fp_ind; /* in bytes */
00541         n_filetypes = -1;
00542         flag = 0;
00543         while (!flag) {
00544             n_filetypes++;
00545             for (i=0; i<flat_file->count; i++) {
00546                 if (disp + flat_file->indices[i] + 
00547                     (ADIO_Offset) n_filetypes*filetype_extent +
00548                       flat_file->blocklens[i] >= offset) {
00549                   st_index = i;
00550                   fwr_size = disp + flat_file->indices[i] + 
00551                     (ADIO_Offset) n_filetypes*filetype_extent
00552                     + flat_file->blocklens[i] - offset;
00553                   flag = 1;
00554                   break;
00555                 }
00556             }
00557         } /* while (!flag) */
00558     } /* if (file_ptr_type == ADIOI_INDIVIDUAL) */
00559     else {
00560         n_etypes_in_filetype = filetype_size/etype_size;
00561         n_filetypes = (int) (offset / n_etypes_in_filetype);
00562         etype_in_filetype = (int) (offset % n_etypes_in_filetype);
00563         size_in_filetype = etype_in_filetype * etype_size;
00564         
00565         sum = 0;
00566         for (i=0; i<flat_file->count; i++) {
00567             sum += flat_file->blocklens[i];
00568             if (sum > size_in_filetype) {
00569                 st_index = i;
00570                 fwr_size = sum - size_in_filetype;
00571                 abs_off_in_filetype = flat_file->indices[i] +
00572                     size_in_filetype - (sum - flat_file->blocklens[i]);
00573                 break;
00574             }
00575         }
00576 
00577         /* abs. offset in bytes in the file */
00578         offset = disp + (ADIO_Offset) n_filetypes*filetype_extent +
00579             abs_off_in_filetype;
00580     } /* else [file_ptr_type != ADIOI_INDIVIDUAL] */
00581 
00582     start_off = offset;
00583     st_fwr_size = fwr_size;
00584     st_n_filetypes = n_filetypes;
00585     
00586     if (buftype_is_contig && !filetype_is_contig) {
00587 
00588 /* contiguous in memory, noncontiguous in file. should be the most
00589    common case. */
00590 
00591         int mem_lengths;
00592         char *mem_offsets;
00593         
00594         i = 0;
00595         j = st_index;
00596         off = offset;
00597         n_filetypes = st_n_filetypes;
00598         
00599         mem_list_count = 1;
00600         
00601         /* determine how many blocks in file to read */
00602         f_data_wrote = ADIOI_MIN(st_fwr_size, bufsize);
00603         total_blks_to_write = 1;
00604         j++;
00605         while (f_data_wrote < bufsize) {
00606             f_data_wrote += flat_file->blocklens[j];
00607             total_blks_to_write++;
00608             if (j<(flat_file->count-1)) j++;
00609             else j = 0; 
00610         }
00611             
00612         j = st_index;
00613         n_filetypes = st_n_filetypes;
00614         n_write_lists = total_blks_to_write/MAX_ARRAY_SIZE;
00615         extra_blks = total_blks_to_write%MAX_ARRAY_SIZE;
00616         
00617         mem_offsets = buf;
00618         mem_lengths = 0;
00619         
00620         /* if at least one full readlist, allocate file arrays
00621            at max array size and don't free until very end */
00622         if (n_write_lists) {
00623             file_offsets = (int64_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
00624                                                   sizeof(int64_t));
00625             file_lengths = (int32_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
00626                                                   sizeof(int32_t));
00627         }
00628         /* if there's no full readlist allocate file arrays according
00629            to needed size (extra_blks) */
00630         else {
00631             file_offsets = (int64_t*)ADIOI_Malloc(extra_blks*
00632                                                   sizeof(int64_t));
00633             file_lengths = (int32_t*)ADIOI_Malloc(extra_blks*
00634                                                   sizeof(int32_t));
00635         }
00636         
00637         /* for file arrays that are of MAX_ARRAY_SIZE, build arrays */
00638         for (i=0; i<n_write_lists; i++) {
00639             file_list_count = MAX_ARRAY_SIZE;
00640             if(!i) {
00641                 file_offsets[0] = offset;
00642                 file_lengths[0] = st_fwr_size;
00643                 mem_lengths = st_fwr_size;
00644             }
00645             for (k=0; k<MAX_ARRAY_SIZE; k++) {
00646                 if (i || k) {
00647                     file_offsets[k] = disp + n_filetypes*filetype_extent
00648                       + flat_file->indices[j];
00649                     file_lengths[k] = flat_file->blocklens[j];
00650                     mem_lengths += file_lengths[k];
00651                 }
00652                 if (j<(flat_file->count - 1)) j++;
00653                 else {
00654                     j = 0;
00655                     n_filetypes++;
00656                 }
00657             } /* for (k=0; k<MAX_ARRAY_SIZE; k++) */
00658             pvfs_write_list(fd->fd_sys, mem_list_count,
00659                            &mem_offsets, &mem_lengths,
00660                            file_list_count, file_offsets,
00661                            file_lengths);
00662             mem_offsets += mem_lengths;
00663             mem_lengths = 0;
00664         } /* for (i=0; i<n_write_lists; i++) */
00665 
00666         /* for file arrays smaller than MAX_ARRAY_SIZE (last read_list call) */
00667         if (extra_blks) {
00668             file_list_count = extra_blks;
00669             if(!i) {
00670                 file_offsets[0] = offset;
00671                 file_lengths[0] = st_fwr_size;
00672             }
00673             for (k=0; k<extra_blks; k++) {
00674                 if(i || k) {
00675                     file_offsets[k] = disp + n_filetypes*filetype_extent +
00676                       flat_file->indices[j];
00677                     if (k == (extra_blks - 1)) {
00678                         file_lengths[k] = bufsize - (int32_t) mem_lengths
00679                           - (int32_t) mem_offsets + (int32_t)  buf;
00680                     }
00681                     else file_lengths[k] = flat_file->blocklens[j];
00682                 } /* if(i || k) */
00683                 mem_lengths += file_lengths[k];
00684                 if (j<(flat_file->count - 1)) j++;
00685                 else {
00686                     j = 0;
00687                     n_filetypes++;
00688                 }
00689             } /* for (k=0; k<extra_blks; k++) */
00690             pvfs_write_list(fd->fd_sys, mem_list_count, &mem_offsets,
00691                            &mem_lengths, file_list_count, file_offsets,
00692                            file_lengths);
00693         }
00694     } 
00695     else {
00696         /* noncontiguous in memory as well as in file */
00697 
00698         ADIOI_Flatten_datatype(datatype);
00699         flat_buf = ADIOI_Flatlist;
00700         while (flat_buf->type != datatype) flat_buf = flat_buf->next;
00701 
00702         size_wrote = 0;
00703         n_filetypes = st_n_filetypes;
00704         fwr_size = st_fwr_size;
00705         bwr_size = flat_buf->blocklens[0];
00706         buf_count = 0;
00707         start_mem_offset = 0;
00708         start_k = k = 0;
00709         start_j = st_index;
00710         max_mem_list = 0;
00711         max_file_list = 0;
00712 
00713         /* run through and file max_file_list and max_mem_list so that you 
00714            can allocate the file and memory arrays less than MAX_ARRAY_SIZE
00715            if possible */
00716 
00717         while (size_wrote < bufsize) {
00718             k = start_k;
00719             new_buffer_write = 0;
00720             mem_list_count = 0;
00721             while ((mem_list_count < MAX_ARRAY_SIZE) && 
00722                    (new_buffer_write < bufsize-size_wrote)) {
00723                 /* find mem_list_count and file_list_count such that both are
00724                    less than MAX_ARRAY_SIZE, the sum of their lengths are
00725                    equal, and the sum of all the data read and data to be
00726                    read in the next immediate read list is less than
00727                    bufsize */
00728                 if(mem_list_count) {
00729                     if((new_buffer_write + flat_buf->blocklens[k] + 
00730                         size_wrote) > bufsize) {
00731                         end_bwr_size = new_buffer_write + 
00732                             flat_buf->blocklens[k] - (bufsize - size_wrote);
00733                         new_buffer_write = bufsize - size_wrote;
00734                     }
00735                     else {
00736                         new_buffer_write += flat_buf->blocklens[k];
00737                         end_bwr_size = flat_buf->blocklens[k];
00738                     }
00739                 }
00740                 else {
00741                     if (bwr_size > (bufsize - size_wrote)) {
00742                         new_buffer_write = bufsize - size_wrote;
00743                         bwr_size = new_buffer_write;
00744                     }
00745                     else new_buffer_write = bwr_size;
00746                 }
00747                 mem_list_count++;
00748                 k = (k + 1)%flat_buf->count;
00749              } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
00750                (new_buffer_write < bufsize-size_wrote)) */
00751             j = start_j;
00752             new_file_write = 0;
00753             file_list_count = 0;
00754             while ((file_list_count < MAX_ARRAY_SIZE) && 
00755                    (new_file_write < new_buffer_write)) {
00756                 if(file_list_count) {
00757                     if((new_file_write + flat_file->blocklens[j]) > 
00758                        new_buffer_write) {
00759                         end_fwr_size = new_buffer_write - new_file_write;
00760                         new_file_write = new_buffer_write;
00761                         j--;
00762                     }
00763                     else {
00764                         new_file_write += flat_file->blocklens[j];
00765                         end_fwr_size = flat_file->blocklens[j];
00766                     }
00767                 }
00768                 else {
00769                     if (fwr_size > new_buffer_write) {
00770                         new_file_write = new_buffer_write;
00771                         fwr_size = new_file_write;
00772                     }
00773                     else new_file_write = fwr_size;
00774                 }
00775                 file_list_count++;
00776                 if (j < (flat_file->count - 1)) j++;
00777                 else j = 0;
00778                 
00779                 k = start_k;
00780                 if ((new_file_write < new_buffer_write) && 
00781                     (file_list_count == MAX_ARRAY_SIZE)) {
00782                     new_buffer_write = 0;
00783                     mem_list_count = 0;
00784                     while (new_buffer_write < new_file_write) {
00785                         if(mem_list_count) {
00786                             if((new_buffer_write + flat_buf->blocklens[k]) >
00787                                new_file_write) {
00788                                 end_bwr_size = new_file_write - 
00789                                     new_buffer_write;
00790                                 new_buffer_write = new_file_write;
00791                                 k--;
00792                             }
00793                             else {
00794                                 new_buffer_write += flat_buf->blocklens[k];
00795                                 end_bwr_size = flat_buf->blocklens[k];
00796                             }
00797                         }
00798                         else {
00799                             new_buffer_write = bwr_size;
00800                             if (bwr_size > (bufsize - size_wrote)) {
00801                                 new_buffer_write = bufsize - size_wrote;
00802                                 bwr_size = new_buffer_write;
00803                             }
00804                         }
00805                         mem_list_count++;
00806                         k = (k + 1)%flat_buf->count;
00807                     } /* while (new_buffer_write < new_file_write) */
00808                 } /* if ((new_file_write < new_buffer_write) &&
00809                      (file_list_count == MAX_ARRAY_SIZE)) */
00810             } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
00811                  (new_buffer_write < bufsize-size_wrote)) */
00812 
00813             /*  fakes filling the writelist arrays of lengths found above  */
00814             k = start_k;
00815             j = start_j;
00816             for (i=0; i<mem_list_count; i++) {       
00817                 if(i) {
00818                     if (i == (mem_list_count - 1)) {
00819                         if (flat_buf->blocklens[k] == end_bwr_size)
00820                             bwr_size = flat_buf->blocklens[(k+1)%
00821                                                           flat_buf->count];
00822                         else {
00823                             bwr_size = flat_buf->blocklens[k] - end_bwr_size;
00824                             k--;
00825                             buf_count--;
00826                         }
00827                     }
00828                 }
00829                 buf_count++;
00830                 k = (k + 1)%flat_buf->count;
00831             } /* for (i=0; i<mem_list_count; i++) */
00832             for (i=0; i<file_list_count; i++) {
00833                 if (i) {
00834                     if (i == (file_list_count - 1)) {
00835                         if (flat_file->blocklens[j] == end_fwr_size)
00836                             fwr_size = flat_file->blocklens[(j+1)%
00837                                                           flat_file->count];   
00838                         else {
00839                             fwr_size = flat_file->blocklens[j] - end_fwr_size;
00840                             j--;
00841                         }
00842                     }
00843                 }
00844                 if (j < flat_file->count - 1) j++;
00845                 else {
00846                     j = 0;
00847                     n_filetypes++;
00848                 }
00849             } /* for (i=0; i<file_list_count; i++) */
00850             size_wrote += new_buffer_write;
00851             start_k = k;
00852             start_j = j;
00853             if (max_mem_list < mem_list_count)
00854                 max_mem_list = mem_list_count;
00855             if (max_file_list < file_list_count)
00856                 max_file_list = file_list_count;
00857             if (max_mem_list == max_mem_list == MAX_ARRAY_SIZE)
00858                 break;
00859         } /* while (size_wrote < bufsize) */
00860 
00861         mem_offsets = (char **)ADIOI_Malloc(max_mem_list*sizeof(char *));
00862         mem_lengths = (int *)ADIOI_Malloc(max_mem_list*sizeof(int));
00863         file_offsets = (int64_t *)ADIOI_Malloc(max_file_list*sizeof(int64_t));
00864         file_lengths = (int32_t *)ADIOI_Malloc(max_file_list*sizeof(int32_t));
00865             
00866         size_wrote = 0;
00867         n_filetypes = st_n_filetypes;
00868         fwr_size = st_fwr_size;
00869         bwr_size = flat_buf->blocklens[0];
00870         buf_count = 0;
00871         start_mem_offset = 0;
00872         start_k = k = 0;
00873         start_j = st_index;
00874 
00875         /*  this section calculates mem_list_count and file_list_count
00876             and also finds the possibly odd sized last array elements
00877             in new_fwr_size and new_bwr_size  */
00878         
00879         while (size_wrote < bufsize) {
00880             k = start_k;
00881             new_buffer_write = 0;
00882             mem_list_count = 0;
00883             while ((mem_list_count < MAX_ARRAY_SIZE) && 
00884                    (new_buffer_write < bufsize-size_wrote)) {
00885                 /* find mem_list_count and file_list_count such that both are
00886                    less than MAX_ARRAY_SIZE, the sum of their lengths are
00887                    equal, and the sum of all the data read and data to be
00888                    read in the next immediate read list is less than
00889                    bufsize */
00890                 if(mem_list_count) {
00891                     if((new_buffer_write + flat_buf->blocklens[k] + 
00892                         size_wrote) > bufsize) {
00893                         end_bwr_size = new_buffer_write + 
00894                             flat_buf->blocklens[k] - (bufsize - size_wrote);
00895                         new_buffer_write = bufsize - size_wrote;
00896                     }
00897                     else {
00898                         new_buffer_write += flat_buf->blocklens[k];
00899                         end_bwr_size = flat_buf->blocklens[k];
00900                     }
00901                 }
00902                 else {
00903                     if (bwr_size > (bufsize - size_wrote)) {
00904                         new_buffer_write = bufsize - size_wrote;
00905                         bwr_size = new_buffer_write;
00906                     }
00907                     else new_buffer_write = bwr_size;
00908                 }
00909                 mem_list_count++;
00910                 k = (k + 1)%flat_buf->count;
00911              } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
00912                (new_buffer_write < bufsize-size_wrote)) */
00913             j = start_j;
00914             new_file_write = 0;
00915             file_list_count = 0;
00916             while ((file_list_count < MAX_ARRAY_SIZE) && 
00917                    (new_file_write < new_buffer_write)) {
00918                 if(file_list_count) {
00919                     if((new_file_write + flat_file->blocklens[j]) > 
00920                        new_buffer_write) {
00921                         end_fwr_size = new_buffer_write - new_file_write;
00922                         new_file_write = new_buffer_write;
00923                         j--;
00924                     }
00925                     else {
00926                         new_file_write += flat_file->blocklens[j];
00927                         end_fwr_size = flat_file->blocklens[j];
00928                     }
00929                 }
00930                 else {
00931                     if (fwr_size > new_buffer_write) {
00932                         new_file_write = new_buffer_write;
00933                         fwr_size = new_file_write;
00934                     }
00935                     else new_file_write = fwr_size;
00936                 }
00937                 file_list_count++;
00938                 if (j < (flat_file->count - 1)) j++;
00939                 else j = 0;
00940                 
00941                 k = start_k;
00942                 if ((new_file_write < new_buffer_write) && 
00943                     (file_list_count == MAX_ARRAY_SIZE)) {
00944                     new_buffer_write = 0;
00945                     mem_list_count = 0;
00946                     while (new_buffer_write < new_file_write) {
00947                         if(mem_list_count) {
00948                             if((new_buffer_write + flat_buf->blocklens[k]) >
00949                                new_file_write) {
00950                                 end_bwr_size = new_file_write -
00951                                   new_buffer_write;
00952                                 new_buffer_write = new_file_write;
00953                                 k--;
00954                             }
00955                             else {
00956                                 new_buffer_write += flat_buf->blocklens[k];
00957                                 end_bwr_size = flat_buf->blocklens[k];
00958                             }
00959                         }
00960                         else {
00961                             new_buffer_write = bwr_size;
00962                             if (bwr_size > (bufsize - size_wrote)) {
00963                                 new_buffer_write = bufsize - size_wrote;
00964                                 bwr_size = new_buffer_write;
00965                             }
00966                         }
00967                         mem_list_count++;
00968                         k = (k + 1)%flat_buf->count;
00969                     } /* while (new_buffer_write < new_file_write) */
00970                 } /* if ((new_file_write < new_buffer_write) &&
00971                      (file_list_count == MAX_ARRAY_SIZE)) */
00972             } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
00973                  (new_buffer_write < bufsize-size_wrote)) */
00974 
00975             /*  fills the allocated readlist arrays  */
00976             k = start_k;
00977             j = start_j;
00978             for (i=0; i<mem_list_count; i++) {       
00979                 mem_offsets[i] = ((char*)buf + buftype_extent*
00980                                          (buf_count/flat_buf->count) +
00981                                          (int)flat_buf->indices[k]);
00982                 
00983                 if(!i) {
00984                     mem_lengths[0] = bwr_size;
00985                     mem_offsets[0] += flat_buf->blocklens[k] - bwr_size;
00986                 }
00987                 else {
00988                     if (i == (mem_list_count - 1)) {
00989                         mem_lengths[i] = end_bwr_size;
00990                         if (flat_buf->blocklens[k] == end_bwr_size)
00991                             bwr_size = flat_buf->blocklens[(k+1)%
00992                                                           flat_buf->count];
00993                         else {
00994                             bwr_size = flat_buf->blocklens[k] - end_bwr_size;
00995                             k--;
00996                             buf_count--;
00997                         }
00998                     }
00999                     else {
01000                         mem_lengths[i] = flat_buf->blocklens[k];
01001                     }
01002                 }
01003                 buf_count++;
01004                 k = (k + 1)%flat_buf->count;
01005             } /* for (i=0; i<mem_list_count; i++) */
01006             for (i=0; i<file_list_count; i++) {
01007                 file_offsets[i] = disp + flat_file->indices[j] + n_filetypes *
01008                     filetype_extent;
01009                 if (!i) {
01010                     file_lengths[0] = fwr_size;
01011                     file_offsets[0] += flat_file->blocklens[j] - fwr_size;
01012                 }
01013                 else {
01014                     if (i == (file_list_count - 1)) {
01015                         file_lengths[i] = end_fwr_size;
01016                         if (flat_file->blocklens[j] == end_fwr_size)
01017                             fwr_size = flat_file->blocklens[(j+1)%
01018                                                           flat_file->count];   
01019                         else {
01020                             fwr_size = flat_file->blocklens[j] - end_fwr_size;
01021                             j--;
01022                         }
01023                     }
01024                     else file_lengths[i] = flat_file->blocklens[j];
01025                 }
01026                 if (j < flat_file->count - 1) j++;
01027                 else {
01028                     j = 0;
01029                     n_filetypes++;
01030                 }
01031             } /* for (i=0; i<file_list_count; i++) */
01032             pvfs_write_list(fd->fd_sys,mem_list_count, mem_offsets,
01033                            mem_lengths, file_list_count, file_offsets,
01034                            file_lengths);
01035             size_wrote += new_buffer_write;
01036             start_k = k;
01037             start_j = j;
01038         } /* while (size_wrote < bufsize) */
01039         ADIOI_Free(mem_offsets);
01040         ADIOI_Free(mem_lengths);
01041     }
01042     ADIOI_Free(file_offsets);
01043     ADIOI_Free(file_lengths);
01044 
01045     if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
01046 #ifdef PRINT_ERR_MSG
01047     *error_code = (err_flag) ? MPI_ERR_UNKNOWN : MPI_SUCCESS;
01048 #else
01049     if (err_flag) {
01050         *error_code = MPIR_Err_setmsg(MPI_ERR_IO, MPIR_ADIO_ERROR,
01051                                       myname, "I/O Error", "%s",
01052                                       strerror(errno));
01053         ADIOI_Error(fd, *error_code, myname);
01054     }
01055     else *error_code = MPI_SUCCESS;
01056 #endif
01057 
01058     fd->fp_sys_posn = -1;   /* set it to null. */
01059 
01060 #ifdef HAVE_STATUS_SET_BYTES
01061     MPIR_Status_set_bytes(status, datatype, bufsize);
01062 /* This is a temporary way of filling in status. The right way is to 
01063    keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
01064 #endif
01065 
01066     if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
01067 }
01068 #endif /* HAVE_PVFS_LISTIO */
libs/ck-libs/ampi/romio/adio/ad_pvfs/ad_pvfs_write.c
