Charm++: Charm Source Code Documentation

00001 /* -*- Mode: C; c-basic-offset:4 ; -*- 
00002  * vim: ts=8 sts=4 sw=4 noexpandtab 
00003  * 
00004  *   Copyright (C) 2008 University of Chicago. 
00005  *   See COPYRIGHT notice in top-level directory.
00006  */
00007 
00008 #include "adio.h"
00009 #include "adio_extern.h"
00010 #include "ad_pvfs2.h"
00011 
00012 #include "ad_pvfs2_common.h"
00013 
00014 void ADIOI_PVFS2_OldWriteStrided(ADIO_File fd, void *buf, int count,
00015             MPI_Datatype datatype, int file_ptr_type,
00016             ADIO_Offset offset, ADIO_Status *status,
00017             int *error_code)
00018 {
00019     /* as with all the other WriteStrided functions, offset is in units of
00020      * etype relative to the filetype */
00021 
00022     /* Since PVFS2 does not support file locking, can't do buffered writes
00023        as on Unix */
00024 
00025     ADIOI_Flatlist_node *flat_buf, *flat_file;
00026     int i, j, k, bwr_size, fwr_size=0, st_index=0;
00027     int bufsize, sum, n_etypes_in_filetype, size_in_filetype;
00028     int n_filetypes, etype_in_filetype;
00029     ADIO_Offset abs_off_in_filetype=0;
00030     int filetype_size, etype_size, buftype_size;
00031     MPI_Aint filetype_extent, buftype_extent;
00032     int buf_count, buftype_is_contig, filetype_is_contig;
00033     ADIO_Offset off, disp, start_off, initial_off;
00034     int flag, st_fwr_size, st_n_filetypes;
00035     int err_flag=0;
00036 
00037     int mem_list_count, file_list_count;
00038     PVFS_size * mem_offsets;
00039     int64_t *file_offsets;
00040     int *mem_lengths;
00041     int32_t *file_lengths;
00042     int total_blks_to_write;
00043 
00044     int max_mem_list, max_file_list;
00045 
00046     int b_blks_wrote;
00047     int f_data_wrote;
00048     int size_wrote=0, n_write_lists, extra_blks;
00049 
00050     int end_bwr_size, end_fwr_size;
00051     int start_k, start_j, new_file_write, new_buffer_write;
00052     int start_mem_offset;
00053     PVFS_Request mem_req, file_req;
00054     ADIOI_PVFS2_fs * pvfs_fs;
00055     PVFS_sysresp_io resp_io;
00056     MPI_Offset total_bytes_written=0;
00057     static char myname[] = "ADIOI_PVFS2_WRITESTRIDED";
00058 
00059     /* note: don't increase this: several parts of PVFS2 now 
00060      * assume this limit*/
00061 #define MAX_ARRAY_SIZE 64
00062 
00063     /* --BEGIN ERROR HANDLING-- */
00064     if (fd->atomicity) {
00065     *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00066                        MPIR_ERR_RECOVERABLE,
00067                        myname, __LINE__,
00068                        MPI_ERR_ARG,
00069                        "Atomic noncontiguous writes are not supported by PVFS2", 0);
00070     return;
00071     }
00072     /* --END ERROR HANDLING-- */
00073 
00074     ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
00075     ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
00076 
00077     /* the HDF5 tests showed a bug in this list processing code (see many many
00078      * lines down below).  We added a workaround, but common HDF5 file types
00079      * are actually contiguous and do not need the expensive workarond */
00080     if (!filetype_is_contig) {
00081     flat_file = ADIOI_Flatlist;
00082     while (flat_file->type != fd->filetype) flat_file = flat_file->next;
00083     if (flat_file->count == 1 && !buftype_is_contig)
00084         filetype_is_contig = 1;
00085     }
00086 
00087     MPI_Type_size(fd->filetype, &filetype_size);
00088     if ( ! filetype_size ) {
00089     *error_code = MPI_SUCCESS; 
00090     return;
00091     }
00092 
00093     MPI_Type_extent(fd->filetype, &filetype_extent);
00094     MPI_Type_size(datatype, &buftype_size);
00095     MPI_Type_extent(datatype, &buftype_extent);
00096     etype_size = fd->etype_size;
00097     
00098     bufsize = buftype_size * count;
00099 
00100     pvfs_fs = (ADIOI_PVFS2_fs*)fd->fs_ptr;
00101 
00102     if (!buftype_is_contig && filetype_is_contig) {
00103 
00104 /* noncontiguous in memory, contiguous in file.  */
00105         int64_t file_offsets;
00106     int32_t file_lengths;
00107 
00108     ADIOI_Flatten_datatype(datatype);
00109     flat_buf = ADIOI_Flatlist;
00110     while (flat_buf->type != datatype) flat_buf = flat_buf->next;
00111     
00112     if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
00113         off = fd->disp + etype_size * offset;
00114     }
00115     else off = fd->fp_ind;
00116 
00117     file_list_count = 1;
00118     file_offsets = off;
00119     file_lengths = 0;
00120     total_blks_to_write = count*flat_buf->count;
00121     b_blks_wrote = 0;
00122 
00123     /* allocate arrays according to max usage */
00124     if (total_blks_to_write > MAX_ARRAY_SIZE)
00125         mem_list_count = MAX_ARRAY_SIZE;
00126     else mem_list_count = total_blks_to_write;
00127     mem_offsets = (PVFS_size*)ADIOI_Malloc(mem_list_count*sizeof(PVFS_size));
00128     mem_lengths = (int*)ADIOI_Malloc(mem_list_count*sizeof(int));
00129 
00130     j = 0;
00131     /* step through each block in memory, filling memory arrays */
00132     while (b_blks_wrote < total_blks_to_write) {
00133         for (i=0; i<flat_buf->count; i++) {
00134         mem_offsets[b_blks_wrote % MAX_ARRAY_SIZE] = 
00135             /* TODO: fix this warning by casting to an integer that's
00136              * the same size as a char * and /then/ casting to
00137              * PVFS_size */
00138             ((PVFS_size)buf + j*buftype_extent + flat_buf->indices[i]);
00139         mem_lengths[b_blks_wrote % MAX_ARRAY_SIZE] = 
00140             flat_buf->blocklens[i];
00141         file_lengths += flat_buf->blocklens[i];
00142         b_blks_wrote++;
00143         if (!(b_blks_wrote % MAX_ARRAY_SIZE) ||
00144             (b_blks_wrote == total_blks_to_write)) {
00145 
00146             /* in the case of the last write list call,
00147                adjust mem_list_count */
00148             if (b_blks_wrote == total_blks_to_write) {
00149                 mem_list_count = total_blks_to_write % MAX_ARRAY_SIZE;
00150             /* in case last write list call fills max arrays */
00151             if (!mem_list_count) mem_list_count = MAX_ARRAY_SIZE;
00152             }
00153             err_flag = PVFS_Request_hindexed(mem_list_count, 
00154                              mem_lengths, mem_offsets,
00155                              PVFS_BYTE, &mem_req);
00156             /* --BEGIN ERROR HANDLING-- */
00157             if (err_flag != 0) {
00158             *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00159                                MPIR_ERR_RECOVERABLE,
00160                                myname, __LINE__,
00161                                ADIOI_PVFS2_error_convert(err_flag),
00162                                "Error in PVFS_Request_hindexed (memory)", 0);
00163             break;
00164             }
00165             /* --END ERROR HANDLING-- */
00166 
00167             err_flag = PVFS_Request_contiguous(file_lengths, 
00168                                PVFS_BYTE, &file_req);
00169             /* --BEGIN ERROR HANDLING-- */
00170             if (err_flag != 0) {
00171             *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00172                                MPIR_ERR_RECOVERABLE,
00173                                myname, __LINE__,
00174                                ADIOI_PVFS2_error_convert(err_flag),
00175                                "Error in PVFS_Request_contiguous (file)", 0);
00176             break;
00177             }
00178             /* --END ERROR HANDLING-- */
00179 
00180 #ifdef ADIOI_MPE_LOGGING
00181                     MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
00182 #endif
00183             err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req, 
00184                           file_offsets, PVFS_BOTTOM,
00185                           mem_req, 
00186                           &(pvfs_fs->credentials),
00187                           &resp_io);
00188 #ifdef ADIOI_MPE_LOGGING
00189                     MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
00190 #endif
00191             total_bytes_written += resp_io.total_completed;
00192           
00193             /* in the case of error or the last write list call, 
00194              * leave here */
00195             /* --BEGIN ERROR HANDLING-- */
00196             if (err_flag) {
00197             *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00198                                MPIR_ERR_RECOVERABLE,
00199                                myname, __LINE__,
00200                                ADIOI_PVFS2_error_convert(err_flag),
00201                                "Error in PVFS_sys_write", 0);
00202             break;
00203             }
00204             /* --END ERROR HANDLING-- */
00205             if (b_blks_wrote == total_blks_to_write) break;
00206 
00207             file_offsets += file_lengths;
00208             file_lengths = 0;
00209             PVFS_Request_free(&mem_req);
00210             PVFS_Request_free(&file_req);
00211         } 
00212         } /* for (i=0; i<flat_buf->count; i++) */
00213         j++;
00214     } /* while (b_blks_wrote < total_blks_to_write) */
00215     ADIOI_Free(mem_offsets);
00216     ADIOI_Free(mem_lengths);
00217 
00218     if (file_ptr_type == ADIO_INDIVIDUAL) 
00219         fd->fp_ind += total_bytes_written;
00220 
00221     if (!err_flag)  *error_code = MPI_SUCCESS;
00222 
00223     fd->fp_sys_posn = -1;   /* clear this. */
00224 
00225 #ifdef HAVE_STATUS_SET_BYTES
00226     MPIR_Status_set_bytes(status, datatype, bufsize);
00227 /* This is a temporary way of filling in status. The right way is to 
00228    keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
00229 #endif
00230 
00231     ADIOI_Delete_flattened(datatype);
00232     return;
00233     } /* if (!buftype_is_contig && filetype_is_contig) */
00234 
00235     /* already know that file is noncontiguous from above */
00236     /* noncontiguous in file */
00237 
00238 /* filetype already flattened in ADIO_Open */
00239     flat_file = ADIOI_Flatlist;
00240     while (flat_file->type != fd->filetype) flat_file = flat_file->next;
00241 
00242     disp = fd->disp;
00243     initial_off = offset;
00244 
00245     /* for each case - ADIO_Individual pointer or explicit, find offset
00246        (file offset in bytes), n_filetypes (how many filetypes into file 
00247        to start), fwr_size (remaining amount of data in present file
00248        block), and st_index (start point in terms of blocks in starting
00249        filetype) */
00250     if (file_ptr_type == ADIO_INDIVIDUAL) {
00251         offset = fd->fp_ind; /* in bytes */
00252     n_filetypes = -1;
00253     flag = 0;
00254     while (!flag) {
00255         n_filetypes++;
00256         for (i=0; i<flat_file->count; i++) {
00257             if (disp + flat_file->indices[i] + 
00258             ((ADIO_Offset) n_filetypes)*filetype_extent +
00259               flat_file->blocklens[i] >= offset) {
00260           st_index = i;
00261           fwr_size = disp + flat_file->indices[i] + 
00262             ((ADIO_Offset) n_filetypes)*filetype_extent
00263             + flat_file->blocklens[i] - offset;
00264           flag = 1;
00265           break;
00266         }
00267         }
00268     } /* while (!flag) */
00269     } /* if (file_ptr_type == ADIO_INDIVIDUAL) */
00270     else {
00271         n_etypes_in_filetype = filetype_size/etype_size;
00272     n_filetypes = (int) (offset / n_etypes_in_filetype);
00273     etype_in_filetype = (int) (offset % n_etypes_in_filetype);
00274     size_in_filetype = etype_in_filetype * etype_size;
00275     
00276     sum = 0;
00277     for (i=0; i<flat_file->count; i++) {
00278         sum += flat_file->blocklens[i];
00279         if (sum > size_in_filetype) {
00280             st_index = i;
00281         fwr_size = sum - size_in_filetype;
00282         abs_off_in_filetype = flat_file->indices[i] +
00283             size_in_filetype - (sum - flat_file->blocklens[i]);
00284         break;
00285         }
00286     }
00287 
00288     /* abs. offset in bytes in the file */
00289     offset = disp + ((ADIO_Offset) n_filetypes)*filetype_extent +
00290         abs_off_in_filetype;
00291     } /* else [file_ptr_type != ADIO_INDIVIDUAL] */
00292 
00293     start_off = offset;
00294     st_fwr_size = fwr_size;
00295     st_n_filetypes = n_filetypes;
00296     
00297     if (buftype_is_contig && !filetype_is_contig) {
00298 
00299 /* contiguous in memory, noncontiguous in file. should be the most
00300    common case. */
00301 
00302         int mem_lengths;
00303     char *mem_offsets;
00304         
00305     i = 0;
00306     j = st_index;
00307     off = offset;
00308     n_filetypes = st_n_filetypes;
00309         
00310     mem_list_count = 1;
00311         
00312     /* determine how many blocks in file to write */
00313     f_data_wrote = ADIOI_MIN(st_fwr_size, bufsize);
00314     total_blks_to_write = 1;
00315     if (j < (flat_file->count -1)) j++;
00316     else {
00317         j = 0;
00318         n_filetypes++;
00319     }
00320     while (f_data_wrote < bufsize) {
00321         f_data_wrote += flat_file->blocklens[j];
00322         total_blks_to_write++;
00323         if (j<(flat_file->count-1)) j++;
00324         else j = 0; 
00325     }
00326         
00327     j = st_index;
00328     n_filetypes = st_n_filetypes;
00329     n_write_lists = total_blks_to_write/MAX_ARRAY_SIZE;
00330     extra_blks = total_blks_to_write%MAX_ARRAY_SIZE;
00331         
00332     mem_offsets = buf;
00333     mem_lengths = 0;
00334         
00335     /* if at least one full writelist, allocate file arrays
00336        at max array size and don't free until very end */
00337     if (n_write_lists) {
00338         file_offsets = (int64_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
00339                           sizeof(int64_t));
00340         file_lengths = (int32_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
00341                           sizeof(int32_t));
00342     }
00343     /* if there's no full writelist allocate file arrays according
00344        to needed size (extra_blks) */
00345     else {
00346         file_offsets = (int64_t*)ADIOI_Malloc(extra_blks*
00347                                                   sizeof(int64_t));
00348             file_lengths = (int32_t*)ADIOI_Malloc(extra_blks*
00349                                                   sizeof(int32_t));
00350         }
00351         
00352         /* for file arrays that are of MAX_ARRAY_SIZE, build arrays */
00353         for (i=0; i<n_write_lists; i++) {
00354             file_list_count = MAX_ARRAY_SIZE;
00355             if(!i) {
00356                 file_offsets[0] = offset;
00357                 file_lengths[0] = st_fwr_size;
00358                 mem_lengths = st_fwr_size;
00359             }
00360             for (k=0; k<MAX_ARRAY_SIZE; k++) {
00361                 if (i || k) {
00362                     file_offsets[k] = disp + 
00363             ((ADIO_Offset)n_filetypes)*filetype_extent
00364             + flat_file->indices[j];
00365                     file_lengths[k] = flat_file->blocklens[j];
00366                     mem_lengths += file_lengths[k];
00367                 }
00368                 if (j<(flat_file->count - 1)) j++;
00369                 else {
00370                     j = 0;
00371                     n_filetypes++;
00372                 }
00373             } /* for (k=0; k<MAX_ARRAY_SIZE; k++) */
00374 
00375         err_flag = PVFS_Request_contiguous(mem_lengths, 
00376                            PVFS_BYTE, &mem_req);
00377         /* --BEGIN ERROR HANDLING-- */
00378         if (err_flag != 0) {
00379         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00380                            MPIR_ERR_RECOVERABLE,
00381                            myname, __LINE__,
00382                            ADIOI_PVFS2_error_convert(err_flag),
00383                            "Error in PVFS_Request_contiguous (memory)", 0);
00384         goto error_state;
00385         }
00386         /* --END ERROR HANDLING-- */
00387 
00388         err_flag = PVFS_Request_hindexed(file_list_count, file_lengths, 
00389                          file_offsets, PVFS_BYTE,
00390                          &file_req);
00391         /* --BEGIN ERROR HANDLING-- */
00392         if (err_flag != 0) {
00393         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00394                            MPIR_ERR_RECOVERABLE,
00395                            myname, __LINE__,
00396                            ADIOI_PVFS2_error_convert(err_flag),
00397                            "Error in PVFS_Request_hindexed (file)", 0);
00398         goto error_state;
00399         }
00400         /* --END ERROR HANDLING-- */
00401 
00402         /* PVFS_Request_hindexed already expresses the offsets into the
00403          * file, so we should not pass in an offset if we are using
00404          * hindexed for the file type */
00405 #ifdef ADIOI_MPE_LOGGING
00406             MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
00407 #endif
00408         err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req, 0, 
00409                       mem_offsets, mem_req,
00410                       &(pvfs_fs->credentials), &resp_io);
00411 #ifdef ADIOI_MPE_LOGGING
00412             MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
00413 #endif
00414         /* --BEGIN ERROR HANDLING-- */
00415         if (err_flag != 0) {
00416         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00417                            MPIR_ERR_RECOVERABLE,
00418                            myname, __LINE__,
00419                            ADIOI_PVFS2_error_convert(err_flag),
00420                            "Error in PVFS_sys_write", 0);
00421         goto error_state;
00422         }
00423         /* --END ERROR HANDLING-- */
00424         total_bytes_written += resp_io.total_completed;
00425 
00426             mem_offsets += mem_lengths;
00427             mem_lengths = 0;
00428         PVFS_Request_free(&file_req);
00429         PVFS_Request_free(&mem_req);
00430 
00431         } /* for (i=0; i<n_write_lists; i++) */
00432 
00433         /* for file arrays smaller than MAX_ARRAY_SIZE (last write_list call) */
00434         if (extra_blks) {
00435             file_list_count = extra_blks;
00436             if(!i) {
00437                 file_offsets[0] = offset;
00438                 file_lengths[0] = ADIOI_MIN(st_fwr_size, bufsize);
00439             }
00440             for (k=0; k<extra_blks; k++) {
00441                 if(i || k) {
00442                     file_offsets[k] = disp + 
00443             ((ADIO_Offset)n_filetypes)*filetype_extent +
00444             flat_file->indices[j];
00445                     if (k == (extra_blks - 1)) {
00446                         file_lengths[k] = bufsize - (int32_t) mem_lengths
00447                           - (int32_t) mem_offsets + (int32_t)  buf;
00448                     }
00449                     else file_lengths[k] = flat_file->blocklens[j];
00450                 } /* if(i || k) */
00451                 mem_lengths += file_lengths[k];
00452                 if (j<(flat_file->count - 1)) j++;
00453                 else {
00454                     j = 0;
00455                     n_filetypes++;
00456                 }
00457             } /* for (k=0; k<extra_blks; k++) */
00458 
00459         err_flag = PVFS_Request_contiguous(mem_lengths, 
00460                            PVFS_BYTE, &mem_req);
00461         /* --BEGIN ERROR HANDLING-- */
00462         if (err_flag != 0) {
00463         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00464                            MPIR_ERR_RECOVERABLE,
00465                            myname, __LINE__,
00466                            ADIOI_PVFS2_error_convert(err_flag),
00467                            "Error in PVFS_Request_contiguous (memory)", 0);
00468         goto error_state;
00469         }
00470         /* --END ERROR HANDLING-- */
00471 
00472         err_flag = PVFS_Request_hindexed(file_list_count, file_lengths, 
00473                          file_offsets, PVFS_BYTE,
00474                          &file_req);
00475         /* --BEGIN ERROR HANDLING-- */
00476         if (err_flag != 0) {
00477         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00478                            MPIR_ERR_RECOVERABLE,
00479                            myname, __LINE__,
00480                            ADIOI_PVFS2_error_convert(err_flag),
00481                            "Error in PVFS_Request_hindexed(file)", 0);
00482         goto error_state;
00483         }
00484         /* --END ERROR HANDLING-- */
00485 
00486         /* as above, use 0 for 'offset' when using hindexed file type*/
00487 #ifdef ADIOI_MPE_LOGGING
00488             MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
00489 #endif
00490         err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req, 0, 
00491                       mem_offsets, mem_req,
00492                       &(pvfs_fs->credentials), &resp_io);
00493 #ifdef ADIOI_MPE_LOGGING
00494             MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
00495 #endif
00496         /* --BEGIN ERROR HANDLING-- */
00497         if (err_flag != 0) {
00498         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00499                            MPIR_ERR_RECOVERABLE,
00500                            myname, __LINE__,
00501                            ADIOI_PVFS2_error_convert(err_flag),
00502                            "Error in PVFS_sys_write", 0);
00503         goto error_state;
00504         }
00505         /* --END ERROR HANDLING-- */
00506         total_bytes_written += resp_io.total_completed;
00507         PVFS_Request_free(&mem_req);
00508         PVFS_Request_free(&file_req);
00509         }
00510     } 
00511     else {
00512         /* noncontiguous in memory as well as in file */
00513 
00514         ADIOI_Flatten_datatype(datatype);
00515     flat_buf = ADIOI_Flatlist;
00516     while (flat_buf->type != datatype) flat_buf = flat_buf->next;
00517 
00518     size_wrote = 0;
00519     n_filetypes = st_n_filetypes;
00520     fwr_size = st_fwr_size;
00521     bwr_size = flat_buf->blocklens[0];
00522     buf_count = 0;
00523     start_mem_offset = 0;
00524     start_k = k = 0;
00525     start_j = st_index;
00526     max_mem_list = 0;
00527     max_file_list = 0;
00528 
00529     /* run through and file max_file_list and max_mem_list so that you 
00530        can allocate the file and memory arrays less than MAX_ARRAY_SIZE
00531        if possible */
00532 
00533     while (size_wrote < bufsize) {
00534         k = start_k;
00535         new_buffer_write = 0;
00536         mem_list_count = 0;
00537         while ((mem_list_count < MAX_ARRAY_SIZE) && 
00538            (new_buffer_write < bufsize-size_wrote)) {
00539             /* find mem_list_count and file_list_count such that both are
00540            less than MAX_ARRAY_SIZE, the sum of their lengths are
00541            equal, and the sum of all the data written and data to be
00542            written in the next immediate write list is less than
00543            bufsize */
00544             if(mem_list_count) {
00545             if((new_buffer_write + flat_buf->blocklens[k] + 
00546             size_wrote) > bufsize) {
00547                 end_bwr_size = new_buffer_write + 
00548                 flat_buf->blocklens[k] - (bufsize - size_wrote);
00549             new_buffer_write = bufsize - size_wrote;
00550             }
00551             else {
00552                 new_buffer_write += flat_buf->blocklens[k];
00553             end_bwr_size = flat_buf->blocklens[k];
00554             }
00555         }
00556         else {
00557             if (bwr_size > (bufsize - size_wrote)) {
00558                 new_buffer_write = bufsize - size_wrote;
00559             bwr_size = new_buffer_write;
00560             }
00561             else new_buffer_write = bwr_size;
00562         }
00563         mem_list_count++;
00564         k = (k + 1)%flat_buf->count;
00565          } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
00566            (new_buffer_write < bufsize-size_wrote)) */
00567         j = start_j;
00568         new_file_write = 0;
00569         file_list_count = 0;
00570         while ((file_list_count < MAX_ARRAY_SIZE) && 
00571            (new_file_write < new_buffer_write)) { 
00572             if(file_list_count) {
00573             if((new_file_write + flat_file->blocklens[j]) > 
00574                new_buffer_write) {
00575                 end_fwr_size = new_buffer_write - new_file_write;
00576             new_file_write = new_buffer_write;
00577             j--;
00578             }
00579             else {
00580                 new_file_write += flat_file->blocklens[j];
00581             end_fwr_size = flat_file->blocklens[j];
00582             }
00583         }
00584         else {
00585             if (fwr_size > new_buffer_write) {
00586                 new_file_write = new_buffer_write;
00587             fwr_size = new_file_write;
00588             }
00589             else new_file_write = fwr_size;
00590         }
00591         file_list_count++;
00592         if (j < (flat_file->count - 1)) j++;
00593         else j = 0;
00594         
00595         k = start_k;
00596         if ((new_file_write < new_buffer_write) && 
00597             (file_list_count == MAX_ARRAY_SIZE)) {
00598             new_buffer_write = 0;
00599             mem_list_count = 0;
00600             while (new_buffer_write < new_file_write) {
00601                 if(mem_list_count) {
00602                 if((new_buffer_write + flat_buf->blocklens[k]) >
00603                    new_file_write) {
00604                     end_bwr_size = new_file_write - 
00605                     new_buffer_write;
00606                 new_buffer_write = new_file_write;
00607                 k--;
00608                 }
00609                 else {
00610                     new_buffer_write += flat_buf->blocklens[k];
00611                 end_bwr_size = flat_buf->blocklens[k];
00612                 }
00613             }
00614             else {
00615                 new_buffer_write = bwr_size;
00616                 if (bwr_size > (bufsize - size_wrote)) {
00617                     new_buffer_write = bufsize - size_wrote;
00618                 bwr_size = new_buffer_write;
00619                 }
00620             }
00621             mem_list_count++;
00622             k = (k + 1)%flat_buf->count;
00623             } /* while (new_buffer_write < new_file_write) */
00624         } /* if ((new_file_write < new_buffer_write) &&
00625              (file_list_count == MAX_ARRAY_SIZE)) */
00626         } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
00627          (new_buffer_write < bufsize-size_wrote)) */
00628 
00629         /*  fakes filling the writelist arrays of lengths found above  */
00630         k = start_k;
00631         j = start_j;
00632         for (i=0; i<mem_list_count; i++) {       
00633         if(i) {
00634             if (i == (mem_list_count - 1)) {
00635             if (flat_buf->blocklens[k] == end_bwr_size)
00636                 bwr_size = flat_buf->blocklens[(k+1)%
00637                               flat_buf->count];
00638             else {
00639                 bwr_size = flat_buf->blocklens[k] - end_bwr_size;
00640                 k--;
00641                 buf_count--;
00642             }
00643             }
00644         }
00645         buf_count++;
00646         k = (k + 1)%flat_buf->count;
00647         } /* for (i=0; i<mem_list_count; i++) */
00648         for (i=0; i<file_list_count; i++) {
00649         if (i) {
00650             if (i == (file_list_count - 1)) {
00651             if (flat_file->blocklens[j] == end_fwr_size)
00652                 fwr_size = flat_file->blocklens[(j+1)%
00653                               flat_file->count];   
00654             else {
00655                 fwr_size = flat_file->blocklens[j] - end_fwr_size;
00656                 j--;
00657             }
00658             }
00659         }
00660         if (j < flat_file->count - 1) j++;
00661         else {
00662             j = 0;
00663             n_filetypes++;
00664         }
00665         } /* for (i=0; i<file_list_count; i++) */
00666         size_wrote += new_buffer_write;
00667         start_k = k;
00668         start_j = j;
00669         if (max_mem_list < mem_list_count)
00670             max_mem_list = mem_list_count;
00671         if (max_file_list < file_list_count)
00672             max_file_list = file_list_count;
00673     } /* while (size_wrote < bufsize) */
00674 
00675     /* one last check before we actually carry out the operation:
00676      * this code has hard-to-fix bugs when a noncontiguous file type has
00677      * such large pieces that the sum of the lengths of the memory type is
00678      * not larger than one of those pieces (and vice versa for large memory
00679      * types and many pices of file types.  In these cases, give up and
00680      * fall back to naive reads and writes.  The testphdf5 test created a
00681      * type with two very large memory regions and 600 very small file
00682      * regions.  The same test also created a type with one very large file
00683      * region and many (700) very small memory regions.  both cases caused
00684      * problems for this code */
00685 
00686     if ( ( (file_list_count == 1) && 
00687             (new_file_write < flat_file->blocklens[0] ) ) ||
00688         ((mem_list_count == 1) && 
00689             (new_buffer_write < flat_buf->blocklens[0]) ) ||
00690         ((file_list_count == MAX_ARRAY_SIZE) && 
00691             (new_file_write < flat_buf->blocklens[0]) ) ||
00692         ( (mem_list_count == MAX_ARRAY_SIZE) &&
00693             (new_buffer_write < flat_file->blocklens[0])) )
00694     {
00695         ADIOI_Delete_flattened(datatype);
00696         ADIOI_GEN_WriteStrided_naive(fd, buf, count, datatype,
00697             file_ptr_type, initial_off, status, error_code);
00698         return;
00699     }
00700 
00701 
00702     mem_offsets = (PVFS_size*)ADIOI_Malloc(max_mem_list*sizeof(PVFS_size));
00703     mem_lengths = (int *)ADIOI_Malloc(max_mem_list*sizeof(int));
00704     file_offsets = (int64_t *)ADIOI_Malloc(max_file_list*sizeof(int64_t));
00705     file_lengths = (int32_t *)ADIOI_Malloc(max_file_list*sizeof(int32_t));
00706         
00707     size_wrote = 0;
00708     n_filetypes = st_n_filetypes;
00709     fwr_size = st_fwr_size;
00710     bwr_size = flat_buf->blocklens[0];
00711     buf_count = 0;
00712     start_mem_offset = 0;
00713     start_k = k = 0;
00714     start_j = st_index;
00715 
00716     /*  this section calculates mem_list_count and file_list_count
00717         and also finds the possibly odd sized last array elements
00718         in new_fwr_size and new_bwr_size  */
00719     
00720     while (size_wrote < bufsize) {
00721         k = start_k;
00722         new_buffer_write = 0;
00723         mem_list_count = 0;
00724         while ((mem_list_count < MAX_ARRAY_SIZE) && 
00725            (new_buffer_write < bufsize-size_wrote)) {
00726             /* find mem_list_count and file_list_count such that both are
00727            less than MAX_ARRAY_SIZE, the sum of their lengths are
00728            equal, and the sum of all the data written and data to be
00729            written in the next immediate write list is less than
00730            bufsize */
00731             if(mem_list_count) {
00732             if((new_buffer_write + flat_buf->blocklens[k] + 
00733             size_wrote) > bufsize) {
00734                 end_bwr_size = new_buffer_write + 
00735                 flat_buf->blocklens[k] - (bufsize - size_wrote);
00736             new_buffer_write = bufsize - size_wrote;
00737             }
00738             else {
00739                 new_buffer_write += flat_buf->blocklens[k];
00740             end_bwr_size = flat_buf->blocklens[k];
00741             }
00742         }
00743         else {
00744             if (bwr_size > (bufsize - size_wrote)) {
00745                 new_buffer_write = bufsize - size_wrote;
00746             bwr_size = new_buffer_write;
00747             }
00748             else new_buffer_write = bwr_size;
00749         }
00750         mem_list_count++;
00751         k = (k + 1)%flat_buf->count;
00752          } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
00753            (new_buffer_write < bufsize-size_wrote)) */
00754         j = start_j;
00755         new_file_write = 0;
00756         file_list_count = 0;
00757         while ((file_list_count < MAX_ARRAY_SIZE) && 
00758            (new_file_write < new_buffer_write)) {
00759             if(file_list_count) {
00760             if((new_file_write + flat_file->blocklens[j]) > 
00761                new_buffer_write) {
00762                 end_fwr_size = new_buffer_write - new_file_write;
00763             new_file_write = new_buffer_write;
00764             j--;
00765             }
00766             else {
00767                 new_file_write += flat_file->blocklens[j];
00768             end_fwr_size = flat_file->blocklens[j];
00769             }
00770         }
00771         else {
00772             if (fwr_size > new_buffer_write) {
00773                 new_file_write = new_buffer_write;
00774             fwr_size = new_file_write;
00775             }
00776             else new_file_write = fwr_size;
00777         }
00778         file_list_count++;
00779         if (j < (flat_file->count - 1)) j++;
00780         else j = 0;
00781         
00782         k = start_k;
00783         if ((new_file_write < new_buffer_write) && 
00784             (file_list_count == MAX_ARRAY_SIZE)) {
00785             new_buffer_write = 0;
00786             mem_list_count = 0;
00787             while (new_buffer_write < new_file_write) {
00788                 if(mem_list_count) {
00789                 if((new_buffer_write + flat_buf->blocklens[k]) >
00790                    new_file_write) {
00791                     end_bwr_size = new_file_write -
00792                   new_buffer_write;
00793                 new_buffer_write = new_file_write;
00794                 k--;
00795                 }
00796                 else {
00797                     new_buffer_write += flat_buf->blocklens[k];
00798                 end_bwr_size = flat_buf->blocklens[k];
00799                 }
00800             }
00801             else {
00802                 new_buffer_write = bwr_size;
00803                 if (bwr_size > (bufsize - size_wrote)) {
00804                     new_buffer_write = bufsize - size_wrote;
00805                 bwr_size = new_buffer_write;
00806                 }
00807             }
00808             mem_list_count++;
00809             k = (k + 1)%flat_buf->count;
00810             } /* while (new_buffer_write < new_file_write) */
00811         } /* if ((new_file_write < new_buffer_write) &&
00812              (file_list_count == MAX_ARRAY_SIZE)) */
00813         } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
00814          (new_buffer_write < bufsize-size_wrote)) */
00815 
00816         /*  fills the allocated writelist arrays  */
00817         k = start_k;
00818         j = start_j;
00819         for (i=0; i<mem_list_count; i++) {       
00820         /* TODO: fix this warning by casting to an integer that's the
00821          * same size as a char * and /then/ casting to PVFS_size */
00822             mem_offsets[i] = ((PVFS_size)buf + buftype_extent*
00823                   (buf_count/flat_buf->count) +
00824                   (int)flat_buf->indices[k]);
00825         
00826         if(!i) {
00827             mem_lengths[0] = bwr_size;
00828             mem_offsets[0] += flat_buf->blocklens[k] - bwr_size;
00829         }
00830         else {
00831             if (i == (mem_list_count - 1)) {
00832                 mem_lengths[i] = end_bwr_size;
00833             if (flat_buf->blocklens[k] == end_bwr_size)
00834                 bwr_size = flat_buf->blocklens[(k+1)%
00835                               flat_buf->count];
00836             else {
00837                 bwr_size = flat_buf->blocklens[k] - end_bwr_size;
00838                 k--;
00839                 buf_count--;
00840             }
00841             }
00842             else {
00843                 mem_lengths[i] = flat_buf->blocklens[k];
00844             }
00845         }
00846         buf_count++;
00847         k = (k + 1)%flat_buf->count;
00848         } /* for (i=0; i<mem_list_count; i++) */
00849         for (i=0; i<file_list_count; i++) {
00850             file_offsets[i] = disp + flat_file->indices[j] + 
00851             ((ADIO_Offset)n_filetypes) * filetype_extent;
00852             if (!i) {
00853             file_lengths[0] = fwr_size;
00854             file_offsets[0] += flat_file->blocklens[j] - fwr_size;
00855         }
00856         else {
00857             if (i == (file_list_count - 1)) {
00858                 file_lengths[i] = end_fwr_size;
00859             if (flat_file->blocklens[j] == end_fwr_size)
00860                 fwr_size = flat_file->blocklens[(j+1)%
00861                               flat_file->count];   
00862             else {
00863                 fwr_size = flat_file->blocklens[j] - end_fwr_size;
00864                 j--;
00865             }
00866             }
00867             else file_lengths[i] = flat_file->blocklens[j];
00868         }
00869         if (j < flat_file->count - 1) j++;
00870         else {
00871             j = 0;
00872             n_filetypes++;
00873         }
00874         } /* for (i=0; i<file_list_count; i++) */
00875 
00876         err_flag = PVFS_Request_hindexed(mem_list_count, mem_lengths, 
00877                          mem_offsets, PVFS_BYTE, &mem_req);
00878         /* --BEGIN ERROR HANDLING-- */
00879         if (err_flag != 0 ) {
00880         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00881                            MPIR_ERR_RECOVERABLE,
00882                            myname, __LINE__,
00883                            ADIOI_PVFS2_error_convert(err_flag),
00884                            "Error in PVFS_Request_hindexed (memory)", 0);
00885         goto error_state;
00886         }
00887         /* --END ERROR HANDLING-- */
00888 
00889         err_flag = PVFS_Request_hindexed(file_list_count, file_lengths, 
00890                          file_offsets, PVFS_BYTE,
00891                          &file_req);
00892         /* --BEGIN ERROR HANDLING-- */
00893         if (err_flag != 0) {
00894         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00895                            MPIR_ERR_RECOVERABLE,
00896                            myname, __LINE__,
00897                            ADIOI_PVFS2_error_convert(err_flag),
00898                            "Error in PVFS_Request_hindexed", 0);
00899         goto error_state;
00900         }
00901         /* --END ERROR HANDLING-- */
00902 
00903         /* offset will be expressed in memory and file datatypes */
00904 
00905 #ifdef ADIOI_MPE_LOGGING
00906             MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
00907 #endif
00908         err_flag = PVFS_sys_write(pvfs_fs->object_ref, file_req, 0, 
00909                       PVFS_BOTTOM, mem_req,
00910                       &(pvfs_fs->credentials), &resp_io);
00911 #ifdef ADIOI_MPE_LOGGING
00912             MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
00913 #endif
00914         /* --BEGIN ERROR HANDLING-- */
00915         if (err_flag != 0) {
00916         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00917                            MPIR_ERR_RECOVERABLE,
00918                            myname, __LINE__,
00919                            ADIOI_PVFS2_error_convert(err_flag),
00920                            "Error in PVFS_sys_write", 0);
00921         goto error_state;
00922         }
00923         /* --END ERROR HANDLING-- */
00924 
00925         size_wrote += new_buffer_write;
00926         total_bytes_written += resp_io.total_completed;
00927         start_k = k;
00928         start_j = j;
00929         PVFS_Request_free(&mem_req);
00930         PVFS_Request_free(&file_req);
00931     } /* while (size_wrote < bufsize) */
00932     ADIOI_Free(mem_offsets);
00933     ADIOI_Free(mem_lengths);
00934     }
00935     /* when incrementing fp_ind, need to also take into account the file type:
00936      * consider an N-element 1-d subarray with a lb and ub: ( |---xxxxx-----|
00937      * if we wrote N elements, offset needs to point at beginning of type, not
00938      * at empty region at offset N+1).  
00939      *
00940      * As we discussed on mpich-discuss in may/june 2009, the code below might
00941      * look wierd, but by putting fp_ind at the last byte written, the next
00942      * time we run through the strided code we'll update the fp_ind to the
00943      * right location. */
00944     if (file_ptr_type == ADIO_INDIVIDUAL) {
00945     fd->fp_ind = file_offsets[file_list_count-1]+
00946         file_lengths[file_list_count-1];
00947     }
00948     ADIOI_Free(file_offsets);
00949     ADIOI_Free(file_lengths);
00950 
00951     *error_code = MPI_SUCCESS;
00952 
00953 error_state:
00954     fd->fp_sys_posn = -1;   /* set it to null. */
00955 
00956 #ifdef HAVE_STATUS_SET_BYTES
00957     MPIR_Status_set_bytes(status, datatype, bufsize);
00958 /* This is a temporary way of filling in status. The right way is to 
00959    keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
00960 #endif
00961 
00962     if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
00963 }
libs/ck-libs/ampi/romio/adio/ad_pvfs2/ad_pvfs2_write_list_classic.c
