Charm++: Charm Source Code Documentation

00001 /* -*- Mode: C; c-basic-offset:4 ; -*- 
00002  * vim: ts=8 sts=4 sw=4 noexpandtab 
00003  * 
00004  *   Copyright (C) 2008 University of Chicago. 
00005  *   See COPYRIGHT notice in top-level directory.
00006  */
00007 
00008 #include "adio.h"
00009 #include "adio_extern.h"
00010 #include "ad_zoidfs.h"
00011 
00012 #include "ad_zoidfs_common.h"
00013 
00014 /* Copied from ADIOI_PVFS2_OldWriteStrided.  It would be good to have fewer
00015  * copies of this code... */
00016 void ADIOI_ZOIDFS_WriteStrided(ADIO_File fd, void *buf, int count,
00017             MPI_Datatype datatype, int file_ptr_type,
00018             ADIO_Offset offset, ADIO_Status *status,
00019             int *error_code)
00020 {
00021     /* as with all the other WriteStrided functions, offset is in units of
00022      * etype relative to the filetype */
00023 
00024     /* Since zoidfs does not support file locking, can't do buffered writes
00025        as on Unix */
00026 
00027     ADIOI_Flatlist_node *flat_buf, *flat_file;
00028     int i, j, k, bwr_size, fwr_size=0, st_index=0;
00029     int bufsize, sum, n_etypes_in_filetype, size_in_filetype;
00030     int n_filetypes, etype_in_filetype;
00031     ADIO_Offset abs_off_in_filetype=0;
00032     int filetype_size, etype_size, buftype_size;
00033     MPI_Aint filetype_extent, buftype_extent;
00034     int buf_count, buftype_is_contig, filetype_is_contig;
00035     ADIO_Offset off, disp, start_off, initial_off;
00036     int flag, st_fwr_size, st_n_filetypes;
00037     int err_flag=0;
00038 
00039     size_t mem_list_count, file_list_count;
00040     const void ** mem_offsets;
00041     uint64_t *file_offsets;
00042     size_t *mem_lengths;
00043     uint64_t *file_lengths;
00044     int total_blks_to_write;
00045 
00046     int max_mem_list, max_file_list;
00047 
00048     int b_blks_wrote;
00049     int f_data_wrote;
00050     int size_wrote=0, n_write_lists, extra_blks;
00051 
00052     int end_bwr_size, end_fwr_size;
00053     int start_k, start_j, new_file_write, new_buffer_write;
00054     int start_mem_offset;
00055     ADIOI_ZOIDFS_object *zoidfs_obj_ptr;
00056     MPI_Offset total_bytes_written=0;
00057     static char myname[] = "ADIOI_ZOIDFS_WRITESTRIDED";
00058 
00059     /* note: I don't know what zoidfs will do if you pass it a super-long list,
00060      * so let's keep with the PVFS limit for now */
00061 #define MAX_ARRAY_SIZE 64
00062 
00063     /* --BEGIN ERROR HANDLING-- */
00064     if (fd->atomicity) {
00065     *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00066                        MPIR_ERR_RECOVERABLE,
00067                        myname, __LINE__,
00068                        MPI_ERR_ARG,
00069                        "Atomic noncontiguous writes are not supported by ZOIDFS", 0);
00070     return;
00071     }
00072     /* --END ERROR HANDLING-- */
00073 
00074     ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
00075     ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
00076 
00077     /* the HDF5 tests showed a bug in this list processing code (see many many
00078      * lines down below).  We added a workaround, but common HDF5 file types
00079      * are actually contiguous and do not need the expensive workarond */
00080     if (!filetype_is_contig) {
00081     flat_file = ADIOI_Flatlist;
00082     while (flat_file->type != fd->filetype) flat_file = flat_file->next;
00083     if (flat_file->count == 1 && !buftype_is_contig)
00084         filetype_is_contig = 1;
00085     }
00086 
00087     MPI_Type_size(fd->filetype, &filetype_size);
00088     if ( ! filetype_size ) {
00089     *error_code = MPI_SUCCESS; 
00090     return;
00091     }
00092 
00093     MPI_Type_extent(fd->filetype, &filetype_extent);
00094     MPI_Type_size(datatype, &buftype_size);
00095     MPI_Type_extent(datatype, &buftype_extent);
00096     etype_size = fd->etype_size;
00097     
00098     bufsize = buftype_size * count;
00099 
00100     zoidfs_obj_ptr = (ADIOI_ZOIDFS_object*)fd->fs_ptr;
00101 
00102     if (!buftype_is_contig && filetype_is_contig) {
00103 
00104 /* noncontiguous in memory, contiguous in file.  */
00105         uint64_t file_offsets;
00106     uint64_t file_lengths;
00107 
00108     ADIOI_Flatten_datatype(datatype);
00109     flat_buf = ADIOI_Flatlist;
00110     while (flat_buf->type != datatype) flat_buf = flat_buf->next;
00111     
00112     if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
00113         off = fd->disp + etype_size * offset;
00114     }
00115     else off = fd->fp_ind;
00116 
00117     file_list_count = 1;
00118     file_offsets = off;
00119     file_lengths = 0;
00120     total_blks_to_write = count*flat_buf->count;
00121     b_blks_wrote = 0;
00122 
00123     /* allocate arrays according to max usage */
00124     if (total_blks_to_write > MAX_ARRAY_SIZE)
00125         mem_list_count = MAX_ARRAY_SIZE;
00126     else mem_list_count = total_blks_to_write;
00127     mem_offsets = (void*)ADIOI_Malloc(mem_list_count*sizeof(void*));
00128     mem_lengths = (size_t*)ADIOI_Malloc(mem_list_count*sizeof(size_t));
00129 
00130     j = 0;
00131     /* step through each block in memory, filling memory arrays */
00132     while (b_blks_wrote < total_blks_to_write) {
00133         for (i=0; i<flat_buf->count; i++) {
00134         mem_offsets[b_blks_wrote % MAX_ARRAY_SIZE] = 
00135             buf + 
00136              j*buftype_extent + 
00137              flat_buf->indices[i];
00138         mem_lengths[b_blks_wrote % MAX_ARRAY_SIZE] = 
00139             flat_buf->blocklens[i];
00140         file_lengths += flat_buf->blocklens[i];
00141         b_blks_wrote++;
00142         if (!(b_blks_wrote % MAX_ARRAY_SIZE) ||
00143             (b_blks_wrote == total_blks_to_write)) {
00144 
00145             /* in the case of the last write list call,
00146                adjust mem_list_count */
00147             if (b_blks_wrote == total_blks_to_write) {
00148                 mem_list_count = total_blks_to_write % MAX_ARRAY_SIZE;
00149             /* in case last write list call fills max arrays */
00150             if (!mem_list_count) mem_list_count = MAX_ARRAY_SIZE;
00151             }
00152 #ifdef ADIOI_MPE_LOGGING
00153                     MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
00154 #endif
00155             NO_STALE(err_flag, fd, zoidfs_obj_ptr,
00156                     zoidfs_write(zoidfs_obj_ptr, 
00157                         mem_list_count,
00158                         mem_offsets, mem_lengths, 
00159                         1, &file_offsets, &file_lengths, ZOIDFS_NO_OP_HINT));
00160 
00161             /* --BEGIN ERROR HANDLING-- */
00162             if (err_flag != ZFS_OK) {
00163             *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00164                                MPIR_ERR_RECOVERABLE,
00165                                myname, __LINE__,
00166                                ADIOI_ZOIDFS_error_convert(err_flag),
00167                                "Error in zoidfs_write", 0);
00168             break;
00169             }
00170 #ifdef ADIOI_MPE_LOGGING
00171                     MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
00172 #endif
00173             total_bytes_written += file_lengths;
00174           
00175             /* in the case of error or the last write list call, 
00176              * leave here */
00177             /* --BEGIN ERROR HANDLING-- */
00178             if (err_flag) {
00179             *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00180                                MPIR_ERR_RECOVERABLE,
00181                                myname, __LINE__,
00182                                ADIOI_ZOIDFS_error_convert(err_flag),
00183                                "Error in zoidfs_write", 0);
00184             break;
00185             }
00186             /* --END ERROR HANDLING-- */
00187             if (b_blks_wrote == total_blks_to_write) break;
00188 
00189             file_offsets += file_lengths;
00190             file_lengths = 0;
00191         } 
00192         } /* for (i=0; i<flat_buf->count; i++) */
00193         j++;
00194     } /* while (b_blks_wrote < total_blks_to_write) */
00195     ADIOI_Free(mem_offsets);
00196     ADIOI_Free(mem_lengths);
00197 
00198     if (file_ptr_type == ADIO_INDIVIDUAL) 
00199         fd->fp_ind += total_bytes_written;
00200 
00201     if (!err_flag)  *error_code = MPI_SUCCESS;
00202 
00203     fd->fp_sys_posn = -1;   /* clear this. */
00204 
00205 #ifdef HAVE_STATUS_SET_BYTES
00206     MPIR_Status_set_bytes(status, datatype, bufsize);
00207 /* This is a temporary way of filling in status. The right way is to 
00208    keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
00209 #endif
00210 
00211     ADIOI_Delete_flattened(datatype);
00212     return;
00213     } /* if (!buftype_is_contig && filetype_is_contig) */
00214 
00215     /* already know that file is noncontiguous from above */
00216     /* noncontiguous in file */
00217 
00218 /* filetype already flattened in ADIO_Open */
00219     flat_file = ADIOI_Flatlist;
00220     while (flat_file->type != fd->filetype) flat_file = flat_file->next;
00221 
00222     disp = fd->disp;
00223     initial_off = offset;
00224 
00225     /* for each case - ADIO_Individual pointer or explicit, find offset
00226        (file offset in bytes), n_filetypes (how many filetypes into file 
00227        to start), fwr_size (remaining amount of data in present file
00228        block), and st_index (start point in terms of blocks in starting
00229        filetype) */
00230     if (file_ptr_type == ADIO_INDIVIDUAL) {
00231         offset = fd->fp_ind; /* in bytes */
00232     n_filetypes = -1;
00233     flag = 0;
00234     while (!flag) {
00235         n_filetypes++;
00236         for (i=0; i<flat_file->count; i++) {
00237             if (disp + flat_file->indices[i] + 
00238             ((ADIO_Offset) n_filetypes)*filetype_extent +
00239               flat_file->blocklens[i] >= offset) {
00240           st_index = i;
00241           fwr_size = disp + flat_file->indices[i] + 
00242             ((ADIO_Offset) n_filetypes)*filetype_extent
00243             + flat_file->blocklens[i] - offset;
00244           flag = 1;
00245           break;
00246         }
00247         }
00248     } /* while (!flag) */
00249     } /* if (file_ptr_type == ADIO_INDIVIDUAL) */
00250     else {
00251         n_etypes_in_filetype = filetype_size/etype_size;
00252     n_filetypes = (int) (offset / n_etypes_in_filetype);
00253     etype_in_filetype = (int) (offset % n_etypes_in_filetype);
00254     size_in_filetype = etype_in_filetype * etype_size;
00255     
00256     sum = 0;
00257     for (i=0; i<flat_file->count; i++) {
00258         sum += flat_file->blocklens[i];
00259         if (sum > size_in_filetype) {
00260             st_index = i;
00261         fwr_size = sum - size_in_filetype;
00262         abs_off_in_filetype = flat_file->indices[i] +
00263             size_in_filetype - (sum - flat_file->blocklens[i]);
00264         break;
00265         }
00266     }
00267 
00268     /* abs. offset in bytes in the file */
00269     offset = disp + ((ADIO_Offset) n_filetypes)*filetype_extent +
00270         abs_off_in_filetype;
00271     } /* else [file_ptr_type != ADIO_INDIVIDUAL] */
00272 
00273     start_off = offset;
00274     st_fwr_size = fwr_size;
00275     st_n_filetypes = n_filetypes;
00276     
00277     if (buftype_is_contig && !filetype_is_contig) {
00278 
00279 /* contiguous in memory, noncontiguous in file. should be the most
00280    common case. */
00281 
00282     /* only one memory off-len pair, so no array */
00283         size_t mem_lengths;
00284     size_t mem_offsets;
00285         
00286     i = 0;
00287     j = st_index;
00288     off = offset;
00289     n_filetypes = st_n_filetypes;
00290         
00291     mem_list_count = 1;
00292         
00293     /* determine how many blocks in file to write */
00294     f_data_wrote = ADIOI_MIN(st_fwr_size, bufsize);
00295     total_blks_to_write = 1;
00296     if (j < (flat_file->count -1)) j++;
00297     else {
00298         j = 0;
00299         n_filetypes++;
00300     }
00301     while (f_data_wrote < bufsize) {
00302         f_data_wrote += flat_file->blocklens[j];
00303         total_blks_to_write++;
00304         if (j<(flat_file->count-1)) j++;
00305         else j = 0; 
00306     }
00307         
00308     j = st_index;
00309     n_filetypes = st_n_filetypes;
00310     n_write_lists = total_blks_to_write/MAX_ARRAY_SIZE;
00311     extra_blks = total_blks_to_write%MAX_ARRAY_SIZE;
00312         
00313     mem_offsets = (size_t)buf;
00314     mem_lengths = 0;
00315         
00316     /* if at least one full writelist, allocate file arrays
00317        at max array size and don't free until very end */
00318     if (n_write_lists) {
00319         file_offsets = (int64_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
00320                           sizeof(int64_t));
00321         file_lengths = (uint64_t*)ADIOI_Malloc(MAX_ARRAY_SIZE*
00322                           sizeof(uint64_t));
00323     }
00324     /* if there's no full writelist allocate file arrays according
00325        to needed size (extra_blks) */
00326     else {
00327         file_offsets = (int64_t*)ADIOI_Malloc(extra_blks*
00328                                                   sizeof(int64_t));
00329             file_lengths = (uint64_t*)ADIOI_Malloc(extra_blks*
00330                                                   sizeof(uint64_t));
00331         }
00332         
00333         /* for file arrays that are of MAX_ARRAY_SIZE, build arrays */
00334         for (i=0; i<n_write_lists; i++) {
00335             file_list_count = MAX_ARRAY_SIZE;
00336             if(!i) {
00337                 file_offsets[0] = offset;
00338                 file_lengths[0] = st_fwr_size;
00339                 mem_lengths = st_fwr_size;
00340             }
00341             for (k=0; k<MAX_ARRAY_SIZE; k++) {
00342                 if (i || k) {
00343                     file_offsets[k] = disp + 
00344             ((ADIO_Offset)n_filetypes)*filetype_extent
00345             + flat_file->indices[j];
00346                     file_lengths[k] = flat_file->blocklens[j];
00347                     mem_lengths += file_lengths[k];
00348                 }
00349                 if (j<(flat_file->count - 1)) j++;
00350                 else {
00351                     j = 0;
00352                     n_filetypes++;
00353                 }
00354             } /* for (k=0; k<MAX_ARRAY_SIZE; k++) */
00355 #ifdef ADIOI_MPE_LOGGING
00356             MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
00357 #endif
00358         NO_STALE(err_flag, fd, zoidfs_obj_ptr,
00359                 zoidfs_write(zoidfs_obj_ptr,
00360                     1, buf, &mem_lengths,
00361                     file_list_count, 
00362                     file_offsets, file_lengths, ZOIDFS_NO_OP_HINT));
00363 
00364 #ifdef ADIOI_MPE_LOGGING
00365             MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
00366 #endif
00367         /* --BEGIN ERROR HANDLING-- */
00368         if (err_flag != ZFS_OK) {
00369         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00370                            MPIR_ERR_RECOVERABLE,
00371                            myname, __LINE__,
00372                            ADIOI_ZOIDFS_error_convert(err_flag),
00373                            "Error in zoidfs_write", 0);
00374         goto error_state;
00375         }
00376         /* --END ERROR HANDLING-- */
00377         total_bytes_written += mem_lengths;
00378 
00379             mem_offsets += mem_lengths;
00380             mem_lengths = 0;
00381 
00382         } /* for (i=0; i<n_write_lists; i++) */
00383 
00384         /* for file arrays smaller than MAX_ARRAY_SIZE (last write_list call) */
00385         if (extra_blks) {
00386             file_list_count = extra_blks;
00387             if(!i) {
00388                 file_offsets[0] = offset;
00389                 file_lengths[0] = ADIOI_MIN(st_fwr_size, bufsize);
00390             }
00391             for (k=0; k<extra_blks; k++) {
00392                 if(i || k) {
00393                     file_offsets[k] = disp + 
00394             ((ADIO_Offset)n_filetypes)*filetype_extent +
00395             flat_file->indices[j];
00396             /* XXX: double-check these casts  */
00397                     if (k == (extra_blks - 1)) {
00398                         file_lengths[k] = bufsize 
00399                 - mem_lengths - mem_offsets +  (size_t)buf;
00400                     }
00401                     else file_lengths[k] = flat_file->blocklens[j];
00402                 } /* if(i || k) */
00403                 mem_lengths += file_lengths[k];
00404                 if (j<(flat_file->count - 1)) j++;
00405                 else {
00406                     j = 0;
00407                     n_filetypes++;
00408                 }
00409             } /* for (k=0; k<extra_blks; k++) */
00410 
00411 #ifdef ADIOI_MPE_LOGGING
00412             MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
00413 #endif
00414         NO_STALE(err_flag, fd, zoidfs_obj_ptr, 
00415                 zoidfs_write(zoidfs_obj_ptr, 1, 
00416                     (const void **)&mem_offsets, 
00417                     &mem_lengths,
00418                     file_list_count, 
00419                     file_offsets, file_lengths, ZOIDFS_NO_OP_HINT));
00420 
00421 #ifdef ADIOI_MPE_LOGGING
00422             MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
00423 #endif
00424         /* --BEGIN ERROR HANDLING-- */
00425         if (err_flag != 0) {
00426         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00427                            MPIR_ERR_RECOVERABLE,
00428                            myname, __LINE__,
00429                            ADIOI_ZOIDFS_error_convert(err_flag),
00430                            "Error in zoidfs_write", 0);
00431         goto error_state;
00432         }
00433         /* --END ERROR HANDLING-- */
00434         total_bytes_written += mem_lengths;
00435         }
00436     } 
00437     else {
00438         /* noncontiguous in memory as well as in file */
00439 
00440         ADIOI_Flatten_datatype(datatype);
00441     flat_buf = ADIOI_Flatlist;
00442     while (flat_buf->type != datatype) flat_buf = flat_buf->next;
00443 
00444     size_wrote = 0;
00445     n_filetypes = st_n_filetypes;
00446     fwr_size = st_fwr_size;
00447     bwr_size = flat_buf->blocklens[0];
00448     buf_count = 0;
00449     start_mem_offset = 0;
00450     start_k = k = 0;
00451     start_j = st_index;
00452     max_mem_list = 0;
00453     max_file_list = 0;
00454 
00455     /* run through and file max_file_list and max_mem_list so that you 
00456        can allocate the file and memory arrays less than MAX_ARRAY_SIZE
00457        if possible */
00458 
00459     while (size_wrote < bufsize) {
00460         k = start_k;
00461         new_buffer_write = 0;
00462         mem_list_count = 0;
00463         while ((mem_list_count < MAX_ARRAY_SIZE) && 
00464            (new_buffer_write < bufsize-size_wrote)) {
00465             /* find mem_list_count and file_list_count such that both are
00466            less than MAX_ARRAY_SIZE, the sum of their lengths are
00467            equal, and the sum of all the data written and data to be
00468            written in the next immediate write list is less than
00469            bufsize */
00470             if(mem_list_count) {
00471             if((new_buffer_write + flat_buf->blocklens[k] + 
00472             size_wrote) > bufsize) {
00473                 end_bwr_size = new_buffer_write + 
00474                 flat_buf->blocklens[k] - (bufsize - size_wrote);
00475             new_buffer_write = bufsize - size_wrote;
00476             }
00477             else {
00478                 new_buffer_write += flat_buf->blocklens[k];
00479             end_bwr_size = flat_buf->blocklens[k];
00480             }
00481         }
00482         else {
00483             if (bwr_size > (bufsize - size_wrote)) {
00484                 new_buffer_write = bufsize - size_wrote;
00485             bwr_size = new_buffer_write;
00486             }
00487             else new_buffer_write = bwr_size;
00488         }
00489         mem_list_count++;
00490         k = (k + 1)%flat_buf->count;
00491          } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
00492            (new_buffer_write < bufsize-size_wrote)) */
00493         j = start_j;
00494         new_file_write = 0;
00495         file_list_count = 0;
00496         while ((file_list_count < MAX_ARRAY_SIZE) && 
00497            (new_file_write < new_buffer_write)) { 
00498             if(file_list_count) {
00499             if((new_file_write + flat_file->blocklens[j]) > 
00500                new_buffer_write) {
00501                 end_fwr_size = new_buffer_write - new_file_write;
00502             new_file_write = new_buffer_write;
00503             j--;
00504             }
00505             else {
00506                 new_file_write += flat_file->blocklens[j];
00507             end_fwr_size = flat_file->blocklens[j];
00508             }
00509         }
00510         else {
00511             if (fwr_size > new_buffer_write) {
00512                 new_file_write = new_buffer_write;
00513             fwr_size = new_file_write;
00514             }
00515             else new_file_write = fwr_size;
00516         }
00517         file_list_count++;
00518         if (j < (flat_file->count - 1)) j++;
00519         else j = 0;
00520         
00521         k = start_k;
00522         if ((new_file_write < new_buffer_write) && 
00523             (file_list_count == MAX_ARRAY_SIZE)) {
00524             new_buffer_write = 0;
00525             mem_list_count = 0;
00526             while (new_buffer_write < new_file_write) {
00527                 if(mem_list_count) {
00528                 if((new_buffer_write + flat_buf->blocklens[k]) >
00529                    new_file_write) {
00530                     end_bwr_size = new_file_write - 
00531                     new_buffer_write;
00532                 new_buffer_write = new_file_write;
00533                 k--;
00534                 }
00535                 else {
00536                     new_buffer_write += flat_buf->blocklens[k];
00537                 end_bwr_size = flat_buf->blocklens[k];
00538                 }
00539             }
00540             else {
00541                 new_buffer_write = bwr_size;
00542                 if (bwr_size > (bufsize - size_wrote)) {
00543                     new_buffer_write = bufsize - size_wrote;
00544                 bwr_size = new_buffer_write;
00545                 }
00546             }
00547             mem_list_count++;
00548             k = (k + 1)%flat_buf->count;
00549             } /* while (new_buffer_write < new_file_write) */
00550         } /* if ((new_file_write < new_buffer_write) &&
00551              (file_list_count == MAX_ARRAY_SIZE)) */
00552         } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
00553          (new_buffer_write < bufsize-size_wrote)) */
00554 
00555         /*  fakes filling the writelist arrays of lengths found above  */
00556         k = start_k;
00557         j = start_j;
00558         for (i=0; i<mem_list_count; i++) {       
00559         if(i) {
00560             if (i == (mem_list_count - 1)) {
00561             if (flat_buf->blocklens[k] == end_bwr_size)
00562                 bwr_size = flat_buf->blocklens[(k+1)%
00563                               flat_buf->count];
00564             else {
00565                 bwr_size = flat_buf->blocklens[k] - end_bwr_size;
00566                 k--;
00567                 buf_count--;
00568             }
00569             }
00570         }
00571         buf_count++;
00572         k = (k + 1)%flat_buf->count;
00573         } /* for (i=0; i<mem_list_count; i++) */
00574         for (i=0; i<file_list_count; i++) {
00575         if (i) {
00576             if (i == (file_list_count - 1)) {
00577             if (flat_file->blocklens[j] == end_fwr_size)
00578                 fwr_size = flat_file->blocklens[(j+1)%
00579                               flat_file->count];   
00580             else {
00581                 fwr_size = flat_file->blocklens[j] - end_fwr_size;
00582                 j--;
00583             }
00584             }
00585         }
00586         if (j < flat_file->count - 1) j++;
00587         else {
00588             j = 0;
00589             n_filetypes++;
00590         }
00591         } /* for (i=0; i<file_list_count; i++) */
00592         size_wrote += new_buffer_write;
00593         start_k = k;
00594         start_j = j;
00595         if (max_mem_list < mem_list_count)
00596             max_mem_list = mem_list_count;
00597         if (max_file_list < file_list_count)
00598             max_file_list = file_list_count;
00599     } /* while (size_wrote < bufsize) */
00600 
00601     /* one last check before we actually carry out the operation:
00602      * this code has hard-to-fix bugs when a noncontiguous file type has
00603      * such large pieces that the sum of the lengths of the memory type is
00604      * not larger than one of those pieces (and vice versa for large memory
00605      * types and many pices of file types.  In these cases, give up and
00606      * fall back to naive reads and writes.  The testphdf5 test created a
00607      * type with two very large memory regions and 600 very small file
00608      * regions.  The same test also created a type with one very large file
00609      * region and many (700) very small memory regions.  both cases caused
00610      * problems for this code */
00611 
00612     if ( ( (file_list_count == 1) && 
00613             (new_file_write < flat_file->blocklens[0] ) ) ||
00614         ((mem_list_count == 1) && 
00615             (new_buffer_write < flat_buf->blocklens[0]) ) ||
00616         ((file_list_count == MAX_ARRAY_SIZE) && 
00617             (new_file_write < flat_buf->blocklens[0]) ) ||
00618         ( (mem_list_count == MAX_ARRAY_SIZE) &&
00619             (new_buffer_write < flat_file->blocklens[0])) )
00620     {
00621         ADIOI_Delete_flattened(datatype);
00622         ADIOI_GEN_WriteStrided_naive(fd, buf, count, datatype,
00623             file_ptr_type, initial_off, status, error_code);
00624         return;
00625     }
00626 
00627 
00628     mem_offsets = (void *)ADIOI_Malloc(max_mem_list*sizeof(void *));
00629     mem_lengths = (size_t*)ADIOI_Malloc(max_mem_list*sizeof(size_t));
00630     file_offsets = (uint64_t *)ADIOI_Malloc(max_file_list*sizeof(uint64_t));
00631     file_lengths = (uint64_t*)ADIOI_Malloc(max_file_list*sizeof(uint64_t));
00632         
00633     size_wrote = 0;
00634     n_filetypes = st_n_filetypes;
00635     fwr_size = st_fwr_size;
00636     bwr_size = flat_buf->blocklens[0];
00637     buf_count = 0;
00638     start_mem_offset = 0;
00639     start_k = k = 0;
00640     start_j = st_index;
00641 
00642     /*  this section calculates mem_list_count and file_list_count
00643         and also finds the possibly odd sized last array elements
00644         in new_fwr_size and new_bwr_size  */
00645     
00646     while (size_wrote < bufsize) {
00647         k = start_k;
00648         new_buffer_write = 0;
00649         mem_list_count = 0;
00650         while ((mem_list_count < MAX_ARRAY_SIZE) && 
00651            (new_buffer_write < bufsize-size_wrote)) {
00652             /* find mem_list_count and file_list_count such that both are
00653            less than MAX_ARRAY_SIZE, the sum of their lengths are
00654            equal, and the sum of all the data written and data to be
00655            written in the next immediate write list is less than
00656            bufsize */
00657             if(mem_list_count) {
00658             if((new_buffer_write + flat_buf->blocklens[k] + 
00659             size_wrote) > bufsize) {
00660                 end_bwr_size = new_buffer_write + 
00661                 flat_buf->blocklens[k] - (bufsize - size_wrote);
00662             new_buffer_write = bufsize - size_wrote;
00663             }
00664             else {
00665                 new_buffer_write += flat_buf->blocklens[k];
00666             end_bwr_size = flat_buf->blocklens[k];
00667             }
00668         }
00669         else {
00670             if (bwr_size > (bufsize - size_wrote)) {
00671                 new_buffer_write = bufsize - size_wrote;
00672             bwr_size = new_buffer_write;
00673             }
00674             else new_buffer_write = bwr_size;
00675         }
00676         mem_list_count++;
00677         k = (k + 1)%flat_buf->count;
00678          } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
00679            (new_buffer_write < bufsize-size_wrote)) */
00680         j = start_j;
00681         new_file_write = 0;
00682         file_list_count = 0;
00683         while ((file_list_count < MAX_ARRAY_SIZE) && 
00684            (new_file_write < new_buffer_write)) {
00685             if(file_list_count) {
00686             if((new_file_write + flat_file->blocklens[j]) > 
00687                new_buffer_write) {
00688                 end_fwr_size = new_buffer_write - new_file_write;
00689             new_file_write = new_buffer_write;
00690             j--;
00691             }
00692             else {
00693                 new_file_write += flat_file->blocklens[j];
00694             end_fwr_size = flat_file->blocklens[j];
00695             }
00696         }
00697         else {
00698             if (fwr_size > new_buffer_write) {
00699                 new_file_write = new_buffer_write;
00700             fwr_size = new_file_write;
00701             }
00702             else new_file_write = fwr_size;
00703         }
00704         file_list_count++;
00705         if (j < (flat_file->count - 1)) j++;
00706         else j = 0;
00707         
00708         k = start_k;
00709         if ((new_file_write < new_buffer_write) && 
00710             (file_list_count == MAX_ARRAY_SIZE)) {
00711             new_buffer_write = 0;
00712             mem_list_count = 0;
00713             while (new_buffer_write < new_file_write) {
00714                 if(mem_list_count) {
00715                 if((new_buffer_write + flat_buf->blocklens[k]) >
00716                    new_file_write) {
00717                     end_bwr_size = new_file_write -
00718                   new_buffer_write;
00719                 new_buffer_write = new_file_write;
00720                 k--;
00721                 }
00722                 else {
00723                     new_buffer_write += flat_buf->blocklens[k];
00724                 end_bwr_size = flat_buf->blocklens[k];
00725                 }
00726             }
00727             else {
00728                 new_buffer_write = bwr_size;
00729                 if (bwr_size > (bufsize - size_wrote)) {
00730                     new_buffer_write = bufsize - size_wrote;
00731                 bwr_size = new_buffer_write;
00732                 }
00733             }
00734             mem_list_count++;
00735             k = (k + 1)%flat_buf->count;
00736             } /* while (new_buffer_write < new_file_write) */
00737         } /* if ((new_file_write < new_buffer_write) &&
00738              (file_list_count == MAX_ARRAY_SIZE)) */
00739         } /* while ((mem_list_count < MAX_ARRAY_SIZE) && 
00740          (new_buffer_write < bufsize-size_wrote)) */
00741 
00742         /*  fills the allocated writelist arrays  */
00743         k = start_k;
00744         j = start_j;
00745         for (i=0; i<mem_list_count; i++) {       
00746             mem_offsets[i] = buf + 
00747             buftype_extent* (buf_count/flat_buf->count) +
00748                   flat_buf->indices[k];
00749         
00750         if(!i) {
00751             mem_lengths[0] = bwr_size;
00752             mem_offsets[0] += flat_buf->blocklens[k] - bwr_size;
00753         }
00754         else {
00755             if (i == (mem_list_count - 1)) {
00756                 mem_lengths[i] = end_bwr_size;
00757             if (flat_buf->blocklens[k] == end_bwr_size)
00758                 bwr_size = flat_buf->blocklens[(k+1)%
00759                               flat_buf->count];
00760             else {
00761                 bwr_size = flat_buf->blocklens[k] - end_bwr_size;
00762                 k--;
00763                 buf_count--;
00764             }
00765             }
00766             else {
00767                 mem_lengths[i] = flat_buf->blocklens[k];
00768             }
00769         }
00770         buf_count++;
00771         k = (k + 1)%flat_buf->count;
00772         } /* for (i=0; i<mem_list_count; i++) */
00773         for (i=0; i<file_list_count; i++) {
00774             file_offsets[i] = disp + flat_file->indices[j] + 
00775             ((ADIO_Offset)n_filetypes) * filetype_extent;
00776             if (!i) {
00777             file_lengths[0] = fwr_size;
00778             file_offsets[0] += flat_file->blocklens[j] - fwr_size;
00779         }
00780         else {
00781             if (i == (file_list_count - 1)) {
00782                 file_lengths[i] = end_fwr_size;
00783             if (flat_file->blocklens[j] == end_fwr_size)
00784                 fwr_size = flat_file->blocklens[(j+1)%
00785                               flat_file->count];   
00786             else {
00787                 fwr_size = flat_file->blocklens[j] - end_fwr_size;
00788                 j--;
00789             }
00790             }
00791             else file_lengths[i] = flat_file->blocklens[j];
00792         }
00793         if (j < flat_file->count - 1) j++;
00794         else {
00795             j = 0;
00796             n_filetypes++;
00797         }
00798         } /* for (i=0; i<file_list_count; i++) */
00799 
00800 #ifdef ADIOI_MPE_LOGGING
00801             MPE_Log_event( ADIOI_MPE_write_a, 0, NULL );
00802 #endif
00803         NO_STALE(err_flag, fd, zoidfs_obj_ptr,
00804                 zoidfs_write(zoidfs_obj_ptr, 
00805                     mem_list_count, mem_offsets, mem_lengths, 
00806                     file_list_count, 
00807                     file_offsets, file_lengths, ZOIDFS_NO_OP_HINT));
00808         /* --BEGIN ERROR HANDLING-- */
00809         if (err_flag != ZFS_OK) {
00810         *error_code = MPIO_Err_create_code(MPI_SUCCESS,
00811                            MPIR_ERR_RECOVERABLE,
00812                            myname, __LINE__,
00813                            ADIOI_ZOIDFS_error_convert(err_flag),
00814                            "Error in zoidfs_write", 0);
00815         goto error_state;
00816         }
00817         /* --END ERROR HANDLING-- */
00818 #ifdef ADIOI_MPE_LOGGING
00819             MPE_Log_event( ADIOI_MPE_write_b, 0, NULL );
00820 #endif
00821         size_wrote += new_buffer_write;
00822         total_bytes_written += new_buffer_write; /* XXX: is this right? */
00823         start_k = k;
00824         start_j = j;
00825     } /* while (size_wrote < bufsize) */
00826     ADIOI_Free(mem_offsets);
00827     ADIOI_Free(mem_lengths);
00828     }
00829     /* when incrementing fp_ind, need to also take into account the file type:
00830      * consider an N-element 1-d subarray with a lb and ub: ( |---xxxxx-----|
00831      * if we wrote N elements, offset needs to point at beginning of type, not
00832      * at empty region at offset N+1).  
00833      *
00834      * As we discussed on mpich-discuss in may/june 2009, the code below might
00835      * look wierd, but by putting fp_ind at the last byte written, the next
00836      * time we run through the strided code we'll update the fp_ind to the
00837      * right location. */
00838     if (file_ptr_type == ADIO_INDIVIDUAL) {
00839     fd->fp_ind = file_offsets[file_list_count-1]+
00840         file_lengths[file_list_count-1];
00841     }
00842     ADIOI_Free(file_offsets);
00843     ADIOI_Free(file_lengths);
00844 
00845     *error_code = MPI_SUCCESS;
00846 
00847 error_state:
00848     fd->fp_sys_posn = -1;   /* set it to null. */
00849 
00850 #ifdef HAVE_STATUS_SET_BYTES
00851     MPIR_Status_set_bytes(status, datatype, bufsize);
00852 /* This is a temporary way of filling in status. The right way is to 
00853    keep track of how much data was actually written by ADIOI_BUFFERED_WRITE. */
00854 #endif
00855 
00856     if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
00857 }
libs/ck-libs/ampi/romio/adio/ad_zoidfs/ad_zoidfs_write_list.c
