HTTP/1.1 -1 Read error in cache disk data: SuccessContent-Type: text/plain
Last-Modified: Fri, 06 Aug 2010 20:11:42 GMT
ETag: "6a196-5ac7-48d2d4683b990"
Content-length: 23239
Connection: Close
Proxy-Connection: Close
X-Cache: HIT from web1.osuosl.org
Server: ProxyTrack 0.5 (HTTrack 3.49.2)

Index: src/gc/alloc_resources.c
===================================================================
--- src/gc/alloc_resources.c	(revision 48320)
+++ src/gc/alloc_resources.c	(working copy)
@@ -281,19 +281,9 @@
 - reduce alignment to a reasonable value i.e. MALLOC_ALIGNMENT
   aka 2*sizeof (size_t) or just 8 (TODO make a config hint)
 
-Buffer memory layout:
+See pobj.h for a discussion of the Buffer descriptor and the buffer itself,
+including its header.
 
-                    +-------------------+
-                    | flags # GC header |
-  obj->bufstart  -> +-------------------+
-                    |  data             |
-                    v                   v
-
- * if PObj_is_COWable is set, then we have space for flags.
-
- * if PObj_align_FLAG is set, obj->bufstart is aligned like discussed above
- * obj->buflen is the usable length excluding the optional GC part.
-
 =cut
 
 */
@@ -612,7 +602,7 @@
     /* we can't perform the math all the time, because
         * strstart might be in unallocated memory */
     if (PObj_is_COWable_TEST(old_buf)) {
-        flags = Buffer_bufrefcountptr(old_buf);
+        flags = Buffer_bufflagsptr(old_buf);
         old_block = Buffer_pool(old_buf);
 
         if (PObj_is_string_TEST(old_buf)) {
@@ -631,7 +621,7 @@
 
         /* Make sure they know that we own it too */
         /* Set Buffer_shared_FLAG in new buffer */
-        *Buffer_bufrefcountptr(hdr) |= Buffer_shared_FLAG;
+        *Buffer_bufflagsptr(hdr) |= Buffer_shared_FLAG;
 
         /* Now make sure we point to where the other guy does */
         Buffer_bufstart(old_buf) = Buffer_bufstart(hdr);
Index: src/gc/gc_ms.c
===================================================================
--- src/gc/gc_ms.c	(revision 48320)
+++ src/gc/gc_ms.c	(working copy)
@@ -1256,7 +1256,7 @@
     PARROT_ASSERT(PObj_is_movable_TESTALL(str));
 
     /* We must not reallocate shared buffers! */
-    PARROT_ASSERT(!(*Buffer_bufrefcountptr(str) & Buffer_shared_FLAG));
+    PARROT_ASSERT(!(*Buffer_bufflagsptr(str) & Buffer_shared_FLAG));
 
     /* Decrease usage */
     PARROT_ASSERT(Buffer_pool(str));
Index: src/gc/gc_private.h
===================================================================
--- src/gc/gc_private.h	(revision 48320)
+++ src/gc/gc_private.h	(working copy)
@@ -77,6 +77,7 @@
    larger then sizeof(PObj), thus creating overflow. However PObjs are never
    used by themselves, things like PMCs and STRINGs are cast to PObj in the
    GC, so we should have plenty of space. */
+
 typedef struct GC_MS_PObj_Wrapper {
     size_t flags;
     struct GC_MS_PObj_Wrapper * next_ptr;
@@ -164,90 +165,120 @@
      */
 } GC_Subsystem;
 
+/* This header structure describes a block of memory that is part of a
+   variable-size pool. The allocatable memory follows the header. */
+
 typedef struct Memory_Block {
-    size_t free;
-    size_t size;
-    struct Memory_Block *prev;
-    struct Memory_Block *next;
-    char *start;
-    char *top;
-
-    /* Amount of freed memory. Used in compact_pool */
-    size_t freed;
+    size_t free;                        /* Remaining free space. */
+    size_t size;                        /* Size of memory. */
+    struct Memory_Block *prev;          /* Pointer to previous block. */
+    struct Memory_Block *next;          /* Pointer to next block. */
+    char *start;                        /* Pointer to start of memory. */
+    char *top;                          /* Pointer to free space in memory. */
+    size_t freed;                       /* Amount of freed memory.
+                                           Used in compact_pool */
 } Memory_Block;
 
+/* This structure describes a variable-size memory pool. Various such pools
+   hang off the Memory_Pools root structure. */
+
 typedef struct Variable_Size_Pool {
-    Memory_Block *top_block;
-    void (*compact)(PARROT_INTERP, struct Memory_Pools *, struct Variable_Size_Pool *);
-    size_t minimum_block_size;
-    size_t total_allocated; /* total bytes allocated to this pool */
-    size_t guaranteed_reclaimable;     /* bytes that can definitely be reclaimed*/
-    size_t possibly_reclaimable;     /* bytes that can possibly be reclaimed
-                                      * (above plus COW-freed bytes) */
-    FLOATVAL reclaim_factor; /* minimum percentage we will reclaim */
+    Memory_Block *top_block;            /* Pointer to most recent memory block. */
+                                        /* Pool compactor, or NULL. */
+    void (*compact)(PARROT_INTERP, struct Memory_Pools *,
+                                   struct Variable_Size_Pool *);
+    size_t minimum_block_size;          /* Minimum allocation size, to
+                                           prevent fragmentation. */
+    size_t total_allocated;             /* Total bytes allocated to this pool. */
+    size_t guaranteed_reclaimable;      /* Bytes that can definitely be reclaimed. */
+    size_t possibly_reclaimable;        /* Bytes that can possibly be reclaimed
+                                           (above plus COW-freed bytes). */
+    FLOATVAL reclaim_factor;            /* Minimum percentage we will reclaim. */
 } Variable_Size_Pool;
 
+/* This header structure describes an arena: a block of memory that is part of a
+   fixed-sized pool. The arena has enough memory for 'total_objects' objects
+   of a particular size specified in the pool. */
+
 typedef struct Fixed_Size_Arena {
-    size_t                     used;
-    size_t                     total_objects;
-    struct Fixed_Size_Arena *prev;
-    struct Fixed_Size_Arena *next;
-    void                      *start_objects;
+    size_t used;                        /* Number of objects slots used. */
+    size_t total_objects;               /* Total number of object slots. */
+    struct Fixed_Size_Arena *prev;      /* Pointer to previous arena. */
+    struct Fixed_Size_Arena *next;      /* Pointer to next arena. */
+    void *start_objects;                /* Pointer to arena memory. */
 } Fixed_Size_Arena;
 
+/* This simple header structure describes a free PMC attribute object.
+   A list of them hang off the PMC attribute pool. */
+
 typedef struct PMC_Attribute_Free_List {
-    struct PMC_Attribute_Free_List * next;
+    struct PMC_Attribute_Free_List * next; /* Pointer to next free object. */
 } PMC_Attribute_Free_List;
 
+/* This header structure describes a PMC attribute arena: A block of memory
+   that is part of a PMC attribute pool. The allocatable memory follows
+   this header. */
+
 typedef struct PMC_Attribute_Arena {
-    struct PMC_Attribute_Arena * next;
-    struct PMC_Attribute_Arena * prev;
+    struct PMC_Attribute_Arena * next;  /* Pointer to next arena. */
+    struct PMC_Attribute_Arena * prev;  /* Pointer to previous arena. */
 } PMC_Attribute_Arena;
 
+/* This structure describes a PMC attribute pool. A vector of them hang
+   off the Memory_Pools root structure. */
+
 typedef struct PMC_Attribute_Pool {
-    size_t attr_size;
-    size_t total_objects;
-    size_t objects_per_alloc;
-    size_t num_free_objects;
-    PMC_Attribute_Free_List * free_list;
-    PMC_Attribute_Arena     * top_arena;
+    size_t attr_size;                   /* Size of attribute object. */
+    size_t total_objects;               /* Total objects in the pool. */
+    size_t objects_per_alloc;           /* Number of object slots to allocate. */
+    size_t num_free_objects;            /* Number of objects on the free list. */
+    PMC_Attribute_Free_List * free_list; /* List of free object slots, or NULL. */
+    PMC_Attribute_Arena     * top_arena; /* Pointer to most recent arena. */
+
 #if GC_USE_LAZY_ALLOCATOR
-    PMC_Attribute_Free_List * newfree;
-    PMC_Attribute_Free_List * newlast;
+                                        /* Using lazy allocator: */
+    PMC_Attribute_Free_List * newfree;  /* Pointer to next object slot in
+                                           latest arena, or NULL (slots weren't
+                                           put on free list). */
+    PMC_Attribute_Free_List * newlast;  /* High water mark in arena. */
 #endif
 } PMC_Attribute_Pool;
 
-/* Tracked resource pool */
+/* This structure describes a fixed-size memory pool. Various such pools
+   hang off the Memory_Pools root structure. */
+
 typedef struct Fixed_Size_Pool {
 
-    struct Variable_Size_Pool *mem_pool;
-   /* Size in bytes of an individual pool item. This size may include
-    * a GC-system specific GC header. */
-    size_t object_size;
+    struct Variable_Size_Pool *mem_pool; /* Pointer to associated variable-size
+                                            pool, or NULL. */
+    size_t object_size;                 /* Size in bytes of an individual pool 
+                                           object. This size may include
+                                           a GC system-specific GC header. */
 
-    size_t start_arena_memory;
-    size_t end_arena_memory;
+    size_t start_arena_memory;          /* Address of the lowest arena. */
+    size_t end_arena_memory;            /* And the highest one. */
 
-    Fixed_Size_Arena *last_Arena;
-    GC_MS_PObj_Wrapper * free_list;
-    size_t num_free_objects;    /* number of resources in the free pool */
-    size_t total_objects;
+    Fixed_Size_Arena *last_Arena;       /* Pointer to most recent arena. */
+    GC_MS_PObj_Wrapper * free_list;     /* List of free object slots, or NULL. */
+    size_t num_free_objects;            /* Number of objects on the free list. */
+    size_t total_objects;               /* Total objects in the pool. */
 
-    PARROT_OBSERVER const char *name;
+    PARROT_OBSERVER const char *name;   /* Name of pool. */
 
-    size_t objects_per_alloc;
+    size_t objects_per_alloc;           /* Number of object slots to allocate. */
 
-    int skip;
-    size_t replenish_level;
+    int skip;                           /* How often to skip full GC
+                                           (see gc_skip_type_enum). */
+    size_t replenish_level;             /* Replenish pool when free object slots
+                                           goes below this level. */
 
-    add_free_object_fn_type     add_free_object; /* adds a free object to
-                                                    the pool's free list  */
-    get_free_object_fn_type     get_free_object; /* gets and removes a free
-                                                    object from the pool's
-                                                    free list */
-    alloc_objects_fn_type       alloc_objects;  /* allocates more objects */
-    alloc_objects_fn_type       more_objects;
-    gc_object_fn_type           gc_object;
+    add_free_object_fn_type add_free_object; /* Adds a free object to
+                                                the pool's free list  */
+    get_free_object_fn_type get_free_object; /* Gets and removes a free object
+                                                from the pool's free list. */
+    alloc_objects_fn_type   alloc_objects;   /* Allocates an arena for objects. */
+    alloc_objects_fn_type   more_objects;    /* Obtain more free objects. */
+    gc_object_fn_type       gc_object;       /* GCs object during sweep. */
 
     /* Contains GC system-specific data structures ... unused at the moment,
      * but this is where it should go when we need it ...
@@ -256,26 +287,39 @@
     */
 
 #if GC_USE_LAZY_ALLOCATOR
-    void *newfree;
-    void *newlast;
+                                        /* Using lazy allocator: */
+    void *newfree;                      /* Pointer to next object slot in
+                                           latest arena, or NULL (slots weren't
+                                           put on free list). */
+    void *newlast;                      /* High water mark in arena. */
 #endif
 
 } Fixed_Size_Pool;
 
+/* This structure acts as the root for all the various memory pools:
+   variable-sized, fixed-size, and PMC attributes. It also contains
+   various GC-related items. It hangs off the Interp structure. */
+
 typedef struct Memory_Pools {
-    Variable_Size_Pool  *memory_pool;
-    Variable_Size_Pool  *constant_string_pool;
-    Fixed_Size_Pool     *string_header_pool;
-    Fixed_Size_Pool     *pmc_pool;
-    Fixed_Size_Pool     *constant_pmc_pool;
-    Fixed_Size_Pool     *constant_string_header_pool;
-    Fixed_Size_Pool    **sized_header_pools;
-    size_t               num_sized;
+    /* Pointers to pools */
+    Variable_Size_Pool  *memory_pool;           /* General memory pool. */
+    Variable_Size_Pool  *constant_string_pool;  /* Constant string pool (not
+                                                   compacted). */
+    Fixed_Size_Pool     *string_header_pool;    /* String header pool. */
+    Fixed_Size_Pool     *pmc_pool;              /* PMC object pool. */
+    Fixed_Size_Pool     *constant_pmc_pool;     /* And one for constant PMCs. */
+    Fixed_Size_Pool     *constant_string_header_pool; /* And a constant string
+                                                         header pool. */
 
-    PMC_Attribute_Pool **attrib_pools;
-    size_t               num_attribs;
+    Fixed_Size_Pool    **sized_header_pools;    /* Vector of pools for other
+                                                  fixed-size headers. */
+    size_t               num_sized;             /* Length of that vector. */
 
-    /** statistics for GC **/
+    PMC_Attribute_Pool **attrib_pools;          /* Vector of pools for PMC
+                                                   attributes. */
+    size_t               num_attribs;           /* Length of that vector. */
+
+    /* statistics for GC */
     size_t  gc_mark_runs;       /* Number of times we've done a mark run */
     size_t  gc_lazy_mark_runs;  /* Number of successful lazy mark runs */
     size_t  gc_collect_runs;    /* Number of times we've done a memory
@@ -288,11 +332,9 @@
                                                  * blocks allocated from
                                                  * the system since the last
                                                  * GC run */
-    size_t  memory_allocated;     /* The total amount of
-                                   * allocatable memory
-                                   * allocated. Doesn't count
-                                   * memory for headers or
-                                   * internal structures or
+    size_t  memory_allocated;     /* The total amount of allocatable memory
+                                   * allocated. Doesn't count memory for
+                                   * headers or internal structures or
                                    * anything */
     UINTVAL memory_collected;     /* Total amount of memory copied
                                      during collection */
@@ -311,7 +353,7 @@
                                      requests are there? */
 
     /* private data for the GC subsystem */
-    void *gc_private;           /* gc subsystem data */
+    void *gc_private;             /* GC subsystem data */
 } Memory_Pools;
 
 /* HEADERIZER BEGIN: src/gc/system.c */
Index: src/gc/mark_sweep.c
===================================================================
--- src/gc/mark_sweep.c	(revision 48320)
+++ src/gc/mark_sweep.c	(working copy)
@@ -681,7 +681,7 @@
     if (mem_pool) {
         /* Update Memory_Block usage */
         if (PObj_is_movable_TESTALL(b)) {
-            INTVAL *buffer_flags = Buffer_bufrefcountptr(b);
+            INTVAL *buffer_flags = Buffer_bufflagsptr(b);
 
             /* Mask low 2 bits used for flags */
             Memory_Block * block = Buffer_pool(b);
Index: src/string/api.c
===================================================================
--- src/string/api.c	(revision 48320)
+++ src/string/api.c	(working copy)
@@ -395,7 +395,7 @@
     /* Now check that buffer allocated from pool and affected by compacting */
     if (is_movable && Buffer_bufstart(s)) {
         /* If so, mark it as shared */
-        INTVAL * const buffer_flags = Buffer_bufrefcountptr(d);
+        INTVAL * const buffer_flags = Buffer_bufflagsptr(d);
         *buffer_flags |= Buffer_shared_FLAG;
     }
 
Index: CREDITS
===================================================================
--- CREDITS	(revision 48320)
+++ CREDITS	(working copy)
@@ -810,6 +810,11 @@
 D: Pynie
 E: pmichaud@pobox.com
 
+N: Paul C. Anagnostopoulos
+U: Paul C. Anagnostopoulos
+E: paul@windfall.com
+D: Improving memory management documentation
+
 N: Paul Cochrane
 U: paultcochrane
 D: Changing the newspaper in the parrot cage
Index: include/parrot/pobj.h
===================================================================
--- include/parrot/pobj.h	(revision 48320)
+++ include/parrot/pobj.h	(working copy)
@@ -7,7 +7,7 @@
  *  Data Structure and Algorithms:
  *  History:
  *  Notes:
- *  References: memory_internals.pod
+ *  References: memory_internals.pod (out of date as of 8/2010).
  */
 
 #ifndef PARROT_POBJ_H_GUARD
@@ -15,53 +15,67 @@
 
 #include "parrot/config.h"
 
-/* Parrot Object - base class for all others */
+/* This is the base Parrot object structure. Every object begins with
+   this slot, then has additional slots as required. */
+
 typedef struct pobj_t {
-    Parrot_UInt flags;
+    Parrot_UInt flags;                  /* Lots of flags (see below). */
 } PObj;
 
+/* This is a buffer header object, "inheriting" from PObj. */
+
 typedef struct buffer_t {
     Parrot_UInt flags;
-    void *     _bufstart;
-    size_t     _buflen;
+    void *     _bufstart;               /* Pointer to start of buffer data
+                                           (not buffer prolog). */
+    size_t     _buflen;                 /* Length of buffer data. */
 } Buffer;
 
+/* Use these macros to access the two buffer header slots. */
+
 #define Buffer_bufstart(buffer)    (buffer)->_bufstart
 #define Buffer_buflen(buffer)      (buffer)->_buflen
 
-/* See src/gc/alloc_resources.c. the basic idea is that buffer memory is
-   set up as follows:
-                    +-----------------+
-                    |  ref_count   |f |    # GC header
-  obj->bufstart  -> +-----------------+
-                    |  data           |
-                    v                 v
+/* A buffer header object points to a buffer in a Memory_Block.
+   The buffer includes a prolog, but _bufstart points to the data
+   portion. Here is how it works:
 
-The actual set-up is more involved because of padding.  obj->bufstart must
-be suitably aligned. The start of the memory region (as returned by malloc())
-is suitably aligned for any use.  If, for example, malloc() returns
-objects aligned on 8-byte boundaries, and obj->bufstart is also aligned
-on 8-byte boundaries, then there should be 4 bytes of padding.
+    Buffer header                         buffer
+   +-------------------+                 +------------------------+
+   |       flags       |                 |  (possible padding)    | }
+   +-------------------+                 +---------------------+--+  > prolog
+   |      _bufstart    | ------+         |    *Memory_Block    |fl| }
+   +-------------------+       |         +---------------------+--+
+   |      _buflen      |       +-------> |    data portion        |
+   +-------------------+                 |                        |
+                                         ~                        ~
+                                         |                        |
+                                         +------------------------+
 
-ptr from malloc ->  +------------------+
-                      [other blocks?]  |
-                    | INTVAL ref_count |
-obj->bufstart   ->  +------------------+
-                    |     data         |
-                    v                  v
-
+   The buffer prolog consists of possible padding and a pointer to the
+   Memory_Block containing the buffer. There are two flags in the low-order
+   bits of the pointer (see string.h). Padding is only required if the
+   alignment of the data portion is higher than that of a pointer.
+   This was not the case as of 8/2010.
 */
 
-/* Given a pointer to the buffer, find the ref_count and the actual start of
-   the allocated space. Setting ref_count is clunky because we avoid lvalue
-   casts. */
-#define Buffer_alloc_offset sizeof (void*)
-#define Buffer_bufallocstart(b)  ((char *)Buffer_bufstart(b) - Buffer_alloc_offset)
-#define Buffer_bufrefcountptr(b) ((INTVAL *)Buffer_bufallocstart(b))
-#define Buffer_pool(b) ((Memory_Block *)( *(INTVAL*)(Buffer_bufallocstart(b)) & ~3 ))
-#define Buffer_poolptr(b) ((Memory_Block **)Buffer_bufallocstart(b))
+/* These macros let us address the prolog of a buffer. */
 
+#define Buffer_prolog_offset (sizeof (void*))
+#define Buffer_bufprolog(b) ((char *)Buffer_bufstart(b) - Buffer_prolog_offset) 
 
+/* This macro gives us the address of the buffer prolog treated as
+   a pointer to the flags. */
+
+#define Buffer_bufflagsptr(b) ((INTVAL *)Buffer_bufprolog(b))
+
+/* These macros give us the Memory_Block pointer and pointer-pointer,
+   eliminating the flags. */
+
+#define Buffer_pool(b) ((Memory_Block *)( *(INTVAL*)(Buffer_bufprolog(b)) & ~3 ))
+#define Buffer_poolptr(b) ((Memory_Block **)Buffer_bufprolog(b))
+
+
 typedef enum {
     enum_stringrep_unknown = 0,
     enum_stringrep_one     = 1,
@@ -69,29 +83,34 @@
     enum_stringrep_four    = 4
 } parrot_string_representation_t;
 
+/* Here is the Parrot string header object, "inheriting" from Buffer. */
+
 struct parrot_string_t {
     Parrot_UInt flags;
     void *     _bufstart;
     size_t     _buflen;
-    char       *strstart;
-    UINTVAL     bufused;
-    UINTVAL     strlen;
-    UINTVAL     hashval; /* cached hash value computation */
+    char       *strstart;               /* Pointer to start of string
+                                           (not necessarily at _bufstart). */
+    UINTVAL     bufused;                /* Length of string in bytes. */
+    UINTVAL     strlen;                 /* Length of string in characters. */
+    UINTVAL     hashval;                /* Cached hash value. */
 
     /*    parrot_string_representation_t representation;*/
-    const struct _encoding *encoding;
-    const struct _charset  *charset;
+    const struct _encoding *encoding;   /* Pointer to encoding structure. */
+    const struct _charset  *charset;    /* Pointer to charset structure. */
 };
 
-/* note that cache and flags are isomorphic with Buffer and PObj */
+/* Here is the Parrot PMC object, "inheriting" from PObj. */
+
 struct PMC {
-    Parrot_UInt     flags;
-    VTABLE         *vtable;
-    DPOINTER       *data;
-
-    PMC *_metadata;      /* properties */
+    Parrot_UInt    flags;
+    VTABLE         *vtable;             /* Pointer to vtable. */
+    DPOINTER       *data;               /* Pointer to attribute structure. */
+    PMC            *_metadata;          /* Pointer to metadata PMC. */
 };
 
+/* Use these macros to access the data and metadata. */
+
 #define PMC_data(pmc)                   (pmc)->data
 #define PMC_data_typed(pmc, type) (type)(pmc)->data
 #define PMC_metadata(pmc)         ((pmc)->_metadata)
icket">
              <a href="/parrot/tick