Ruby  2.7.2p137(2020-10-01revision5445e0435260b449decf2ac16f9d09bae3cafe72)
transient_heap.c
Go to the documentation of this file.
1 /**********************************************************************
2 
3  transient_heap.c - implement transient_heap.
4 
5  Copyright (C) 2018 Koichi Sasada
6 
7 **********************************************************************/
8 
9 #include "ruby/ruby.h"
10 #include "ruby/debug.h"
11 #include "vm_debug.h"
12 #include "gc.h"
13 #include "internal.h"
14 #include "ruby_assert.h"
15 #include "transient_heap.h"
16 #include "debug_counter.h"
17 
18 #if USE_TRANSIENT_HEAP /* USE_TRANSIENT_HEAP */
19 /*
20  * 1: enable assertions
21  * 2: enable verify all transient heaps
22  */
23 #ifndef TRANSIENT_HEAP_CHECK_MODE
24 #define TRANSIENT_HEAP_CHECK_MODE 0
25 #endif
26 #define TH_ASSERT(expr) RUBY_ASSERT_MESG_WHEN(TRANSIENT_HEAP_CHECK_MODE > 0, expr, #expr)
27 
28 /*
29  * 1: show events
30  * 2: show dump at events
31  * 3: show all operations
32  */
33 #define TRANSIENT_HEAP_DEBUG 0
34 
35 /* For Debug: Provide blocks infinitely.
36  * This mode generates blocks unlimitedly
37  * and prohibit access free'ed blocks to check invalid access.
38  */
39 #define TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK 0
40 
41 #if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
42 #include <sys/mman.h>
43 #include <errno.h>
44 #endif
45 
46 /* For Debug: Prohibit promoting to malloc space.
47  */
48 #define TRANSIENT_HEAP_DEBUG_DONT_PROMOTE 0
49 
50 /* size configuration */
51 #define TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE 1024
52 
53  /* K M */
54 #define TRANSIENT_HEAP_BLOCK_SIZE (1024 * 32 ) /* 32KB int16_t */
55 #define TRANSIENT_HEAP_TOTAL_SIZE (1024 * 1024 * 32) /* 32 MB */
56 #define TRANSIENT_HEAP_ALLOC_MAX (1024 * 2 ) /* 2 KB */
57 #define TRANSIENT_HEAP_BLOCK_NUM (TRANSIENT_HEAP_TOTAL_SIZE / TRANSIENT_HEAP_BLOCK_SIZE)
58 
59 #define TRANSIENT_HEAP_ALLOC_MAGIC 0xfeab
60 #define TRANSIENT_HEAP_ALLOC_ALIGN RUBY_ALIGNOF(void *)
61 
62 #define TRANSIENT_HEAP_ALLOC_MARKING_LAST -1
63 #define TRANSIENT_HEAP_ALLOC_MARKING_FREE -2
64 
69 };
70 
73  int16_t size; /* sizeof(block) = TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header) */
78  } info;
80 };
81 
90 
94 
96  int arena_index; /* increment only */
97 };
98 
105 };
106 
107 static struct transient_heap global_transient_heap;
108 
109 static void transient_heap_promote_add(struct transient_heap* theap, VALUE obj);
110 static const void *transient_heap_ptr(VALUE obj, int error);
111 static int transient_header_managed_ptr_p(struct transient_heap* theap, const void *ptr);
112 
113 #define ROUND_UP(v, a) (((size_t)(v) + (a) - 1) & ~((a) - 1))
114 
115 static void
116 transient_heap_block_dump(struct transient_heap* theap, struct transient_heap_block *block)
117 {
118  int i=0, n=0;
119 
120  while (i<block->info.index) {
121  void *ptr = &block->buff[i];
122  struct transient_alloc_header *header = ptr;
123  fprintf(stderr, "%4d %8d %p size:%4d next:%4d %s\n", n, i, ptr, header->size, header->next_marked_index, rb_obj_info(header->obj));
124  i += header->size;
125  n++;
126  }
127 }
128 
129 static void
130 transient_heap_blocks_dump(struct transient_heap* theap, struct transient_heap_block *block, const char *type_str)
131 {
132  while (block) {
133  fprintf(stderr, "- transient_heap_dump: %s:%p index:%d objects:%d last_marked_index:%d next:%p\n",
134  type_str, (void *)block, block->info.index, block->info.objects, block->info.last_marked_index, (void *)block->info.next_block);
135 
136  transient_heap_block_dump(theap, block);
137  block = block->info.next_block;
138  }
139 }
140 
141 static void
142 transient_heap_dump(struct transient_heap* theap)
143 {
144  fprintf(stderr, "transient_heap_dump objects:%d marked_objects:%d blocks:%d\n", theap->total_objects, theap->total_marked_objects, theap->total_blocks);
145  transient_heap_blocks_dump(theap, theap->using_blocks, "using_blocks");
146  transient_heap_blocks_dump(theap, theap->marked_blocks, "marked_blocks");
147  transient_heap_blocks_dump(theap, theap->free_blocks, "free_blocks");
148 }
149 
150 /* Debug: dump all tarnsient_heap blocks */
151 void
153 {
154  transient_heap_dump(&global_transient_heap);
155 }
156 
157 #if TRANSIENT_HEAP_CHECK_MODE >= 2
158 ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static void transient_heap_ptr_check(struct transient_heap *theap, VALUE obj));
159 static void
160 transient_heap_ptr_check(struct transient_heap *theap, VALUE obj)
161 {
162  if (obj != Qundef) {
163  const void *ptr = transient_heap_ptr(obj, FALSE);
164  TH_ASSERT(ptr == NULL || transient_header_managed_ptr_p(theap, ptr));
165  }
166 }
167 
168 ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static int transient_heap_block_verify(struct transient_heap *theap, struct transient_heap_block *block));
169 static int
170 transient_heap_block_verify(struct transient_heap *theap, struct transient_heap_block *block)
171 {
172  int i=0, n=0;
173  struct transient_alloc_header *header;
174 
175  while (i<block->info.index) {
176  header = (void *)&block->buff[i];
178  transient_heap_ptr_check(theap, header->obj);
179  n ++;
180  i += header->size;
181  }
182  TH_ASSERT(block->info.objects == n);
183 
184  return n;
185 }
186 
187 static int
188 transient_heap_blocks_verify(struct transient_heap *theap, struct transient_heap_block *blocks, int *block_num_ptr)
189 {
190  int n = 0;
191  struct transient_heap_block *block = blocks;
192  while (block) {
193  n += transient_heap_block_verify(theap, block);
194  *block_num_ptr += 1;
195  block = block->info.next_block;
196  }
197 
198  return n;
199 }
200 #endif
201 
202 static void
203 transient_heap_verify(struct transient_heap *theap)
204 {
205 #if TRANSIENT_HEAP_CHECK_MODE >= 2
206  int n=0, block_num=0;
207 
208  n += transient_heap_blocks_verify(theap, theap->using_blocks, &block_num);
209  n += transient_heap_blocks_verify(theap, theap->marked_blocks, &block_num);
210 
211  TH_ASSERT(n == theap->total_objects);
212  TH_ASSERT(n >= theap->total_marked_objects);
213  TH_ASSERT(block_num == theap->total_blocks);
214 #endif
215 }
216 
217 /* Debug: check assertions for all transient_heap blocks */
218 void
220 {
221  transient_heap_verify(&global_transient_heap);
222 }
223 
224 static struct transient_heap*
225 transient_heap_get(void)
226 {
227  struct transient_heap* theap = &global_transient_heap;
228  transient_heap_verify(theap);
229  return theap;
230 }
231 
232 static void
233 reset_block(struct transient_heap_block *block)
234 {
235  __msan_allocated_memory(block, sizeof block);
236  block->info.size = TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header);
237  block->info.index = 0;
238  block->info.objects = 0;
240  block->info.next_block = NULL;
241  __asan_poison_memory_region(&block->buff, sizeof block->buff);
242 }
243 
244 static void
245 connect_to_free_blocks(struct transient_heap *theap, struct transient_heap_block *block)
246 {
247  block->info.next_block = theap->free_blocks;
248  theap->free_blocks = block;
249 }
250 
251 static void
252 connect_to_using_blocks(struct transient_heap *theap, struct transient_heap_block *block)
253 {
254  block->info.next_block = theap->using_blocks;
255  theap->using_blocks = block;
256 }
257 
258 #if 0
259 static void
260 connect_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *block)
261 {
262  block->info.next_block = theap->marked_blocks;
263  theap->marked_blocks = block;
264 }
265 #endif
266 
267 static void
268 append_to_marked_blocks(struct transient_heap *theap, struct transient_heap_block *append_blocks)
269 {
270  if (theap->marked_blocks) {
271  struct transient_heap_block *block = theap->marked_blocks, *last_block = NULL;
272  while (block) {
273  last_block = block;
274  block = block->info.next_block;
275  }
276 
277  TH_ASSERT(last_block->info.next_block == NULL);
278  last_block->info.next_block = append_blocks;
279  }
280  else {
281  theap->marked_blocks = append_blocks;
282  }
283 }
284 
285 static struct transient_heap_block *
286 transient_heap_block_alloc(struct transient_heap* theap)
287 {
288  struct transient_heap_block *block;
289 #if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
290  block = mmap(NULL, TRANSIENT_HEAP_BLOCK_SIZE, PROT_READ | PROT_WRITE,
291  MAP_PRIVATE | MAP_ANONYMOUS,
292  -1, 0);
293  if (block == MAP_FAILED) rb_bug("transient_heap_block_alloc: err:%d\n", errno);
294 #else
295  if (theap->arena == NULL) {
297  if (theap->arena == NULL) {
298  rb_bug("transient_heap_block_alloc: failed\n");
299  }
300  }
301 
303  block = &theap->arena[theap->arena_index++];
304  TH_ASSERT(((intptr_t)block & (TRANSIENT_HEAP_BLOCK_SIZE - 1)) == 0);
305 #endif
306  reset_block(block);
307 
308  TH_ASSERT(((intptr_t)block->buff & (TRANSIENT_HEAP_ALLOC_ALIGN-1)) == 0);
309  if (0) fprintf(stderr, "transient_heap_block_alloc: %4d %p\n", theap->total_blocks, (void *)block);
310  return block;
311 }
312 
313 
314 static struct transient_heap_block *
315 transient_heap_allocatable_block(struct transient_heap* theap)
316 {
317  struct transient_heap_block *block;
318 
319 #if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
320  block = transient_heap_block_alloc(theap);
321  theap->total_blocks++;
322 #else
323  /* get one block from free_blocks */
324  block = theap->free_blocks;
325  if (block) {
326  theap->free_blocks = block->info.next_block;
327  block->info.next_block = NULL;
328  theap->total_blocks++;
329  }
330 #endif
331 
332  return block;
333 }
334 
335 static struct transient_alloc_header *
336 transient_heap_allocatable_header(struct transient_heap* theap, size_t size)
337 {
338  struct transient_heap_block *block = theap->using_blocks;
339 
340  while (block) {
341  TH_ASSERT(block->info.size >= block->info.index);
342 
343  if (block->info.size - block->info.index >= (int32_t)size) {
344  struct transient_alloc_header *header = (void *)&block->buff[block->info.index];
345  block->info.index += size;
346  block->info.objects++;
347  return header;
348  }
349  else {
350  block = transient_heap_allocatable_block(theap);
351  if (block) connect_to_using_blocks(theap, block);
352  }
353  }
354 
355  return NULL;
356 }
357 
358 void *
360 {
361  struct transient_heap* theap = transient_heap_get();
362  size_t size = ROUND_UP(req_size + sizeof(struct transient_alloc_header), TRANSIENT_HEAP_ALLOC_ALIGN);
363 
365  RB_TYPE_P(obj, T_OBJECT) ||
366  RB_TYPE_P(obj, T_STRUCT) ||
367  RB_TYPE_P(obj, T_HASH)); /* supported types */
368 
370  if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [too big: %ld] %s\n", (long)size, rb_obj_info(obj));
371  return NULL;
372  }
373 #if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE == 0
374  else if (RB_OBJ_PROMOTED_RAW(obj)) {
375  if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [promoted object] %s\n", rb_obj_info(obj));
376  return NULL;
377  }
378 #else
379  else if (RBASIC_CLASS(obj) == 0) {
380  if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [hidden object] %s\n", rb_obj_info(obj));
381  return NULL;
382  }
383 #endif
384  else {
385  struct transient_alloc_header *header = transient_heap_allocatable_header(theap, size);
386  if (header) {
387  void *ptr;
388 
389  /* header is poisoned to prevent buffer overflow, should
390  * unpoison first... */
391  asan_unpoison_memory_region(header, sizeof *header, true);
392 
393  header->size = size;
396  header->obj = obj; /* TODO: can we eliminate it? */
397 
398  /* header is fixed; shall poison again */
399  asan_poison_memory_region(header, sizeof *header);
400  ptr = header + 1;
401 
402  theap->total_objects++; /* statistics */
403 
404 #if TRANSIENT_HEAP_DEBUG_DONT_PROMOTE
405  if (RB_OBJ_PROMOTED_RAW(obj)) {
406  transient_heap_promote_add(theap, obj);
407  }
408 #endif
409  if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: header:%p ptr:%p size:%d obj:%s\n", (void *)header, ptr, (int)size, rb_obj_info(obj));
410 
411  RB_DEBUG_COUNTER_INC(theap_alloc);
412 
413  /* ptr is set up; OK to unpoison. */
414  asan_unpoison_memory_region(ptr, size - sizeof *header, true);
415  return ptr;
416  }
417  else {
418  if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_alloc: [no enough space: %ld] %s\n", (long)size, rb_obj_info(obj));
419  RB_DEBUG_COUNTER_INC(theap_alloc_fail);
420  return NULL;
421  }
422  }
423 }
424 
425 void
427 {
428  int i, block_num;
429  struct transient_heap* theap = transient_heap_get();
430 
431 #if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
432  block_num = 0;
433 #else
435  block_num = TRANSIENT_HEAP_BLOCK_NUM;
436 #endif
437  for (i=0; i<block_num; i++) {
438  connect_to_free_blocks(theap, transient_heap_block_alloc(theap));
439  }
440  theap->using_blocks = transient_heap_allocatable_block(theap);
441 
443  theap->promoted_objects_index = 0;
444  /* should not use ALLOC_N to be free from GC */
445  theap->promoted_objects = malloc(sizeof(VALUE) * theap->promoted_objects_size);
447  integer_overflow,
449  if (theap->promoted_objects == NULL) rb_bug("Init_TransientHeap: malloc failed.");
450 }
451 
452 static struct transient_heap_block *
453 blocks_alloc_header_to_block(struct transient_heap *theap, struct transient_heap_block *blocks, struct transient_alloc_header *header)
454 {
455  struct transient_heap_block *block = blocks;
456 
457  while (block) {
458  if (block->buff <= (char *)header && (char *)header < block->buff + block->info.size) {
459  return block;
460  }
461  block = block->info.next_block;
462  }
463 
464  return NULL;
465 }
466 
467 static struct transient_heap_block *
468 alloc_header_to_block_verbose(struct transient_heap *theap, struct transient_alloc_header *header)
469 {
470  struct transient_heap_block *block;
471 
472  if ((block = blocks_alloc_header_to_block(theap, theap->marked_blocks, header)) != NULL) {
473  if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in marked_blocks\n");
474  return block;
475  }
476  else if ((block = blocks_alloc_header_to_block(theap, theap->using_blocks, header)) != NULL) {
477  if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "alloc_header_to_block: found in using_blocks\n");
478  return block;
479  }
480  else {
481  return NULL;
482  }
483 }
484 
485 static struct transient_alloc_header *
486 ptr_to_alloc_header(const void *ptr)
487 {
488  struct transient_alloc_header *header = (void *)ptr;
489  header -= 1;
490  return header;
491 }
492 
493 static int
494 transient_header_managed_ptr_p(struct transient_heap* theap, const void *ptr)
495 {
496  if (alloc_header_to_block_verbose(theap, ptr_to_alloc_header(ptr))) {
497  return TRUE;
498  }
499  else {
500  return FALSE;
501  }
502 }
503 
504 
505 int
507 {
508  return transient_header_managed_ptr_p(transient_heap_get(), ptr);
509 }
510 
511 static struct transient_heap_block *
512 alloc_header_to_block(struct transient_heap *theap, struct transient_alloc_header *header)
513 {
514  struct transient_heap_block *block;
515 #if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
516  block = alloc_header_to_block_verbose(theap, header);
517  if (block == NULL) {
518  transient_heap_dump(theap);
519  rb_bug("alloc_header_to_block: not found in mark_blocks (%p)\n", header);
520  }
521 #else
522  block = (void *)((intptr_t)header & ~(TRANSIENT_HEAP_BLOCK_SIZE-1));
523  TH_ASSERT(block == alloc_header_to_block_verbose(theap, header));
524 #endif
525  return block;
526 }
527 
528 void
530 {
531  struct transient_alloc_header *header = ptr_to_alloc_header(ptr);
532  asan_unpoison_memory_region(header, sizeof *header, false);
533  if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) rb_bug("rb_transient_heap_mark: wrong header, %s (%p)", rb_obj_info(obj), ptr);
534  if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_mark: %s (%p)\n", rb_obj_info(obj), ptr);
535 
536 #if TRANSIENT_HEAP_CHECK_MODE > 0
537  {
538  struct transient_heap* theap = transient_heap_get();
540  TH_ASSERT(transient_header_managed_ptr_p(theap, ptr));
541 
542  if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) {
543  transient_heap_dump(theap);
544  rb_bug("rb_transient_heap_mark: magic is broken");
545  }
546  else if (header->obj != obj) {
547  // transient_heap_dump(theap);
548  rb_bug("rb_transient_heap_mark: unmatch (%s is stored, but %s is given)\n",
549  rb_obj_info(header->obj), rb_obj_info(obj));
550  }
551  }
552 #endif
553 
555  /* already marked */
556  return;
557  }
558  else {
559  struct transient_heap* theap = transient_heap_get();
560  struct transient_heap_block *block = alloc_header_to_block(theap, header);
561  __asan_unpoison_memory_region(&block->info, sizeof block->info);
562  header->next_marked_index = block->info.last_marked_index;
563  block->info.last_marked_index = (int)((char *)header - block->buff);
564  theap->total_marked_objects++;
565 
566  transient_heap_verify(theap);
567  }
568 }
569 
570 ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static const void *transient_heap_ptr(VALUE obj, int error));
571 static const void *
572 transient_heap_ptr(VALUE obj, int error)
573 {
574  const void *ptr = NULL;
575 
576  switch (BUILTIN_TYPE(obj)) {
577  case T_ARRAY:
578  if (RARRAY_TRANSIENT_P(obj)) {
580  ptr = RARRAY(obj)->as.heap.ptr;
581  }
582  break;
583  case T_OBJECT:
584  if (ROBJ_TRANSIENT_P(obj)) {
585  ptr = ROBJECT_IVPTR(obj);
586  }
587  break;
588  case T_STRUCT:
589  if (RSTRUCT_TRANSIENT_P(obj)) {
590  ptr = rb_struct_const_heap_ptr(obj);
591  }
592  break;
593  case T_HASH:
594  if (RHASH_TRANSIENT_P(obj)) {
596  ptr = (VALUE *)(RHASH(obj)->as.ar);
597  }
598  else {
599  ptr = NULL;
600  }
601  break;
602  default:
603  if (error) {
604  rb_bug("transient_heap_ptr: unknown obj %s\n", rb_obj_info(obj));
605  }
606  }
607 
608  return ptr;
609 }
610 
611 static void
612 transient_heap_promote_add(struct transient_heap* theap, VALUE obj)
613 {
614  if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, "rb_transient_heap_promote: %s\n", rb_obj_info(obj));
615 
617  /* duplicate check */
618  int i;
619  for (i=0; i<theap->promoted_objects_index; i++) {
620  if (theap->promoted_objects[i] == obj) return;
621  }
622  }
623 
624  if (theap->promoted_objects_size <= theap->promoted_objects_index) {
625  theap->promoted_objects_size *= 2;
626  if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "rb_transient_heap_promote: expand table to %d\n", theap->promoted_objects_size);
627  if (UNLIKELY((size_t)theap->promoted_objects_size > SIZE_MAX / sizeof(VALUE))) {
628  /* realloc failure due to integer overflow */
629  theap->promoted_objects = NULL;
630  }
631  else {
632  theap->promoted_objects = realloc(theap->promoted_objects, theap->promoted_objects_size * sizeof(VALUE));
633  }
634  if (theap->promoted_objects == NULL) rb_bug("rb_transient_heap_promote: realloc failed");
635  }
636  theap->promoted_objects[theap->promoted_objects_index++] = obj;
637 }
638 
639 void
641 {
642  if (transient_heap_ptr(obj, FALSE)) {
643  struct transient_heap* theap = transient_heap_get();
644  transient_heap_promote_add(theap, obj);
645  }
646  else {
647  /* ignore */
648  }
649 }
650 
651 static struct transient_alloc_header *
652 alloc_header(struct transient_heap_block* block, int index)
653 {
654  return (void *)&block->buff[index];
655 }
656 
657 static void
658 transient_heap_reset(void)
659 {
660  struct transient_heap* theap = transient_heap_get();
661  struct transient_heap_block* block;
662 
663  if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset\n");
664 
665  block = theap->marked_blocks;
666  while (block) {
667  struct transient_heap_block *next_block = block->info.next_block;
668  theap->total_objects -= block->info.objects;
669 #if TRANSIENT_HEAP_DEBUG_INFINITE_BLOCK
670  if (madvise(block, TRANSIENT_HEAP_BLOCK_SIZE, MADV_DONTNEED) != 0) {
671  rb_bug("madvise err:%d", errno);
672  }
673  if (mprotect(block, TRANSIENT_HEAP_BLOCK_SIZE, PROT_NONE) != 0) {
674  rb_bug("mprotect err:%d", errno);
675  }
676 #else
677  reset_block(block);
678  connect_to_free_blocks(theap, block);
679 #endif
680  theap->total_blocks--;
681  block = next_block;
682  }
683 
684  if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_reset block_num:%d\n", theap->total_blocks);
685 
686  theap->marked_blocks = NULL;
687  theap->total_marked_objects = 0;
688 }
689 
690 static void
691 transient_heap_block_evacuate(struct transient_heap* theap, struct transient_heap_block* block)
692 {
693  int marked_index = block->info.last_marked_index;
695 
696  while (marked_index >= 0) {
697  struct transient_alloc_header *header = alloc_header(block, marked_index);
698  VALUE obj = header->obj;
700  if (header->magic != TRANSIENT_HEAP_ALLOC_MAGIC) rb_bug("rb_transient_heap_mark: wrong header %s\n", rb_obj_info(obj));
701 
702  if (TRANSIENT_HEAP_DEBUG >= 3) fprintf(stderr, " * transient_heap_block_evacuate %p %s\n", (void *)header, rb_obj_info(obj));
703 
704  if (obj != Qnil) {
705  RB_DEBUG_COUNTER_INC(theap_evacuate);
706 
707  switch (BUILTIN_TYPE(obj)) {
708  case T_ARRAY:
710  break;
711  case T_OBJECT:
713  break;
714  case T_STRUCT:
716  break;
717  case T_HASH:
719  break;
720  default:
721  rb_bug("unsupporeted: %s\n", rb_obj_info(obj));
722  }
723  header->obj = Qundef; /* for debug */
724  }
725  marked_index = header->next_marked_index;
726  }
727 }
728 
729 static void
730 transient_heap_update_status(struct transient_heap* theap, enum transient_heap_status status)
731 {
732  TH_ASSERT(theap->status != status);
733  theap->status = status;
734 }
735 
736 static void
737 transient_heap_evacuate(void *dmy)
738 {
739  struct transient_heap* theap = transient_heap_get();
740 
741  if (theap->status == transient_heap_marking) {
742  if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! transient_heap_evacuate: skip while transient_heap_marking\n");
743  }
744  else {
745  VALUE gc_disabled = rb_gc_disable_no_rest();
746  struct transient_heap_block* block;
747 
748  if (TRANSIENT_HEAP_DEBUG >= 1) {
749  int i;
750  fprintf(stderr, "!! transient_heap_evacuate start total_blocks:%d\n", theap->total_blocks);
751  if (TRANSIENT_HEAP_DEBUG >= 4) {
752  for (i=0; i<theap->promoted_objects_index; i++) fprintf(stderr, "%4d %s\n", i, rb_obj_info(theap->promoted_objects[i]));
753  }
754  }
755  if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
756 
758  transient_heap_update_status(theap, transient_heap_escaping);
759 
760  /* evacuate from marked blocks */
761  block = theap->marked_blocks;
762  while (block) {
763  transient_heap_block_evacuate(theap, block);
764  block = block->info.next_block;
765  }
766 
767  /* evacuate from using blocks
768  only affect incremental marking */
769  block = theap->using_blocks;
770  while (block) {
771  transient_heap_block_evacuate(theap, block);
772  block = block->info.next_block;
773  }
774 
775  /* all objects in marked_objects are escaped. */
776  transient_heap_reset();
777 
778  if (TRANSIENT_HEAP_DEBUG > 0) {
779  fprintf(stderr, "!! transient_heap_evacuate end total_blocks:%d\n", theap->total_blocks);
780  }
781 
782  transient_heap_verify(theap);
783  transient_heap_update_status(theap, transient_heap_none);
784  if (gc_disabled != Qtrue) rb_gc_enable();
785  }
786 }
787 
788 static void
789 clear_marked_index(struct transient_heap_block* block)
790 {
791  int marked_index = block->info.last_marked_index;
792 
793  while (marked_index != TRANSIENT_HEAP_ALLOC_MARKING_LAST) {
794  struct transient_alloc_header *header = alloc_header(block, marked_index);
795  /* header is poisoned to prevent buffer overflow, should
796  * unpoison first... */
797  asan_unpoison_memory_region(header, sizeof *header, false);
799  if (0) fprintf(stderr, "clear_marked_index - block:%p mark_index:%d\n", (void *)block, marked_index);
800 
801  marked_index = header->next_marked_index;
803  }
804 
806 }
807 
808 static void
809 blocks_clear_marked_index(struct transient_heap_block* block)
810 {
811  while (block) {
812  clear_marked_index(block);
813  block = block->info.next_block;
814  }
815 }
816 
817 static void
818 transient_heap_block_update_refs(struct transient_heap* theap, struct transient_heap_block* block)
819 {
820  int i=0, n=0;
821 
822  while (i<block->info.index) {
823  void *ptr = &block->buff[i];
824  struct transient_alloc_header *header = ptr;
825 
826  asan_unpoison_memory_region(header, sizeof *header, false);
827 
828  void *poisoned = __asan_region_is_poisoned((void *)header->obj, SIZEOF_VALUE);
829  asan_unpoison_object(header->obj, false);
830 
831  header->obj = rb_gc_location(header->obj);
832 
833  if (poisoned) {
834  asan_poison_object(header->obj);
835  }
836 
837  i += header->size;
838  asan_poison_memory_region(header, sizeof *header);
839  n++;
840  }
841 }
842 
843 static void
844 transient_heap_blocks_update_refs(struct transient_heap* theap, struct transient_heap_block *block, const char *type_str)
845 {
846  while (block) {
847  transient_heap_block_update_refs(theap, block);
848  block = block->info.next_block;
849  }
850 }
851 
852 void
854 {
855  struct transient_heap* theap = transient_heap_get();
856  int i;
857 
858  transient_heap_blocks_update_refs(theap, theap->using_blocks, "using_blocks");
859  transient_heap_blocks_update_refs(theap, theap->marked_blocks, "marked_blocks");
860 
861  for (i=0; i<theap->promoted_objects_index; i++) {
862  VALUE obj = theap->promoted_objects[i];
864  }
865 }
866 
867 void
869 {
870  struct transient_heap* theap = transient_heap_get();
871 
872  if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! rb_transient_heap_start_marking objects:%d blocks:%d promtoed:%d full_marking:%d\n",
873  theap->total_objects, theap->total_blocks, theap->promoted_objects_index, full_marking);
874  if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
875 
876  blocks_clear_marked_index(theap->marked_blocks);
877  blocks_clear_marked_index(theap->using_blocks);
878 
879  if (theap->using_blocks) {
880  if (theap->using_blocks->info.objects > 0) {
881  append_to_marked_blocks(theap, theap->using_blocks);
882  theap->using_blocks = NULL;
883  }
884  else {
885  append_to_marked_blocks(theap, theap->using_blocks->info.next_block);
886  theap->using_blocks->info.next_block = NULL;
887  }
888  }
889 
890  if (theap->using_blocks == NULL) {
891  theap->using_blocks = transient_heap_allocatable_block(theap);
892  }
893 
895  transient_heap_update_status(theap, transient_heap_marking);
896  theap->total_marked_objects = 0;
897 
898  if (full_marking) {
899  theap->promoted_objects_index = 0;
900  }
901  else { /* mark promoted objects */
902  int i;
903  for (i=0; i<theap->promoted_objects_index; i++) {
904  VALUE obj = theap->promoted_objects[i];
905  const void *ptr = transient_heap_ptr(obj, TRUE);
906  if (ptr) {
908  }
909  }
910  }
911 
912  transient_heap_verify(theap);
913 }
914 
915 void
917 {
918  struct transient_heap* theap = transient_heap_get();
919 
920  if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "!! rb_transient_heap_finish_marking objects:%d, marked:%d\n",
921  theap->total_objects,
922  theap->total_marked_objects);
923  if (TRANSIENT_HEAP_DEBUG >= 2) transient_heap_dump(theap);
924 
925  TH_ASSERT(theap->total_objects >= theap->total_marked_objects);
926 
928  transient_heap_update_status(theap, transient_heap_none);
929 
930  if (theap->total_marked_objects > 0) {
931  if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "-> rb_transient_heap_finish_marking register escape func.\n");
932  rb_postponed_job_register_one(0, transient_heap_evacuate, NULL);
933  }
934  else {
935  transient_heap_reset();
936  }
937 
938  transient_heap_verify(theap);
939 }
940 #endif /* USE_TRANSIENT_HEAP */
rb_transient_heap_managed_ptr_p
int rb_transient_heap_managed_ptr_p(const void *ptr)
Definition: transient_heap.c:506
RARRAY_TRANSIENT_P
#define RARRAY_TRANSIENT_P(ary)
Definition: ruby.h:1076
UNLIKELY
#define UNLIKELY(x)
Definition: ffi_common.h:126
transient_heap_block::transient_heap_block_header::next_block
struct transient_heap_block * next_block
Definition: transient_heap.c:77
TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE
#define TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE
Definition: transient_heap.c:51
TRUE
#define TRUE
Definition: nkf.h:175
rb_transient_heap_mark
void rb_transient_heap_mark(VALUE obj, const void *ptr)
Definition: transient_heap.c:529
error
const rb_iseq_t const char * error
Definition: rb_mjit_min_header-2.7.2.h:13471
transient_heap::total_marked_objects
int total_marked_objects
Definition: transient_heap.c:87
ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS
ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS(static const void *transient_heap_ptr(VALUE obj, int error))
int16_t
__int16_t int16_t
Definition: rb_mjit_min_header-2.7.2.h:1169
transient_alloc_header
Definition: transient_heap.c:99
transient_heap
Definition: transient_heap.c:82
rb_postponed_job_register_one
int rb_postponed_job_register_one(unsigned int flags, rb_postponed_job_func_t func, void *data)
Definition: vm_trace.c:1614
gc.h
int
__inline__ int
Definition: rb_mjit_min_header-2.7.2.h:2845
__asan_unpoison_memory_region
#define __asan_unpoison_memory_region(x, y)
Definition: internal.h:109
TRANSIENT_HEAP_ALLOC_ALIGN
#define TRANSIENT_HEAP_ALLOC_ALIGN
Definition: transient_heap.c:60
rb_obj_info
MJIT_FUNC_EXPORTED const char * rb_obj_info(VALUE obj)
Definition: gc.c:11683
transient_heap::free_blocks
struct transient_heap_block * free_blocks
Definition: transient_heap.c:85
ROUND_UP
#define ROUND_UP(v, a)
Definition: transient_heap.c:113
__asan_poison_memory_region
#define __asan_poison_memory_region(x, y)
Definition: internal.h:108
i
uint32_t i
Definition: rb_mjit_min_header-2.7.2.h:5460
SIZE_MAX
#define SIZE_MAX
Definition: ruby.h:307
Init_TransientHeap
void Init_TransientHeap(void)
Definition: transient_heap.c:426
transient_heap::total_objects
int total_objects
Definition: transient_heap.c:86
transient_heap::marked_blocks
struct transient_heap_block * marked_blocks
Definition: transient_heap.c:84
VALUE
unsigned long VALUE
Definition: ruby.h:102
RB_TYPE_P
#define RB_TYPE_P(obj, type)
Definition: ruby.h:560
rb_gc_location
VALUE rb_gc_location(VALUE value)
Definition: gc.c:8114
rb_aligned_malloc
void * rb_aligned_malloc(size_t alignment, size_t size)
Definition: gc.c:9630
rb_ary_transient_heap_evacuate
void rb_ary_transient_heap_evacuate(VALUE ary, int promote)
Definition: array.c:402
TRANSIENT_HEAP_BLOCK_NUM
#define TRANSIENT_HEAP_BLOCK_NUM
Definition: transient_heap.c:57
rb_transient_heap_dump
void rb_transient_heap_dump(void)
Definition: transient_heap.c:152
transient_alloc_header::magic
uint16_t magic
Definition: transient_heap.c:100
TRANSIENT_HEAP_ALLOC_MAX
#define TRANSIENT_HEAP_ALLOC_MAX
Definition: transient_heap.c:56
transient_heap_block::transient_heap_block_header::objects
int16_t objects
Definition: transient_heap.c:76
Qundef
#define Qundef
Definition: ruby.h:470
ptr
struct RIMemo * ptr
Definition: debug.c:65
RB_OBJ_PROMOTED_RAW
#define RB_OBJ_PROMOTED_RAW(x)
Definition: ruby.h:1482
transient_heap::total_blocks
int total_blocks
Definition: transient_heap.c:88
__asan_region_is_poisoned
#define __asan_region_is_poisoned(x, y)
Definition: internal.h:110
TRANSIENT_HEAP_TOTAL_SIZE
#define TRANSIENT_HEAP_TOTAL_SIZE
Definition: transient_heap.c:55
TRANSIENT_HEAP_ALLOC_MARKING_FREE
#define TRANSIENT_HEAP_ALLOC_MARKING_FREE
Definition: transient_heap.c:63
transient_heap::arena
struct transient_heap_block * arena
Definition: transient_heap.c:95
transient_heap_block::transient_heap_block_header::size
int16_t size
Definition: transient_heap.c:73
NULL
#define NULL
Definition: _sdbm.c:101
vm_debug.h
TRANSIENT_HEAP_ALLOC_MARKING_LAST
#define TRANSIENT_HEAP_ALLOC_MARKING_LAST
Definition: transient_heap.c:62
transient_heap::promoted_objects
VALUE * promoted_objects
Definition: transient_heap.c:91
ruby.h
rb_gc_enable
VALUE rb_gc_enable(void)
Definition: gc.c:9212
T_OBJECT
#define T_OBJECT
Definition: ruby.h:523
TRANSIENT_HEAP_BLOCK_SIZE
#define TRANSIENT_HEAP_BLOCK_SIZE
Definition: transient_heap.c:54
TRANSIENT_HEAP_DEBUG
#define TRANSIENT_HEAP_DEBUG
Definition: transient_heap.c:33
TH_ASSERT
#define TH_ASSERT(expr)
Definition: transient_heap.c:26
transient_alloc_header::dummy
int16_t dummy
Definition: transient_heap.c:103
rb_transient_heap_update_references
void rb_transient_heap_update_references(void)
Definition: transient_heap.c:853
transient_heap_block
Definition: transient_heap.c:71
transient_heap_block::buff
char buff[TRANSIENT_HEAP_BLOCK_SIZE - sizeof(struct transient_heap_block_header)]
Definition: transient_heap.c:79
transient_heap_block::transient_heap_block_header::last_marked_index
int16_t last_marked_index
Definition: transient_heap.c:75
SIZEOF_VALUE
#define SIZEOF_VALUE
Definition: ruby.h:105
rb_transient_heap_finish_marking
void rb_transient_heap_finish_marking(void)
Definition: transient_heap.c:916
RSTRUCT_TRANSIENT_P
#define RSTRUCT_TRANSIENT_P(st)
Definition: internal.h:933
rb_hash_transient_heap_evacuate
void rb_hash_transient_heap_evacuate(VALUE hash, int promote)
Definition: hash.c:1244
__msan_allocated_memory
#define __msan_allocated_memory(x, y)
Definition: internal.h:120
TRANSIENT_HEAP_ALLOC_MAGIC
#define TRANSIENT_HEAP_ALLOC_MAGIC
Definition: transient_heap.c:59
int32_t
__int32_t int32_t
Definition: rb_mjit_min_header-2.7.2.h:1174
rb_struct_transient_heap_evacuate
void rb_struct_transient_heap_evacuate(VALUE obj, int promote)
Definition: struct.c:681
transient_heap_block::transient_heap_block_header::index
int16_t index
Definition: transient_heap.c:74
transient_heap.h
rb_transient_heap_promote
void rb_transient_heap_promote(VALUE obj)
Definition: transient_heap.c:640
RBASIC_CLASS
#define RBASIC_CLASS(obj)
Definition: ruby.h:906
RARRAY_EMBED_FLAG
@ RARRAY_EMBED_FLAG
Definition: ruby.h:1029
transient_alloc_header::size
uint16_t size
Definition: transient_heap.c:101
transient_heap_marking
@ transient_heap_marking
Definition: transient_heap.c:67
RHASH_TRANSIENT_P
#define RHASH_TRANSIENT_P(hash)
Definition: internal.h:870
rb_transient_heap_alloc
void * rb_transient_heap_alloc(VALUE obj, size_t req_size)
Definition: transient_heap.c:359
transient_heap::promoted_objects_size
int promoted_objects_size
Definition: transient_heap.c:92
ROBJECT_IVPTR
#define ROBJECT_IVPTR(o)
Definition: ruby.h:937
size
int size
Definition: encoding.c:58
FALSE
#define FALSE
Definition: nkf.h:174
transient_heap::status
enum transient_heap_status status
Definition: transient_heap.c:89
RHASH
#define RHASH(obj)
Definition: internal.h:859
T_HASH
#define T_HASH
Definition: ruby.h:531
transient_heap_none
@ transient_heap_none
Definition: transient_heap.c:66
transient_heap_escaping
@ transient_heap_escaping
Definition: transient_heap.c:68
FL_TEST_RAW
#define FL_TEST_RAW(x, f)
Definition: ruby.h:1352
transient_heap_block::info
struct transient_heap_block::transient_heap_block_header info
obj
const VALUE VALUE obj
Definition: rb_mjit_min_header-2.7.2.h:5738
rb_bug
void rb_bug(const char *fmt,...)
Definition: error.c:636
internal.h
T_ARRAY
#define T_ARRAY
Definition: ruby.h:530
ROBJ_TRANSIENT_P
#define ROBJ_TRANSIENT_P(obj)
Definition: internal.h:2262
STATIC_ASSERT
#define STATIC_ASSERT(name, expr)
Definition: internal.h:230
RARRAY
#define RARRAY(obj)
Definition: ruby.h:1273
debug_counter.h
transient_heap::using_blocks
struct transient_heap_block * using_blocks
Definition: transient_heap.c:83
transient_alloc_header::obj
VALUE obj
Definition: transient_heap.c:104
BUILTIN_TYPE
#define BUILTIN_TYPE(x)
Definition: ruby.h:551
Qtrue
#define Qtrue
Definition: ruby.h:468
errno
int errno
rb_obj_transient_heap_evacuate
void rb_obj_transient_heap_evacuate(VALUE obj, int promote)
Definition: variable.c:1215
transient_heap::promoted_objects_index
int promoted_objects_index
Definition: transient_heap.c:93
transient_heap_status
transient_heap_status
Definition: transient_heap.c:65
RB_DEBUG_COUNTER_INC
#define RB_DEBUG_COUNTER_INC(type)
Definition: debug_counter.h:375
transient_heap_block::transient_heap_block_header
Definition: transient_heap.c:72
intptr_t
int intptr_t
Definition: win32.h:90
stderr
#define stderr
Definition: rb_mjit_min_header-2.7.2.h:1516
transient_heap::arena_index
int arena_index
Definition: transient_heap.c:96
TRANSIENT_HEAP_DEBUG_DONT_PROMOTE
#define TRANSIENT_HEAP_DEBUG_DONT_PROMOTE
Definition: transient_heap.c:48
index
int index
Definition: rb_mjit_min_header-2.7.2.h:11214
malloc
void * malloc(size_t) __attribute__((__malloc__)) __attribute__((__warn_unused_result__)) __attribute__((__alloc_size__(1)))
rb_gc_disable_no_rest
VALUE rb_gc_disable_no_rest(void)
Definition: gc.c:9234
uint16_t
__uint16_t uint16_t
Definition: rb_mjit_min_header-2.7.2.h:1171
Qnil
#define Qnil
Definition: ruby.h:469
T_STRUCT
#define T_STRUCT
Definition: ruby.h:532
realloc
void * realloc(void *, size_t) __attribute__((__warn_unused_result__)) __attribute__((__alloc_size__(2)))
fprintf
int fprintf(FILE *__restrict, const char *__restrict,...) __attribute__((__format__(__printf__
RHASH_AR_TABLE_P
#define RHASH_AR_TABLE_P(hash)
Definition: internal.h:854
rb_transient_heap_verify
void rb_transient_heap_verify(void)
Definition: transient_heap.c:219
ruby_assert.h
debug.h
rb_transient_heap_start_marking
void rb_transient_heap_start_marking(int full_marking)
Definition: transient_heap.c:868
transient_alloc_header::next_marked_index
int16_t next_marked_index
Definition: transient_heap.c:102
n
const char size_t n
Definition: rb_mjit_min_header-2.7.2.h:5452