Ruby 3.2.1p31 (2023-02-08 revision 31819e82c88c6f8ecfaeb162519bfa26a14b21fd)
mjit.c
1/**********************************************************************
2
3 mjit.c - MRI method JIT compiler functions
4
5 Copyright (C) 2017 Vladimir Makarov <vmakarov@redhat.com>.
6 Copyright (C) 2017 Takashi Kokubun <k0kubun@ruby-lang.org>.
7
8**********************************************************************/
9
10/* We utilize widely used C compilers (GCC and LLVM Clang) to
11 implement MJIT. We feed them a C code generated from ISEQ. The
12 industrial C compilers are slower than regular JIT engines.
13 Generated code performance of the used C compilers has a higher
14 priority over the compilation speed.
15
16 So our major goal is to minimize the ISEQ compilation time when we
17 use widely optimization level (-O2). It is achieved by
18
19 o Using a precompiled version of the header
20 o Keeping all files in `/tmp`. On modern Linux `/tmp` is a file
21 system in memory. So it is pretty fast
22 o Implementing MJIT as a multi-threaded code because we want to
23 compile ISEQs in parallel with iseq execution to speed up Ruby
24 code execution. MJIT has one thread (*worker*) to do
25 parallel compilations:
26 o It prepares a precompiled code of the minimized header.
27 It starts at the MRI execution start
28 o It generates PIC object files of ISEQs
29 o It takes one JIT unit from a priority queue unless it is empty.
30 o It translates the JIT unit ISEQ into C-code using the precompiled
31 header, calls CC and load PIC code when it is ready
32 o Currently MJIT put ISEQ in the queue when ISEQ is called
33 o MJIT can reorder ISEQs in the queue if some ISEQ has been called
34 many times and its compilation did not start yet
35 o MRI reuses the machine code if it already exists for ISEQ
36 o The machine code we generate can stop and switch to the ISEQ
37 interpretation if some condition is not satisfied as the machine
38 code can be speculative or some exception raises
39 o Speculative machine code can be canceled.
40
41 Here is a diagram showing the MJIT organization:
42
43 _______
44 |header |
45 |_______|
46 | MRI building
47 --------------|----------------------------------------
48 | MRI execution
49 |
50 _____________|_____
51 | | |
52 | ___V__ | CC ____________________
53 | | |----------->| precompiled header |
54 | | | | |____________________|
55 | | | | |
56 | | MJIT | | |
57 | | | | |
58 | | | | ____V___ CC __________
59 | |______|----------->| C code |--->| .so file |
60 | | |________| |__________|
61 | | |
62 | | |
63 | MRI machine code |<-----------------------------
64 |___________________| loading
65
66*/
67
68#include "ruby/internal/config.h" // defines USE_MJIT
69
70#if USE_MJIT
71
72#include "constant.h"
73#include "id_table.h"
74#include "internal.h"
75#include "internal/class.h"
76#include "internal/cmdlineopt.h"
77#include "internal/cont.h"
78#include "internal/file.h"
79#include "internal/hash.h"
80#include "internal/process.h"
81#include "internal/warnings.h"
82#include "vm_sync.h"
83#include "ractor_core.h"
84
85#ifdef __sun
86#define __EXTENSIONS__ 1
87#endif
88
89#include "vm_core.h"
90#include "vm_callinfo.h"
91#include "mjit.h"
92#include "mjit_c.h"
93#include "gc.h"
94#include "ruby_assert.h"
95#include "ruby/debug.h"
96#include "ruby/thread.h"
97#include "ruby/version.h"
98#include "builtin.h"
99#include "insns.inc"
100#include "insns_info.inc"
101#include "internal/compile.h"
102
103#include <sys/wait.h>
104#include <sys/time.h>
105#include <dlfcn.h>
106#include <errno.h>
107#ifdef HAVE_FCNTL_H
108#include <fcntl.h>
109#endif
110#ifdef HAVE_SYS_PARAM_H
111# include <sys/param.h>
112#endif
113#include "dln.h"
114
115#include "ruby/util.h"
116#undef strdup // ruby_strdup may trigger GC
117
118#ifndef MAXPATHLEN
119# define MAXPATHLEN 1024
120#endif
121
122// Atomically set function pointer if possible.
123#define MJIT_ATOMIC_SET(var, val) (void)ATOMIC_PTR_EXCHANGE(var, val)
124
125#define MJIT_TMP_PREFIX "_ruby_mjit_"
126
131
132// process.c
133extern void mjit_add_waiting_pid(rb_vm_t *vm, rb_pid_t pid);
134
135// A copy of MJIT portion of MRI options since MJIT initialization. We
136// need them as MJIT threads still can work when the most MRI data were
137// freed.
138struct mjit_options mjit_opts;
139
140// true if MJIT is enabled.
141bool mjit_enabled = false;
142// true if JIT-ed code should be called. When `ruby_vm_event_enabled_global_flags & ISEQ_TRACE_EVENTS`
143// and `mjit_call_p == false`, any JIT-ed code execution is cancelled as soon as possible.
144bool mjit_call_p = false;
145// A flag to communicate that mjit_call_p should be disabled while it's temporarily false.
146bool mjit_cancel_p = false;
147// There's an ISEQ in unit_queue whose total_calls reached 2 * call_threshold.
148// If this is true, check_unit_queue will start compiling ISEQs in unit_queue.
149static bool mjit_compile_p = false;
150// The actual number of units in active_units
151static int active_units_length = 0;
152
153// Priority queue of iseqs waiting for JIT compilation.
154// This variable is a pointer to head unit of the queue.
155static struct rb_mjit_unit_list unit_queue = { CCAN_LIST_HEAD_INIT(unit_queue.head) };
156// List of units which are successfully compiled.
157static struct rb_mjit_unit_list active_units = { CCAN_LIST_HEAD_INIT(active_units.head) };
158// List of compacted so files which will be cleaned up by `free_list()` in `mjit_finish()`.
159static struct rb_mjit_unit_list compact_units = { CCAN_LIST_HEAD_INIT(compact_units.head) };
160// List of units before recompilation and just waiting for dlclose().
161static struct rb_mjit_unit_list stale_units = { CCAN_LIST_HEAD_INIT(stale_units.head) };
162// The number of so far processed ISEQs, used to generate unique id.
163static int current_unit_num;
164// A mutex for conitionals and critical sections.
165static rb_nativethread_lock_t mjit_engine_mutex;
166// Set to true to stop worker.
167static bool stop_worker_p;
168// Set to true if worker is stopped.
169static bool worker_stopped = true;
170
171// Path of "/tmp", which is different on Windows or macOS. See: system_default_tmpdir()
172static char *tmp_dir;
173
174// Used C compiler path.
175static const char *cc_path;
176// Used C compiler flags.
177static const char **cc_common_args;
178// Used C compiler flags added by --mjit-debug=...
179static char **cc_added_args;
180// Name of the precompiled header file.
181static char *pch_file;
182// The process id which should delete the pch_file on mjit_finish.
183static rb_pid_t pch_owner_pid;
184// Status of the precompiled header creation. The status is
185// shared by the workers and the pch thread.
186static enum {PCH_NOT_READY, PCH_FAILED, PCH_SUCCESS} pch_status;
187
188// The start timestamp of current compilation
189static double current_cc_ms = 0.0; // TODO: make this part of unit?
190// Currently compiling MJIT unit
191static struct rb_mjit_unit *current_cc_unit = NULL;
192// PID of currently running C compiler process. 0 if nothing is running.
193static pid_t current_cc_pid = 0; // TODO: make this part of unit?
194
195// Name of the header file.
196static char *header_file;
197
198#include "mjit_config.h"
199
200#if defined(__GNUC__) && \
201 (!defined(__clang__) || \
202 (defined(__clang__) && (defined(__FreeBSD__) || defined(__GLIBC__))))
203# define GCC_PIC_FLAGS "-Wfatal-errors", "-fPIC", "-shared", "-w", "-pipe",
204# define MJIT_CFLAGS_PIPE 1
205#else
206# define GCC_PIC_FLAGS /* empty */
207# define MJIT_CFLAGS_PIPE 0
208#endif
209
210// Use `-nodefaultlibs -nostdlib` for GCC where possible, which does not work on cygwin, AIX, and OpenBSD.
211// This seems to improve MJIT performance on GCC.
212#if defined __GNUC__ && !defined __clang__ && !defined(__CYGWIN__) && !defined(_AIX) && !defined(__OpenBSD__)
213# define GCC_NOSTDLIB_FLAGS "-nodefaultlibs", "-nostdlib",
214#else
215# define GCC_NOSTDLIB_FLAGS // empty
216#endif
217
218static const char *const CC_COMMON_ARGS[] = {
219 MJIT_CC_COMMON MJIT_CFLAGS GCC_PIC_FLAGS
220 NULL
221};
222
223static const char *const CC_DEBUG_ARGS[] = {MJIT_DEBUGFLAGS NULL};
224static const char *const CC_OPTIMIZE_ARGS[] = {MJIT_OPTFLAGS NULL};
225
226static const char *const CC_LDSHARED_ARGS[] = {MJIT_LDSHARED MJIT_CFLAGS GCC_PIC_FLAGS NULL};
227static const char *const CC_DLDFLAGS_ARGS[] = {MJIT_DLDFLAGS NULL};
228// `CC_LINKER_ARGS` are linker flags which must be passed to `-c` as well.
229static const char *const CC_LINKER_ARGS[] = {
230#if defined __GNUC__ && !defined __clang__ && !defined(__OpenBSD__)
231 "-nostartfiles",
232#endif
233 GCC_NOSTDLIB_FLAGS NULL
234};
235
236static const char *const CC_LIBS[] = {
237#if defined(__CYGWIN__)
238 MJIT_LIBS // mswin, cygwin
239#endif
240#if defined __GNUC__ && !defined __clang__
241 "-lgcc", // cygwin, and GCC platforms using `-nodefaultlibs -nostdlib`
242#endif
243#if defined __ANDROID__
244 "-lm", // to avoid 'cannot locate symbol "modf" referenced by .../_ruby_mjit_XXX.so"'
245#endif
246 NULL
247};
248
249#define CC_CODEFLAG_ARGS (mjit_opts.debug ? CC_DEBUG_ARGS : CC_OPTIMIZE_ARGS)
250
251// Print the arguments according to FORMAT to stderr only if MJIT
252// verbose option value is more or equal to LEVEL.
253PRINTF_ARGS(static void, 2, 3)
254verbose(int level, const char *format, ...)
255{
256 if (mjit_opts.verbose >= level) {
257 va_list args;
258 size_t len = strlen(format);
259 char *full_format = alloca(sizeof(char) * (len + 2));
260
261 // Creating `format + '\n'` to atomically print format and '\n'.
262 memcpy(full_format, format, len);
263 full_format[len] = '\n';
264 full_format[len+1] = '\0';
265
266 va_start(args, format);
267 vfprintf(stderr, full_format, args);
268 va_end(args);
269 }
270}
271
272PRINTF_ARGS(static void, 1, 2)
273mjit_warning(const char *format, ...)
274{
275 if (mjit_opts.warnings || mjit_opts.verbose) {
276 va_list args;
277
278 fprintf(stderr, "MJIT warning: ");
279 va_start(args, format);
280 vfprintf(stderr, format, args);
281 va_end(args);
282 fprintf(stderr, "\n");
283 }
284}
285
286// Add unit node to the tail of doubly linked `list`. It should be not in
287// the list before.
288static void
289add_to_list(struct rb_mjit_unit *unit, struct rb_mjit_unit_list *list)
290{
291 ccan_list_add_tail(&list->head, &unit->unode);
292 list->length++;
293}
294
295static void
296remove_from_list(struct rb_mjit_unit *unit, struct rb_mjit_unit_list *list)
297{
298 ccan_list_del(&unit->unode);
299 list->length--;
300}
301
302static void
303remove_file(const char *filename)
304{
305 if (remove(filename)) {
306 mjit_warning("failed to remove \"%s\": %s", filename, strerror(errno));
307 }
308}
309
310// This is called in the following situations:
311// 1) On dequeue or `unload_units()`, associated ISeq is already GCed.
312// 2) The unit is not called often and unloaded by `unload_units()`.
313// 3) Freeing lists on `mjit_finish()`.
314//
315// `jit_func` value does not matter for 1 and 3 since the unit won't be used anymore.
316// For the situation 2, this sets the ISeq's JIT state to MJIT_FUNC_FAILED
317// to prevent the situation that the same methods are continuously compiled.
318static void
319free_unit(struct rb_mjit_unit *unit)
320{
321 if (unit->iseq) { // ISeq is not GCed
322 ISEQ_BODY(unit->iseq)->jit_func = (jit_func_t)MJIT_FUNC_FAILED;
323 ISEQ_BODY(unit->iseq)->mjit_unit = NULL;
324 }
325 if (unit->cc_entries) {
326 void *entries = (void *)unit->cc_entries;
327 free(entries);
328 }
329 if (unit->handle && dlclose(unit->handle)) { // handle is NULL if it's in queue
330 mjit_warning("failed to close handle for u%d: %s", unit->id, dlerror());
331 }
332 xfree(unit);
333}
334
335// Start a critical section. Use message `msg` to print debug info at `level`.
336static inline void
337CRITICAL_SECTION_START(int level, const char *msg)
338{
339 verbose(level, "Locking %s", msg);
340 rb_native_mutex_lock(&mjit_engine_mutex);
341 verbose(level, "Locked %s", msg);
342}
343
344// Finish the current critical section. Use message `msg` to print
345// debug info at `level`.
346static inline void
347CRITICAL_SECTION_FINISH(int level, const char *msg)
348{
349 verbose(level, "Unlocked %s", msg);
350 rb_native_mutex_unlock(&mjit_engine_mutex);
351}
352
353static pid_t mjit_pid = 0;
354
355static int
356sprint_uniq_filename(char *str, size_t size, unsigned long id, const char *prefix, const char *suffix)
357{
358 return snprintf(str, size, "%s/%sp%"PRI_PIDT_PREFIX"uu%lu%s", tmp_dir, prefix, mjit_pid, id, suffix);
359}
360
361// Return time in milliseconds as a double.
362#ifdef __APPLE__
363double ruby_real_ms_time(void);
364# define real_ms_time() ruby_real_ms_time()
365#else
366static double
367real_ms_time(void)
368{
369# ifdef HAVE_CLOCK_GETTIME
370 struct timespec tv;
371# ifdef CLOCK_MONOTONIC
372 const clockid_t c = CLOCK_MONOTONIC;
373# else
374 const clockid_t c = CLOCK_REALTIME;
375# endif
376
377 clock_gettime(c, &tv);
378 return tv.tv_nsec / 1000000.0 + tv.tv_sec * 1000.0;
379# else
380 struct timeval tv;
381
382 gettimeofday(&tv, NULL);
383 return tv.tv_usec / 1000.0 + tv.tv_sec * 1000.0;
384# endif
385}
386#endif
387
388// Return the best unit from list. The best is the first
389// high priority unit or the unit whose iseq has the biggest number
390// of calls so far.
391static struct rb_mjit_unit *
392get_from_list(struct rb_mjit_unit_list *list)
393{
394 // Find iseq with max total_calls
395 struct rb_mjit_unit *unit = NULL, *next, *best = NULL;
396 ccan_list_for_each_safe(&list->head, unit, next, unode) {
397 if (unit->iseq == NULL) { // ISeq is GCed.
398 remove_from_list(unit, list);
399 free_unit(unit);
400 continue;
401 }
402
403 if (best == NULL || ISEQ_BODY(best->iseq)->total_calls < ISEQ_BODY(unit->iseq)->total_calls) {
404 best = unit;
405 }
406 }
407
408 if (best) {
409 remove_from_list(best, list);
410 }
411 return best;
412}
413
414// Return length of NULL-terminated array `args` excluding the NULL marker.
415static size_t
416args_len(char *const *args)
417{
418 size_t i;
419
420 for (i = 0; (args[i]) != NULL;i++)
421 ;
422 return i;
423}
424
425// Concatenate `num` passed NULL-terminated arrays of strings, put the
426// result (with NULL end marker) into the heap, and return the result.
427static char **
428form_args(int num, ...)
429{
430 va_list argp;
431 size_t len, n;
432 int i;
433 char **args, **res, **tmp;
434
435 va_start(argp, num);
436 res = NULL;
437 for (i = len = 0; i < num; i++) {
438 args = va_arg(argp, char **);
439 n = args_len(args);
440 if ((tmp = (char **)realloc(res, sizeof(char *) * (len + n + 1))) == NULL) {
441 free(res);
442 res = NULL;
443 break;
444 }
445 res = tmp;
446 MEMCPY(res + len, args, char *, n + 1);
447 len += n;
448 }
449 va_end(argp);
450 return res;
451}
452
453COMPILER_WARNING_PUSH
454#if __has_warning("-Wdeprecated-declarations") || RBIMPL_COMPILER_IS(GCC)
455COMPILER_WARNING_IGNORED(-Wdeprecated-declarations)
456#endif
457// Start an OS process of absolute executable path with arguments `argv`.
458// Return PID of the process.
459static pid_t
460start_process(const char *abspath, char *const *argv)
461{
462 // Not calling non-async-signal-safe functions between vfork
463 // and execv for safety
464 int dev_null = rb_cloexec_open(ruby_null_device, O_WRONLY, 0);
465 if (dev_null < 0) {
466 verbose(1, "MJIT: Failed to open a null device: %s", strerror(errno));
467 return -1;
468 }
469 if (mjit_opts.verbose >= 2) {
470 const char *arg;
471 fprintf(stderr, "Starting process: %s", abspath);
472 for (int i = 0; (arg = argv[i]) != NULL; i++)
473 fprintf(stderr, " %s", arg);
474 fprintf(stderr, "\n");
475 }
476
477 pid_t pid;
478 if ((pid = vfork()) == 0) { /* TODO: reuse some function in process.c */
479 umask(0077);
480 if (mjit_opts.verbose == 0) {
481 // CC can be started in a thread using a file which has been
482 // already removed while MJIT is finishing. Discard the
483 // messages about missing files.
484 dup2(dev_null, STDERR_FILENO);
485 dup2(dev_null, STDOUT_FILENO);
486 }
487 (void)close(dev_null);
488 pid = execv(abspath, argv); // Pid will be negative on an error
489 // Even if we successfully found CC to compile PCH we still can
490 // fail with loading the CC in very rare cases for some reasons.
491 // Stop the forked process in this case.
492 verbose(1, "MJIT: Error in execv: %s", abspath);
493 _exit(1);
494 }
495 (void)close(dev_null);
496 return pid;
497}
498COMPILER_WARNING_POP
499
500// Execute an OS process of executable PATH with arguments ARGV.
501// Return -1 or -2 if failed to execute, otherwise exit code of the process.
502// TODO: Use a similar function in process.c
503static int
504exec_process(const char *path, char *const argv[])
505{
506 int stat, exit_code = -2;
507 pid_t pid = start_process(path, argv);
508 for (;pid > 0;) {
509 pid_t r = waitpid(pid, &stat, 0);
510 if (r == -1) {
511 if (errno == EINTR) continue;
512 fprintf(stderr, "[%"PRI_PIDT_PREFIX"d] waitpid(%lu): %s (SIGCHLD=%d,%u)\n",
513 getpid(), (unsigned long)pid, strerror(errno),
514 RUBY_SIGCHLD, SIGCHLD_LOSSY);
515 break;
516 }
517 else if (r == pid) {
518 if (WIFEXITED(stat)) {
519 exit_code = WEXITSTATUS(stat);
520 break;
521 }
522 else if (WIFSIGNALED(stat)) {
523 exit_code = -1;
524 break;
525 }
526 }
527 }
528 return exit_code;
529}
530
531static void
532remove_so_file(const char *so_file, struct rb_mjit_unit *unit)
533{
534 remove_file(so_file);
535}
536
537// Print _mjitX, but make a human-readable funcname when --mjit-debug is used
538static void
539sprint_funcname(char *funcname, size_t funcname_size, const struct rb_mjit_unit *unit)
540{
541 const rb_iseq_t *iseq = unit->iseq;
542 if (iseq == NULL || (!mjit_opts.debug && !mjit_opts.debug_flags)) {
543 snprintf(funcname, funcname_size, "_mjit%d", unit->id);
544 return;
545 }
546
547 // Generate a short path
548 const char *path = RSTRING_PTR(rb_iseq_path(iseq));
549 const char *lib = "/lib/";
550 const char *version = "/" STRINGIZE(RUBY_API_VERSION_MAJOR) "." STRINGIZE(RUBY_API_VERSION_MINOR) "." STRINGIZE(RUBY_API_VERSION_TEENY) "/";
551 while (strstr(path, lib)) // skip "/lib/"
552 path = strstr(path, lib) + strlen(lib);
553 while (strstr(path, version)) // skip "/x.y.z/"
554 path = strstr(path, version) + strlen(version);
555
556 // Annotate all-normalized method names
557 const char *method = RSTRING_PTR(ISEQ_BODY(iseq)->location.label);
558 if (!strcmp(method, "[]")) method = "AREF";
559 if (!strcmp(method, "[]=")) method = "ASET";
560
561 // Print and normalize
562 snprintf(funcname, funcname_size, "_mjit%d_%s_%s", unit->id, path, method);
563 for (size_t i = 0; i < strlen(funcname); i++) {
564 char c = funcname[i];
565 if (!(('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || ('0' <= c && c <= '9') || c == '_')) {
566 funcname[i] = '_';
567 }
568 }
569}
570
571static const int c_file_access_mode =
572#ifdef O_BINARY
573 O_BINARY|
574#endif
575 O_WRONLY|O_EXCL|O_CREAT;
576
577#define append_str2(p, str, len) ((char *)memcpy((p), str, (len))+(len))
578#define append_str(p, str) append_str2(p, str, sizeof(str)-1)
579#define append_lit(p, str) append_str2(p, str, rb_strlen_lit(str))
580
581// The function producing the pre-compiled header.
582static void
583make_pch(void)
584{
585 const char *rest_args[] = {
586# ifdef __clang__
587 "-emit-pch",
588 "-c",
589# endif
590 // -nodefaultlibs is a linker flag, but it may affect cc1 behavior on Gentoo, which should NOT be changed on pch:
591 // https://gitweb.gentoo.org/proj/gcc-patches.git/tree/7.3.0/gentoo/13_all_default-ssp-fix.patch
592 GCC_NOSTDLIB_FLAGS
593 "-o", pch_file, header_file,
594 NULL,
595 };
596
597 verbose(2, "Creating precompiled header");
598 char **args = form_args(4, cc_common_args, CC_CODEFLAG_ARGS, cc_added_args, rest_args);
599 if (args == NULL) {
600 mjit_warning("making precompiled header failed on forming args");
601 pch_status = PCH_FAILED;
602 return;
603 }
604
605 int exit_code = exec_process(cc_path, args);
606 free(args);
607
608 if (exit_code == 0) {
609 pch_status = PCH_SUCCESS;
610 }
611 else {
612 mjit_warning("Making precompiled header failed on compilation. Stopping MJIT worker...");
613 pch_status = PCH_FAILED;
614 }
615}
616
617static int
618c_compile(const char *c_file, const char *so_file)
619{
620 const char *so_args[] = {
621 "-o", so_file,
622# ifdef __clang__
623 "-include-pch", pch_file,
624# endif
625 c_file, NULL
626 };
627
628# if defined(__MACH__)
629 extern VALUE rb_libruby_selfpath;
630 const char *loader_args[] = {"-bundle_loader", StringValuePtr(rb_libruby_selfpath), NULL};
631# else
632 const char *loader_args[] = {NULL};
633# endif
634
635 char **args = form_args(8, CC_LDSHARED_ARGS, CC_CODEFLAG_ARGS, cc_added_args,
636 so_args, loader_args, CC_LIBS, CC_DLDFLAGS_ARGS, CC_LINKER_ARGS);
637 if (args == NULL) return 1;
638
639 int exit_code = exec_process(cc_path, args);
640 if (!mjit_opts.save_temps)
641 remove_file(c_file);
642
643 free(args);
644 return exit_code;
645}
646
647static int
648c_compile_unit(struct rb_mjit_unit *unit)
649{
650 static const char c_ext[] = ".c";
651 static const char so_ext[] = DLEXT;
652 char c_file[MAXPATHLEN], so_file[MAXPATHLEN];
653
654 sprint_uniq_filename(c_file, (int)sizeof(c_file), unit->id, MJIT_TMP_PREFIX, c_ext);
655 sprint_uniq_filename(so_file, (int)sizeof(so_file), unit->id, MJIT_TMP_PREFIX, so_ext);
656
657 return c_compile(c_file, so_file);
658}
659
660static void compile_prelude(FILE *f);
661
662static bool
663mjit_batch(struct rb_mjit_unit *unit)
664{
665 VM_ASSERT(unit->type == MJIT_UNIT_BATCH);
666 static const char c_ext[] = ".c";
667 static const char so_ext[] = DLEXT;
668 char c_file[MAXPATHLEN], so_file[MAXPATHLEN];
669
670 sprint_uniq_filename(c_file, (int)sizeof(c_file), unit->id, MJIT_TMP_PREFIX, c_ext);
671 sprint_uniq_filename(so_file, (int)sizeof(so_file), unit->id, MJIT_TMP_PREFIX, so_ext);
672
673 FILE *f;
674 int fd = rb_cloexec_open(c_file, c_file_access_mode, 0600);
675 if (fd < 0 || (f = fdopen(fd, "w")) == NULL) {
676 int e = errno;
677 if (fd >= 0) (void)close(fd);
678 verbose(1, "Failed to fopen '%s', giving up JIT for it (%s)", c_file, strerror(e));
679 return false;
680 }
681
682 compile_prelude(f);
683
684 bool success = true;
685 struct rb_mjit_unit *child_unit = 0;
686 ccan_list_for_each(&unit->units.head, child_unit, unode) {
687 if (!success) continue;
688 if (child_unit->iseq == NULL) continue; // ISEQ is GCed
689
690 char funcname[MAXPATHLEN];
691 sprint_funcname(funcname, sizeof(funcname), child_unit);
692
693 int iseq_lineno = ISEQ_BODY(child_unit->iseq)->location.first_lineno;
694 const char *sep = "@";
695 const char *iseq_label = RSTRING_PTR(ISEQ_BODY(child_unit->iseq)->location.label);
696 const char *iseq_path = RSTRING_PTR(rb_iseq_path(child_unit->iseq));
697 if (!iseq_label) iseq_label = sep = "";
698 fprintf(f, "\n/* %s%s%s:%d */\n", iseq_label, sep, iseq_path, iseq_lineno);
699 success &= mjit_compile(f, child_unit->iseq, funcname, child_unit->id);
700 }
701
702 fclose(f);
703 return success;
704}
705
706// Compile all cached .c files and build a single .so file. Reload all JIT func from it.
707// This improves the code locality for better performance in terms of iTLB and iCache.
708static bool
709mjit_compact(struct rb_mjit_unit *unit)
710{
711 VM_ASSERT(unit->type == MJIT_UNIT_COMPACT);
712 static const char c_ext[] = ".c";
713 static const char so_ext[] = DLEXT;
714 char c_file[MAXPATHLEN], so_file[MAXPATHLEN];
715
716 sprint_uniq_filename(c_file, (int)sizeof(c_file), unit->id, MJIT_TMP_PREFIX, c_ext);
717 sprint_uniq_filename(so_file, (int)sizeof(so_file), unit->id, MJIT_TMP_PREFIX, so_ext);
718
719 FILE *f;
720 int fd = rb_cloexec_open(c_file, c_file_access_mode, 0600);
721 if (fd < 0 || (f = fdopen(fd, "w")) == NULL) {
722 int e = errno;
723 if (fd >= 0) (void)close(fd);
724 verbose(1, "Failed to fopen '%s', giving up JIT for it (%s)", c_file, strerror(e));
725 return false;
726 }
727
728 compile_prelude(f);
729
730 bool success = true;
731 struct rb_mjit_unit *batch_unit = 0, *child_unit = 0;
732 ccan_list_for_each(&active_units.head, batch_unit, unode) {
733 ccan_list_for_each(&batch_unit->units.head, child_unit, unode) {
734 if (!success) continue;
735 if (child_unit->iseq == NULL) continue; // ISEQ is GCed
736
737 char funcname[MAXPATHLEN];
738 sprint_funcname(funcname, sizeof(funcname), child_unit);
739
740 int iseq_lineno = ISEQ_BODY(child_unit->iseq)->location.first_lineno;
741 const char *sep = "@";
742 const char *iseq_label = RSTRING_PTR(ISEQ_BODY(child_unit->iseq)->location.label);
743 const char *iseq_path = RSTRING_PTR(rb_iseq_path(child_unit->iseq));
744 if (!iseq_label) iseq_label = sep = "";
745 fprintf(f, "\n/* %s%s%s:%d */\n", iseq_label, sep, iseq_path, iseq_lineno);
746 success &= mjit_compile(f, child_unit->iseq, funcname, child_unit->id);
747 }
748 }
749
750 fclose(f);
751 return success;
752}
753
754static void
755load_batch_funcs_from_so(struct rb_mjit_unit *unit, char *c_file, char *so_file)
756{
757 double end_time = real_ms_time();
758
759 void *handle = dlopen(so_file, RTLD_NOW);
760 if (handle == NULL) {
761 mjit_warning("failure in loading code from batched '%s': %s", so_file, dlerror());
762 xfree(unit);
763 return;
764 }
765 unit->handle = handle;
766
767 // lazily dlclose handle on `mjit_finish()`.
768 add_to_list(unit, &active_units);
769 active_units_length += unit->units.length;
770
771 if (!mjit_opts.save_temps)
772 remove_so_file(so_file, unit);
773
774 struct rb_mjit_unit *child_unit = 0;
775 ccan_list_for_each(&unit->units.head, child_unit, unode) {
776 char funcname[MAXPATHLEN];
777 sprint_funcname(funcname, sizeof(funcname), child_unit);
778
779 void *func;
780 if ((func = dlsym(handle, funcname)) == NULL) {
781 mjit_warning("skipping to load '%s' from '%s': %s", funcname, so_file, dlerror());
782 continue;
783 }
784
785 if (child_unit->iseq) { // Check whether GCed or not
786 // Usage of jit_code might be not in a critical section.
787 const rb_iseq_t *iseq = child_unit->iseq;
788 MJIT_ATOMIC_SET(ISEQ_BODY(iseq)->jit_func, (jit_func_t)func);
789
790 verbose(1, "JIT success: %s@%s:%d",
791 RSTRING_PTR(ISEQ_BODY(iseq)->location.label),
792 RSTRING_PTR(rb_iseq_path(iseq)), ISEQ_BODY(iseq)->location.first_lineno);
793 }
794 else {
795 verbose(1, "JIT skip: A compiled method has been GCed");
796 }
797 }
798 verbose(1, "JIT batch (%.1fms): Batched %d methods %s -> %s", end_time - current_cc_ms, unit->units.length, c_file, so_file);
799}
800
801static void
802load_compact_funcs_from_so(struct rb_mjit_unit *unit, char *c_file, char *so_file)
803{
804 double end_time = real_ms_time();
805
806 void *handle = dlopen(so_file, RTLD_NOW);
807 if (handle == NULL) {
808 mjit_warning("failure in loading code from compacted '%s': %s", so_file, dlerror());
809 xfree(unit);
810 return;
811 }
812 unit->handle = handle;
813
814 // lazily dlclose handle on `mjit_finish()`.
815 add_to_list(unit, &compact_units);
816
817 if (!mjit_opts.save_temps)
818 remove_so_file(so_file, unit);
819
820 struct rb_mjit_unit *batch_unit = 0, *child_unit = 0;
821 ccan_list_for_each(&active_units.head, batch_unit, unode) {
822 ccan_list_for_each(&batch_unit->units.head, child_unit, unode) {
823 if (child_unit->iseq == NULL) continue; // ISEQ is GCed
824
825 char funcname[MAXPATHLEN];
826 sprint_funcname(funcname, sizeof(funcname), child_unit);
827
828 void *func;
829 if ((func = dlsym(handle, funcname)) == NULL) {
830 mjit_warning("skipping to reload '%s' from '%s': %s", funcname, so_file, dlerror());
831 continue;
832 }
833
834 if (child_unit->iseq) { // Check whether GCed or not
835 // Usage of jit_code might be not in a critical section.
836 MJIT_ATOMIC_SET(ISEQ_BODY(child_unit->iseq)->jit_func, (jit_func_t)func);
837 }
838 }
839 }
840 verbose(1, "JIT compaction (%.1fms): Compacted %d methods %s -> %s", end_time - current_cc_ms, active_units_length, c_file, so_file);
841}
842
843#ifndef __clang__
844static const char *
845header_name_end(const char *s)
846{
847 const char *e = s + strlen(s);
848# ifdef __GNUC__ // don't chomp .pch for mswin
849 static const char suffix[] = ".gch";
850
851 // chomp .gch suffix
852 if (e > s+sizeof(suffix)-1 && strcmp(e-sizeof(suffix)+1, suffix) == 0) {
853 e -= sizeof(suffix)-1;
854 }
855# endif
856 return e;
857}
858#endif
859
860// Print platform-specific prerequisites in generated code.
861static void
862compile_prelude(FILE *f)
863{
864#ifndef __clang__ // -include-pch is used for Clang
865 const char *s = pch_file;
866 const char *e = header_name_end(s);
867
868 fprintf(f, "#include \"");
869 // print pch_file except .gch for gcc, but keep .pch for mswin
870 for (; s < e; s++) {
871 switch (*s) {
872 case '\\': case '"':
873 fputc('\\', f);
874 }
875 fputc(*s, f);
876 }
877 fprintf(f, "\"\n");
878#endif
879}
880
881static pid_t
882start_c_compile_unit(struct rb_mjit_unit *unit)
883{
884 extern pid_t rb_mjit_fork();
885 pid_t pid = rb_mjit_fork();
886 if (pid == 0) {
887 int exit_code = c_compile_unit(unit);
888 exit(exit_code);
889 }
890 else {
891 return pid;
892 }
893}
894
895// Capture cc entries of `captured_iseq` and append them to `compiled_iseq->mjit_unit->cc_entries`.
896// This is needed when `captured_iseq` is inlined by `compiled_iseq` and GC needs to mark inlined cc.
897//
898// Index to refer to `compiled_iseq->mjit_unit->cc_entries` is returned instead of the address
899// because old addresses may be invalidated by `realloc` later. -1 is returned on failure.
900//
901// This assumes that it's safe to reference cc without acquiring GVL.
902int
903mjit_capture_cc_entries(const struct rb_iseq_constant_body *compiled_iseq, const struct rb_iseq_constant_body *captured_iseq)
904{
905 VM_ASSERT(compiled_iseq != NULL);
906 VM_ASSERT(compiled_iseq->mjit_unit != NULL);
907 VM_ASSERT(captured_iseq != NULL);
908
909 struct rb_mjit_unit *unit = compiled_iseq->mjit_unit;
910 unsigned int new_entries_size = unit->cc_entries_size + captured_iseq->ci_size;
911 VM_ASSERT(captured_iseq->ci_size > 0);
912
913 // Allocate new cc_entries and append them to unit->cc_entries
914 const struct rb_callcache **cc_entries;
915 int cc_entries_index = unit->cc_entries_size;
916 if (unit->cc_entries_size == 0) {
917 VM_ASSERT(unit->cc_entries == NULL);
918 unit->cc_entries = cc_entries = malloc(sizeof(struct rb_callcache *) * new_entries_size);
919 if (cc_entries == NULL) return -1;
920 }
921 else {
922 void *cc_ptr = (void *)unit->cc_entries; // get rid of bogus warning by VC
923 cc_entries = realloc(cc_ptr, sizeof(struct rb_callcache *) * new_entries_size);
924 if (cc_entries == NULL) return -1;
925 unit->cc_entries = cc_entries;
926 cc_entries += cc_entries_index;
927 }
928 unit->cc_entries_size = new_entries_size;
929
930 // Capture cc to cc_enties
931 for (unsigned int i = 0; i < captured_iseq->ci_size; i++) {
932 cc_entries[i] = captured_iseq->call_data[i].cc;
933 }
934
935 return cc_entries_index;
936}
937
938static void mjit_add_iseq_to_process(const rb_iseq_t *iseq, const struct rb_mjit_compile_info *compile_info);
939
940// Return an unique file name in /tmp with PREFIX and SUFFIX and
941// number ID. Use getpid if ID == 0. The return file name exists
942// until the next function call.
943static char *
944get_uniq_filename(unsigned long id, const char *prefix, const char *suffix)
945{
946 char buff[70], *str = buff;
947 int size = sprint_uniq_filename(buff, sizeof(buff), id, prefix, suffix);
948 str = 0;
949 ++size;
950 str = xmalloc(size);
951 if (size <= (int)sizeof(buff)) {
952 memcpy(str, buff, size);
953 }
954 else {
955 sprint_uniq_filename(str, size, id, prefix, suffix);
956 }
957 return str;
958}
959
960// Prohibit calling JIT-ed code and let existing JIT-ed frames exit before the next insn.
961void
962mjit_cancel_all(const char *reason)
963{
964 if (!mjit_enabled)
965 return;
966
967 mjit_call_p = false;
968 mjit_cancel_p = true;
969 if (mjit_opts.warnings || mjit_opts.verbose) {
970 fprintf(stderr, "JIT cancel: Disabled JIT-ed code because %s\n", reason);
971 }
972}
973
974// Deal with ISeq movement from compactor
975void
976mjit_update_references(const rb_iseq_t *iseq)
977{
978 if (!mjit_enabled)
979 return;
980
981 CRITICAL_SECTION_START(4, "mjit_update_references");
982 if (ISEQ_BODY(iseq)->mjit_unit) {
983 ISEQ_BODY(iseq)->mjit_unit->iseq = (rb_iseq_t *)rb_gc_location((VALUE)ISEQ_BODY(iseq)->mjit_unit->iseq);
984 // We need to invalidate JIT-ed code for the ISeq because it embeds pointer addresses.
985 // To efficiently do that, we use the same thing as TracePoint and thus everything is cancelled for now.
986 // See mjit.h and tool/ruby_vm/views/_mjit_compile_insn.erb for how `mjit_call_p` is used.
987 mjit_cancel_all("GC.compact is used"); // TODO: instead of cancelling all, invalidate only this one and recompile it with some threshold.
988 }
989
990 // Units in stale_units (list of over-speculated and invalidated code) are not referenced from
991 // `ISEQ_BODY(iseq)->mjit_unit` anymore (because new one replaces that). So we need to check them too.
992 // TODO: we should be able to reduce the number of units checked here.
993 struct rb_mjit_unit *unit = NULL;
994 ccan_list_for_each(&stale_units.head, unit, unode) {
995 if (unit->iseq == iseq) {
996 unit->iseq = (rb_iseq_t *)rb_gc_location((VALUE)unit->iseq);
997 }
998 }
999 CRITICAL_SECTION_FINISH(4, "mjit_update_references");
1000}
1001
1002// Iseqs can be garbage collected. This function should call when it
1003// happens. It removes iseq from the unit.
1004void
1005mjit_free_iseq(const rb_iseq_t *iseq)
1006{
1007 if (!mjit_enabled)
1008 return;
1009
1010 if (ISEQ_BODY(iseq)->mjit_unit) {
1011 // mjit_unit is not freed here because it may be referred by multiple
1012 // lists of units. `get_from_list` and `mjit_finish` do the job.
1013 ISEQ_BODY(iseq)->mjit_unit->iseq = NULL;
1014 }
1015 // Units in stale_units (list of over-speculated and invalidated code) are not referenced from
1016 // `ISEQ_BODY(iseq)->mjit_unit` anymore (because new one replaces that). So we need to check them too.
1017 // TODO: we should be able to reduce the number of units checked here.
1018 struct rb_mjit_unit *unit = NULL;
1019 ccan_list_for_each(&stale_units.head, unit, unode) {
1020 if (unit->iseq == iseq) {
1021 unit->iseq = NULL;
1022 }
1023 }
1024}
1025
1026// Free unit list. This should be called only when worker is finished
1027// because node of unit_queue and one of active_units may have the same unit
1028// during proceeding unit.
1029static void
1030free_list(struct rb_mjit_unit_list *list, bool close_handle_p)
1031{
1032 struct rb_mjit_unit *unit = 0, *next;
1033
1034 ccan_list_for_each_safe(&list->head, unit, next, unode) {
1035 ccan_list_del(&unit->unode);
1036 if (!close_handle_p) unit->handle = NULL; /* Skip dlclose in free_unit() */
1037
1038 if (list == &stale_units) { // `free_unit(unit)` crashes after GC.compact on `stale_units`
1039 /*
1040 * TODO: REVERT THIS BRANCH
1041 * Debug the crash on stale_units w/ GC.compact and just use `free_unit(unit)`!!
1042 */
1043 if (unit->handle && dlclose(unit->handle)) {
1044 mjit_warning("failed to close handle for u%d: %s", unit->id, dlerror());
1045 }
1046 xfree(unit);
1047 }
1048 else {
1049 free_unit(unit);
1050 }
1051 }
1052 list->length = 0;
1053}
1054
1055static struct rb_mjit_unit*
1056create_unit(enum rb_mjit_unit_type type)
1057{
1058 struct rb_mjit_unit *unit = ZALLOC_N(struct rb_mjit_unit, 1);
1059 unit->id = current_unit_num++;
1060 unit->type = type;
1061 if (type == MJIT_UNIT_BATCH) {
1062 ccan_list_head_init(&unit->units.head);
1063 }
1064 return unit;
1065}
1066
1067static struct rb_mjit_unit*
1068create_iseq_unit(const rb_iseq_t *iseq)
1069{
1070 struct rb_mjit_unit *unit = create_unit(MJIT_UNIT_ISEQ);
1071 unit->iseq = (rb_iseq_t *)iseq;
1072 ISEQ_BODY(iseq)->mjit_unit = unit;
1073 return unit;
1074}
1075
1076static void mjit_wait(struct rb_mjit_unit *unit);
1077
1078// Check the unit queue and start mjit_compile if nothing is in progress.
1079static void
1080check_unit_queue(void)
1081{
1082 if (mjit_opts.custom) return; // Custom RubyVM::MJIT.compile is in use
1083 if (worker_stopped) return;
1084 if (current_cc_pid != 0) return; // still compiling
1085
1086 // TODO: resurrect unload_units
1087 if (active_units_length >= mjit_opts.max_cache_size) return; // wait until unload_units makes a progress
1088
1089 // No ISEQ in unit_queue has enough calls to trigger JIT
1090 if (!mjit_compile_p) return;
1091 mjit_compile_p = false;
1092
1093 // Compile all ISEQs in unit_queue together
1094 struct rb_mjit_unit *unit = create_unit(MJIT_UNIT_BATCH);
1095 struct rb_mjit_unit *child_unit = NULL;
1096 VM_ASSERT(unit_queue.length > 0);
1097 while ((child_unit = get_from_list(&unit_queue)) != NULL && (active_units_length + unit->units.length) < mjit_opts.max_cache_size) {
1098 add_to_list(child_unit, &unit->units);
1099 ISEQ_BODY(child_unit->iseq)->jit_func = (jit_func_t)MJIT_FUNC_COMPILING;
1100 }
1101
1102 // Run the MJIT compiler synchronously
1103 current_cc_ms = real_ms_time();
1104 current_cc_unit = unit;
1105 bool success = mjit_batch(unit);
1106 if (!success) {
1107 mjit_notify_waitpid(1);
1108 return;
1109 }
1110
1111 // Run the C compiler asynchronously (unless --mjit-wait)
1112 if (mjit_opts.wait) {
1113 int exit_code = c_compile_unit(unit);
1114 mjit_notify_waitpid(exit_code);
1115 }
1116 else {
1117 current_cc_pid = start_c_compile_unit(unit);
1118 if (current_cc_pid == -1) { // JIT failure
1119 mjit_notify_waitpid(1);
1120 }
1121 }
1122}
1123
1124// Check if it should compact all JIT code and start it as needed
1125static void
1126check_compaction(void)
1127{
1128 // Allow only `max_cache_size / 100` times (default: 100) of compaction.
1129 // Note: GC of compacted code has not been implemented yet.
1130 int max_compact_size = mjit_opts.max_cache_size / 100;
1131 if (max_compact_size < 10) max_compact_size = 10;
1132
1133 if (active_units_length == mjit_opts.max_cache_size) {
1134 struct rb_mjit_unit *unit = create_unit(MJIT_UNIT_COMPACT);
1135
1136 // Run the MJIT compiler synchronously
1137 current_cc_ms = real_ms_time();
1138 current_cc_unit = unit;
1139 bool success = mjit_compact(unit);
1140 if (!success) {
1141 mjit_notify_waitpid(1);
1142 return;
1143 }
1144
1145 // Run the C compiler asynchronously (unless --mjit-wait)
1146 if (mjit_opts.wait) {
1147 int exit_code = c_compile_unit(unit);
1148 mjit_notify_waitpid(exit_code);
1149 }
1150 else {
1151 current_cc_pid = start_c_compile_unit(unit);
1152 if (current_cc_pid == -1) { // JIT failure
1153 mjit_notify_waitpid(1);
1154 }
1155 }
1156 }
1157}
1158
1159// Check the current CC process if any, and start a next C compiler process as needed.
1160void
1161mjit_notify_waitpid(int exit_code)
1162{
1163 VM_ASSERT(mjit_opts.wait || current_cc_pid != 0);
1164 current_cc_pid = 0;
1165
1166 // Delete .c file
1167 char c_file[MAXPATHLEN];
1168 sprint_uniq_filename(c_file, (int)sizeof(c_file), current_cc_unit->id, MJIT_TMP_PREFIX, ".c");
1169
1170 // Check the result
1171 if (exit_code != 0) {
1172 verbose(2, "Failed to generate so");
1173 // TODO: set MJIT_FUNC_FAILED to unit->units
1174 // TODO: free list of unit->units
1175 free_unit(current_cc_unit);
1176 current_cc_unit = NULL;
1177 return;
1178 }
1179
1180 // Load .so file
1181 char so_file[MAXPATHLEN];
1182 sprint_uniq_filename(so_file, (int)sizeof(so_file), current_cc_unit->id, MJIT_TMP_PREFIX, DLEXT);
1183 switch (current_cc_unit->type) {
1184 case MJIT_UNIT_ISEQ:
1185 rb_bug("unreachable: current_cc_unit->type must not be MJIT_UNIT_ISEQ");
1186 case MJIT_UNIT_BATCH:
1187 load_batch_funcs_from_so(current_cc_unit, c_file, so_file);
1188 current_cc_unit = NULL;
1189
1190 // Run compaction if it should
1191 if (!stop_worker_p) {
1192 check_compaction();
1193 }
1194 break;
1195 case MJIT_UNIT_COMPACT:
1196 load_compact_funcs_from_so(current_cc_unit, c_file, so_file);
1197 current_cc_unit = NULL;
1198 break;
1199 }
1200
1201 // Skip further compilation if mjit_finish is trying to stop it
1202 if (!stop_worker_p) {
1203 // Start the next one as needed
1204 check_unit_queue();
1205 }
1206}
1207
1208// Return true if given ISeq body should be compiled by MJIT
1209static inline int
1210mjit_target_iseq_p(const rb_iseq_t *iseq)
1211{
1212 struct rb_iseq_constant_body *body = ISEQ_BODY(iseq);
1213 return (body->type == ISEQ_TYPE_METHOD || body->type == ISEQ_TYPE_BLOCK)
1214 && !body->builtin_inline_p
1215 && strcmp("<internal:mjit>", RSTRING_PTR(rb_iseq_path(iseq))) != 0;
1216}
1217
1218// RubyVM::MJIT
1219static VALUE rb_mMJIT = 0;
1220// RubyVM::MJIT::C
1221static VALUE rb_mMJITC = 0;
1222// RubyVM::MJIT::Compiler
1223static VALUE rb_cMJITCompiler = 0;
1224// RubyVM::MJIT::CPointer::Struct_rb_iseq_t
1225static VALUE rb_cMJITIseqPtr = 0;
1226// RubyVM::MJIT::CPointer::Struct_IC
1227static VALUE rb_cMJITICPtr = 0;
1228// RubyVM::MJIT::Compiler
1229static VALUE rb_mMJITHooks = 0;
1230
1231#define WITH_MJIT_DISABLED(stmt) do { \
1232 bool original_call_p = mjit_call_p; \
1233 mjit_call_p = false; \
1234 stmt; \
1235 mjit_call_p = original_call_p; \
1236 if (mjit_cancel_p) mjit_call_p = false; \
1237} while (0);
1238
1239// Hook MJIT when BOP is redefined.
1240MJIT_FUNC_EXPORTED void
1241rb_mjit_bop_redefined(int redefined_flag, enum ruby_basic_operators bop)
1242{
1243 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks) return;
1244 WITH_MJIT_DISABLED({
1245 rb_funcall(rb_mMJITHooks, rb_intern("on_bop_redefined"), 2, INT2NUM(redefined_flag), INT2NUM((int)bop));
1246 });
1247}
1248
1249// Hook MJIT when CME is invalidated.
1250MJIT_FUNC_EXPORTED void
1251rb_mjit_cme_invalidate(rb_callable_method_entry_t *cme)
1252{
1253 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks) return;
1254 WITH_MJIT_DISABLED({
1255 VALUE cme_klass = rb_funcall(rb_mMJITC, rb_intern("rb_callable_method_entry_struct"), 0);
1256 VALUE cme_ptr = rb_funcall(cme_klass, rb_intern("new"), 1, SIZET2NUM((size_t)cme));
1257 rb_funcall(rb_mMJITHooks, rb_intern("on_cme_invalidate"), 1, cme_ptr);
1258 });
1259}
1260
1261// Hook MJIT when Ractor is spawned.
1262void
1263rb_mjit_before_ractor_spawn(void)
1264{
1265 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks) return;
1266 WITH_MJIT_DISABLED({
1267 rb_funcall(rb_mMJITHooks, rb_intern("on_ractor_spawn"), 0);
1268 });
1269}
1270
1271static void
1272mjit_constant_state_changed(void *data)
1273{
1274 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks) return;
1275 ID id = (ID)data;
1276 WITH_MJIT_DISABLED({
1277 rb_funcall(rb_mMJITHooks, rb_intern("on_constant_state_changed"), 1, ID2SYM(id));
1278 });
1279}
1280
1281// Hook MJIT when constant state is changed.
1282MJIT_FUNC_EXPORTED void
1283rb_mjit_constant_state_changed(ID id)
1284{
1285 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks) return;
1286 // Asynchronously hook the Ruby code since this is hooked during a "Ruby critical section".
1287 extern int rb_workqueue_register(unsigned flags, rb_postponed_job_func_t func, void *data);
1288 rb_workqueue_register(0, mjit_constant_state_changed, (void *)id);
1289}
1290
1291// Hook MJIT when constant IC is updated.
1292MJIT_FUNC_EXPORTED void
1293rb_mjit_constant_ic_update(const rb_iseq_t *const iseq, IC ic, unsigned insn_idx)
1294{
1295 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks) return;
1296 WITH_MJIT_DISABLED({
1297 VALUE iseq_ptr = rb_funcall(rb_cMJITIseqPtr, rb_intern("new"), 1, SIZET2NUM((size_t)iseq));
1298 VALUE ic_ptr = rb_funcall(rb_cMJITICPtr, rb_intern("new"), 1, SIZET2NUM((size_t)ic));
1299 rb_funcall(rb_mMJITHooks, rb_intern("on_constant_ic_update"), 3, iseq_ptr, ic_ptr, UINT2NUM(insn_idx));
1300 });
1301}
1302
1303// Hook MJIT when TracePoint is enabled.
1304MJIT_FUNC_EXPORTED void
1305rb_mjit_tracing_invalidate_all(rb_event_flag_t new_iseq_events)
1306{
1307 if (!mjit_enabled || !mjit_call_p || !rb_mMJITHooks) return;
1308 WITH_MJIT_DISABLED({
1309 rb_funcall(rb_mMJITHooks, rb_intern("on_tracing_invalidate_all"), 1, UINT2NUM(new_iseq_events));
1310 });
1311}
1312
1313// [experimental] Call custom RubyVM::MJIT.compile if defined
1314static void
1315mjit_hook_custom_compile(const rb_iseq_t *iseq)
1316{
1317 WITH_MJIT_DISABLED({
1318 VALUE iseq_class = rb_funcall(rb_mMJITC, rb_intern("rb_iseq_t"), 0);
1319 VALUE iseq_ptr = rb_funcall(iseq_class, rb_intern("new"), 1, ULONG2NUM((size_t)iseq));
1320 VALUE jit_func = rb_funcall(rb_mMJIT, rb_intern("compile"), 1, iseq_ptr);
1321 ISEQ_BODY(iseq)->jit_func = (jit_func_t)NUM2ULONG(jit_func);
1322 });
1323}
1324
1325static void
1326mjit_add_iseq_to_process(const rb_iseq_t *iseq, const struct rb_mjit_compile_info *compile_info)
1327{
1328 if (!mjit_enabled) return;
1329 if (mjit_opts.custom) { // Hook custom RubyVM::MJIT.compile if defined
1330 mjit_hook_custom_compile(iseq);
1331 return;
1332 }
1333 if (pch_status != PCH_SUCCESS || !rb_ractor_main_p()) // TODO: Support non-main Ractors
1334 return;
1335 if (!mjit_target_iseq_p(iseq)) {
1336 ISEQ_BODY(iseq)->jit_func = (jit_func_t)MJIT_FUNC_FAILED; // skip mjit_wait
1337 return;
1338 }
1339
1340 // For batching multiple ISEQs, we only enqueue ISEQs when total_calls reaches call_threshold,
1341 // and compile all enqueued ISEQs when any ISEQ reaches call_threshold * 2.
1342 bool recompile_p = !MJIT_FUNC_STATE_P(ISEQ_BODY(iseq)->jit_func);
1343 if (!ISEQ_BODY(iseq)->mjit_unit || recompile_p) { // call_threshold, or recompile
1344 // Discard an old unit with recompile_p
1345 if (recompile_p) {
1346 ISEQ_BODY(iseq)->mjit_unit->iseq = NULL; // Ignore this from compaction
1347 ISEQ_BODY(iseq)->jit_func = (jit_func_t)MJIT_FUNC_NOT_COMPILED;
1348 active_units_length--;
1349 }
1350
1351 // Create a new unit and enqueue it
1352 struct rb_mjit_unit *unit = create_iseq_unit(iseq);
1353 if (recompile_p) {
1354 VM_ASSERT(compile_info != NULL);
1355 unit->compile_info = *compile_info;
1356 }
1357 add_to_list(unit, &unit_queue);
1358 ISEQ_BODY(iseq)->total_calls = 0; // come here again :)
1359 }
1360 else { // call_threshold * 2
1361 VM_ASSERT(compile_info == NULL);
1362 mjit_compile_p = true; // compile all ISEQs in unit_queue
1363 }
1364}
1365
1366// Add ISEQ to be JITed in parallel with the current thread.
1367// Unload some JIT codes if there are too many of them.
1368void
1369rb_mjit_add_iseq_to_process(const rb_iseq_t *iseq)
1370{
1371 mjit_add_iseq_to_process(iseq, NULL);
1372 check_unit_queue();
1373}
1374
1375// For this timeout seconds, mjit_finish will wait for JIT compilation finish.
1376#define MJIT_WAIT_TIMEOUT_SECONDS 5
1377
1378static void
1379mjit_wait(struct rb_mjit_unit *unit)
1380{
1381 pid_t initial_pid = current_cc_pid;
1382 if (initial_pid == 0) {
1383 mjit_warning("initial_pid was 0 on mjit_wait");
1384 return;
1385 }
1386 if (pch_status == PCH_FAILED) return;
1387
1388 int tries = 0;
1389 struct timeval tv = { .tv_sec = 0, .tv_usec = 1000 };
1390 while (current_cc_pid == initial_pid) {
1391 tries++;
1392 if (tries / 1000 > MJIT_WAIT_TIMEOUT_SECONDS) {
1393 if (unit->type == MJIT_UNIT_ISEQ) {
1394 unit->iseq->body->jit_func = (jit_func_t)MJIT_FUNC_FAILED; // C compiler was too slow. Give up.
1395 }
1396 mjit_warning("timed out to wait for JIT finish");
1397 break;
1398 }
1399
1401 }
1402}
1403
1405rb_mjit_iseq_compile_info(const struct rb_iseq_constant_body *body)
1406{
1407 VM_ASSERT(body->mjit_unit != NULL);
1408 return &body->mjit_unit->compile_info;
1409}
1410
1411static void
1412mjit_recompile(const rb_iseq_t *iseq)
1413{
1414 if (MJIT_FUNC_STATE_P(ISEQ_BODY(iseq)->jit_func))
1415 return;
1416
1417 verbose(1, "JIT recompile: %s@%s:%d", RSTRING_PTR(ISEQ_BODY(iseq)->location.label),
1418 RSTRING_PTR(rb_iseq_path(iseq)), ISEQ_BODY(iseq)->location.first_lineno);
1419 VM_ASSERT(ISEQ_BODY(iseq)->mjit_unit != NULL);
1420
1421 mjit_add_iseq_to_process(iseq, &ISEQ_BODY(iseq)->mjit_unit->compile_info);
1422 check_unit_queue();
1423}
1424
1425// Recompile iseq, disabling send optimization
1426void
1427rb_mjit_recompile_send(const rb_iseq_t *iseq)
1428{
1429 rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_send_cache = true;
1430 mjit_recompile(iseq);
1431}
1432
1433// Recompile iseq, disabling ivar optimization
1434void
1435rb_mjit_recompile_ivar(const rb_iseq_t *iseq)
1436{
1437 rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_ivar_cache = true;
1438 mjit_recompile(iseq);
1439}
1440
1441// Recompile iseq, disabling exivar optimization
1442void
1443rb_mjit_recompile_exivar(const rb_iseq_t *iseq)
1444{
1445 rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_exivar_cache = true;
1446 mjit_recompile(iseq);
1447}
1448
1449// Recompile iseq, disabling method inlining
1450void
1451rb_mjit_recompile_inlining(const rb_iseq_t *iseq)
1452{
1453 rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_inlining = true;
1454 mjit_recompile(iseq);
1455}
1456
1457// Recompile iseq, disabling getconstant inlining
1458void
1459rb_mjit_recompile_const(const rb_iseq_t *iseq)
1460{
1461 rb_mjit_iseq_compile_info(ISEQ_BODY(iseq))->disable_const_cache = true;
1462 mjit_recompile(iseq);
1463}
1464
1465extern VALUE ruby_archlibdir_path, ruby_prefix_path;
1466
1467// Initialize header_file, pch_file, libruby_pathflag. Return true on success.
1468static bool
1469init_header_filename(void)
1470{
1471 int fd;
1472#ifdef LOAD_RELATIVE
1473 // Root path of the running ruby process. Equal to RbConfig::TOPDIR.
1474 VALUE basedir_val;
1475#endif
1476 const char *basedir = "";
1477 size_t baselen = 0;
1478 char *p;
1479
1480#ifdef LOAD_RELATIVE
1481 basedir_val = ruby_prefix_path;
1482 basedir = StringValuePtr(basedir_val);
1483 baselen = RSTRING_LEN(basedir_val);
1484#else
1485 if (getenv("MJIT_SEARCH_BUILD_DIR")) {
1486 // This path is not intended to be used on production, but using build directory's
1487 // header file here because people want to run `make test-all` without running
1488 // `make install`. Don't use $MJIT_SEARCH_BUILD_DIR except for test-all.
1489
1490 struct stat st;
1491 const char *hdr = dlsym(RTLD_DEFAULT, "MJIT_HEADER");
1492 if (!hdr) {
1493 verbose(1, "No MJIT_HEADER");
1494 }
1495 else if (hdr[0] != '/') {
1496 verbose(1, "Non-absolute header file path: %s", hdr);
1497 }
1498 else if (stat(hdr, &st) || !S_ISREG(st.st_mode)) {
1499 verbose(1, "Non-file header file path: %s", hdr);
1500 }
1501 else if ((st.st_uid != getuid()) || (st.st_mode & 022) ||
1502 !rb_path_check(hdr)) {
1503 verbose(1, "Unsafe header file: uid=%ld mode=%#o %s",
1504 (long)st.st_uid, (unsigned)st.st_mode, hdr);
1505 return FALSE;
1506 }
1507 else {
1508 // Do not pass PRELOADENV to child processes, on
1509 // multi-arch environment
1510 verbose(3, "PRELOADENV("PRELOADENV")=%s", getenv(PRELOADENV));
1511 // assume no other PRELOADENV in test-all
1512 unsetenv(PRELOADENV);
1513 verbose(3, "MJIT_HEADER: %s", hdr);
1514 header_file = ruby_strdup(hdr);
1515 if (!header_file) return false;
1516 }
1517 }
1518 else
1519#endif
1520 {
1521 // A name of the header file included in any C file generated by MJIT for iseqs.
1522 static const char header_name[] = MJIT_HEADER_INSTALL_DIR "/" MJIT_MIN_HEADER_NAME;
1523 const size_t header_name_len = sizeof(header_name) - 1;
1524
1525 header_file = xmalloc(baselen + header_name_len + 1);
1526 p = append_str2(header_file, basedir, baselen);
1527 p = append_str2(p, header_name, header_name_len + 1);
1528
1529 if ((fd = rb_cloexec_open(header_file, O_RDONLY, 0)) < 0) {
1530 verbose(1, "Cannot access header file: %s", header_file);
1531 xfree(header_file);
1532 header_file = NULL;
1533 return false;
1534 }
1535 (void)close(fd);
1536 }
1537
1538 pch_file = get_uniq_filename(0, MJIT_TMP_PREFIX "h", ".h.gch");
1539
1540 return true;
1541}
1542
1543static char *
1544system_default_tmpdir(void)
1545{
1546 // c.f. ext/etc/etc.c:etc_systmpdir()
1547#if defined _CS_DARWIN_USER_TEMP_DIR
1548 char path[MAXPATHLEN];
1549 size_t len = confstr(_CS_DARWIN_USER_TEMP_DIR, path, sizeof(path));
1550 if (len > 0) {
1551 char *tmpdir = xmalloc(len);
1552 if (len > sizeof(path)) {
1553 confstr(_CS_DARWIN_USER_TEMP_DIR, tmpdir, len);
1554 }
1555 else {
1556 memcpy(tmpdir, path, len);
1557 }
1558 return tmpdir;
1559 }
1560#endif
1561 return 0;
1562}
1563
1564static int
1565check_tmpdir(const char *dir)
1566{
1567 struct stat st;
1568
1569 if (!dir) return FALSE;
1570 if (stat(dir, &st)) return FALSE;
1571#ifndef S_ISDIR
1572# define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
1573#endif
1574 if (!S_ISDIR(st.st_mode)) return FALSE;
1575#ifndef S_IWOTH
1576# define S_IWOTH 002
1577#endif
1578 if (st.st_mode & S_IWOTH) {
1579#ifdef S_ISVTX
1580 if (!(st.st_mode & S_ISVTX)) return FALSE;
1581#else
1582 return FALSE;
1583#endif
1584 }
1585 if (access(dir, W_OK)) return FALSE;
1586 return TRUE;
1587}
1588
1589static char *
1590system_tmpdir(void)
1591{
1592 char *tmpdir;
1593# define RETURN_ENV(name) \
1594 if (check_tmpdir(tmpdir = getenv(name))) return ruby_strdup(tmpdir)
1595 RETURN_ENV("TMPDIR");
1596 RETURN_ENV("TMP");
1597 tmpdir = system_default_tmpdir();
1598 if (check_tmpdir(tmpdir)) return tmpdir;
1599 return ruby_strdup("/tmp");
1600# undef RETURN_ENV
1601}
1602
1603// Minimum value for JIT cache size.
1604#define MIN_CACHE_SIZE 10
1605// Default permitted number of units with a JIT code kept in memory.
1606#define DEFAULT_MAX_CACHE_SIZE 100
1607// A default threshold used to add iseq to JIT.
1608#define DEFAULT_CALL_THRESHOLD 10000
1609
1610// Start MJIT worker. Return TRUE if worker is successfully started.
1611static bool
1612start_worker(void)
1613{
1614 stop_worker_p = false;
1615 worker_stopped = false;
1616 return true;
1617}
1618
1619// There's no strndup on Windows
1620static char*
1621ruby_strndup(const char *str, size_t n)
1622{
1623 char *ret = xmalloc(n + 1);
1624 memcpy(ret, str, n);
1625 ret[n] = '\0';
1626 return ret;
1627}
1628
1629// Convert "foo bar" to {"foo", "bar", NULL} array. Caller is responsible for
1630// freeing a returned buffer and its elements.
1631static char **
1632split_flags(const char *flags)
1633{
1634 char *buf[MAXPATHLEN];
1635 int i = 0;
1636 char *next;
1637 for (; flags != NULL; flags = next) {
1638 next = strchr(flags, ' ');
1639 if (next == NULL) {
1640 if (strlen(flags) > 0)
1641 buf[i++] = strdup(flags);
1642 }
1643 else {
1644 if (next > flags)
1645 buf[i++] = ruby_strndup(flags, next - flags);
1646 next++; // skip space
1647 }
1648 }
1649
1650 char **ret = xmalloc(sizeof(char *) * (i + 1));
1651 memcpy(ret, buf, sizeof(char *) * i);
1652 ret[i] = NULL;
1653 return ret;
1654}
1655
1656#define opt_match_noarg(s, l, name) \
1657 opt_match(s, l, name) && (*(s) ? (rb_warn("argument to --mjit-" name " is ignored"), 1) : 1)
1658#define opt_match_arg(s, l, name) \
1659 opt_match(s, l, name) && (*(s) ? 1 : (rb_raise(rb_eRuntimeError, "--mjit-" name " needs an argument"), 0))
1660
1661void
1662mjit_setup_options(const char *s, struct mjit_options *mjit_opt)
1663{
1664 const size_t l = strlen(s);
1665 if (l == 0) {
1666 return;
1667 }
1668 else if (opt_match_noarg(s, l, "warnings")) {
1669 mjit_opt->warnings = true;
1670 }
1671 else if (opt_match(s, l, "debug")) {
1672 if (*s)
1673 mjit_opt->debug_flags = strdup(s + 1);
1674 else
1675 mjit_opt->debug = true;
1676 }
1677 else if (opt_match_noarg(s, l, "wait")) {
1678 mjit_opt->wait = true;
1679 }
1680 else if (opt_match_noarg(s, l, "save-temps")) {
1681 mjit_opt->save_temps = true;
1682 }
1683 else if (opt_match(s, l, "verbose")) {
1684 mjit_opt->verbose = *s ? atoi(s + 1) : 1;
1685 }
1686 else if (opt_match_arg(s, l, "max-cache")) {
1687 mjit_opt->max_cache_size = atoi(s + 1);
1688 }
1689 else if (opt_match_arg(s, l, "call-threshold")) {
1690 mjit_opt->call_threshold = atoi(s + 1);
1691 }
1692 // --mjit=pause is an undocumented feature for experiments
1693 else if (opt_match_noarg(s, l, "pause")) {
1694 mjit_opt->pause = true;
1695 }
1696 else {
1698 "invalid MJIT option `%s' (--help will show valid MJIT options)", s);
1699 }
1700}
1701
1702#define M(shortopt, longopt, desc) RUBY_OPT_MESSAGE(shortopt, longopt, desc)
1703const struct ruby_opt_message mjit_option_messages[] = {
1704 M("--mjit-warnings", "", "Enable printing JIT warnings"),
1705 M("--mjit-debug", "", "Enable JIT debugging (very slow), or add cflags if specified"),
1706 M("--mjit-wait", "", "Wait until JIT compilation finishes every time (for testing)"),
1707 M("--mjit-save-temps", "", "Save JIT temporary files in $TMP or /tmp (for testing)"),
1708 M("--mjit-verbose=num", "", "Print JIT logs of level num or less to stderr (default: 0)"),
1709 M("--mjit-max-cache=num", "", "Max number of methods to be JIT-ed in a cache (default: "
1710 STRINGIZE(DEFAULT_MAX_CACHE_SIZE) ")"),
1711 M("--mjit-call-threshold=num", "", "Number of calls to trigger JIT (for testing, default: "
1712 STRINGIZE(DEFAULT_CALL_THRESHOLD) ")"),
1713 {0}
1714};
1715#undef M
1716
1717// Initialize MJIT. Start a thread creating the precompiled header and
1718// processing ISeqs. The function should be called first for using MJIT.
1719// If everything is successful, MJIT_INIT_P will be TRUE.
1720void
1721mjit_init(const struct mjit_options *opts)
1722{
1723 VM_ASSERT(mjit_enabled);
1724 mjit_opts = *opts;
1725
1726 // MJIT doesn't support miniruby, but it might reach here by MJIT_FORCE_ENABLE.
1727 rb_mMJIT = rb_const_get(rb_cRubyVM, rb_intern("MJIT"));
1728 if (!rb_const_defined(rb_mMJIT, rb_intern("Compiler"))) {
1729 verbose(1, "Disabling MJIT because RubyVM::MJIT::Compiler is not defined");
1730 mjit_enabled = false;
1731 return;
1732 }
1733 rb_mMJITC = rb_const_get(rb_mMJIT, rb_intern("C"));
1734 rb_cMJITCompiler = rb_funcall(rb_const_get(rb_mMJIT, rb_intern("Compiler")), rb_intern("new"), 0);
1735 rb_cMJITIseqPtr = rb_funcall(rb_mMJITC, rb_intern("rb_iseq_t"), 0);
1736 rb_cMJITICPtr = rb_funcall(rb_mMJITC, rb_intern("IC"), 0);
1737 rb_funcall(rb_cMJITICPtr, rb_intern("new"), 1, SIZET2NUM(0)); // Trigger no-op constant events before enabling hooks
1738 rb_mMJITHooks = rb_const_get(rb_mMJIT, rb_intern("Hooks"));
1739
1740 mjit_call_p = true;
1741 mjit_pid = getpid();
1742
1743 // Normalize options
1744 if (mjit_opts.call_threshold == 0)
1745 mjit_opts.call_threshold = DEFAULT_CALL_THRESHOLD;
1746 if (mjit_opts.call_threshold % 2 == 1) {
1747 mjit_opts.call_threshold += 1;
1748 mjit_warning("--mjit-call-threshold must be an even number. Using %d instead.", mjit_opts.call_threshold);
1749 }
1750 mjit_opts.call_threshold /= 2; // Half for enqueue, half for trigger
1751 if (mjit_opts.max_cache_size <= 0)
1752 mjit_opts.max_cache_size = DEFAULT_MAX_CACHE_SIZE;
1753 if (mjit_opts.max_cache_size < MIN_CACHE_SIZE)
1754 mjit_opts.max_cache_size = MIN_CACHE_SIZE;
1755
1756 // Initialize variables for compilation
1757 pch_status = PCH_NOT_READY;
1758 cc_path = CC_COMMON_ARGS[0];
1759 verbose(2, "MJIT: CC defaults to %s", cc_path);
1760 cc_common_args = xmalloc(sizeof(CC_COMMON_ARGS));
1761 memcpy((void *)cc_common_args, CC_COMMON_ARGS, sizeof(CC_COMMON_ARGS));
1762 cc_added_args = split_flags(opts->debug_flags);
1763 xfree(opts->debug_flags);
1764#if MJIT_CFLAGS_PIPE
1765 // Filter out `-save-temps`. It's a C compiler flag used by update-deps and not compatible with `-pipe`.
1766 for (size_t i = 0, j = 0; i < sizeof(CC_COMMON_ARGS) / sizeof(char *); i++) {
1767 if (CC_COMMON_ARGS[i] && strncmp("-save-temps", CC_COMMON_ARGS[i], strlen("-save-temps")) == 0)
1768 continue; // Skip `-save-temps`
1769 cc_common_args[j] = CC_COMMON_ARGS[i];
1770 j++;
1771 }
1772#endif
1773
1774 tmp_dir = system_tmpdir();
1775 verbose(2, "MJIT: tmp_dir is %s", tmp_dir);
1776
1777 if (!init_header_filename()) {
1778 mjit_enabled = false;
1779 verbose(1, "Failure in MJIT header file name initialization\n");
1780 return;
1781 }
1782 pch_owner_pid = getpid();
1783
1784 // Initialize mutex
1785 rb_native_mutex_initialize(&mjit_engine_mutex);
1786
1787 // If --mjit=pause is given, lazily start MJIT when RubyVM::MJIT.resume is called.
1788 // You can use it to control MJIT warmup, or to customize the JIT implementation.
1789 if (!mjit_opts.pause) {
1790 // TODO: Consider running C compiler asynchronously
1791 make_pch();
1792
1793 // Enable MJIT compilation
1794 start_worker();
1795 }
1796}
1797
1798static void
1799stop_worker(void)
1800{
1801 stop_worker_p = true;
1802 if (current_cc_unit != NULL) {
1803 mjit_wait(current_cc_unit);
1804 }
1805 worker_stopped = true;
1806}
1807
1808// Stop JIT-compiling methods but compiled code is kept available.
1809VALUE
1810mjit_pause(bool wait_p)
1811{
1812 if (!mjit_enabled) {
1813 rb_raise(rb_eRuntimeError, "MJIT is not enabled");
1814 }
1815 if (worker_stopped) {
1816 return Qfalse;
1817 }
1818
1819 // Flush all queued units with no option or `wait: true`
1820 if (wait_p) {
1821 while (current_cc_unit != NULL) {
1822 mjit_wait(current_cc_unit);
1823 }
1824 }
1825
1826 stop_worker();
1827 return Qtrue;
1828}
1829
1830// Restart JIT-compiling methods after mjit_pause.
1831VALUE
1832mjit_resume(void)
1833{
1834 if (!mjit_enabled) {
1835 rb_raise(rb_eRuntimeError, "MJIT is not enabled");
1836 }
1837 if (!worker_stopped) {
1838 return Qfalse;
1839 }
1840
1841 // Lazily prepare PCH when --mjit=pause is given
1842 if (pch_status == PCH_NOT_READY) {
1843 if (rb_respond_to(rb_mMJIT, rb_intern("compile"))) {
1844 // [experimental] defining RubyVM::MJIT.compile allows you to replace JIT
1845 mjit_opts.custom = true;
1846 pch_status = PCH_SUCCESS;
1847 }
1848 else {
1849 // Lazy MJIT boot
1850 make_pch();
1851 }
1852 }
1853
1854 if (!start_worker()) {
1855 rb_raise(rb_eRuntimeError, "Failed to resume MJIT worker");
1856 }
1857 return Qtrue;
1858}
1859
1860// This is called after fork initiated by Ruby's method to launch MJIT worker thread
1861// for child Ruby process.
1862//
1863// In multi-process Ruby applications, child Ruby processes do most of the jobs.
1864// Thus we want child Ruby processes to enqueue ISeqs to MJIT worker's queue and
1865// call the JIT-ed code.
1866//
1867// But unfortunately current MJIT-generated code is process-specific. After the fork,
1868// JIT-ed code created by parent Ruby process cannot be used in child Ruby process
1869// because the code could rely on inline cache values (ivar's IC, send's CC) which
1870// may vary between processes after fork or embed some process-specific addresses.
1871//
1872// So child Ruby process can't request parent process to JIT an ISeq and use the code.
1873// Instead of that, MJIT worker thread is created for all child Ruby processes, even
1874// while child processes would end up with compiling the same ISeqs.
1875void
1876mjit_child_after_fork(void)
1877{
1878 if (!mjit_enabled)
1879 return;
1880
1881 /* MJIT worker thread is not inherited on fork. Start it for this child process. */
1882 start_worker();
1883}
1884
1885// Finish the threads processing units and creating PCH, finalize
1886// and free MJIT data. It should be called last during MJIT
1887// life.
1888//
1889// If close_handle_p is true, it calls dlclose() for JIT-ed code. So it should be false
1890// if the code can still be on stack. ...But it means to leak JIT-ed handle forever (FIXME).
1891void
1892mjit_finish(bool close_handle_p)
1893{
1894 if (!mjit_enabled)
1895 return;
1896
1897 // Stop worker
1898 verbose(2, "Stopping worker thread");
1899 stop_worker();
1900
1901 rb_native_mutex_destroy(&mjit_engine_mutex);
1902
1903 if (!mjit_opts.save_temps && getpid() == pch_owner_pid && pch_status == PCH_SUCCESS && !mjit_opts.custom)
1904 remove_file(pch_file);
1905
1906 xfree(header_file); header_file = NULL;
1907 xfree((void *)cc_common_args); cc_common_args = NULL;
1908 for (char **flag = cc_added_args; *flag != NULL; flag++)
1909 xfree(*flag);
1910 xfree((void *)cc_added_args); cc_added_args = NULL;
1911 xfree(tmp_dir); tmp_dir = NULL;
1912 xfree(pch_file); pch_file = NULL;
1913
1914 mjit_call_p = false;
1915 free_list(&unit_queue, close_handle_p);
1916 free_list(&active_units, close_handle_p);
1917 free_list(&compact_units, close_handle_p);
1918 free_list(&stale_units, close_handle_p);
1919
1920 mjit_enabled = false;
1921 verbose(1, "Successful MJIT finish");
1922}
1923
1924// Called by rb_vm_mark().
1925//
1926// Mark active_units so that we do not GC ISeq which may still be
1927// referenced by mjit_recompile() or mjit_compact().
1928void
1929mjit_mark(void)
1930{
1931 if (!mjit_enabled)
1932 return;
1933 RUBY_MARK_ENTER("mjit");
1934
1935 // Mark objects used by the MJIT compiler
1936 rb_gc_mark(rb_cMJITCompiler);
1937 rb_gc_mark(rb_cMJITIseqPtr);
1938 rb_gc_mark(rb_cMJITICPtr);
1939 rb_gc_mark(rb_mMJITHooks);
1940
1941 // Mark JIT-compiled ISEQs
1942 struct rb_mjit_unit *unit = NULL;
1943 ccan_list_for_each(&active_units.head, unit, unode) {
1944 rb_gc_mark((VALUE)unit->iseq);
1945 }
1946
1947 RUBY_MARK_LEAVE("mjit");
1948}
1949
1950// Called by rb_iseq_mark() to mark cc_entries captured for MJIT
1951void
1952mjit_mark_cc_entries(const struct rb_iseq_constant_body *const body)
1953{
1954 const struct rb_callcache **cc_entries;
1955 if (body->mjit_unit && (cc_entries = body->mjit_unit->cc_entries) != NULL) {
1956 // It must be `body->mjit_unit->cc_entries_size` instead of `body->ci_size` to mark children's cc_entries
1957 for (unsigned int i = 0; i < body->mjit_unit->cc_entries_size; i++) {
1958 const struct rb_callcache *cc = cc_entries[i];
1959 if (cc != NULL && vm_cc_markable(cc)) {
1960 // Pin `cc` and `cc->cme` against GC.compact as their addresses may be written in JIT-ed code.
1961 rb_gc_mark((VALUE)cc);
1962 rb_gc_mark((VALUE)vm_cc_cme(cc));
1963 }
1964 }
1965 }
1966}
1967
1968// Compile ISeq to C code in `f`. It returns true if it succeeds to compile.
1969bool
1970mjit_compile(FILE *f, const rb_iseq_t *iseq, const char *funcname, int id)
1971{
1972 bool original_call_p = mjit_call_p;
1973 mjit_call_p = false; // Avoid impacting JIT metrics by itself
1974
1975 VALUE iseq_ptr = rb_funcall(rb_cMJITIseqPtr, rb_intern("new"), 1, ULONG2NUM((size_t)iseq));
1976 VALUE src = rb_funcall(rb_cMJITCompiler, rb_intern("compile"), 3,
1977 iseq_ptr, rb_str_new_cstr(funcname), INT2NUM(id));
1978 if (!NIL_P(src)) {
1979 fprintf(f, "%s", RSTRING_PTR(src));
1980 }
1981
1982 mjit_call_p = original_call_p;
1983 return !NIL_P(src);
1984}
1985
1986#include "mjit.rbinc"
1987
1988#endif // USE_MJIT
void(* rb_postponed_job_func_t)(void *arg)
Type of postponed jobs.
Definition debug.h:608
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:103
#define NUM2ULONG
Old name of RB_NUM2ULONG.
Definition long.h:52
#define xfree
Old name of ruby_xfree.
Definition xmalloc.h:58
#define ID2SYM
Old name of RB_ID2SYM.
Definition symbol.h:44
#define ULONG2NUM
Old name of RB_ULONG2NUM.
Definition long.h:60
#define SIZET2NUM
Old name of RB_SIZE2NUM.
Definition size_t.h:62
#define xmalloc
Old name of ruby_xmalloc.
Definition xmalloc.h:53
#define ZALLOC_N
Old name of RB_ZALLOC_N.
Definition memory.h:395
#define Qtrue
Old name of RUBY_Qtrue.
#define INT2NUM
Old name of RB_INT2NUM.
Definition int.h:43
#define Qfalse
Old name of RUBY_Qfalse.
#define NIL_P
Old name of RB_NIL_P.
#define UINT2NUM
Old name of RB_UINT2NUM.
Definition int.h:46
void rb_raise(VALUE exc, const char *fmt,...)
Exception entry point.
Definition error.c:3148
void rb_bug(const char *fmt,...)
Interpreter panic switch.
Definition error.c:794
VALUE rb_eRuntimeError
RuntimeError exception.
Definition error.c:1089
int rb_cloexec_open(const char *pathname, int flags, mode_t mode)
Opens a file that closes on exec.
Definition io.c:310
#define rb_str_new_cstr(str)
Identical to rb_str_new, except it assumes the passed pointer is a pointer to a C string.
Definition string.h:1514
void rb_thread_wait_for(struct timeval time)
Identical to rb_thread_sleep(), except it takes struct timeval instead.
Definition thread.c:1404
VALUE rb_const_get(VALUE space, ID name)
Identical to rb_const_defined(), except it returns the actual defined value.
Definition variable.c:2883
int rb_const_defined(VALUE space, ID name)
Queries if the constant is defined at the namespace.
Definition variable.c:3191
int rb_respond_to(VALUE obj, ID mid)
Queries if the object responds to the method.
Definition vm_method.c:2823
char * ruby_strdup(const char *str)
This is our own version of strdup(3) that uses ruby_xmalloc() instead of system malloc (benefits our ...
Definition util.c:538
#define strdup(s)
Just another name of ruby_strdup.
Definition util.h:176
#define RUBY_API_VERSION_TEENY
Teeny version.
Definition version.h:76
#define RUBY_API_VERSION_MAJOR
Major version.
Definition version.h:64
#define RUBY_API_VERSION_MINOR
Minor version.
Definition version.h:70
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
Definition memory.h:366
VALUE type(ANYARGS)
ANYARGS-ed function type.
#define PRI_PIDT_PREFIX
A rb_sprintf() format prefix to be used for a pid_t parameter.
Definition pid_t.h:38
#define StringValuePtr(v)
Identical to StringValue, except it returns a char*.
Definition rstring.h:82
Definition method.h:62
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_destroy.
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52