Mingw make patch for parallel build of ruby

0-timing_result.md

30% speed up of make mingw ruby on Windows 7 with Intel Core i5 2.40GHz CPU

Build trunk without ext/tk

$ gnumake clean
$ time gnumake

real    9m14.244s
user    0m0.030s
sys     0m0.031s


$ gnumake clean
$ time gnumake -j6

real    6m27.588s
user    0m0.031s
sys     0m0.016s

1-build_make.sh

#!/bin/sh
#
# Build make from cvs repository.
#
# Requirements:
#   Install MinGW/MSYS environment.
#   See: http://www.mingw.org/wiki/Getting_Started
# 
# Run MinGW shell and run this shell script.
#   wget --no-check-certificate -O - https://raw.github.com/gist/3888567/1-build_make.sh | sh
# make/gnumake.exe will be generated.
#

mingw-get install gcc msys-cvs msys-wget msys-patch 

cvs -z3 -d:pserver:[email protected]:/sources/make co make
cd make
wget --no-check-certificate -O - https://raw.github.com/gist/3888567/2-make_mingw_espace.patch | patch -p0
cmd /c "build_w32.bat gcc"

2-make_mingw_espace.patch

Index: build_w32.bat
===================================================================
RCS file: /sources/make/make/build_w32.bat,v
retrieving revision 2.23
diff -u -r2.23 build_w32.bat
--- build_w32.bat	5 Mar 2012 14:10:41 -0000	2.23
+++ build_w32.bat	5 Nov 2012 12:50:14 -0000
@@ -237,6 +237,7 @@
 gcc -mthreads -Wall -gdwarf-2 -g3 -O2 -I. -I./glob -I./w32/include -DWINDOWS32 -DHAVE_CONFIG_H -c function.c
 gcc -mthreads -Wall -gdwarf-2 -g3 -O2 -I. -I./glob -I./w32/include -DWINDOWS32 -DHAVE_CONFIG_H -c vpath.c
 gcc -mthreads -Wall -gdwarf-2 -g3 -O2 -I. -I./glob -I./w32/include -DWINDOWS32 -DHAVE_CONFIG_H -c implicit.c
+gcc -mthreads -Wall -gdwarf-2 -g3 -O2 -I. -I./glob -I./w32/include -DWINDOWS32 -DHAVE_CONFIG_H -c load.c
 gcc -mthreads -Wall -gdwarf-2 -g3 -O2 -I. -I./glob -I./w32/include -DWINDOWS32 -DHAVE_CONFIG_H -c ./glob/glob.c -o glob.o
 gcc -mthreads -Wall -gdwarf-2 -g3 -O2 -I. -I./glob -I./w32/include -DWINDOWS32 -DHAVE_CONFIG_H -c ./glob/fnmatch.c -o fnmatch.o
 gcc -mthreads -Wall -gdwarf-2 -g3 -O2 -I. -I./glob -I./w32/include -DWINDOWS32 -DHAVE_CONFIG_H -c ./w32/pathstuff.c -o pathstuff.o
@@ -248,7 +249,7 @@
 gcc -mthreads -Wall -gdwarf-2 -g3 -O2 %GUILECFLAGS% -I. -I./glob -I./w32/include -DWINDOWS32 -DHAVE_CONFIG_H -c guile.c
 :LinkGCC
 @echo on
-gcc -mthreads -gdwarf-2 -g3 -o gnumake.exe variable.o rule.o remote-stub.o commands.o file.o getloadavg.o default.o signame.o expand.o dir.o main.o getopt1.o %GUILEOBJ% job.o read.o version.o getopt.o arscan.o remake.o misc.o hash.o strcache.o ar.o function.o vpath.o implicit.o glob.o fnmatch.o pathstuff.o w32_misc.o sub_proc.o w32err.o %GUILELIBS% -lkernel32 -luser32 -lgdi32 -lwinspool -lcomdlg32 -ladvapi32 -lshell32 -lole32 -loleaut32 -luuid -lodbc32 -lodbccp32
+gcc -mthreads -gdwarf-2 -g3 -o gnumake.exe variable.o rule.o remote-stub.o commands.o file.o getloadavg.o default.o signame.o expand.o dir.o main.o getopt1.o %GUILEOBJ% job.o read.o version.o getopt.o arscan.o remake.o misc.o hash.o strcache.o ar.o function.o vpath.o implicit.o load.o glob.o fnmatch.o pathstuff.o w32_misc.o sub_proc.o w32err.o %GUILELIBS% -lkernel32 -luser32 -lgdi32 -lwinspool -lcomdlg32 -ladvapi32 -lshell32 -lole32 -loleaut32 -luuid -lodbc32 -lodbccp32
 :BuildEnd
 @echo off
 set GUILEOBJ=
Index: job.c
===================================================================
RCS file: /sources/make/make/job.c,v
retrieving revision 1.220
diff -u -r1.220 job.c
--- job.c	9 Sep 2012 23:25:07 -0000	1.220
+++ job.c	5 Nov 2012 12:50:15 -0000
@@ -2672,6 +2672,11 @@
 	/* '...' is a wildcard in DJGPP.  */
 	goto slow;
 #endif
+#ifdef  WINDOWS32
+      else if (unixy_shell && p == line && *p == '/')
+	/* It seems like posix path. Use shell. */
+	goto slow;
+#endif
       else
 	/* Not a special char.  */
 	switch (*p)
Index: w32/subproc/sub_proc.c
===================================================================
RCS file: /sources/make/make/w32/subproc/sub_proc.c,v
retrieving revision 1.30
diff -u -r1.30 sub_proc.c
--- w32/subproc/sub_proc.c	5 Mar 2012 14:10:48 -0000	1.30
+++ w32/subproc/sub_proc.c	5 Nov 2012 12:50:16 -0000
@@ -1117,6 +1117,9 @@
 #ifdef HAVE_CYGWIN_SHELL
 	have_sh = (shell_name != NULL || strstr(full_exec_path, "sh.exe"));
 	cygwin_mode = 1;
+#else
+	have_sh = ((shell_name != NULL && strstr(shell_name, "sh.exe")) ||
+		   strstr(full_exec_path, "sh.exe"));
 #endif
 
 	if (shell_name && full_exec_path) {
@@ -1153,6 +1156,7 @@
 	while(*argvi) {
 		char* p = *argvi;
 		unsigned int backslash_count = 0;
+		unsigned int head_spaces = 1;
 
 		/*
 		 * We have to enclose empty arguments in ".
@@ -1188,14 +1192,26 @@
 	 * and the _setargv in SETARGV.OBJ would _not_ expand the *.
 	 */
 			case ' ':
+				/* ignore head and trailing spaces */
+				if (!head_spaces &&
+				    (*(p + 1) != ' ' || *(p + 1) != '\0')) {
+					*enclose_in_quotes_i = 1;
+				}
+				bytes_required += (backslash_count + 1);
+				backslash_count = 0;
+				break;
 			case '\t':
 				*enclose_in_quotes_i = 1;
 				/* fall through */
 
 			default:
+				bytes_required += backslash_count;
 				backslash_count = 0;
 				break;
 			}
+			if (head_spaces && *p != ' ') {
+				head_spaces = 0;
+			}
 
 			/*
 			 * Add one for each character in argv[i].
@@ -1257,17 +1273,63 @@
 	while(*argvi) {
 		char* p = *argvi;
 		unsigned int backslash_count = 0;
+		int in_quoted_value = 0;
+		int quoted_variable = 0;
+		int skip_next_quote = 0;
+		int in_single_quote = 0;
 
 		if (*enclose_in_quotes_i) {
 			*(command_line_i++) = '\"';
 		}
 
 		while(*p) {
+			int skip_quote = 0;
+			int skip_space = 0;
+
 			if (*p == '\"') {
-				if (cygwin_mode && have_sh) { /* HAVE_CYGWIN_SHELL */
+			    	if (!cygwin_mode && have_sh) {
+					/*
+					 * If variables are quoted by double quotations,
+					 * we remove " and espace spaces.
+					 *
+					 * example:
+					 *  NAME="a b"  => NAME=a\ b
+					 *  --opt="a b"  => --opt=a\ b
+					 *   "a b"  => a\ b
+					 */
+					if (backslash_count % 2 == 0) {
+						/* not escaped */
+						in_quoted_value = 1 - in_quoted_value;
+					}
+					if (!in_quoted_value && skip_next_quote) {
+						skip_quote = 1;
+						skip_next_quote = 0;
+					}
+					if (!in_quoted_value) {
+						quoted_variable = 0;
+						in_single_quote = 0;
+					}
+					if (in_quoted_value && *(p - 1) == '=') {
+						char *q = p - 1;
+						while(q != *argvi && *q != ' ') {
+							q--;
+						}
+						quoted_variable = 0;
+						if (*q == ' ')
+							quoted_variable = 1;
+						if (q == *argvi && *q != '-')
+							quoted_variable = 1;
+						if (quoted_variable) {
+							skip_quote = 1;
+							skip_next_quote = 1;
+						}
+					}
+				}
+
+				if (!skip_quote && cygwin_mode && have_sh) { /* HAVE_CYGWIN_SHELL */
 					/* instead of a \", cygwin likes "" */
 					*(command_line_i++) = '\"';
-				} else {
+				} else if (!skip_quote) {
 
 				/*
 				 * We have to insert a backslash for the "
@@ -1280,10 +1342,42 @@
 					backslash_count--;
 				};
 				}
+			} else if (!cygwin_mode && have_sh && *p == ' ') {
+				while(backslash_count) {
+					*(command_line_i++) = '\\';
+					backslash_count--;
+				}
+				if (in_quoted_value && *(p + 1) == '\"') {
+				    	char *q = p - 1;
+					skip_space = 1;
+					/* removes trailing spaces */
+					while(q != *argvi && *q == ' ') {
+					    command_line_i--;
+					    q--;
+					}
+				}
+				/* escape space */
+				if (!skip_space && !in_single_quote &&
+				    in_quoted_value && quoted_variable) {
+					*(command_line_i++) = '\\';
+				}
+				/* append a space to work quoted empty string */
+				if (!in_quoted_value &&
+				    p != *argvi && *(p - 1) == '\"')
+					*(command_line_i++) = ' ';
 #if !defined(HAVE_MKS_SHELL) && !defined(HAVE_CYGWIN_SHELL)
-			} else if (*p == '\\') {
+			} else if (*p == '\\' && *(p + 1) != '\r' && *(p + 1) != '\n') {
 				backslash_count++;
+			} else if (*p == '\'') {
+			    	if(in_quoted_value)
+				    in_single_quote = 1 - in_single_quote;
 			} else {
+			    	if (!cygwin_mode && have_sh) {
+					while(backslash_count) {
+						*(command_line_i++) = '\\';
+						backslash_count--;
+					};
+				}
 				backslash_count = 0;
 #endif
 			}
@@ -1291,7 +1385,18 @@
 			/*
 			 * Copy the character.
 			 */
-			*(command_line_i++) = *(p++);
+			if (skip_space) {
+				p++;
+			} else if (skip_quote && *(p + 1) == '\"') {
+				quoted_variable = 0;
+				in_quoted_value = 0;
+				*(command_line_i++) = ' ';
+				p += 2;
+			} else if (skip_quote) {
+				p++;
+			} else {
+				*(command_line_i++) = *(p++);
+			}
 		}
 
 		if (*enclose_in_quotes_i) {

3-parallel_make_ruby.patch

diff --git a/common.mk b/common.mk
index d2818f6..3585e12 100644
--- a/common.mk
+++ b/common.mk
@@ -212,7 +212,7 @@ Doxyfile: $(srcdir)/template/Doxyfile.tmpl $(PREP) $(srcdir)/tool/generic_erb.rb
 program: showflags $(PROGRAM)
 wprogram: showflags $(WPROGRAM)
 
-$(PROGRAM): $(LIBRUBY) $(MAINOBJ) $(OBJS) $(EXTOBJS) $(SETUP) $(PREP)
+$(PROGRAM) $(WPROGRAM): $(LIBRUBY) $(MAINOBJ) $(OBJS) $(EXTOBJS) $(SETUP) $(PREP)
 
 $(LIBRUBY_A):	$(OBJS) $(MAINOBJ) $(DMYEXT) $(ARCHFILE)
 
diff --git a/cygwin/GNUmakefile.in b/cygwin/GNUmakefile.in
index 19d1727..2b0a806 100644
--- a/cygwin/GNUmakefile.in
+++ b/cygwin/GNUmakefile.in
@@ -1,5 +1,4 @@
 include Makefile
--include uncommon.mk
 
 ENABLE_SHARED=@ENABLE_SHARED@
 DLLWRAP = @DLLWRAP@ --target=@target_os@ --driver-name="$(CC)"
@@ -35,6 +34,8 @@ EXTOBJS += $(if $(filter-out $(RUBYW_INSTALL_NAME),$(@:$(EXEEXT)=)),$(RUBY_INSTA
 RCFILES = $(RUBY_INSTALL_NAME).rc $(RUBYW_INSTALL_NAME).rc $(DLL_BASE_NAME).rc
 RUBYDEF = $(DLL_BASE_NAME).def
 
+-include uncommon.mk
+
 ruby: $(PROGRAM)
 rubyw: $(WPROGRAM)
 stub: $(STUBPROGRAM)
diff --git a/ext/extmk.rb b/ext/extmk.rb
index df86c0f..40c3f40 100755
--- a/ext/extmk.rb
+++ b/ext/extmk.rb
@@ -656,13 +656,11 @@ if $configure_only and $command_output
       mf.puts "#{tgt}: $(extensions:/.=/#{tgt})"
     end
     mf.puts
-    mf.puts "all: #{rubies.join(' ')}"
-    mf.puts "static: #{rubies.join(' ')}"
     mf.puts "clean:\n\t-$(Q)$(RM) ext/extinit.#{$OBJEXT}"
     mf.puts "distclean:\n\t-$(Q)$(RM) ext/extinit.c"
     mf.puts
     mf.puts "#{rubies.join(' ')}: $(extensions:/.=/#{$force_static ? 'static' : 'all'})"
-    rubies.each do |tgt|
+    (["all static"] + rubies).each_with_index do |tgt, i|
       mf.print "#{tgt}:\n\t$(Q)$(MAKE) "
       mf.print "$(MFLAGS) "
       if enable_config("shared", $enable_shared)
@@ -671,7 +669,12 @@ if $configure_only and $command_output
       else
         mf.print %[EXTOBJS="$(EXTOBJS) $(ENCOBJS)" EXTLIBS="$(EXTLIBS)" ]
       end
-      mf.puts 'EXTLDFLAGS="$(EXTLDFLAGS)" $@'
+      mf.print 'EXTLDFLAGS="$(EXTLDFLAGS)" '
+      if i == 0
+        mf.puts rubies.join(' ')
+      else
+        mf.puts '$@'
+      end
     end
     mf.puts
     exec = config_string("exec") {|str| str + " "}