True color (24-bit) terminal support for Emacs 24.4

emacs-24.4-24bit.diff

This diff is a modified version of a diff written by Rüdiger Sonderfeld.

The original diff can be found here:

http://emacs.1067599.n5.nabble.com/RFC-Add-tty-True-Color-support-tt299962.html

To enable the feature one must set one of 2 environment variables either
ITERM_24BIT or KONSOLE_DBUS_SESSION. The former will enable ITU T.416 mode, the
latter will use semi-colon seperators used by other terminals (and also
supported by iterm development branch).

I've added ITERM_24BIT as an environment variable to set that will cause emacs
to use the : seperated values as specified by ITU T.416. These are apparently
better to use and supported by iterm development branch.

Other chagnes:
- modified map_tty_color to set the color into the pixel field.
- modified the colors defined to be the standard X11 list.
- modified tty-color-desc to add missing colors on demand.

*** ./lisp/term/xterm.el.orig	2014-06-20 13:28:33.000000000 -0400
--- ./lisp/term/xterm.el	2015-02-23 07:01:48.000000000 -0500
***************
*** 674,679 ****
--- 674,688 ----
      ;; are more colors to support, compute them now.
      (when (> ncolors 0)
        (cond
+        ((= (display-color-cells (selected-frame)) 16777216)	; 24-bit xterm
+         (let ((idx (length xterm-standard-colors)))
+           ;; Insert standard X colors after the standard xterm ones
+           (mapc (lambda (color)
+                        (if (not (assoc (car color) xterm-standard-colors))
+                            (progn
+                              (tty-color-define (car color) idx (cdr color))
+                              (setq idx (1+ idx)))))
+                 color-name-rgb-alist)))
         ((= ncolors 240)	; 256-color xterm
  	;; 216 non-gray colors first
  	(let ((r 0) (g 0) (b 0))
*** ./src/dispextern.h.orig	2014-09-15 23:13:46.000000000 -0400
--- ./src/dispextern.h	2015-02-23 07:01:48.000000000 -0500
***************
*** 1739,1747 ****
  INLINE bool
  face_tty_specified_color (unsigned long color)
  {
!   return color < FACE_TTY_DEFAULT_BG_COLOR;
  }
  
  /* Non-zero if FACE was realized for unibyte use.  */
  
  #define FACE_UNIBYTE_P(FACE) ((FACE)->charset < 0)
--- 1739,1753 ----
  INLINE bool
  face_tty_specified_color (unsigned long color)
  {
!   return (color < FACE_TTY_DEFAULT_BG_COLOR);
  }
  
+ INLINE bool
+ face_tty_specified_24_bit_color (unsigned long color)
+ {
+   /* 24 bit colors have 24th but not 25th bit set */
+   return ((color & (0x03 << 24)) == (0x01 << 24));
+ }
  /* Non-zero if FACE was realized for unibyte use.  */
  
  #define FACE_UNIBYTE_P(FACE) ((FACE)->charset < 0)
*** ./src/term.c.orig	2014-06-04 11:47:40.000000000 -0400
--- ./src/term.c	2015-02-23 07:01:48.000000000 -0500
***************
*** 1915,1932 ****
        const char *ts;
        char *p;
  
!       ts = tty->standout_mode ? tty->TS_set_background : tty->TS_set_foreground;
        if (face_tty_specified_color (fg) && ts)
  	{
!           p = tparam (ts, NULL, 0, fg, 0, 0, 0);
  	  OUTPUT (tty, p);
  	  xfree (p);
  	}
  
!       ts = tty->standout_mode ? tty->TS_set_foreground : tty->TS_set_background;
        if (face_tty_specified_color (bg) && ts)
  	{
!           p = tparam (ts, NULL, 0, bg, 0, 0, 0);
  	  OUTPUT (tty, p);
  	  xfree (p);
  	}
--- 1915,1954 ----
        const char *ts;
        char *p;
  
!       if (face_tty_specified_24_bit_color(fg))
!         ts = tty->standout_mode ? tty->TS_set_rgb_background : tty->TS_set_rgb_foreground;
!       else
!         ts = tty->standout_mode ? tty->TS_set_background : tty->TS_set_foreground;
        if (face_tty_specified_color (fg) && ts)
  	{
!           if (!face_tty_specified_24_bit_color(fg))
!             p = tparam (ts, NULL, 0, fg, 0, 0, 0);
!           else
!             {
!               const unsigned char r = (fg >> 16) & 0xFF,
!                 g = (fg >> 8) & 0xFF,
!                 b = fg & 0xFF;
!               p = tparam (ts, NULL, 0, (int)r, (int)g, (int)b, 0);
!             }
  	  OUTPUT (tty, p);
  	  xfree (p);
  	}
  
!       if (face_tty_specified_24_bit_color(bg))
!         ts = tty->standout_mode ? tty->TS_set_rgb_foreground : tty->TS_set_rgb_background;
!       else
!         ts = tty->standout_mode ? tty->TS_set_foreground : tty->TS_set_background;
        if (face_tty_specified_color (bg) && ts)
  	{
!           if (!face_tty_specified_24_bit_color(bg))
!             p = tparam (ts, NULL, 0, bg, 0, 0, 0);
!           else
!             {
!               const unsigned char r = (bg >> 16) & 0xFF,
!                 g = (bg >> 8) & 0xFF,
!                 b = bg & 0xFF;
!               p = tparam (ts, NULL, 0, (int)r, (int)g, (int)b, 0);
!           }
  	  OUTPUT (tty, p);
  	  xfree (p);
  	}
***************
*** 2028,2033 ****
--- 2050,2057 ----
    struct terminal *t = get_tty_terminal (terminal, 0);
    if (!t)
      return make_number (0);
+   else if (t->display_info.tty->TS_set_rgb_foreground)
+     return make_number (16777216); /* 24 bit True Color */
    else
      return make_number (t->display_info.tty->TN_max_colors);
  }
***************
*** 2043,2048 ****
--- 2067,2074 ----
  static char *default_orig_pair;
  static char *default_set_foreground;
  static char *default_set_background;
+ static char *default_set_rgb_foreground;
+ static char *default_set_rgb_background;
  
  /* Save or restore the default color-related capabilities of this
     terminal.  */
***************
*** 2055,2060 ****
--- 2081,2088 ----
        dupstring (&default_orig_pair, tty->TS_orig_pair);
        dupstring (&default_set_foreground, tty->TS_set_foreground);
        dupstring (&default_set_background, tty->TS_set_background);
+       dupstring (&default_set_rgb_foreground, tty->TS_set_rgb_foreground);
+       dupstring (&default_set_rgb_background, tty->TS_set_rgb_background);
        default_max_colors = tty->TN_max_colors;
        default_max_pairs = tty->TN_max_pairs;
        default_no_color_video = tty->TN_no_color_video;
***************
*** 2064,2069 ****
--- 2092,2099 ----
        tty->TS_orig_pair = default_orig_pair;
        tty->TS_set_foreground = default_set_foreground;
        tty->TS_set_background = default_set_background;
+       tty->TS_set_rgb_foreground = default_set_rgb_foreground;
+       tty->TS_set_rgb_background = default_set_rgb_background;
        tty->TN_max_colors = default_max_colors;
        tty->TN_max_pairs = default_max_pairs;
        tty->TN_no_color_video = default_no_color_video;
***************
*** 2088,2093 ****
--- 2118,2124 ----
  	tty->TN_max_pairs = 0;
  	tty->TN_no_color_video = 0;
  	tty->TS_set_foreground = tty->TS_set_background = tty->TS_orig_pair = NULL;
+         tty->TS_set_rgb_foreground = tty->TS_set_rgb_background = NULL;
  	break;
        case 0:	 /* default colors, if any */
        default:
***************
*** 2102,2111 ****
--- 2133,2161 ----
  	tty->TS_set_foreground = "\033[3%dm";
  	tty->TS_set_background = "\033[4%dm";
  #endif
+         tty->TS_set_rgb_foreground = NULL;
+         tty->TS_set_rgb_background = NULL;
  	tty->TN_max_colors = 8;
  	tty->TN_max_pairs = 64;
  	tty->TN_no_color_video = 0;
  	break;
+       case 16777216: /* RGB colors */
+         tty->TS_orig_pair = "\033[0m";
+ #ifdef TERMINFO
+ 	tty->TS_set_foreground = "\033[3%p1%dm";
+ 	tty->TS_set_background = "\033[4%p1%dm";
+         tty->TS_set_rgb_foreground = "\033[38;2;%p1%d;%p2%d;%p3%dm";
+         tty->TS_set_rgb_background = "\033[48;2;%p1%d;%p2%d;%p3%dm";
+ #else
+ 	tty->TS_set_foreground = "\033[3%dm";
+ 	tty->TS_set_background = "\033[4%dm";
+         tty->TS_set_rgb_foreground = "\033[38;2;%d;%d;%dm";
+         tty->TS_set_rgb_background = "\033[48;2;%d;%d;%dm";
+ #endif
+         tty->TN_max_colors = 16777216;
+         /*tty->TN_max_pairs = 64; TODO */
+ 	tty->TN_no_color_video = 0;
+         break;
      }
  }
  
***************
*** 4201,4206 ****
--- 4251,4286 ----
        tty->TN_no_color_video = tgetnum ("NC");
        if (tty->TN_no_color_video == -1)
          tty->TN_no_color_video = 0;
+ 
+       /* TODO Reliable way to detect: Konsole, iTerm2, st */
+       if (getenv ("KONSOLE_DBUS_SESSION"))
+         {
+           /* TODO This should be extracted from terminfo/termcap. */
+ #ifdef TERMINFO
+           tty->TS_set_rgb_foreground = "\033[38;2;%p1%d;%p2%d;%p3%dm";
+           tty->TS_set_rgb_background = "\033[48;2;%p1%d;%p2%d;%p3%dm";
+ #else
+           tty->TS_set_rgb_foreground = "\033[38;2;%d;%d;%dm";
+           tty->TS_set_rgb_background = "\033[48;2;%d;%d;%dm";
+ #endif
+         }
+       else if (getenv ("ITERM_24BIT"))
+         {
+           /* XXX chopps use ITU T.421 ':' separator */
+           /* TODO This should be extracted from terminfo/termcap. */
+ #ifdef TERMINFO
+           tty->TS_set_rgb_foreground = "\033[38:2:%p1%d:%p2%d:%p3%dm";
+           tty->TS_set_rgb_background = "\033[48:2:%p1%d:%p2%d:%p3%dm";
+ #else
+           tty->TS_set_rgb_foreground = "\033[38:2:%d:%d:%dm";
+           tty->TS_set_rgb_background = "\033[48:2:%d:%d:%dm";
+ #endif
+         }
+       else
+         {
+           tty->TS_set_rgb_foreground = NULL;
+           tty->TS_set_rgb_background = NULL;
+         }
      }
  
    tty_default_color_capabilities (tty, 1);
*** ./src/termchar.h.orig	2014-03-21 01:34:40.000000000 -0400
--- ./src/termchar.h	2015-02-23 07:01:48.000000000 -0500
***************
*** 157,162 ****
--- 157,166 ----
    const char *TS_set_foreground;
    const char *TS_set_background;
  
+   /* Support for 24bit RGB color terminals. */
+   const char *TS_set_rgb_foreground;
+   const char *TS_set_rgb_background;
+ 
    int TF_hazeltine;             /* termcap hz flag. */
    int TF_insmode_motion;        /* termcap mi flag: can move while in insert mode. */
    int TF_standout_motion;       /* termcap mi flag: can move while in standout mode. */
*** ./src/xfaces.c.orig	2014-10-09 12:07:28.000000000 -0400
--- ./src/xfaces.c	2015-02-23 07:02:06.000000000 -0500
***************
*** 382,388 ****
  
  /* TTY color-related functions (defined in tty-colors.el).  */
  
! static Lisp_Object Qtty_color_desc, Qtty_color_by_index, Qtty_color_standard_values;
  
  /* The name of the function used to compute colors on TTYs.  */
  
--- 382,388 ----
  
  /* TTY color-related functions (defined in tty-colors.el).  */
  
! static Lisp_Object Qtty_color_desc, Qtty_color_by_index, Qtty_color_standard_values, Qtty_color_canonicalize;
  
  /* The name of the function used to compute colors on TTYs.  */
  
***************
*** 943,996 ****
    if (!STRINGP (color) || NILP (Ffboundp (Qtty_color_desc)))
      return 0;
  
!   XSETFRAME (frame, f);
  
!   color_desc = call2 (Qtty_color_desc, color, frame);
!   if (CONSP (color_desc) && CONSP (XCDR (color_desc)))
      {
!       Lisp_Object rgb;
  
!       if (! INTEGERP (XCAR (XCDR (color_desc))))
! 	return 0;
  
!       tty_color->pixel = XINT (XCAR (XCDR (color_desc)));
  
!       rgb = XCDR (XCDR (color_desc));
!       if (! parse_rgb_list (rgb, tty_color))
! 	return 0;
  
!       /* Should we fill in STD_COLOR too?  */
!       if (std_color)
! 	{
! 	  /* Default STD_COLOR to the same as TTY_COLOR.  */
! 	  *std_color = *tty_color;
  
! 	  /* Do a quick check to see if the returned descriptor is
! 	     actually _exactly_ equal to COLOR, otherwise we have to
! 	     lookup STD_COLOR separately.  If it's impossible to lookup
! 	     a standard color, we just give up and use TTY_COLOR.  */
! 	  if ((!STRINGP (XCAR (color_desc))
! 	       || NILP (Fstring_equal (color, XCAR (color_desc))))
! 	      && !NILP (Ffboundp (Qtty_color_standard_values)))
! 	    {
! 	      /* Look up STD_COLOR separately.  */
! 	      rgb = call1 (Qtty_color_standard_values, color);
! 	      if (! parse_rgb_list (rgb, std_color))
! 		return 0;
! 	    }
! 	}
  
!       return 1;
      }
-   else if (NILP (Fsymbol_value (intern ("tty-defined-color-alist"))))
-     /* We were called early during startup, and the colors are not
-        yet set up in tty-defined-color-alist.  Don't return a failure
-        indication, since this produces the annoying "Unable to
-        load color" messages in the *Messages* buffer.  */
-     return 1;
-   else
-     /* tty-color-desc seems to have returned a bad value.  */
-     return 0;
  }
  
  /* A version of defined_color for non-X frames.  */
--- 943,1022 ----
    if (!STRINGP (color) || NILP (Ffboundp (Qtty_color_desc)))
      return 0;
  
!   if (f->output_method == output_termcap
!       && f->output_data.tty->display_info->TS_set_rgb_foreground
!       && !NILP (Ffboundp (Qtty_color_standard_values)))
!     {
!       /* Terminal supports 3 byte RGB colors. */
!       if (!NILP (Ffboundp (Qtty_color_canonicalize)))
!         color = call1(Qtty_color_canonicalize, color);
! 
!       color_desc = call1 (Qtty_color_standard_values, color);
!       if (! parse_rgb_list (color_desc, tty_color))
!         return 0;
! 
!       /* Map XColor to 3 byte values. */
!       tty_color->pixel = 1 << 24 /* Set bit 24 to mark RGB values. */
!         | (tty_color->red / 256) << 16
!         | (tty_color->green / 256) << 8
!         | (tty_color->blue / 256);
! 
!       if (std_color)
!         *std_color = *tty_color;
  
!       return 1;
!     }
!   else
      {
!       XSETFRAME (frame, f);
  
!       color_desc = call2 (Qtty_color_desc, color, frame);
!       if (CONSP (color_desc) && CONSP (XCDR (color_desc)))
!         {
!           Lisp_Object rgb;
  
!           if (! INTEGERP (XCAR (XCDR (color_desc))))
!             return 0;
  
!           tty_color->pixel = XINT (XCAR (XCDR (color_desc)));
  
!           rgb = XCDR (XCDR (color_desc));
!           if (! parse_rgb_list (rgb, tty_color))
!             return 0;
  
!           /* Should we fill in STD_COLOR too?  */
!           if (std_color)
!             {
!               /* Default STD_COLOR to the same as TTY_COLOR.  */
!               *std_color = *tty_color;
  
!               /* Do a quick check to see if the returned descriptor is
!                  actually _exactly_ equal to COLOR, otherwise we have to
!                  lookup STD_COLOR separately.  If it's impossible to lookup
!                  a standard color, we just give up and use TTY_COLOR.  */
!               if ((!STRINGP (XCAR (color_desc))
!                    || NILP (Fstring_equal (color, XCAR (color_desc))))
!                   && !NILP (Ffboundp (Qtty_color_standard_values)))
!                 {
!                   /* Look up STD_COLOR separately.  */
!                   rgb = call1 (Qtty_color_standard_values, color);
!                   if (! parse_rgb_list (rgb, std_color))
!                     return 0;
!                 }
!             }
! 
!           return 1;
!         }
!       else if (NILP (Fsymbol_value (intern ("tty-defined-color-alist"))))
!         /* We were called early during startup, and the colors are not
!            yet set up in tty-defined-color-alist.  Don't return a failure
!            indication, since this produces the annoying "Unable to
!            load color" messages in the *Messages* buffer.  */
!         return 1;
!       else
!         /* tty-color-desc seems to have returned a bad value.  */
!         return 0;
      }
  }
  
  /* A version of defined_color for non-X frames.  */
***************
*** 1008,1014 ****
    color_def->green = 0;
  
    if (*color_name)
!     status = tty_lookup_color (f, build_string (color_name), color_def, NULL);
  
    if (color_def->pixel == FACE_TTY_DEFAULT_COLOR && *color_name)
      {
--- 1034,1042 ----
    color_def->green = 0;
  
    if (*color_name)
!     {
!       status = tty_lookup_color (f, build_string (color_name), color_def, NULL);
!     }
  
    if (color_def->pixel == FACE_TTY_DEFAULT_COLOR && *color_name)
      {
***************
*** 5780,5785 ****
--- 5808,5814 ----
    unsigned long default_pixel =
      foreground_p ? FACE_TTY_DEFAULT_FG_COLOR : FACE_TTY_DEFAULT_BG_COLOR;
    unsigned long pixel = default_pixel;
+   XColor true_color;
  #ifdef MSDOS
    unsigned long default_other_pixel =
      foreground_p ? FACE_TTY_DEFAULT_BG_COLOR : FACE_TTY_DEFAULT_FG_COLOR;
***************
*** 5798,5804 ****
      {
        /* Associations in tty-defined-color-alist are of the form
  	 (NAME INDEX R G B).  We need the INDEX part.  */
!       pixel = XINT (XCAR (XCDR (def)));
      }
  
    if (pixel == default_pixel && STRINGP (color))
--- 5827,5844 ----
      {
        /* Associations in tty-defined-color-alist are of the form
  	 (NAME INDEX R G B).  We need the INDEX part.  */
!       if (f->output_method == output_termcap
!           && f->output_data.tty->display_info->TS_set_rgb_foreground
!           && parse_rgb_list (XCDR (XCDR(def)), &true_color))
!         {
!           /* Map XColor to 3 byte values. */
!           pixel = 1 << 24 /* Set bit 24 to mark RGB values. */
!             | (true_color.red / 256) << 16
!             | (true_color.green / 256) << 8
!             | (true_color.blue / 256);
!         }
!        else
!          pixel = XINT (XCAR (XCDR (def)));
      }
  
    if (pixel == default_pixel && STRINGP (color))
***************
*** 6460,6465 ****
--- 6500,6506 ----
    DEFSYM (Qwindow_divider, "window-divider");
    DEFSYM (Qwindow_divider_first_pixel, "window-divider-first-pixel");
    DEFSYM (Qwindow_divider_last_pixel, "window-divider-last-pixel");
+   DEFSYM (Qtty_color_canonicalize, "tty-color-canonicalize");
    DEFSYM (Qtty_color_desc, "tty-color-desc");
    DEFSYM (Qtty_color_standard_values, "tty-color-standard-values");
    DEFSYM (Qtty_color_by_index, "tty-color-by-index");

everybody who contributed to this patch is doing God's work

Could you explain why there's

     tty->TS_set_rgb_foreground = "\033[38;2;%p1%d;%p2%d;%p3%dm";

vs

     tty->TS_set_rgb_foreground = "\033[38;2;%d;%d;%dm";

?

It looks like the former also prints some pointers, but I can't imagine why...

It looks like tmux's true color support is hard coded to use the latter snippet: tmux/tmux@427b820

Oh, spoke too soon. Just read src/tparam.c.

So, in the ifdef TERMINFO case, we're using %pN to substitute positional params (corresponding to red, green and blue values). But, if I'm reading this right, when TERMINFO isn't defined, we still print the params in the same order. If I understood that correctly, why bother with the #ifdef?

(I'd like to get this upstreamed, but I think I should first understand the current state of this patch.)

How do you apply the patch? I'm sorry, I'm a noob at this.

What file do I have to apply it to?