path: root/data/vim/patches/8.1.0633

To: vim_dev@googlegroups.com
Subject: Patch 8.1.0633
Fcc: outbox
From: Bram Moolenaar <Bram@moolenaar.net>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
------------

Patch 8.1.0633
Problem:    Crash when out of memory while opening a terminal window.
Solution:   Handle out-of-memory more gracefully.
Files:	    src/terminal.c, src/libvterm/src/vterm.c,
            src/libvterm/src/state.c, src/libvterm/src/termscreen.c


*** ../vim-8.1.0632/src/terminal.c	2018-12-21 20:55:18.892739645 +0100
--- src/terminal.c	2018-12-24 21:22:28.150773840 +0100
***************
*** 3430,3435 ****
--- 3430,3436 ----
  {
      int		index = 0;
      VTermState	*state = vterm_obtain_state(vterm);
+ 
      for (; index < 16; index++)
      {
  	VTermColor	color;
***************
*** 3703,3710 ****
  
  /*
   * Create a new vterm and initialize it.
   */
!     static void
  create_vterm(term_T *term, int rows, int cols)
  {
      VTerm	    *vterm;
--- 3704,3712 ----
  
  /*
   * Create a new vterm and initialize it.
+  * Return FAIL when out of memory.
   */
!     static int
  create_vterm(term_T *term, int rows, int cols)
  {
      VTerm	    *vterm;
***************
*** 3714,3720 ****
--- 3716,3733 ----
  
      vterm = vterm_new_with_allocator(rows, cols, &vterm_allocator, NULL);
      term->tl_vterm = vterm;
+     if (vterm == NULL)
+ 	return FAIL;
+ 
+     // Allocate screen and state here, so we can bail out if that fails.
+     state = vterm_obtain_state(vterm);
      screen = vterm_obtain_screen(vterm);
+     if (state == NULL || screen == NULL)
+     {
+ 	vterm_free(vterm);
+ 	return FAIL;
+     }
+ 
      vterm_screen_set_callbacks(screen, &screen_callbacks, term);
      /* TODO: depends on 'encoding'. */
      vterm_set_utf8(vterm, 1);
***************
*** 3722,3728 ****
      init_default_colors(term);
  
      vterm_state_set_default_colors(
! 	    vterm_obtain_state(vterm),
  	    &term->tl_default_color.fg,
  	    &term->tl_default_color.bg);
  
--- 3735,3741 ----
      init_default_colors(term);
  
      vterm_state_set_default_colors(
! 	    state,
  	    &term->tl_default_color.fg,
  	    &term->tl_default_color.bg);
  
***************
*** 3746,3754 ****
  #else
      value.boolean = 0;
  #endif
-     state = vterm_obtain_state(vterm);
      vterm_state_set_termprop(state, VTERM_PROP_CURSORBLINK, &value);
      vterm_state_set_unrecognised_fallbacks(state, &parser_fallbacks, term);
  }
  
  /*
--- 3759,3768 ----
  #else
      value.boolean = 0;
  #endif
      vterm_state_set_termprop(state, VTERM_PROP_CURSORBLINK, &value);
      vterm_state_set_unrecognised_fallbacks(state, &parser_fallbacks, term);
+ 
+     return OK;
  }
  
  /*
***************
*** 5629,5635 ****
      vim_free(cwd_wchar);
      vim_free(env_wchar);
  
!     create_vterm(term, term->tl_rows, term->tl_cols);
  
  #if defined(FEAT_GUI) || defined(FEAT_TERMGUICOLORS)
      if (opt->jo_set2 & JO2_ANSI_COLORS)
--- 5643,5650 ----
      vim_free(cwd_wchar);
      vim_free(env_wchar);
  
!     if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL)
! 	goto failed;
  
  #if defined(FEAT_GUI) || defined(FEAT_TERMGUICOLORS)
      if (opt->jo_set2 & JO2_ANSI_COLORS)
***************
*** 5710,5716 ****
      char	    in_name[80], out_name[80];
      channel_T	    *channel = NULL;
  
!     create_vterm(term, term->tl_rows, term->tl_cols);
  
      vim_snprintf(in_name, sizeof(in_name), "\\\\.\\pipe\\vim-%d-in-%d",
  	    GetCurrentProcessId(),
--- 5725,5732 ----
      char	    in_name[80], out_name[80];
      channel_T	    *channel = NULL;
  
!     if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL)
! 	return FAIL;
  
      vim_snprintf(in_name, sizeof(in_name), "\\\\.\\pipe\\vim-%d-in-%d",
  	    GetCurrentProcessId(),
***************
*** 5822,5828 ****
  	jobopt_T    *opt,
  	jobopt_T    *orig_opt UNUSED)
  {
!     create_vterm(term, term->tl_rows, term->tl_cols);
  
  #if defined(FEAT_GUI) || defined(FEAT_TERMGUICOLORS)
      if (opt->jo_set2 & JO2_ANSI_COLORS)
--- 5838,5845 ----
  	jobopt_T    *opt,
  	jobopt_T    *orig_opt UNUSED)
  {
!     if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL)
! 	return FAIL;
  
  #if defined(FEAT_GUI) || defined(FEAT_TERMGUICOLORS)
      if (opt->jo_set2 & JO2_ANSI_COLORS)
***************
*** 5844,5850 ****
      static int
  create_pty_only(term_T *term, jobopt_T *opt)
  {
!     create_vterm(term, term->tl_rows, term->tl_cols);
  
      term->tl_job = job_alloc();
      if (term->tl_job == NULL)
--- 5861,5868 ----
      static int
  create_pty_only(term_T *term, jobopt_T *opt)
  {
!     if (create_vterm(term, term->tl_rows, term->tl_cols) == FAIL)
! 	return FAIL;
  
      term->tl_job = job_alloc();
      if (term->tl_job == NULL)
*** ../vim-8.1.0632/src/libvterm/src/vterm.c	2018-12-15 14:49:30.800096933 +0100
--- src/libvterm/src/vterm.c	2018-12-24 21:26:14.100725256 +0100
***************
*** 1,5 ****
--- 1,6 ----
  #define DEFINE_INLINES
  
+ /* vim: set sw=2 : */
  #include "vterm_internal.h"
  
  #include <stdio.h>
***************
*** 41,46 ****
--- 42,49 ----
    /* Need to bootstrap using the allocator function directly */
    VTerm *vt = (*funcs->malloc)(sizeof(VTerm), allocdata);
  
+   if (vt == NULL)
+     return NULL;
    vt->allocator = funcs;
    vt->allocdata = allocdata;
  
***************
*** 55,64 ****
--- 58,78 ----
    vt->parser.strbuffer_len = 500; /* should be able to hold an OSC string */
    vt->parser.strbuffer_cur = 0;
    vt->parser.strbuffer = vterm_allocator_malloc(vt, vt->parser.strbuffer_len);
+   if (vt->parser.strbuffer == NULL)
+   {
+     vterm_allocator_free(vt, vt);
+     return NULL;
+   }
  
    vt->outbuffer_len = 200;
    vt->outbuffer_cur = 0;
    vt->outbuffer = vterm_allocator_malloc(vt, vt->outbuffer_len);
+   if (vt->outbuffer == NULL)
+   {
+     vterm_allocator_free(vt, vt->parser.strbuffer);
+     vterm_allocator_free(vt, vt);
+     return NULL;
+   }
  
    return vt;
  }
***************
*** 82,90 ****
    return (*vt->allocator->malloc)(size, vt->allocdata);
  }
  
  INTERNAL void vterm_allocator_free(VTerm *vt, void *ptr)
  {
!   (*vt->allocator->free)(ptr, vt->allocdata);
  }
  
  void vterm_get_size(const VTerm *vt, int *rowsp, int *colsp)
--- 96,108 ----
    return (*vt->allocator->malloc)(size, vt->allocdata);
  }
  
+ /*
+  * Free "ptr" unless it is NULL.
+  */
  INTERNAL void vterm_allocator_free(VTerm *vt, void *ptr)
  {
!   if (ptr)
!     (*vt->allocator->free)(ptr, vt->allocdata);
  }
  
  void vterm_get_size(const VTerm *vt, int *rowsp, int *colsp)
*** ../vim-8.1.0632/src/libvterm/src/state.c	2018-07-09 20:39:12.824845063 +0200
--- src/libvterm/src/state.c	2018-12-24 20:45:42.697972984 +0100
***************
*** 53,58 ****
--- 53,60 ----
  {
    VTermState *state = vterm_allocator_malloc(vt, sizeof(VTermState));
  
+   if (state == NULL)
+     return NULL;
    state->vt = vt;
  
    state->rows = vt->rows;
***************
*** 1693,1698 ****
--- 1695,1704 ----
    on_resize /* resize */
  };
  
+ /*
+  * Return the existing state or create a new one.
+  * Returns NULL when out of memory.
+  */
  VTermState *vterm_obtain_state(VTerm *vt)
  {
    VTermState *state;
***************
*** 1700,1705 ****
--- 1706,1713 ----
      return vt->state;
  
    state = vterm_state_new(vt);
+   if (state == NULL)
+     return NULL;
    vt->state = state;
  
    state->combine_chars_size = 16;
*** ../vim-8.1.0632/src/libvterm/src/termscreen.c	2018-09-13 17:23:05.169150892 +0200
--- src/libvterm/src/termscreen.c	2018-12-24 21:29:52.730781805 +0100
***************
*** 1,5 ****
--- 1,6 ----
  #include "vterm_internal.h"
  
+ /* vim: set sw=2 : */
  #include <stdio.h>
  #include <string.h>
  
***************
*** 95,102 ****
      }
    }
  
!   if(buffer)
!     vterm_allocator_free(screen->vt, buffer);
  
    return new_buffer;
  }
--- 96,102 ----
      }
    }
  
!   vterm_allocator_free(screen->vt, buffer);
  
    return new_buffer;
  }
***************
*** 518,525 ****
    screen->rows = new_rows;
    screen->cols = new_cols;
  
!   if(screen->sb_buffer)
!     vterm_allocator_free(screen->vt, screen->sb_buffer);
  
    screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * new_cols);
  
--- 518,524 ----
    screen->rows = new_rows;
    screen->cols = new_cols;
  
!   vterm_allocator_free(screen->vt, screen->sb_buffer);
  
    screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * new_cols);
  
***************
*** 619,634 ****
    &setlineinfo /* setlineinfo */
  };
  
  static VTermScreen *screen_new(VTerm *vt)
  {
    VTermState *state = vterm_obtain_state(vt);
    VTermScreen *screen;
    int rows, cols;
  
!   if(!state)
      return NULL;
- 
    screen = vterm_allocator_malloc(vt, sizeof(VTermScreen));
  
    vterm_get_size(vt, &rows, &cols);
  
--- 618,638 ----
    &setlineinfo /* setlineinfo */
  };
  
+ /*
+  * Allocate a new screen and return it.
+  * Return NULL when out of memory.
+  */
  static VTermScreen *screen_new(VTerm *vt)
  {
    VTermState *state = vterm_obtain_state(vt);
    VTermScreen *screen;
    int rows, cols;
  
!   if (state == NULL)
      return NULL;
    screen = vterm_allocator_malloc(vt, sizeof(VTermScreen));
+   if (screen == NULL)
+     return NULL;
  
    vterm_get_size(vt, &rows, &cols);
  
***************
*** 646,655 ****
    screen->cbdata    = NULL;
  
    screen->buffers[0] = realloc_buffer(screen, NULL, rows, cols);
- 
    screen->buffer = screen->buffers[0];
- 
    screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * cols);
  
    vterm_state_set_callbacks(screen->state, &state_cbs, screen);
  
--- 650,662 ----
    screen->cbdata    = NULL;
  
    screen->buffers[0] = realloc_buffer(screen, NULL, rows, cols);
    screen->buffer = screen->buffers[0];
    screen->sb_buffer = vterm_allocator_malloc(screen->vt, sizeof(VTermScreenCell) * cols);
+   if (screen->buffer == NULL || screen->sb_buffer == NULL)
+   {
+     vterm_screen_free(screen);
+     return NULL;
+   }
  
    vterm_state_set_callbacks(screen->state, &state_cbs, screen);
  
***************
*** 659,669 ****
  INTERNAL void vterm_screen_free(VTermScreen *screen)
  {
    vterm_allocator_free(screen->vt, screen->buffers[0]);
!   if(screen->buffers[1])
!     vterm_allocator_free(screen->vt, screen->buffers[1]);
! 
    vterm_allocator_free(screen->vt, screen->sb_buffer);
- 
    vterm_allocator_free(screen->vt, screen);
  }
  
--- 666,673 ----
  INTERNAL void vterm_screen_free(VTermScreen *screen)
  {
    vterm_allocator_free(screen->vt, screen->buffers[0]);
!   vterm_allocator_free(screen->vt, screen->buffers[1]);
    vterm_allocator_free(screen->vt, screen->sb_buffer);
    vterm_allocator_free(screen->vt, screen);
  }
  
*** ../vim-8.1.0632/src/version.c	2018-12-24 20:23:39.440716979 +0100
--- src/version.c	2018-12-24 21:34:22.808414341 +0100
***************
*** 801,802 ****
--- 801,804 ----
  {   /* Add new patch number below this line */
+ /**/
+     633,
  /**/

-- 
"Oh, no!  NOT the Spanish Inquisition!"
"NOBODY expects the Spanish Inquisition!!!"
				-- Monty Python sketch --
"Oh, no!  NOT another option!"
"EVERYBODY expects another option!!!"
				-- Discussion in vim-dev mailing list --

 /// Bram Moolenaar -- Bram@Moolenaar.net -- http://www.Moolenaar.net   \\\
///        sponsor Vim, vote for features -- http://www.Vim.org/sponsor/ \\\
\\\  an exciting new programming language -- http://www.Zimbu.org        ///
 \\\            help me help AIDS victims -- http://ICCF-Holland.org    ///