Introduce rb_parser_dedent_string function

ext/ripper/ripper_init.c.tmpl

@@ -249,6 +249,15 @@ ripper_parser_set_debug_output(VALUE self, VALUE output)
  return output;
 }
 
+static int
+rb_ruby_ripper_dedent_string(rb_parser_t *p, VALUE string, int width)
+{
+ rb_parser_string_t *str = rb_str_to_parser_string(p, string);
+ int result = rb_parser_dedent_string(p, str, width);
+ rb_str_replace(string, rb_str_new_parser_string(str));
+ return result;
+}
+
 #ifdef UNIVERSAL_PARSER
 struct dedent_string_arg {
  struct parser_params *p;

internal/parse.h

@@ -70,6 +70,10 @@ int rb_parser_reg_fragment_check(struct parser_params*, rb_parser_string_t*, int
 int rb_reg_named_capture_assign_iter_impl(struct parser_params *p, const char *s, long len, rb_encoding *enc, NODE **succ_block, const rb_code_location_t *loc);
 int rb_parser_local_defined(struct parser_params *p, ID id, const struct rb_iseq_struct *iseq);
 
+#ifndef UNIVERSAL_PARSER
+int rb_parser_dedent_string(struct parser_params *p, rb_parser_string_t *string, int width);
+#endif
+
 RUBY_SYMBOL_EXPORT_END
 
 #ifndef UNIVERSAL_PARSER
@@ -97,7 +101,6 @@ VALUE rb_ruby_parser_ruby_sourcefile_string(rb_parser_t *p);
 int rb_ruby_parser_ruby_sourceline(rb_parser_t *p);
 int rb_ruby_parser_lex_state(rb_parser_t *p);
 void rb_ruby_ripper_parse0(rb_parser_t *p);
-int rb_ruby_ripper_dedent_string(rb_parser_t *p, VALUE string, int width);
 int rb_ruby_ripper_initialized_p(rb_parser_t *p);
 void rb_ruby_ripper_parser_initialize(rb_parser_t *p);
 long rb_ruby_ripper_column(rb_parser_t *p);

internal/ruby_parser.h

@@ -19,6 +19,7 @@ RUBY_SYMBOL_EXPORT_BEGIN
 #ifdef UNIVERSAL_PARSER
 const rb_parser_config_t *rb_ruby_parser_config(void);
 rb_parser_t *rb_parser_params_new(void);
+int rb_parser_dedent_string(rb_parser_t *p, rb_parser_string_t *string, int width);
 #endif
 VALUE rb_parser_set_context(VALUE, const struct rb_iseq_struct *, int);
 VALUE rb_parser_new(void);

parse.y

@@ -2131,11 +2131,15 @@ rb_char_p_hash(const char *c)
  return parser_memhash((const void *)c, strlen(c));
 }
 
+#ifndef UNIVERSAL_PARSER
+#ifndef RIPPER
 static size_t
 rb_parser_str_capacity(rb_parser_string_t *str, const int termlen)
 {
  return PARSER_STRING_LEN(str);
 }
+#endif
+#endif /* !UNIVERSAL_PARSER */
 
 #ifndef RIPPER
 static char *
@@ -2344,6 +2348,8 @@ rb_parser_str_modify(rb_parser_string_t *str)
  PARSER_ENC_CODERANGE_CLEAR(str);
 }
 
+#ifndef UNIVERSAL_PARSER
+#ifndef RIPPER
 static void
 rb_parser_str_set_len(struct parser_params *p, rb_parser_string_t *str, long len)
 {
@@ -2372,6 +2378,8 @@ rb_parser_str_set_len(struct parser_params *p, rb_parser_string_t *str, long len
  STRING_SET_LEN(str, len);
  STRING_TERM_FILL(str);
 }
+#endif
+#endif /* !UNIVERSAL_PARSER */
 
 static rb_parser_string_t *
 rb_parser_str_buf_cat(struct parser_params *p, rb_parser_string_t *str, const char *ptr, long len)
@@ -9066,6 +9074,8 @@ heredoc_restore(struct parser_params *p, rb_strterm_heredoc_t *here)
  xfree(term);
 }
 
+#ifndef UNIVERSAL_PARSER
+#ifndef RIPPER
 static int
 dedent_string_column(const char *str, long len, int width)
 {
@@ -9088,8 +9098,8 @@ dedent_string_column(const char *str, long len, int width)
  return i;
 }
 
-static int
-dedent_string(struct parser_params *p, rb_parser_string_t *string, int width)
+int
+rb_parser_dedent_string(struct parser_params *p, rb_parser_string_t *string, int width)
 {
  char *str;
  long len;
@@ -9109,6 +9119,8 @@ dedent_string(struct parser_params *p, rb_parser_string_t *string, int width)
  rb_parser_str_set_len(p, string, len - i);
  return i;
 }
+#endif
+#endif /* !UNIVERSAL_PARSER */
 
 static NODE *
 heredoc_dedent(struct parser_params *p, NODE *root)
@@ -9127,7 +9139,7 @@ heredoc_dedent(struct parser_params *p, NODE *root)
  while (str_node) {
  rb_parser_string_t *lit = RNODE_STR(str_node)->string;
  if (nd_fl_newline(str_node)) {
- dedent_string(p, lit, indent);
+ rb_parser_dedent_string(p, lit, indent);
  }
  if (!prev_lit) {
  prev_lit = lit;
@@ -16069,26 +16081,6 @@ rb_ruby_ripper_parse0(rb_parser_t *p)
  p->eval_tree_begin = 0;
 }
 
-int
-rb_ruby_ripper_dedent_string(rb_parser_t *p, VALUE string, int width)
-{
- char *str;
- long len;
- int i;
-
- RSTRING_GETMEM(string, str, len);
- i = dedent_string_column(str, len, width);
- if (!i) return 0;
-
- rb_str_modify(string);
- str = RSTRING_PTR(string);
- if (RSTRING_LEN(string) != len)
- rb_fatal("literal string changed: %+"PRIsVALUE, string);
- MEMMOVE(str, str + i, char, len - i);
- rb_str_set_len(string, len - i);
- return i;
-}
-
 int
 rb_ruby_ripper_initialized_p(rb_parser_t *p)
 {

ruby_parser.c

@@ -250,12 +250,6 @@ syntax_error_new(void)
  return rb_class_new_instance(0, 0, rb_eSyntaxError);
 }
 
-static void *
-memmove2(void *dest, const void *src, size_t t, size_t n)
-{
- return memmove(dest, src, rbimpl_size_mul_or_raise(t, n));
-}
-
 static void *
 nonempty_memcpy(void *dest, const void *src, size_t t, size_t n)
 {
@@ -342,7 +336,6 @@ static const rb_parser_config_t rb_global_parser_config = {
  .alloc = ruby_xmalloc,
  .realloc_n = ruby_xrealloc2,
  .zalloc = zalloc,
- .rb_memmove = memmove2,
  .nonempty_memcpy = nonempty_memcpy,
  .xmalloc_mul_add = rb_xmalloc_mul_add,
 
@@ -454,7 +447,6 @@ static const rb_parser_config_t rb_global_parser_config = {
  .compile_warn = rb_compile_warn,
  .compile_warning = rb_compile_warning,
  .bug = rb_bug,
- .fatal = rb_fatal,
  .verbose = ruby_verbose2,
  .errno_ptr = rb_errno_ptr2,
 
@@ -476,6 +468,97 @@ static const rb_parser_config_t rb_global_parser_config = {
  .static_id2sym = static_id2sym,
  .str_coderange_scan_restartable = str_coderange_scan_restartable,
 };
+
+
+static size_t
+rb_parser_str_capacity(rb_parser_string_t *str, const int termlen)
+{
+ return str->len;
+}
+
+static void
+rb_parser_str_set_len(rb_parser_t *p, rb_parser_string_t *str, long len)
+{
+#define RARSER_STRING_TERM_FILL(str) (str->ptr[str->len] = '\0')
+#define RARSER_STRING_TERM_LEN(str) (1)
+#define RARSER_STRING_SET_LEN(str, n) do { \
+ (str)->len = (n); \
+} while (0)
+ long capa;
+ const int termlen = RARSER_STRING_TERM_LEN(str);
+
+ if (len > (capa = (long)(rb_parser_str_capacity(str, termlen))) || len < 0) {
+ rb_bug("probable buffer overflow: %ld for %ld", len, capa);
+ }
+
+ int cr = str->coderange;
+ if (cr == RB_PARSER_ENC_CODERANGE_UNKNOWN) {
+ /* Leave unknown. */
+ }
+ else if (len > str->len) {
+ str->coderange = RB_PARSER_ENC_CODERANGE_UNKNOWN;
+ }
+ else if (len < str->len) {
+ if (cr != RB_PARSER_ENC_CODERANGE_7BIT) {
+ /* ASCII-only string is keeping after truncated. Valid
+ * and broken may be invalid or valid, leave unknown. */
+ str->coderange = RB_PARSER_ENC_CODERANGE_UNKNOWN;
+ }
+ }
+
+ RARSER_STRING_SET_LEN(str, len);
+ RARSER_STRING_TERM_FILL(str);
+#undef RARSER_STRING_TERM_FILL
+#undef RARSER_STRING_TERM_LEN
+#undef RARSER_STRING_SET_LEN
+}
+
+static int
+dedent_string_column(const char *str, long len, int width)
+{
+#define TAB_WIDTH 8
+ int i, col = 0;
+
+ for (i = 0; i < len && col < width; i++) {
+ if (str[i] == ' ') {
+ col++;
+ }
+ else if (str[i] == '\t') {
+ int n = TAB_WIDTH * (col / TAB_WIDTH + 1);
+ if (n > width) break;
+ col = n;
+ }
+ else {
+ break;
+ }
+ }
+
+ return i;
+#undef TAB_WIDTH
+}
+
+int
+rb_parser_dedent_string(rb_parser_t *p, rb_parser_string_t *string, int width)
+{
+ char *str;
+ long len;
+ int i;
+
+ len = string->len;
+ str = string->ptr;
+
+ i = dedent_string_column(str, len, width);
+ if (!i) return 0;
+
+ string->coderange = RB_PARSER_ENC_CODERANGE_UNKNOWN;
+ str = string->ptr;
+ if (string->len != len)
+ rb_fatal("literal string changed: %s", string->ptr);
+ MEMMOVE(str, str + i, char, len - i);
+ rb_parser_str_set_len(p, string, len - i);
+ return i;
+}
+
 #endif
 
 enum lex_type {

rubyparser.h

@@ -1247,7 +1247,6 @@ typedef struct rb_parser_config_struct {
  void *(*alloc)(size_t elemsiz);
  void *(*realloc_n)(void *ptr, size_t newelems, size_t newsiz);
  void *(*zalloc)(size_t elemsiz);
- void *(*rb_memmove)(void *dest, const void *src, size_t t, size_t n);
  void *(*nonempty_memcpy)(void *dest, const void *src, size_t t, size_t n);
  void *(*xmalloc_mul_add)(size_t x, size_t y, size_t z);
 
@@ -1380,7 +1379,6 @@ typedef struct rb_parser_config_struct {
  void (*compile_warn)(const char *file, int line, const char *fmt, ...) RUBYPARSER_ATTRIBUTE_FORMAT(3, 4);
  void (*compile_warning)(const char *file, int line, const char *fmt, ...) RUBYPARSER_ATTRIBUTE_FORMAT(3, 4);
  void (*bug)(const char *fmt, ...) RUBYPARSER_ATTRIBUTE_FORMAT(1, 2);
- void (*fatal)(const char *fmt, ...) RUBYPARSER_ATTRIBUTE_FORMAT(1, 2);
  VALUE (*verbose)(void);
  int *(*errno_ptr)(void);
 

universal_parser.c

@@ -80,8 +80,6 @@
 #define REALLOC_N(var,type,n) ((var) = (type *)p->config->realloc_n((void *)var, n, sizeof(type)))
 #undef ZALLOC
 #define ZALLOC(type) ((type *)p->config->zalloc(sizeof(type)))
-#undef MEMMOVE
-#define MEMMOVE(p1,p2,type,n) (p->config->rb_memmove((p1), (p2), sizeof(type), (n)))
 #undef MEMCPY
 #define MEMCPY(p1,p2,type,n) (p->config->nonempty_memcpy((p1), (p2), sizeof(type), (n)))
 
@@ -200,7 +198,6 @@
 #define rb_compile_warn p->config->compile_warn
 #define rb_compile_warning p->config->compile_warning
 #define rb_bug p->config->bug
-#define rb_fatal p->config->fatal
 #undef ruby_verbose
 #define ruby_verbose p->config->verbose()
 #undef errno