fix-generators patches

0001-Improve-yield-support-and-restrict-usage-of-strict.patch

From 76698eeb1135148567d999922f1b55e14a734bbe Mon Sep 17 00:00:00 2001
From: Anthony Van de Gejuchte <[email protected]>
Date: Thu, 26 May 2016 17:00:37 +0200
Subject: [PATCH 1/2] Improve yield support and restrict usage of strict

- Partially reverting 91cdb93e57 and eaf3911c31 and reimplement
- Add generators support for objects and classes
- Only classes can have static methods so restrict use of it
---
 lib/ast.js               | 18 ++++++++-
 lib/compress.js          |  2 +-
 lib/output.js            | 36 ++++++++++++++----
 lib/parse.js             | 92 +++++++++++++++++++++++++++++++++++-----------
 test/compress/harmony.js | 95 ++++++++++++++++++++++++++++++++++++++++++++++++
 test/mocha/object.js     | 20 ++++++++++
 test/mocha/yield.js      | 34 +++++++++++++++++
 test/parser.js           |  2 +-
 8 files changed, 267 insertions(+), 32 deletions(-)
 create mode 100644 test/mocha/object.js
 create mode 100644 test/mocha/yield.js

diff --git a/lib/ast.js b/lib/ast.js
index b89a00d..0cb44ac 100644
--- a/lib/ast.js
+++ b/lib/ast.js
@@ -487,8 +487,9 @@ var AST_Arrow = DEFNODE("Arrow", null, {
     $documentation: "An ES6 Arrow function ((a) => b)"
 }, AST_Lambda);
 
-var AST_ConciseMethod = DEFNODE("ConciseMethod", "static", {
+var AST_ConciseMethod = DEFNODE("ConciseMethod", "is_generator static", {
     $propdoc: {
+        is_generator: "is generatorFn or not",
         static: "[boolean] whether this method is static (classes only)",
     },
     $documentation: "An ES6 concise method inside an object or class"
@@ -1241,6 +1242,21 @@ var AST_True = DEFNODE("True", null, {
     value: true
 }, AST_Boolean);
 
+/* -----[ Yield ]----- */
+
+var AST_Yield = DEFNODE("Yield", "value is_star", {
+    $documentation: "A `yield` statement",
+    $propdoc: {
+        value: "[AST_Node?] the value returned or thrown by this statement; could be null",
+        is_star: "[Boolean] Whether this is a yield or yield* statement"
+    },
+    _walk: function(visitor) {
+        return visitor._visit(this, this.value && function(){
+            this.value._walk(visitor);
+        });
+    }
+}, AST_Statement);
+
 /* -----[ TreeWalker ]----- */
 
 function TreeWalker(callback) {
diff --git a/lib/compress.js b/lib/compress.js
index 31e42d5..2a05e40 100644
--- a/lib/compress.js
+++ b/lib/compress.js
@@ -1245,7 +1245,7 @@ merge(Compressor.prototype, {
                 || this.alternative.has_side_effects(compressor);
         });
         def(AST_Unary, function(compressor){
-            return member(this.operator, ["delete", "++", "--", "yield", "yield*"])
+            return member(this.operator, ["delete", "++", "--"])
                 || this.expression.has_side_effects(compressor);
         });
         def(AST_SymbolRef, function(compressor){
diff --git a/lib/output.js b/lib/output.js
index a71cd36..3eb33e1 100644
--- a/lib/output.js
+++ b/lib/output.js
@@ -560,6 +560,20 @@ function OutputStream(options) {
         }
     });
 
+    PARENS(AST_Yield, function(output){
+        var p = output.parent();
+        // (yield 1) + (yield 2)
+        // a = yield 3
+        if (p instanceof AST_Binary && p.operator !== "=")
+            return true;
+        // (yield 1) ? yield 2 : yield 3
+        if (p instanceof AST_Conditional && p.condition === this)
+            return true;
+        // -(yield 4)
+        if (p instanceof AST_Unary)
+            return true;
+    });
+
     PARENS(AST_PropAccess, function(output){
         var p = output.parent();
         if (p instanceof AST_New && p.expression === this) {
@@ -868,6 +882,9 @@ function OutputStream(options) {
             output.print("static");
             output.space();
         }
+        if (self.is_generator) {
+            output.print("*");
+        }
         self._do_print(output, true /* do not print "function" */);
     });
 
@@ -887,6 +904,17 @@ function OutputStream(options) {
         self._do_print(output, "throw");
     });
 
+    /* -----[ yield ]----- */
+
+    DEFPRINT(AST_Yield, function(self, output){
+        var star = self.is_star ? "*" : "";
+        output.print("yield" + star);
+        if (self.value) {
+            output.space();
+            self.value.print(output);
+        }
+    });
+
     /* -----[ loop control ]----- */
     AST_LoopControl.DEFMETHOD("_do_print", function(output, kind){
         output.print(kind);
@@ -1218,13 +1246,8 @@ function OutputStream(options) {
         output.print(self.operator);
     });
     DEFPRINT(AST_Binary, function(self, output){
-        var isYield = (self.left.operator == "yield" || self.left.operator === "yield*");
         var op = self.operator;
-
-        isYield && output.print("(");
         self.left.print(output);
-        isYield && output.print(")");
-
         if (op[0] == ">" /* ">>" ">>>" ">" ">=" */
             && self.left instanceof AST_UnaryPostfix
             && self.left.operator == "--") {
@@ -1234,10 +1257,7 @@ function OutputStream(options) {
             // the space is optional depending on "beautify"
             output.space();
         }
-
-        isYield = (self.right.operator == "yield" || self.right.operator === "yield*");
         output.print(op);
-
         if ((op == "<" || op == "<<")
             && self.right instanceof AST_UnaryPrefix
             && self.right.operator == "!"
diff --git a/lib/parse.js b/lib/parse.js
index a36b9fa..ea38258 100644
--- a/lib/parse.js
+++ b/lib/parse.js
@@ -44,11 +44,11 @@
 
 "use strict";
 
-var KEYWORDS = 'break case catch class const continue debugger default delete do else export extends finally for function if in instanceof new return switch throw try typeof var let void while with import yield';
+var KEYWORDS = 'break case catch class const continue debugger default delete do else export extends finally for function if in instanceof new return switch throw try typeof var let void while with import';
 var KEYWORDS_ATOM = 'false null true';
 var RESERVED_WORDS = 'abstract boolean byte char class double enum export extends final float goto implements import int interface let long native package private protected public short static super synchronized this throws transient volatile'
     + " " + KEYWORDS_ATOM + " " + KEYWORDS;
-var KEYWORDS_BEFORE_EXPRESSION = 'return new delete throw else case';
+var KEYWORDS_BEFORE_EXPRESSION = 'return new delete throw else case yield';
 
 KEYWORDS = makePredicate(KEYWORDS);
 RESERVED_WORDS = makePredicate(RESERVED_WORDS);
@@ -68,7 +68,6 @@ var OPERATORS = makePredicate([
     "instanceof",
     "typeof",
     "new",
-    "yield",
     "void",
     "delete",
     "++",
@@ -113,6 +112,8 @@ var OPERATORS = makePredicate([
 
 var WHITESPACE_CHARS = makePredicate(characters(" \u00a0\n\r\t\f\u000b\u200b\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u202f\u205f\u3000\uFEFF"));
 
+var PUNC_AFTER_EXPRESSION = makePredicate(characters(";]),:"));
+
 var PUNC_BEFORE_EXPRESSION = makePredicate(characters("[{(,.;:"));
 
 var PUNC_CHARS = makePredicate(characters("[]{}(),;:`"));
@@ -647,7 +648,6 @@ var UNARY_PREFIX = makePredicate([
     "typeof",
     "void",
     "delete",
-    "yield",
     "--",
     "++",
     "!",
@@ -711,6 +711,7 @@ function parse($TEXT, options) {
         prev          : null,
         peeked        : null,
         in_function   : 0,
+        in_generator  : -1,
         in_directives : true,
         in_loop       : 0,
         labels        : []
@@ -776,6 +777,10 @@ function parse($TEXT, options) {
         );
     };
 
+    function is_in_generator() {
+        return S.in_generator === S.in_function;
+    }
+
     function semicolon(optional) {
         if (is("punc", ";")) next();
         else if (!optional && !can_insert_semicolon()) unexpected();
@@ -944,6 +949,10 @@ function parse($TEXT, options) {
 
     function labeled_statement() {
         var label = as_symbol(AST_Label);
+        if (label.name === "yield" && is_in_generator()) {
+            // Ecma-262, 12.1.1 Static Semantics: Early Errors
+            croak("Yield cannot be used as label inside generators");
+        }
         if (find_if(function(l){ return l.name == label.name }, S.labels)) {
             // ECMA-262, 12.12: An ECMAScript program is considered
             // syntactically incorrect if it contains a
@@ -1093,7 +1102,7 @@ function parse($TEXT, options) {
             unexpected();
 
         var args = params_or_seq_().as_params(croak);
-        var body = _function_body(true);
+        var body = _function_body(true, is_generator);
         return new ctor({
             start : args.start,
             end   : body.end,
@@ -1131,10 +1140,13 @@ function parse($TEXT, options) {
         });
     }
 
-    function _function_body(block) {
+    function _function_body(block, generator) {
         var loop = S.in_loop;
         var labels = S.labels;
+        var current_generator = S.in_generator;
         ++S.in_function;
+        if (generator)
+            S.in_generator = S.in_function;
         if (block)
             S.in_directives = true;
         S.in_loop = 0;
@@ -1146,9 +1158,39 @@ function parse($TEXT, options) {
         --S.in_function;
         S.in_loop = loop;
         S.labels = labels;
+        S.in_generator = current_generator;
         return a;
     }
 
+    function _yield_expression() {
+        // Previous token must be keyword yield and not be interpret as an identifier
+        if (!is_in_generator()) {
+            throw new JS_Parse_Error("Unexpected yield expression outside generator function",
+                S.prev.file, S.prev.line, S.prev.col, S.prev.pos);
+        }
+        var star = false;
+        var statement = true;
+        var tmp;
+
+        // Get expression behind yield, default to value `undefined` stored as `null` in ast
+        if (is("operator", "*")) {
+            star = true;
+            next();
+            if (S.token.nlb) {
+                unexpected(S.prev);
+            }
+        } else if (can_insert_semicolon()
+            || (is("punc") && PUNC_AFTER_EXPRESSION(S.token.value))
+        ) {
+            statement = false;
+        }
+
+        return new AST_Yield({
+            is_star : star,
+            value   : statement ? expression() : null
+        });
+    }
+
     function if_() {
         var cond = parenthesised(), body = statement(), belse = null;
         if (is("keyword", "else")) {
@@ -1585,7 +1627,7 @@ function parse($TEXT, options) {
         }
 
         if (KindOfClass === AST_DefClass && !class_name) {
-            croak();
+            unexpected();
         }
 
         if (S.token.value == "extends") {
@@ -1599,8 +1641,8 @@ function parse($TEXT, options) {
         while (!is("punc", "}")) {
             start = S.token;
             name = as_property_name();
-            method = concise_method_or_getset(name, start);
-            if (!method) { croak(); }
+            method = concise_method_or_getset(name, start, true);
+            if (!method) { unexpected(); }
             a.push(method);
             if (is("punc", ";")) { next(); }
         }
@@ -1616,21 +1658,28 @@ function parse($TEXT, options) {
         });
     }
 
-    function concise_method_or_getset(name, start) {
+    function concise_method_or_getset(name, start, is_class) {
         var is_static = false;
-        if (name === "static" && !is("punc", "(")) {
+        var is_generator = false;
+        if (is_class && name === "static" && !is("punc", "(")) {
             is_static = true;
             name = S.token.value;
             next();
         }
+        if (name === "*") {
+            is_generator = true;
+            name = S.token.value;
+            next();
+        }
         if (is("punc", "(")) {
             return new AST_ConciseMethod({
-                start    : start,
-                static   : is_static,
-                name     : new AST_SymbolMethod({ name: name }),
-                argnames : params_or_seq_().as_params(croak),
-                body     : _function_body(true),
-                end      : prev()
+                is_generator: is_generator,
+                start       : start,
+                static      : is_static,
+                name        : new AST_SymbolMethod({ name: name }),
+                argnames    : params_or_seq_().as_params(croak),
+                body        : _function_body(true, is_generator),
+                end         : prev()
             });
         }
         if (name == "get") {
@@ -1870,10 +1919,6 @@ function parse($TEXT, options) {
         var start = S.token;
         if (is("operator") && UNARY_PREFIX(start.value)) {
             next();
-            if  (start.type === "operator" && start.value === "yield" && is("operator", "*")) {
-                start.value = "yield*";
-                next();
-            }
             handle_regexp();
             var ex = make_unary(AST_UnaryPrefix, start.value, maybe_unary(allow_calls));
             ex.start = start;
@@ -1947,6 +1992,11 @@ function parse($TEXT, options) {
     var maybe_assign = function(no_in) {
         var start = S.token;
 
+        if (start.type == "name" && start.value == "yield" && is_in_generator()) {
+            next();
+            return _yield_expression();
+        }
+
         if (start.type == "punc" && start.value == "(" && peek().value == ")") {
             next();
             next();
diff --git a/test/compress/harmony.js b/test/compress/harmony.js
index 6ee1b24..f3a5a0f 100644
--- a/test/compress/harmony.js
+++ b/test/compress/harmony.js
@@ -213,6 +213,23 @@ concise_methods_and_mangle_props: {
     }
 }
 
+concise_generators: {
+    input: {
+        x = {
+            *foo(a, b) {
+                return x;
+            }
+        }
+        y = {
+            *foo([{a}]) {
+                yield a;
+            },
+            bar(){}
+        }
+    }
+    expect_exact: "x={*foo(a,b){return x}};y={*foo([{a}]){yield a},bar(){}};"
+}
+
 concise_methods_and_keyword_names: {
     input: {
         x = {
@@ -295,6 +312,21 @@ class_name_can_be_preserved: {
     }
 }
 
+classes_can_have_generators: {
+    input: {
+        class Foo {
+            *bar() {}
+            static *baz() {}
+        }
+    }
+    expect: {
+        class Foo {
+            *bar() {}
+            static *baz() {}
+        }
+    }
+}
+
 new_target: {
     input: {
         new.target;
@@ -444,3 +476,66 @@ generators_yield_assign: {
     }
     expect_exact: "function*fn(){var x={};x.prop=yield 5}"
 }
+
+generator_yield_undefined: {
+    input: {
+        function* fn() {
+            yield;
+        }
+    }
+    expect_exact: "function*fn(){yield}"
+}
+
+yield_statements: {
+    input: {
+        function* fn() {
+            var a = (yield 1) + (yield 2);
+            var b = (yield 3) === (yield 4);
+            var c = (yield 5) << (yield 6);
+            var d = yield 7;
+            var e = (yield 8) ? yield 9 : yield 10;
+            var f = -(yield 11);
+        }
+    }
+    expect_exact: "function*fn(){var a=(yield 1)+(yield 2);var b=(yield 3)===(yield 4);var c=(yield 5)<<(yield 6);var d=yield 7;var e=(yield 8)?yield 9:yield 10;var f=-(yield 11)}"
+}
+
+yield_as_identifier_in_function_in_generator: {
+    input: {
+        var g = function*() {
+            function h() {
+                yield = 1;
+            }
+        };
+    }
+    expect: {
+        var g = function*() {
+            function h() {
+                yield = 1;
+            }
+        };
+    }
+}
+
+yield_before_punctuators: {
+    input: {
+        iter = (function*() {
+            assignmentResult = [ x = yield ] = value;
+        })();
+        function* g1() { (yield) }
+        function* g2() { [yield] }
+        function* g3() { return {yield} } // Added return to avoid {} drop
+        function* g4() { yield, yield; }
+        function* g5() { (yield) ? yield : yield; }
+    }
+    expect: {
+        iter = (function*() {
+            assignmentResult = [ x = yield ] = value;
+        })();
+        function* g1() { (yield) }
+        function* g2() { [yield] }
+        function* g3() { return {yield} }
+        function* g4() { yield, yield; }
+        function* g5() { (yield) ? yield : yield; }
+    }
+}
\ No newline at end of file
diff --git a/test/mocha/object.js b/test/mocha/object.js
new file mode 100644
index 0000000..a937374
--- /dev/null
+++ b/test/mocha/object.js
@@ -0,0 +1,20 @@
+var Uglify = require("../../");
+var assert = require("assert");
+
+describe("Object", function() {
+    it ("Should allow objects to have a methodDefinition as property", function() {
+        var code = "var a = {test() {return true;}}";
+        assert.equal(Uglify.minify(code, {fromString: true}).code, "var a={test(){return!0}};");
+    });
+
+    it ("Should not allow objects to use static keywords like in classes", function() {
+        var code = "{static test() {}}";
+        var parse = function() {
+            Uglify.parse(code);
+        }
+        var expect = function(e) {
+            return e instanceof Uglify.JS_Parse_Error;
+        }
+        assert.throws(parse, expect);
+    });
+});
\ No newline at end of file
diff --git a/test/mocha/yield.js b/test/mocha/yield.js
new file mode 100644
index 0000000..ad02fa1
--- /dev/null
+++ b/test/mocha/yield.js
@@ -0,0 +1,34 @@
+var UglifyJS = require("../../");
+var assert = require("assert");
+
+describe("Yield", function() {
+    it("Should not delete statements after yield", function() {
+        var js = 'function *foo(bar) { yield 1; yield 2; return 3; }';
+        var result = UglifyJS.minify(js, {fromString: true});
+        assert.strictEqual(result.code, 'function*foo(n){return yield 1,yield 2,3}');
+    });
+
+    it("Should not allow yield as labelIdentifier within generators", function() {
+        var js = "function* g() {yield: 1}"
+        var test = function() {
+            UglifyJS.parse(js);
+        }
+        var expect = function(e) {
+            return e instanceof UglifyJS.JS_Parse_Error &&
+                e.message === "Yield cannot be used as label inside generators";
+        }
+        assert.throws(test, expect);
+    });
+
+    it("Should not allow yield* followed by a newline in generators", function() {
+        var js = "function* test() {yield*\n123;}";
+        var test = function() {
+            UglifyJS.parse(js);
+        }
+        var expect = function(e) {
+            return e instanceof UglifyJS.JS_Parse_Error &&
+                e.message === "Unexpected token: operator (*)";
+        }
+        assert.throws(test, expect);
+    });
+});
\ No newline at end of file
diff --git a/test/parser.js b/test/parser.js
index 5f6f62f..c91f1e7 100644
--- a/test/parser.js
+++ b/test/parser.js
@@ -128,7 +128,7 @@ module.exports = function () {
     });
 
     var generators_yield_def = UglifyJS.parse('function* fn() {\nyield remote();\}').body[0].body[0];
-    ok.equal(generators_yield_def.body.operator, 'yield');
+    ok.strictEqual(generators_yield_def.body.is_star, false);
 }
 
 // Run standalone
-- 
2.8.2.windows.1

0002-Various-fixes.diff

From 331f3513b7509114d5e5ba017da1375725bd9ec5 Mon Sep 17 00:00:00 2001
From: Anthony Van de Gejuchte <[email protected]>
Date: Thu, 9 Jun 2016 13:00:18 +0200
Subject: [PATCH 2/2] Various fixes

---
 lib/ast.js          | 10 +++++-----
 lib/output.js       |  4 ++--
 lib/parse.js        | 13 ++++++-------
 lib/transform.js    |  4 ++++
 test/mocha/yield.js |  7 +++++++
 5 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/lib/ast.js b/lib/ast.js
index 0cb44ac..129d2ef 100644
--- a/lib/ast.js
+++ b/lib/ast.js
@@ -1244,18 +1244,18 @@ var AST_True = DEFNODE("True", null, {
 
 /* -----[ Yield ]----- */
 
-var AST_Yield = DEFNODE("Yield", "value is_star", {
+var AST_Yield = DEFNODE("Yield", "expression is_star", {
     $documentation: "A `yield` statement",
     $propdoc: {
-        value: "[AST_Node?] the value returned or thrown by this statement; could be null",
+        expression: "[AST_Node?] the value returned or thrown by this statement; could be null",
         is_star: "[Boolean] Whether this is a yield or yield* statement"
     },
     _walk: function(visitor) {
-        return visitor._visit(this, this.value && function(){
-            this.value._walk(visitor);
+        return visitor._visit(this, this.expression && function(){
+            this.expression._walk(visitor);
         });
     }
-}, AST_Statement);
+});
 
 /* -----[ TreeWalker ]----- */
 
diff --git a/lib/output.js b/lib/output.js
index 3eb33e1..5de1596 100644
--- a/lib/output.js
+++ b/lib/output.js
@@ -909,9 +909,9 @@ function OutputStream(options) {
     DEFPRINT(AST_Yield, function(self, output){
         var star = self.is_star ? "*" : "";
         output.print("yield" + star);
-        if (self.value) {
+        if (self.expression) {
             output.space();
-            self.value.print(output);
+            self.expression.print(output);
         }
     });
 
diff --git a/lib/parse.js b/lib/parse.js
index ea38258..c99497d 100644
--- a/lib/parse.js
+++ b/lib/parse.js
@@ -1169,7 +1169,7 @@ function parse($TEXT, options) {
                 S.prev.file, S.prev.line, S.prev.col, S.prev.pos);
         }
         var star = false;
-        var statement = true;
+        var has_expression = true;
         var tmp;
 
         // Get expression behind yield, default to value `undefined` stored as `null` in ast
@@ -1179,15 +1179,14 @@ function parse($TEXT, options) {
             if (S.token.nlb) {
                 unexpected(S.prev);
             }
-        } else if (can_insert_semicolon()
-            || (is("punc") && PUNC_AFTER_EXPRESSION(S.token.value))
-        ) {
-            statement = false;
+        } else if (can_insert_semicolon() ||
+            (is("punc") && PUNC_AFTER_EXPRESSION(S.token.value))) {
+            has_expression = false;
         }
 
         return new AST_Yield({
-            is_star : star,
-            value   : statement ? expression() : null
+            is_star    : star,
+            expression : has_expression ? expression() : null
         });
     }
 
diff --git a/lib/transform.js b/lib/transform.js
index 3466335..6e3a9d9 100644
--- a/lib/transform.js
+++ b/lib/transform.js
@@ -196,6 +196,10 @@ TreeTransformer.prototype = new TreeWalker;
         self.property = self.property.transform(tw);
     });
 
+    _(AST_Yield, function(self, tw){
+        if (self.expression) self.expression = self.expression.transform(tw);
+    });
+
     _(AST_Unary, function(self, tw){
         self.expression = self.expression.transform(tw);
     });
diff --git a/test/mocha/yield.js b/test/mocha/yield.js
index ad02fa1..e636c65 100644
--- a/test/mocha/yield.js
+++ b/test/mocha/yield.js
@@ -31,4 +31,11 @@ describe("Yield", function() {
         }
         assert.throws(test, expect);
     });
+
+    it ("Should be able to compress it's expression", function() {
+        assert.strictEqual(
+            UglifyJS.minify("function *f() { yield 3-4; }", {fromString: true, compress: true}).code,
+            "function*f(){yield-1}"
+        );
+    });
 });
\ No newline at end of file
-- 
2.8.2.windows.1

Typo on line 116: it's should be its.