0001-Improve-yield-support-and-restrict-usage-of-strict.patch

From 10e7fd5c00d5bc413df7ee680f3ae6b965fdf03a Mon Sep 17 00:00:00 2001
From: Anthony Van de Gejuchte <[email protected]>
Date: Thu, 26 May 2016 17:00:37 +0200
Subject: [PATCH 1/2] Improve yield support and restrict usage of strict

- Partially reverting 91cdb93e57 and eaf3911c31 and reimplement
- Add generators support for objects and classes
- Only classes can have static methods so restrict use of it
---
 lib/ast.js               | 18 ++++++++-
 lib/compress.js          |  2 +-
 lib/output.js            | 36 ++++++++++++++----
 lib/parse.js             | 92 +++++++++++++++++++++++++++++++++++-----------
 lib/transform.js         |  4 ++
 test/compress/harmony.js | 95 ++++++++++++++++++++++++++++++++++++++++++++++++
 test/mocha/object.js     | 20 ++++++++++
 test/mocha/yield.js      | 41 +++++++++++++++++++++
 test/parser.js           |  2 +-
 9 files changed, 278 insertions(+), 32 deletions(-)
 create mode 100644 test/mocha/object.js
 create mode 100644 test/mocha/yield.js

diff --git a/lib/ast.js b/lib/ast.js
index b89a00d..129d2ef 100644
--- a/lib/ast.js
+++ b/lib/ast.js
@@ -487,8 +487,9 @@ var AST_Arrow = DEFNODE("Arrow", null, {
     $documentation: "An ES6 Arrow function ((a) => b)"
 }, AST_Lambda);
 
-var AST_ConciseMethod = DEFNODE("ConciseMethod", "static", {
+var AST_ConciseMethod = DEFNODE("ConciseMethod", "is_generator static", {
     $propdoc: {
+        is_generator: "is generatorFn or not",
         static: "[boolean] whether this method is static (classes only)",
     },
     $documentation: "An ES6 concise method inside an object or class"
@@ -1241,6 +1242,21 @@ var AST_True = DEFNODE("True", null, {
     value: true
 }, AST_Boolean);
 
+/* -----[ Yield ]----- */
+
+var AST_Yield = DEFNODE("Yield", "expression is_star", {
+    $documentation: "A `yield` statement",
+    $propdoc: {
+        expression: "[AST_Node?] the value returned or thrown by this statement; could be null",
+        is_star: "[Boolean] Whether this is a yield or yield* statement"
+    },
+    _walk: function(visitor) {
+        return visitor._visit(this, this.expression && function(){
+            this.expression._walk(visitor);
+        });
+    }
+});
+
 /* -----[ TreeWalker ]----- */
 
 function TreeWalker(callback) {
diff --git a/lib/compress.js b/lib/compress.js
index 31e42d5..2a05e40 100644
--- a/lib/compress.js
+++ b/lib/compress.js
@@ -1245,7 +1245,7 @@ merge(Compressor.prototype, {
                 || this.alternative.has_side_effects(compressor);
         });
         def(AST_Unary, function(compressor){
-            return member(this.operator, ["delete", "++", "--", "yield", "yield*"])
+            return member(this.operator, ["delete", "++", "--"])
                 || this.expression.has_side_effects(compressor);
         });
         def(AST_SymbolRef, function(compressor){
diff --git a/lib/output.js b/lib/output.js
index a71cd36..5de1596 100644
--- a/lib/output.js
+++ b/lib/output.js
@@ -560,6 +560,20 @@ function OutputStream(options) {
         }
     });
 
+    PARENS(AST_Yield, function(output){
+        var p = output.parent();
+        // (yield 1) + (yield 2)
+        // a = yield 3
+        if (p instanceof AST_Binary && p.operator !== "=")
+            return true;
+        // (yield 1) ? yield 2 : yield 3
+        if (p instanceof AST_Conditional && p.condition === this)
+            return true;
+        // -(yield 4)
+        if (p instanceof AST_Unary)
+            return true;
+    });
+
     PARENS(AST_PropAccess, function(output){
         var p = output.parent();
         if (p instanceof AST_New && p.expression === this) {
@@ -868,6 +882,9 @@ function OutputStream(options) {
             output.print("static");
             output.space();
         }
+        if (self.is_generator) {
+            output.print("*");
+        }
         self._do_print(output, true /* do not print "function" */);
     });
 
@@ -887,6 +904,17 @@ function OutputStream(options) {
         self._do_print(output, "throw");
     });
 
+    /* -----[ yield ]----- */
+
+    DEFPRINT(AST_Yield, function(self, output){
+        var star = self.is_star ? "*" : "";
+        output.print("yield" + star);
+        if (self.expression) {
+            output.space();
+            self.expression.print(output);
+        }
+    });
+
     /* -----[ loop control ]----- */
     AST_LoopControl.DEFMETHOD("_do_print", function(output, kind){
         output.print(kind);
@@ -1218,13 +1246,8 @@ function OutputStream(options) {
         output.print(self.operator);
     });
     DEFPRINT(AST_Binary, function(self, output){
-        var isYield = (self.left.operator == "yield" || self.left.operator === "yield*");
         var op = self.operator;
-
-        isYield && output.print("(");
         self.left.print(output);
-        isYield && output.print(")");
-
         if (op[0] == ">" /* ">>" ">>>" ">" ">=" */
             && self.left instanceof AST_UnaryPostfix
             && self.left.operator == "--") {
@@ -1234,10 +1257,7 @@ function OutputStream(options) {
             // the space is optional depending on "beautify"
             output.space();
         }
-
-        isYield = (self.right.operator == "yield" || self.right.operator === "yield*");
         output.print(op);
-
         if ((op == "<" || op == "<<")
             && self.right instanceof AST_UnaryPrefix
             && self.right.operator == "!"
diff --git a/lib/parse.js b/lib/parse.js
index a36b9fa..b776ec1 100644
--- a/lib/parse.js
+++ b/lib/parse.js
@@ -44,11 +44,11 @@
 
 "use strict";
 
-var KEYWORDS = 'break case catch class const continue debugger default delete do else export extends finally for function if in instanceof new return switch throw try typeof var let void while with import yield';
+var KEYWORDS = 'break case catch class const continue debugger default delete do else export extends finally for function if in instanceof new return switch throw try typeof var let void while with import';
 var KEYWORDS_ATOM = 'false null true';
 var RESERVED_WORDS = 'abstract boolean byte char class double enum export extends final float goto implements import int interface let long native package private protected public short static super synchronized this throws transient volatile'
     + " " + KEYWORDS_ATOM + " " + KEYWORDS;
-var KEYWORDS_BEFORE_EXPRESSION = 'return new delete throw else case';
+var KEYWORDS_BEFORE_EXPRESSION = 'return new delete throw else case yield';
 
 KEYWORDS = makePredicate(KEYWORDS);
 RESERVED_WORDS = makePredicate(RESERVED_WORDS);
@@ -68,7 +68,6 @@ var OPERATORS = makePredicate([
     "instanceof",
     "typeof",
     "new",
-    "yield",
     "void",
     "delete",
     "++",
@@ -113,6 +112,8 @@ var OPERATORS = makePredicate([
 
 var WHITESPACE_CHARS = makePredicate(characters(" \u00a0\n\r\t\f\u000b\u200b\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u202f\u205f\u3000\uFEFF"));
 
+var PUNC_AFTER_EXPRESSION = makePredicate(characters(";]),:"));
+
 var PUNC_BEFORE_EXPRESSION = makePredicate(characters("[{(,.;:"));
 
 var PUNC_CHARS = makePredicate(characters("[]{}(),;:`"));
@@ -647,7 +648,6 @@ var UNARY_PREFIX = makePredicate([
     "typeof",
     "void",
     "delete",
-    "yield",
     "--",
     "++",
     "!",
@@ -711,6 +711,7 @@ function parse($TEXT, options) {
         prev          : null,
         peeked        : null,
         in_function   : 0,
+        in_generator  : -1,
         in_directives : true,
         in_loop       : 0,
         labels        : []
@@ -776,6 +777,10 @@ function parse($TEXT, options) {
         );
     };
 
+    function is_in_generator() {
+        return S.in_generator === S.in_function;
+    }
+
     function semicolon(optional) {
         if (is("punc", ";")) next();
         else if (!optional && !can_insert_semicolon()) unexpected();
@@ -944,6 +949,10 @@ function parse($TEXT, options) {
 
     function labeled_statement() {
         var label = as_symbol(AST_Label);
+        if (label.name === "yield" && is_in_generator()) {
+            // Ecma-262, 12.1.1 Static Semantics: Early Errors
+            croak("Yield cannot be used as label inside generators");
+        }
         if (find_if(function(l){ return l.name == label.name }, S.labels)) {
             // ECMA-262, 12.12: An ECMAScript program is considered
             // syntactically incorrect if it contains a
@@ -1093,7 +1102,7 @@ function parse($TEXT, options) {
             unexpected();
 
         var args = params_or_seq_().as_params(croak);
-        var body = _function_body(true);
+        var body = _function_body(true, is_generator);
         return new ctor({
             start : args.start,
             end   : body.end,
@@ -1131,10 +1140,13 @@ function parse($TEXT, options) {
         });
     }
 
-    function _function_body(block) {
+    function _function_body(block, generator) {
         var loop = S.in_loop;
         var labels = S.labels;
+        var current_generator = S.in_generator;
         ++S.in_function;
+        if (generator)
+            S.in_generator = S.in_function;
         if (block)
             S.in_directives = true;
         S.in_loop = 0;
@@ -1146,9 +1158,39 @@ function parse($TEXT, options) {
         --S.in_function;
         S.in_loop = loop;
         S.labels = labels;
+        S.in_generator = current_generator;
         return a;
     }
 
+    function _yield_expression() {
+        // Previous token must be keyword yield and not be interpret as an identifier
+        if (!is_in_generator()) {
+            throw new JS_Parse_Error("Unexpected yield expression outside generator function",
+                S.prev.file, S.prev.line, S.prev.col, S.prev.pos);
+        }
+        var star = false;
+        var has_expression = true;
+        var tmp;
+
+        // Get expression behind yield, default to value `undefined` stored as `null` in ast
+        // Expression must start on same line, yield* has a mandatory expression
+        if (is("operator", "*")) {
+            star = true;
+            next();
+            if (S.token.nlb) {
+                unexpected(S.prev);
+            }
+        } else if (can_insert_semicolon() ||
+            (is("punc") && PUNC_AFTER_EXPRESSION(S.token.value))) {
+            has_expression = false;
+        }
+
+        return new AST_Yield({
+            is_star    : star,
+            expression : has_expression ? expression() : null
+        });
+    }
+
     function if_() {
         var cond = parenthesised(), body = statement(), belse = null;
         if (is("keyword", "else")) {
@@ -1585,7 +1627,7 @@ function parse($TEXT, options) {
         }
 
         if (KindOfClass === AST_DefClass && !class_name) {
-            croak();
+            unexpected();
         }
 
         if (S.token.value == "extends") {
@@ -1599,8 +1641,8 @@ function parse($TEXT, options) {
         while (!is("punc", "}")) {
             start = S.token;
             name = as_property_name();
-            method = concise_method_or_getset(name, start);
-            if (!method) { croak(); }
+            method = concise_method_or_getset(name, start, true);
+            if (!method) { unexpected(); }
             a.push(method);
             if (is("punc", ";")) { next(); }
         }
@@ -1616,21 +1658,28 @@ function parse($TEXT, options) {
         });
     }
 
-    function concise_method_or_getset(name, start) {
+    function concise_method_or_getset(name, start, is_class) {
         var is_static = false;
-        if (name === "static" && !is("punc", "(")) {
+        var is_generator = false;
+        if (is_class && name === "static" && !is("punc", "(")) {
             is_static = true;
             name = S.token.value;
             next();
         }
+        if (name === "*") {
+            is_generator = true;
+            name = S.token.value;
+            next();
+        }
         if (is("punc", "(")) {
             return new AST_ConciseMethod({
-                start    : start,
-                static   : is_static,
-                name     : new AST_SymbolMethod({ name: name }),
-                argnames : params_or_seq_().as_params(croak),
-                body     : _function_body(true),
-                end      : prev()
+                is_generator: is_generator,
+                start       : start,
+                static      : is_static,
+                name        : new AST_SymbolMethod({ name: name }),
+                argnames    : params_or_seq_().as_params(croak),
+                body        : _function_body(true, is_generator),
+                end         : prev()
             });
         }
         if (name == "get") {
@@ -1870,10 +1919,6 @@ function parse($TEXT, options) {
         var start = S.token;
         if (is("operator") && UNARY_PREFIX(start.value)) {
             next();
-            if  (start.type === "operator" && start.value === "yield" && is("operator", "*")) {
-                start.value = "yield*";
-                next();
-            }
             handle_regexp();
             var ex = make_unary(AST_UnaryPrefix, start.value, maybe_unary(allow_calls));
             ex.start = start;
@@ -1947,6 +1992,11 @@ function parse($TEXT, options) {
     var maybe_assign = function(no_in) {
         var start = S.token;
 
+        if (start.type == "name" && start.value == "yield" && is_in_generator()) {
+            next();
+            return _yield_expression();
+        }
+
         if (start.type == "punc" && start.value == "(" && peek().value == ")") {
             next();
             next();
diff --git a/lib/transform.js b/lib/transform.js
index 3466335..6e3a9d9 100644
--- a/lib/transform.js
+++ b/lib/transform.js
@@ -196,6 +196,10 @@ TreeTransformer.prototype = new TreeWalker;
         self.property = self.property.transform(tw);
     });
 
+    _(AST_Yield, function(self, tw){
+        if (self.expression) self.expression = self.expression.transform(tw);
+    });
+
     _(AST_Unary, function(self, tw){
         self.expression = self.expression.transform(tw);
     });
diff --git a/test/compress/harmony.js b/test/compress/harmony.js
index 6ee1b24..f3a5a0f 100644
--- a/test/compress/harmony.js
+++ b/test/compress/harmony.js
@@ -213,6 +213,23 @@ concise_methods_and_mangle_props: {
     }
 }
 
+concise_generators: {
+    input: {
+        x = {
+            *foo(a, b) {
+                return x;
+            }
+        }
+        y = {
+            *foo([{a}]) {
+                yield a;
+            },
+            bar(){}
+        }
+    }
+    expect_exact: "x={*foo(a,b){return x}};y={*foo([{a}]){yield a},bar(){}};"
+}
+
 concise_methods_and_keyword_names: {
     input: {
         x = {
@@ -295,6 +312,21 @@ class_name_can_be_preserved: {
     }
 }
 
+classes_can_have_generators: {
+    input: {
+        class Foo {
+            *bar() {}
+            static *baz() {}
+        }
+    }
+    expect: {
+        class Foo {
+            *bar() {}
+            static *baz() {}
+        }
+    }
+}
+
 new_target: {
     input: {
         new.target;
@@ -444,3 +476,66 @@ generators_yield_assign: {
     }
     expect_exact: "function*fn(){var x={};x.prop=yield 5}"
 }
+
+generator_yield_undefined: {
+    input: {
+        function* fn() {
+            yield;
+        }
+    }
+    expect_exact: "function*fn(){yield}"
+}
+
+yield_statements: {
+    input: {
+        function* fn() {
+            var a = (yield 1) + (yield 2);
+            var b = (yield 3) === (yield 4);
+            var c = (yield 5) << (yield 6);
+            var d = yield 7;
+            var e = (yield 8) ? yield 9 : yield 10;
+            var f = -(yield 11);
+        }
+    }
+    expect_exact: "function*fn(){var a=(yield 1)+(yield 2);var b=(yield 3)===(yield 4);var c=(yield 5)<<(yield 6);var d=yield 7;var e=(yield 8)?yield 9:yield 10;var f=-(yield 11)}"
+}
+
+yield_as_identifier_in_function_in_generator: {
+    input: {
+        var g = function*() {
+            function h() {
+                yield = 1;
+            }
+        };
+    }
+    expect: {
+        var g = function*() {
+            function h() {
+                yield = 1;
+            }
+        };
+    }
+}
+
+yield_before_punctuators: {
+    input: {
+        iter = (function*() {
+            assignmentResult = [ x = yield ] = value;
+        })();
+        function* g1() { (yield) }
+        function* g2() { [yield] }
+        function* g3() { return {yield} } // Added return to avoid {} drop
+        function* g4() { yield, yield; }
+        function* g5() { (yield) ? yield : yield; }
+    }
+    expect: {
+        iter = (function*() {
+            assignmentResult = [ x = yield ] = value;
+        })();
+        function* g1() { (yield) }
+        function* g2() { [yield] }
+        function* g3() { return {yield} }
+        function* g4() { yield, yield; }
+        function* g5() { (yield) ? yield : yield; }
+    }
+}
\ No newline at end of file
diff --git a/test/mocha/object.js b/test/mocha/object.js
new file mode 100644
index 0000000..a937374
--- /dev/null
+++ b/test/mocha/object.js
@@ -0,0 +1,20 @@
+var Uglify = require("../../");
+var assert = require("assert");
+
+describe("Object", function() {
+    it ("Should allow objects to have a methodDefinition as property", function() {
+        var code = "var a = {test() {return true;}}";
+        assert.equal(Uglify.minify(code, {fromString: true}).code, "var a={test(){return!0}};");
+    });
+
+    it ("Should not allow objects to use static keywords like in classes", function() {
+        var code = "{static test() {}}";
+        var parse = function() {
+            Uglify.parse(code);
+        }
+        var expect = function(e) {
+            return e instanceof Uglify.JS_Parse_Error;
+        }
+        assert.throws(parse, expect);
+    });
+});
\ No newline at end of file
diff --git a/test/mocha/yield.js b/test/mocha/yield.js
new file mode 100644
index 0000000..e636c65
--- /dev/null
+++ b/test/mocha/yield.js
@@ -0,0 +1,41 @@
+var UglifyJS = require("../../");
+var assert = require("assert");
+
+describe("Yield", function() {
+    it("Should not delete statements after yield", function() {
+        var js = 'function *foo(bar) { yield 1; yield 2; return 3; }';
+        var result = UglifyJS.minify(js, {fromString: true});
+        assert.strictEqual(result.code, 'function*foo(n){return yield 1,yield 2,3}');
+    });
+
+    it("Should not allow yield as labelIdentifier within generators", function() {
+        var js = "function* g() {yield: 1}"
+        var test = function() {
+            UglifyJS.parse(js);
+        }
+        var expect = function(e) {
+            return e instanceof UglifyJS.JS_Parse_Error &&
+                e.message === "Yield cannot be used as label inside generators";
+        }
+        assert.throws(test, expect);
+    });
+
+    it("Should not allow yield* followed by a newline in generators", function() {
+        var js = "function* test() {yield*\n123;}";
+        var test = function() {
+            UglifyJS.parse(js);
+        }
+        var expect = function(e) {
+            return e instanceof UglifyJS.JS_Parse_Error &&
+                e.message === "Unexpected token: operator (*)";
+        }
+        assert.throws(test, expect);
+    });
+
+    it ("Should be able to compress it's expression", function() {
+        assert.strictEqual(
+            UglifyJS.minify("function *f() { yield 3-4; }", {fromString: true, compress: true}).code,
+            "function*f(){yield-1}"
+        );
+    });
+});
\ No newline at end of file
diff --git a/test/parser.js b/test/parser.js
index 5f6f62f..c91f1e7 100644
--- a/test/parser.js
+++ b/test/parser.js
@@ -128,7 +128,7 @@ module.exports = function () {
     });
 
     var generators_yield_def = UglifyJS.parse('function* fn() {\nyield remote();\}').body[0].body[0];
-    ok.equal(generators_yield_def.body.operator, 'yield');
+    ok.strictEqual(generators_yield_def.body.is_star, false);
 }
 
 // Run standalone
-- 
2.8.2.windows.1

0002-Fix-optimizing-yield.patch

From 1f93b2a6a4346608c08701490497bd69fb292454 Mon Sep 17 00:00:00 2001
From: Anthony Van de Gejuchte <[email protected]>
Date: Thu, 9 Jun 2016 15:30:45 +0200
Subject: [PATCH 2/2] Fix optimizing yield

---
 lib/compress.js          |  7 +++++++
 lib/output.js            |  4 ++++
 test/compress/harmony.js | 17 +++++++++++++++++
 test/mocha/yield.js      | 16 +++++++++++++++-
 4 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/lib/compress.js b/lib/compress.js
index 2a05e40..5801ab2 100644
--- a/lib/compress.js
+++ b/lib/compress.js
@@ -2968,4 +2968,11 @@ merge(Compressor.prototype, {
         return self;
     });
 
+    OPT(AST_Yield, function(self, compressor){
+        if (self.expression instanceof AST_Undefined) {
+            self.expression = null;
+        }
+        return self;
+    });
+
 })();
diff --git a/lib/output.js b/lib/output.js
index 5de1596..88d4ad6 100644
--- a/lib/output.js
+++ b/lib/output.js
@@ -912,6 +912,10 @@ function OutputStream(options) {
         if (self.expression) {
             output.space();
             self.expression.print(output);
+        } else if (self.is_star === true) {
+            // Yield* must have an expression behind to avoid invalid syntax
+            output.space();
+            AST_Undefined.prototype._codegen(self, output);
         }
     });
 
diff --git a/test/compress/harmony.js b/test/compress/harmony.js
index f3a5a0f..50fb47a 100644
--- a/test/compress/harmony.js
+++ b/test/compress/harmony.js
@@ -486,6 +486,23 @@ generator_yield_undefined: {
     expect_exact: "function*fn(){yield}"
 }
 
+yield_optimize_expression: {
+    options = {
+    }
+    input: {
+        function* f1() { yield; }
+        function* f2() { yield undefined; }
+        function* f3() { yield null; }
+        function* f4() { yield* undefined; }
+    }
+    expect: {
+        function* f1() { yield }
+        function* f2() { yield; }
+        function* f3() { yield null; }
+        function* f4() { yield* void 0; }
+    }
+}
+
 yield_statements: {
     input: {
         function* fn() {
diff --git a/test/mocha/yield.js b/test/mocha/yield.js
index e636c65..90ed50d 100644
--- a/test/mocha/yield.js
+++ b/test/mocha/yield.js
@@ -32,10 +32,24 @@ describe("Yield", function() {
         assert.throws(test, expect);
     });
 
-    it ("Should be able to compress it's expression", function() {
+    it("Should be able to compress it's expression", function() {
         assert.strictEqual(
             UglifyJS.minify("function *f() { yield 3-4; }", {fromString: true, compress: true}).code,
             "function*f(){yield-1}"
         );
     });
+
+    it("Should be able to drop undefined after yield if necessary without compression", function() {
+        assert.strictEqual(
+            UglifyJS.minify("function *f() { yield undefined; yield; yield* undefined; yield void 0}", {fromString: true, compress: false}).code,
+            "function*f(){yield undefined;yield;yield*undefined;yield void 0}"
+        );
+    });
+
+    it("Should be able to drop undefined after yield if necessary with compression", function() {
+        assert.strictEqual(
+            UglifyJS.minify("function *f() { yield undefined; yield; yield* undefined; yield void 0}", {fromString: true, compress: true}).code,
+            "function*f(){yield,yield,yield*void 0,yield}"
+        );
+    });
 });
\ No newline at end of file
-- 
2.8.2.windows.1