レイヤーマスク属性の「ぼかし」に値が入ってたらMayaでおかしくなるのでアウトと判定する。参考: https://qiita.com/mizu16/items/85a1bc001db2581528a2

hasMaskFeather.js

var _layerList = [],
    main = function() {
        var document  = activeDocument;
        var layerList = document.layers;

        getLayer(layerList);
        var hasFeather = [];

        for(var i = 0, num = _layerList.length; i < num; i++){
            var layer = _layerList[i];
            document.activeLayer = layer;
            var hasMask = hasLayerMask();
            if (hasMask && hasMaskFeather() ) {
                hasFeather.push(layer);
            }
        }
        if (hasFeather.length != 0) {
            alert( "ぼかしマスクを発見しました。。。アウトです！" );
        } else {
            alert( "ぼかしマスクは、ありませんでした！ 花丸です！" );
        }
    },
    hasLayerMask = function() {
        //レイヤーマスクの有無を判定
        var m_Ref01 = new ActionReference();
        m_Ref01.putEnumerated( sTID( "layer" ), cTID( "Ordn" ), cTID( "Trgt" ));
        var m_Dsc01= executeActionGet( m_Ref01 );
        return m_Dsc01.hasKey(cTID('Usrs'));
    },
    hasMaskFeather = function() {
        var m_Ref01 = new ActionReference();
        m_Ref01.putEnumerated( sTID( "layer" ), cTID( "Ordn" ), cTID( "Trgt" ));
        var m_Dsc01= executeActionGet( m_Ref01 );
        return 0 != m_Dsc01.getUnitDoubleValue( sTID( "userMaskFeather" ) );
    },
    cTID = function(s) {return app.charIDToTypeID(s);},
    sTID = function(s) {return app.stringIDToTypeID(s);},
    getLayer = function(layerList){
        for(var i = 0, num = layerList.length; i < num; i++){
            var layer = layerList[i];
            if(layer.typename == "LayerSet") getLayer(layer.layers);
            _layerList.push(layer);
        }
    };
main();