A high performance Powershell Gallery Module Installer

ModuleFast.ps1

#requires -version 4.0

<#
      .SYNOPSIS
      High Performance Powershell Module Installation

      .DESCRIPTION
      This is a proof of concept for using the Powershell Gallery OData API and HTTPClient to parallel install packages
      It is also a demonstration of using async tasks in powershell appropriately. Who says powershell can't be fast?
      This drastically reduces the bandwidth/load against Powershell Gallery by only requesting the required data
      It also handles dependencies (via Nuget), checks for existing packages, and caches already downloaded packages

      .LINK
      https://gist.github.com/JustinGrote/ecdf96b4179da43fb017dccbd1cc56f6

      .NOTES
      THIS IS NOT FOR PRODUCTION, it should be considered "Fragile" and has very little error handling and type safety
      It also doesn't generate the PowershellGet XML files currently, so PowershellGet will see them as "External" modules
#>

if (-not (Get-Command -Name 'nuget.exe'))
{
   throw 'This module requires nuget.exe to be in your path. Please install it.'
}

function Get-ModuleFast
{
   <#
         .SYNOPSIS
         Add a brief description for Get-ModuleFast 
   
         .DESCRIPTION
         Add a detailed description for Get-NotInstalledModules
   
         .PARAMETER Name
         A list of modules to install, specified either as strings or as hashtables with nuget version style (e.g. @{Name='test';Version='1.0'})
   
         .PARAMETER AllowPrerelease
         Whether to include prerelease modules in the request
   
         .PARAMETER Depth
         How far down the dependency tree to go. This generally does not need to be adjusted and is primarily a dependency loop prevention mechanism.
   
         .EXAMPLE
         PS C:\> Get-ModuleFast
   
         .LINK

         .NOTES
         Any additional information
   #>
   
   [CmdletBinding(ConfirmImpact = 'Low')]
   param
   (
      [Parameter(ValueFromPipeline,
      ValueFromPipelineByPropertyName)]
      [ValidateNotNullOrEmpty()]
      [Object[]]
      $Name,
      [Parameter(ValueFromPipeline,
      ValueFromPipelineByPropertyName)]
      [Switch]
      $AllowPrerelease,
      [Parameter(ValueFromPipeline,
      ValueFromPipelineByPropertyName)]
      [int]
      $Depth = 10
   )
   
   begin
   {
      #region Helpers
      #Check installation
      function Get-NotInstalledModules
      {
         <#
               .SYNOPSIS
               Add a brief description for Get-NotInstalledModules

               .DESCRIPTION
               Add a detailed description for Get-NotInstalledModules

               .PARAMETER Name
               Describe parameter -Name.

               .EXAMPLE
               Get-NotInstalledModules -Name Value
               Describe what this call does

               .LINK
               https://gist.github.com/JustinGrote/ecdf96b4179da43fb017dccbd1cc56f6

               .NOTES
               Any additional information
         #>
         [CmdletBinding(ConfirmImpact = 'None')]
         param
         (
            [Parameter(ValueFromPipeline,
            ValueFromPipelineByPropertyName)]
            [ValidateNotNullOrEmpty()]
            [String[]]
            $Name
         )
         
         process
         {
            $InstalledModules = Get-Module -Name $Name -ListAvailable
            $Name.where{
               $isInstalled = $PSItem -notin $InstalledModules.Name
               if ($isInstalled)
               {
                  Write-Verbose -Message ('{0} is already installed. Skipping...' -f $PSItem)
               }
               return $isInstalled
            }
         }
      }
      
      function Get-PSGalleryModule
      {
         <#
               .SYNOPSIS
               Add a brief description for Get-NotInstalledModules
   
               .DESCRIPTION
               Add a detailed description for Get-NotInstalledModules
   
               .PARAMETER Name
               The Name(s) of the PSGallery Module(s)
   
               .PARAMETER Properties
               A description of the Properties parameter.
   
               .PARAMETER AllowPrerelease
               A description of the AllowPrerelease parameter.
   
               .EXAMPLE
               PS C:\> Get-PSGalleryModule -Name $value1
   
               .LINK
               https://gist.github.com/JustinGrote/ecdf96b4179da43fb017dccbd1cc56f6

               .NOTES
               Any additional information
         #>
         [CmdletBinding(ConfirmImpact = 'None')]
         param
         (
            [Parameter(Mandatory,
                  ValueFromPipeline,
                  ValueFromPipelineByPropertyName,
            HelpMessage = 'The Name(s) of the PSGallery Module(s)')]
            [ValidateNotNullOrEmpty()]
            [Microsoft.PowerShell.Commands.ModuleSpecification[]]
            $Name,
            [string[]]
            $Properties = [string[]]('Id', 'Version', 'NormalizedVersion', 'Dependencies'),
            [Switch]
            $AllowPrerelease
         )
         
         begin
         {
            $null = (Add-Type -AssemblyName System.Web -ErrorAction SilentlyContinue)
         }
         
         process
         {
            $queries = foreach ($ModuleSpecItem in $Name)
            {
               $galleryQuery = [uribuilder]$baseUri
               # Creates a Query Name Value Builder
               $queryBuilder = [web.httputility]::ParseQueryString($null)
               $ModuleId = $ModuleSpecItem.Name
               $FilterSet = @()
               $FilterSet += ("Id eq '{0}'" -f $ModuleId)
               $FilterSet += ('IsPrerelease eq {0}' -f ([String]$AllowPrerelease).tolower())
               
               switch ($true)
               {
                  ([bool]$ModuleSpecItem.Version)
                  {
                     $FilterSet += ("Version eq '{0}'" -f $ModuleSpecItem.Version)
                     # Don't need to add required and minimum if an explicit version was specified, hence the break
                     break
                  }
                  # We use "required" as "minimum" for purposes of the gallery query
                  ([bool]$ModuleSpecItem.RequiredVersion)
                  {
                     $FilterSet += ("Version ge '{0}'" -f $ModuleSpecItem.RequiredVersion)
                  }
                  # We assume for now that if you set the max as "2.0" you really meant "1.99"
                  # TODO: Fix this to handle explicit/implicit dependencies
                  ([bool]$ModuleSpecItem.MaximumVersion)
                  {
                     $FilterSet += ("Version lt '{0}'" -f $ModuleSpecItem.MaximumVersion)
                  }
               }
               
               # Construct the Odata Query
               $Filter = $FilterSet -join ' and '
               $null = $queryBuilder.Add('$top', '1')
               $null = $queryBuilder.Add('$filter', $Filter)
               $null = $queryBuilder.Add('$orderby', 'Version desc')
               $null = $queryBuilder.Add('$select', ($Properties -join ','))
               
               $galleryQuery.Query = $queryBuilder.tostring()
               
               Write-Debug -Message $galleryQuery.uri
               
               $httpClient.GetStringAsync($galleryQuery.Uri)
            }
            
            # Construct a summary object
            foreach ($moduleItem in ($queries.result.foreach{
                     [xml]$PSItem
            }).feed.entry)
            {
               $OutputProperties = $Properties + @{
                  N = 'Source'
                  E = {
                     $moduleItem.content.src
                  }
               }
               $moduleItem.properties | Select-Object -Property $OutputProperties
            }
         }
      }
      
      function Parse-NugetDependency
      {
         <#
               .SYNOPSIS
               Add a brief description for Get-NotInstalledModules
   
               .DESCRIPTION
               Add a detailed description for Get-NotInstalledModules
   
               .PARAMETER DependencyString
               A description of the DependencyString parameter.
   
               .EXAMPLE
               PS C:\> Parse-NugetDependency
   
               .LINK
               https://gist.github.com/JustinGrote/ecdf96b4179da43fb017dccbd1cc56f6

               .NOTES
               RequiredVersion is used for Minimumversion and ModuleVersion is RequiredVersion for purposes of Nuget query
         #>
         [CmdletBinding(ConfirmImpact = 'None')]
         param
         (
            [Parameter(ValueFromPipeline,
            ValueFromPipelineByPropertyName)]
            [string]
            $DependencyString
         )
         
         begin
         {
            $DependencyParts = $DependencyString -split '\:'
            $dep = @{
               ModuleName = $DependencyParts[0]
            }
            $Version = $DependencyParts[1]
         }
         
         process
         {
            if ($Version)
            {
               # If it is an exact match version (has brackets and doesn't have a comma), set version accordingly
               $ExactVersionRegex = '\[([^,]+)\]'
               if ($Version -match $ExactVersionRegex)
               {
                  return $dep.Version = $matches[1]
               }
               
               # Parse all other remainder options. For this purpose we ignore inclusive vs. exclusive
               # TODO: Add inclusive/exclusive parsing
               $Version = $Version -replace '[\[\(\)\]]', '' -split ','
               $requiredVersion = $Version[0].trim()
               $maximumVersion = $Version[1].trim()
               
               if ($requiredVersion -and $maximumVersion -and ($requiredVersion -eq $maximumVersion))
               {
                  $dep.ModuleVersion = $requiredVersion
               }
               elseif ($requiredVersion -or $maximumVersion)
               {
                  if ($requiredVersion)
                  {
                     $dep.RequiredVersion = $requiredVersion
                  }
                  
                  if ($maximumVersion)
                  {
                     $dep.MaximumVersion = $maximumVersion
                  }
               }
               else
               {
                  #If no matching version works, just set dep to a string of the modulename
                  [string]$dep = $DependencyParts[0]
               }
            }
         }
         
         end
         {
            return [Microsoft.PowerShell.Commands.ModuleSpecification]$dep
         }
      }
      #endregion Helpers
      
      #region Main
      # Only need one httpclient for all operations
      if (-not $httpClient)
      {
         $null = (Add-Type -AssemblyName System.Net.Http -ErrorAction SilentlyContinue)
         $SCRIPT:httpClient = [Net.Http.HttpClient]::new()
      }
      
      $baseUri = 'https://www.powershellgallery.com/api/v2/Packages'
      
      Write-Progress -Id 1 -Activity 'Get-ModuleFast' -CurrentOperation 'Fetching module information from Powershell Gallery'
   }
   
   process
   {
      # TODO: Add back Get-NotInstalledModules
      $modulesToInstall = @()
      $modulesToInstall += Get-PSGalleryModule -Name ($Name)
      
      # Loop through dependencies to the expected depth
      $currentDependencies = @($modulesToInstall.dependencies.where{
            $PSItem
      })
      $i = 0
      
      while ($currentDependencies -and ($i -le $Depth))
      {
         Write-Verbose -Message ('{0} modules had additional dependencies, fetching...' -f $currentDependencies.count)
         
         $i++
         $dependencyName = $currentDependencies -split '\|' | ForEach-Object -Process {
            Parse-NugetDependency -DependencyString $PSItem
         } | Sort-Object -Unique
         
         if ($dependencyName)
         {
            $dependentModules = (Get-PSGalleryModule -Name $dependencyName)
            $modulesToInstall += $dependentModules
            $currentDependencies = $dependentModules.dependencies.where{
               $PSItem
            }
         }
         else
         {
            $currentDependencies = $false
         }
      }
      
      $modulesToInstall = ($modulesToInstall | Sort-Object -Property id, version -Unique)
   }

   end
   {
      return $modulesToInstall
   }
}

function New-NuGetPackageConfig
{
   <#
         .SYNOPSIS
         Add a brief description for Get-NotInstalledModules

         .DESCRIPTION
         Add a detailed description for Get-NotInstalledModules

         .PARAMETER modulesToInstall
         Name(s) of the PowerShell Module(s) to Install

         .PARAMETER Path
         Tem File

         .EXAMPLE
         PS C:\> New-NuGetPackageConfig -modulesToInstall $value1

         .LINK
         https://gist.github.com/JustinGrote/ecdf96b4179da43fb017dccbd1cc56f6

         .NOTES
         Any additional information
   #>
   [CmdletBinding(ConfirmImpact = 'Low')]
   param
   (
      [Parameter(Mandatory,
            ValueFromPipeline,
            ValueFromPipelineByPropertyName,
      HelpMessage = 'Name of the PowerShell Module to Install')]
      [ValidateNotNullOrEmpty()]
      [string[]]
      $modulesToInstall,
      [Parameter(ValueFromPipeline,
      ValueFromPipelineByPropertyName)]
      [string]
      $Path = [io.path]::GetTempFileName()
   )
   
   begin
   {
      $packageConfig = [xml.xmlwriter]::Create([string]$Path)
      $packageConfig.WriteStartDocument()
      $packageConfig.WriteStartElement('packages')
   }
   
   process
   {
      foreach ($moduleItem in $modulesToInstall)
      {
         $packageConfig.WriteStartElement('package')
         $packageConfig.WriteAttributeString('id', $null, $moduleItem.id)
         $packageConfig.WriteAttributeString('version', $null, $moduleItem.Version)
         $packageConfig.WriteEndElement()
      }
      
      $packageConfig.WriteEndElement()
      $packageConfig.WriteEndDocument()
      $packageConfig.Flush()
      $packageConfig.Close()
   }
   
   end
   {
      return $Path
   }
}

function Install-Modulefast
{
   <#
         .SYNOPSIS
         Add a brief description for Get-NotInstalledModules

         .DESCRIPTION
         Add a detailed description for Get-NotInstalledModules

         .PARAMETER modulesToInstall
         Name(s) of the PowerShell Module(s) to install

         .PARAMETER Path
         Where to install

         .PARAMETER ModuleCache
         Where to cache

         .PARAMETER NuGetCache
         Where to cache

         .PARAMETER Force
         Enforce?

         .EXAMPLE
         PS C:\> Install-Modulefast -modulesToInstall $value1 -Path 'Value2'

         .LINK
         https://gist.github.com/JustinGrote/ecdf96b4179da43fb017dccbd1cc56f6

         .NOTES
         Any additional information
   #>
   [CmdletBinding(ConfirmImpact = 'Low')]
   param
   (
      [Parameter(Mandatory,
            ValueFromPipeline,
            ValueFromPipelineByPropertyName,
      HelpMessage = 'Name(s) of the PowerShell Module(s) to install')]
      [ValidateNotNullOrEmpty()]
      [string[]]
      $modulesToInstall,
      [Parameter(Mandatory,
            ValueFromPipeline,
            ValueFromPipelineByPropertyName,
      HelpMessage = 'Where to install')]
      [ValidateNotNullOrEmpty()]
      [string]
      $Path,
      [Parameter(ValueFromPipeline,
      ValueFromPipelineByPropertyName)]
      [string]
      $ModuleCache = (New-Item -ItemType Directory -Force -Path (Join-Path -Path ([io.path]::GetTempPath()) -ChildPath 'ModuleFastCache')),
      [Parameter(ValueFromPipeline,
      ValueFromPipelineByPropertyName)]
      [string]
      $NuGetCache = [io.path]::Combine([string[]]($HOME, '.nuget', 'psgallery')),
      [Parameter(ValueFromPipeline,
      ValueFromPipelineByPropertyName)]
      [Switch]
      $Force
   )
   
   process
   {
      # Do a really crappy guess for the current user modules folder.
      # TODO: "Scope CurrentUser" type logic
      if (-not $Path)
      {
         if (-not $env:PSModulePath)
         {
            throw 'PSModulePath is not defined, therefore the -Path parameter is mandatory'
         }
         
         $envSeparator = ';'
         
         if ($isLinux)
         {
            $envSeparator = ':'
         }
         
         $Path = ($env:PSModulePath -split $envSeparator)[0]
      }
      
      if (-not $httpClient)
      {
         $null = (Add-Type -AssemblyName System.Net.Http -ErrorAction SilentlyContinue)
         $SCRIPT:httpClient = [Net.Http.HttpClient]::new()
      }
      
      $baseUri = 'https://www.powershellgallery.com/api/v2/package/'
      
      Write-Progress -Id 1 -Activity 'Install-Modulefast' -Status ('Creating Download Tasks for {0} modules' -f $modulesToInstall.count)
      
      $DownloadTasks = foreach ($moduleItem in $modulesToInstall)
      {
         $ModulePackageName = @($moduleItem.Id, $moduleItem.Version, 'nupkg') -join '.'
         $ModuleCachePath = [io.path]::Combine(
            [string[]](
               $NuGetCache, 
               $moduleItem.Id, 
               $moduleItem.Version, 
               $ModulePackageName
            )
         )
         
         # TODO: Remove Me
         $ModuleCachePath = ('{0}/{1}' -f $ModuleCache, $ModulePackageName)
         #$uri = $baseuri + $ModuleName + '/' + $ModuleVersion
         $null = [io.directory]::CreateDirectory((Split-Path -Path $ModuleCachePath))
         $ModulePackageTempFile = [io.file]::Create($ModuleCachePath)
         $DownloadTask = $httpClient.GetStreamAsync($moduleItem.Source)
         
         # Return a hashtable with the task and file handle which we will need later
         @{
            DownloadTask = $DownloadTask
            FileHandle   = $ModulePackageTempFile
         }
      }
      
      # NOTE: This seems to go much faster when it's not in the same foreach as above, no idea why, seems to be blocking on the call
      $DownloadTasks = $DownloadTasks.Foreach{
         $PSItem.CopyTask = $PSItem.DownloadTask.result.CopyToAsync($PSItem.FileHandle)
         return $PSItem
      }
      
      # TODO: Add timeout via Stopwatch
      [array]$CopyTasks = $DownloadTasks.CopyTask
      while ($false -in $CopyTasks.iscompleted)
      {
         [int]$remainingTasks = (($CopyTasks | Where-Object -Property iscompleted -EQ -Value $false).count)
         $progressParams = @{
            Id               = 1
            Activity         = 'Install-Modulefast'
            Status           = ('Downloading {0} Modules' -f $CopyTasks.count)
            CurrentOperation = ('{0} Modules Remaining' -f $remainingTasks)
            PercentComplete  = [int](($CopyTasks.count - $remainingTasks) / $CopyTasks.count * 100)
         }
         
         Write-Progress @progressParams
         
         Start-Sleep -Seconds 0.2
      }
      
      $failedDownloads = $DownloadTasks.downloadtask.where{
         $PSItem.isfaulted
      }
      
      if ($failedDownloads)
      {
         # TODO: More comprehensive error message
         throw ('{0} files failed to download. Aborting' -f $failedDownloads.count)
      }
      
      $failedCopyTasks = $DownloadTasks.copytask.where{
         $PSItem.isfaulted
      }
      
      if ($failedCopyTasks)
      {
         # TODO: More comprehensive error message
         throw ('{0} files failed to copy. Aborting' -f $failedCopyTasks.count)
      }
      
      # Release the files once done downloading. If you don't do this powershell may keep a file locked.
      $DownloadTasks.FileHandle.close()
      
      # Cleanup
      # TODO: Cleanup should be in a trap or try/catch
      $DownloadTasks.DownloadTask.dispose()
      $DownloadTasks.CopyTask.dispose()
      $DownloadTasks.FileHandle.dispose()
      
      # Unpack the files
      Write-Progress -Id 1 -Activity 'Install-Modulefast' -Status ('Extracting {0} Modules' -f $modulesToInstall.id.count)
      
      $packageConfigPath = (Join-Path -Path $ModuleCache -ChildPath 'packages.config')
      
      if (Test-Path -Path $packageConfigPath -ErrorAction SilentlyContinue)
      {
         $null = (Remove-Item -Path $packageConfigPath -Force -Confirm:$false -ErrorAction SilentlyContinue)
      }
      
      $packageConfig = (New-NuGetPackageConfig -modulesToInstall $modulesToInstall -path $packageConfigPath)
      $timer = [diagnostics.stopwatch]::startnew()
      $moduleCount = $modulesToInstall.id.count
      $ipackage = 0
      # Initialize the files in the repository, if relevant
      & nuget.exe init $ModuleCache $NuGetCache | Where-Object {
         $PSItem -match 'already exists|installing'
      } | ForEach-Object {
         if ($ipackage -lt $moduleCount)
         {
            $ipackage++
         }
         
         # Write-Progress has a performance issue if run too frequently
         if ($timer.elapsedmilliseconds -gt 200)
         {
            $progressParams = @{
               id               = 1
               Activity         = 'Install-Modulefast'
               Status           = ('Extracting {0} Modules' -f $moduleCount)
               CurrentOperation = ('{0} of {1} Remaining' -f $ipackage, $moduleCount)
               PercentComplete  = [int]($ipackage / $moduleCount * 100)
            }
            
            Write-Progress @progressParams
            
            $timer.restart()
         }
      }
      
      if ($LASTEXITCODE)
      {
         throw 'There was a problem with nuget.exe'
      }
      
      # Create symbolic links from the nuget repository to "install" the packages
      foreach ($moduleItem in $modulesToInstall)
      {
         $moduleRelativePath = [io.path]::Combine($moduleItem.id, $moduleItem.version)
         # nuget saves as lowercase, matching to avoid Linux case issues
         $moduleNugetPath = (Join-Path -Path $NuGetCache -ChildPath $moduleRelativePath).tolower()
         $moduleTargetPath = (Join-Path -Path $Path -ChildPath $moduleRelativePath)
         
         if (-not (Test-Path -Path $moduleNugetPath))
         {
            Write-Error -Message ("{0} doesn't exist" -f $moduleNugetPath)
            continue
         }
         
         if (-not (Test-Path -Path $moduleTargetPath -ErrorAction SilentlyContinue) -and -not $Force)
         {
            $ModuleFolder = (Split-Path -Path $moduleTargetPath)
            # Create the parent target folder (as well as any hierarchy) if it doesn't exist
            $null = [io.directory]::createdirectory($ModuleFolder)
            
            # Create a symlink to the module in the package repository
            if ($PSCmdlet.ShouldProcess($moduleTargetPath, ('Install Powershell Module {0} {1}' -f $moduleItem.id, $moduleItem.version)))
            {
               $null = (New-Item -ItemType SymbolicLink -Path $ModuleFolder -Name $moduleItem.version -Value $moduleNugetPath)
            }
         }
         else
         {
            Write-Verbose -Message ('{0} already exists' -f $moduleTargetPath)
         }
         
         # Create the parent target folder if it doesn't exist
         #[io.directory]::createdirectory($moduleTargetPath)
      }
   }
}