Powershell script to copy files from FTP server to blob store.

CopyFromFtpToBlob.ps1

# Script used to copy files from specified FTP server.
# Requires the WinSCP .NET Assembly and WinSCP.exe: https://winscp.net/download
# Requires Azure CLI:  https://learn.microsoft.com/en-us/cli/azure/install-azure-cli-windows?tabs=azure-cli
# Usage:
# CopyFromFtpToBlob.ps1 <ftp_hostname> <ftp_username> <ftp_password> <ftp_directory_path> <source_file_names> <storage_account_connection_string> <container> <destination_path_under container>
# The <ftp_password> is expected to bee a secure string:  
#   https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.security/convertto-securestring?view=powershell-7.3
# The <source_file_names> is an array of filenames.  If empty, all files in <ftp_directory_path> will be used.
# Additional
# Read from file into array:
# $files = gc .\your_file_list.
# Join to use in SQL query IN clause:
# $files | % {"'$_',"}
# Set SecureString from string for testing:
# ConvertTo-SecureString 'your_string' -AsPlainText -Force
# Get string from SecureString:
# (New-Object PSCredential 0, $Password).GetNetworkCredential().Password
# You can also just debug with the $session object directly after initializing it.


param(
    [Parameter(Mandatory=$true, Position=0, ValueFromPipeline=$false)]
    [string]
    $HostName,

    [Parameter(Mandatory=$true, Position=1, ValueFromPipeline=$false)]
    [string]
    $UserName,

    [Parameter(Mandatory=$true, Position=2, ValueFromPipeline=$false)]
    [System.Security.SecureString]
    $Password,
    
    [Parameter(Mandatory=$true, Position=3, ValueFromPipeline=$false)]
    [String]
    $FtpDirectory,
    
    [Parameter(Mandatory=$true, Position=4, ValueFromPipeline=$false)]
    [string[]]
    $SourceFileNames,
    
    [Parameter(Mandatory=$false, Position=5, ValueFromPipeline=$false)]
    [System.Security.SecureString]
    $StorageAccountConnectionString,
    
    [Parameter(Mandatory=$false, Position=6, ValueFromPipeline=$false)]
    [string]
    $Container,
    
    [Parameter(Mandatory=$false, Position=7, ValueFromPipeline=$false)]
    [string]
    $DestinationPath # BlobPath everything, but filename.
)

$ErrorActionPreference = "Stop"


# Replace if you want your locally downloaded files to be somewhere else.
$localPath = Join-Path -Path (Get-Location).Path  -ChildPath "downloaded"
New-Item -ItemType Directory -Force -Path $localPath | Out-Null


# Load WinSCP .NET assembly
Add-Type -Path "WinSCPnet.dll"

if ($Password.Length -eq 0) {
  Write-Output "FTP password was empty."
  $Password = Read-Host -AsSecureString
}

function InitializeFtp()
{
  # Setup session options
  $sessionOptions = New-Object WinSCP.SessionOptions -Property @{
      Protocol = [WinSCP.Protocol]::Sftp
      HostName = $HostName
      UserName = $UserName
      #Password = (New-Object PSCredential 0, $Password).GetNetworkCredential().Password
      SecurePassword = $Password
  }

  $sessionOptions.SshHostKeyFingerprint = ''
  
  # You don't want to do this if you can help it (host not in trusted network):
  # $sessionOptions.GiveUpSecurityAndAcceptAnySshHostKey = "true"

  # Get the SshHostKeyFingerprint for whatever host you're connected to.
  #$session.ScanFingerprint($sessionOptions, 'SHA-256')

  return $sessionOptions
}


function DownloadFromFtp() 
{
  param(
    [Parameter(Mandatory=$true, Position=0)]
    [WinSCP.SessionOptions] $sessionOptions,

    [Parameter(Mandatory=$true, Position=1)]
    [string] $ftpDirectory,

    [Parameter(Mandatory=$true, Position=2)]
    [string[]] $sourceFileNames,

    [Parameter(Mandatory=$true,Position=3)]
    [string] $localPath
  )

  $session = New-Object WinSCP.Session
  try
  {
      # Connect
      $session.Open($sessionOptions)

      #$session.ListDirectory($ftpDirectory)

      # Download files
      if ($sourceFileNames.Length -eq 0)
      {
        $path = $ftpDirectory + '/*'
        Write-Host "Downloading all files from '$path' to '$localPath'..."
        $session.GetFilesToDirectory($path, $localPath)
        return
      }

      foreach ($filename in $sourceFileNames)
      {
        # Note: Files could have had a .done extension after processing.
        # Make sure you know exactly the filename or use glob wildcard (*).
        #$path = $ftpDirectory + '/' + $filename + '.done'
        $path = $ftpDirectory + '/' + $filename
        Write-Host "Downloading from '$path' to '$localPath'..."
        $session.GetFileToDirectory($path, $localPath)
      }

      # If we need performance use C# and GetFile(s) which would return a Stream.
  }
  finally
  {
      # Disconnect
      $session.Dispose()
  }
}


function StripExtension()
{
  param(
    [Parameter(Mandatory=$true, Position=0)]
    [string] $localPath,
    
    [Parameter(Mandatory=$false, Position=1)]
    [string] $extension = '.done'
  )

  Write-Host 'Renaming files under $localPath...'
  Get-ChildItem $localPath | ForEach-Object {
    if ($_.Name.EndsWith($extension))
    {
      #$filePath = $localPath + '\' + $_.Name
      $filePath = Join-Path -Path $localPath  -ChildPath $_.Name
      $splitName = $filePath.Split('.')
      $newFilePath = (($splitName | Select-Object -SkipLast 1) -Join '.')
      Write-Host "Renaming '$filePath ' to '$newFilePath'"

      Rename-Item -Path $filePath -NewName $newFilePath
    }
  }
}


function UploadToStorageAccount()
{
  param(
    [Parameter(Mandatory=$true, Position=0)]
    [string] $localPath,

    [Parameter(Mandatory=$true, Position=1)]
    [string] $blobPath,
    
    [Parameter(Mandatory=$true, Position=2)]
    [System.Security.SecureString] $connectionString
  )

  # Firewall has IP allowed or is temporarily disabled.

  Write-Host "Uploading files from '$localPath' to storage account at '$Container/$DestinationPath'..."
  # Get-ChildItem $localPath | ForEach-Object {
  #   $filePath = Join-Path -Path $localPath -ChildPath $_.Name
  #   $blobName = Join-Path -Path $blobPath -ChildPath $_.Name
  #   az storage blob upload --connection-string  ((New-Object PSCredential 0, $connectionString).GetNetworkCredential().Password)`
  #     -c $Container -f $filePath -n $blobName
  # }

  az storage blob upload-batch  --connection-string ((New-Object PSCredential 0, $connectionString).GetNetworkCredential().Password) `
    -d $Container -s $localPath --destination-path $blobPath  # --dryrun
}



try 
{
  [WinSCP.SessionOptions] $sessionOptions = InitializeFtp
  DownloadFromFtp $sessionOptions $FtpDirectory $SourceFileNames $localPath

  StripExtension $localPath '.done'

  UploadToStorageAccount $localPath $DestinationPath $StorageAccountConnectionString 
}
catch
{
    Write-Host "Error: $($_.Exception.Message)" -ForegroundColor Red
    exit 1
}