Environment Variable Synchronization Script

sync-env.js

#!/usr/bin/env node

/**
 * Environment Variable Synchronization Script
 * 
 * Reads environment-specific YAML files and distributes variables to packages
 * based on configuration in env.config.json
 * 
 * Usage:
 *   node scripts/sync-env.js                      # Sync all packages (dev)
 *   node scripts/sync-env.js --env prod           # Sync all packages (production)
 *   node scripts/sync-env.js --env dev --dry-run  # Preview changes
 *   node scripts/sync-env.js --package web        # Sync specific package
 *   node scripts/sync-env.js --validate           # Validate only
 */

const fs = require('fs').promises
const path = require('path')
const yaml = require('yaml')

class EnvSynchronizer {
  constructor(environment = 'dev') {
    this.environment = environment
    this.globalEnvPath = `env.global.${environment}.yaml`
    this.configPath = 'env.config.json'
    this.globalEnv = {}
    this.config = {}
    this.flattenedVars = {}
    
    // Standard environment variable names that shouldn't be prefixed
    this.standardVarNames = [
      'DATABASE_URL', 
      'DIRECT_URL', 
      'SUPABASE_URL', 
      'SUPABASE_ANON_KEY', 
      'SUPABASE_SERVICE_ROLE_KEY',
      'NEXTAUTH_URL', 
      'NEXTAUTH_SECRET', 
      'GOOGLE_CLIENT_ID', 
      'GOOGLE_CLIENT_SECRET',
      'STRIPE_SECRET_KEY', 
      'STRIPE_PUBLIC_KEY', 
      'STRIPE_WEBHOOK_SECRET',
      'EMAIL_FROM',
      'RESEND_API_KEY', 
      'MAILGUN_API_KEY', 
      'SENDGRID_API_KEY',
      'SMTP_HOST',
      'SMTP_PORT',
      'SMTP_USER',
      'SMTP_PASS',
      'SMTP_FROM',
      'POSTMARK_API_KEY',
      'OPENAI_API_KEY', 
      'ANTHROPIC_API_KEY',
      'AWS_ACCESS_KEY_ID',
      'AWS_SECRET_ACCESS_KEY',
      'AWS_REGION',
      'AWS_S3_BUCKET',
      'AWS_S3_ENDPOINT',
      'AWS_S3_FORCE_PATH_STYLE',
      'AWS_S3_DEFAULT_ACL',
      'CLOUDFLARE_R2_BUCKET',
      'CLOUDFLARE_R2_ACCOUNT_ID',
      'CLOUDFLARE_R2_ACCESS_KEY_ID',
      'CLOUDFLARE_R2_SECRET_ACCESS_KEY',
      'CLOUDFLARE_R2_REGION',
      'CLOUDFLARE_R2_ENDPOINT',
      'CLOUDFLARE_R2_PUBLIC_DOMAIN',
      'SUPABASE_STORAGE_BUCKET',
      'SUPABASE_STORAGE_PUBLIC',
      'NODE_ENV', 
      'DEBUG', 
      'VERBOSE_LOGGING'
    ]
  }

  async init() {
    try {
      // Load global environment configuration from the selected environment file
      // (e.g., env.global.dev.yaml or env.global.prod.yaml)
      const globalEnvContent = await fs.readFile(this.globalEnvPath, 'utf8')
      this.globalEnv = yaml.parse(globalEnvContent)
      
      // Load package configuration that defines which variables each package needs
      const configContent = await fs.readFile(this.configPath, 'utf8')
      this.config = JSON.parse(configContent)
      
      // Flatten nested YAML structure (e.g., auth.NEXTAUTH_URL -> "auth.NEXTAUTH_URL")
      // This makes it easier to match against package variable patterns
      this.flattenedVars = this.flattenObject(this.globalEnv)
      
      console.log(`✅ Loaded ${Object.keys(this.flattenedVars).length} environment variables`)
      console.log(`✅ Found ${Object.keys(this.config.packages).length} package configurations`)
      
    } catch (error) {
      console.error('❌ Failed to initialize:', error.message)
      process.exit(1)
    }
  }

  /**
   * Recursively flattens a nested object structure using dot notation
   * 
   * Example:
   *   Input:  { auth: { NEXTAUTH_URL: "http://localhost" } }
   *   Output: { "auth.NEXTAUTH_URL": "http://localhost" }
   * 
   * This allows pattern matching like "auth.*" in package configurations
   */
  flattenObject(obj, prefix = '') {
    const flattened = {}
    
    for (const [key, value] of Object.entries(obj)) {
      const newKey = prefix ? `${prefix}.${key}` : key
      
      // If the value is a nested object, recurse deeper
      if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
        Object.assign(flattened, this.flattenObject(value, newKey))
      } else {
        // Store the final key-value pair with dot notation
        flattened[newKey] = value
      }
    }
    
    return flattened
  }

  /**
   * Checks if a variable name matches a pattern from package configuration
   * 
   * Supports wildcard patterns:
   *   - "auth.*" matches "auth.NEXTAUTH_URL", "auth.GOOGLE_CLIENT_ID", etc.
   *   - "auth.NEXTAUTH_URL" matches exactly "auth.NEXTAUTH_URL"
   */
  matchPattern(pattern, varName) {
    if (pattern.endsWith('.*')) {
      // Handle wildcard patterns like "auth.*"
      const prefix = pattern.slice(0, -2)
      return varName.startsWith(prefix + '.')
    }
    // Handle exact matches
    return pattern === varName
  }

  /**
   * Determines if a variable should keep its original name without section prefix
   * 
   * For example:
   *   - "NEXTAUTH_URL" stays as "NEXTAUTH_URL" (not "AUTH_NEXTAUTH_URL")
   *   - "CUSTOM_VAR" becomes "AUTH_CUSTOM_VAR" if not in standard list
   * 
   * This prevents duplication like "AUTH_AUTH_NEXTAUTH_URL"
   */
  shouldUseStandardName(upperActualVarName, sectionPrefix) {
    // Variables that should use their original names without section prefix
    return this.standardVarNames.includes(upperActualVarName) ||
           upperActualVarName.startsWith('NEXT_PUBLIC_') ||
           upperActualVarName.startsWith('NEXTAUTH_') ||
           upperActualVarName.startsWith('GOOGLE_') ||
           upperActualVarName.startsWith('STRIPE_') ||
           upperActualVarName.startsWith('SUPABASE_') ||
           upperActualVarName.startsWith('AWS_') ||
           upperActualVarName.startsWith('CLOUDFLARE_') ||
           upperActualVarName.includes('_' + sectionPrefix + '_') ||
           upperActualVarName.startsWith(sectionPrefix + '_')
  }

  /**
   * Finds all environment variables that a specific package needs
   * 
   * Uses pattern matching to select variables from the global environment:
   *   - "auth.*" gets all auth-related variables
   *   - "database.DATABASE_URL" gets only that specific variable
   * 
   * Returns both matched variables and any missing patterns for debugging
   */
  getVariablesForPackage(packageName) {
    const packageConfig = this.config.packages[packageName]
    if (!packageConfig) {
      throw new Error(`Package ${packageName} not found in configuration`)
    }

    const matchedVars = {}
    const missingVars = []

    // Process each variable pattern defined for this package
    for (const pattern of packageConfig.variables) {
      let matched = false
      
      // Check all flattened variables against this pattern
      for (const [varName, value] of Object.entries(this.flattenedVars)) {
        if (this.matchPattern(pattern, varName)) {
          matchedVars[varName] = value
          matched = true
        }
      }
      
      // Track patterns that didn't match anything (but ignore wildcards)
      if (!matched && !pattern.includes('*')) {
        missingVars.push(pattern)
      }
    }

    return { matchedVars, missingVars }
  }

  /**
   * Generates the actual .env file content for a package
   * 
   * Process:
   * 1. Adds header with timestamp and package description
   * 2. Groups variables by section (auth, database, etc.)
   * 3. Applies smart naming to avoid duplicates (e.g., NEXTAUTH_URL not AUTH_NEXTAUTH_URL)
   * 4. Outputs in standard .env format: VARIABLE_NAME="value"
   */
  generateEnvContent(packageName, variables, missingVars = []) {
    const packageConfig = this.config.packages[packageName]
    const timestamp = new Date().toISOString()
    
    // Start with header template (auto-generated warning)
    let content = this.config.templates.header
      .replace('{timestamp}', timestamp)
    
    // Add package-specific information
    if (packageConfig.description) {
      content += `# Package: ${packageName}\n# ${packageConfig.description}\n`
    }

    // Group variables by their section (first part before the dot)
    // e.g., "auth.NEXTAUTH_URL" and "auth.GOOGLE_CLIENT_ID" both go in "auth" section
    const sections = {}
    for (const [varName, value] of Object.entries(variables)) {
      const section = varName.split('.')[0]
      if (!sections[section]) sections[section] = {}
      sections[section][varName] = value
    }

    // Generate content organized by section
    for (const [sectionName, sectionVars] of Object.entries(sections)) {
      content += this.config.templates.section_header
        .replace('{section_name}', sectionName.toUpperCase())
      
      for (const [varName, value] of Object.entries(sectionVars)) {
        // Transform "auth.NEXTAUTH_URL" -> "NEXTAUTH_URL" or "AUTH_CUSTOM_VAR"
        const parts = varName.split('.')
        const actualVarName = parts.slice(1).join('_') // Remove section prefix
        const sectionPrefix = parts[0].toUpperCase()
        
        // Smart naming to prevent duplication (e.g., AUTH_AUTH_NEXTAUTH_URL)
        let envVarName
        const upperActualVarName = actualVarName.toUpperCase()
        
        if (this.shouldUseStandardName(upperActualVarName, sectionPrefix)) {
          // Use variable name as-is (NEXTAUTH_URL stays NEXTAUTH_URL)
          envVarName = upperActualVarName
        } else {
          // Add section prefix for custom variables (CUSTOM_VAR becomes AUTH_CUSTOM_VAR)
          envVarName = `${sectionPrefix}_${upperActualVarName}`
        }
        
        // Output in standard .env format with quotes
        content += `${envVarName}="${value}"\n`
      }
    }

    // Add missing variables as comments
    if (missingVars.length > 0) {
      content += '\n# MISSING VARIABLES (add to env.global.yaml)\n'
      for (const missingVar of missingVars) {
        // Apply same logic for missing variables to avoid duplication
        const parts = missingVar.split('.')
        const actualVarName = parts.slice(1).join('_')
        const sectionPrefix = parts[0].toUpperCase()
        
        // Smart handling of variable names to avoid duplication  
        let envVarName
        const upperActualVarName = actualVarName.toUpperCase()
        
        if (this.shouldUseStandardName(upperActualVarName, sectionPrefix)) {
          // Use variable name as-is without section prefix
          envVarName = upperActualVarName
        } else {
          // Add section prefix for custom variables
          envVarName = `${sectionPrefix}_${upperActualVarName}`
        }
        
        content += this.config.templates.missing_var_comment
          .replace('{var_name}', envVarName)
        content += '\n'
      }
    }

    return content
  }

  async validateEnvironment() {
    console.log('🔍 Validating environment configuration...')
    
    const errors = []
    const warnings = []

    // Check required variables
    for (const requiredVar of this.config.validation.required) {
      if (!this.flattenedVars[requiredVar]) {
        errors.push(`Required variable missing: ${requiredVar}`)
      }
    }

    // Check patterns
    for (const [varName, pattern] of Object.entries(this.config.validation.patterns)) {
      const value = this.flattenedVars[varName]
      if (value && !new RegExp(pattern).test(value)) {
        warnings.push(`Variable ${varName} doesn't match expected pattern: ${pattern}`)
      }
    }

    // Report results
    if (errors.length > 0) {
      console.error('❌ Validation errors:')
      errors.forEach(error => console.error(`  • ${error}`))
    }

    if (warnings.length > 0) {
      console.warn('⚠️  Validation warnings:')
      warnings.forEach(warning => console.warn(`  • ${warning}`))
    }

    if (errors.length === 0 && warnings.length === 0) {
      console.log('✅ Environment validation passed')
    }

    return { errors, warnings }
  }

  /**
   * Processes a single package: finds its variables and writes its .env file
   * 
   * Steps:
   * 1. Get variables that match this package's patterns
   * 2. Generate .env file content with proper formatting
   * 3. Write to the package's designated output file (or preview if dry run)
   */
  async syncPackage(packageName, dryRun = false) {
    console.log(`📦 Processing package: ${packageName}`)
    
    const packageConfig = this.config.packages[packageName]
    const { matchedVars, missingVars } = this.getVariablesForPackage(packageName)
    
    console.log(`  • Found ${Object.keys(matchedVars).length} environment variables`)
    if (missingVars.length > 0) {
      console.warn(`  • Warning: ${missingVars.length} patterns didn't match any variables`)
    }

    // Generate the .env file content
    const envContent = this.generateEnvContent(packageName, matchedVars, missingVars)
    
    if (dryRun) {
      // Preview mode: show what would be written without actually writing
      console.log(`  • Would write to: ${packageConfig.outputFile}`)
      console.log(`  • Content preview:\n${envContent.split('\n').slice(0, 10).join('\n')}...`)
    } else {
      // Ensure the output directory exists before writing
      const outputDir = path.dirname(packageConfig.outputFile)
      await fs.mkdir(outputDir, { recursive: true })
      
      // Write the generated content to the package's .env file
      await fs.writeFile(packageConfig.outputFile, envContent)
      console.log(`  ✅ Updated: ${packageConfig.outputFile}`)
    }

    return { matchedVars, missingVars }
  }

  /**
   * Main orchestration function that syncs environment variables for all (or filtered) packages
   * 
   * Process:
   * 1. Validate configuration first (required variables, patterns, etc.)
   * 2. Process each package individually 
   * 3. Provide summary statistics
   * 
   * Supports options for dry runs, single package sync, and validation-only mode
   */
  async syncAll(options = {}) {
    const { dryRun = false, packageFilter = null, validateOnly = false } = options
    
    // Always validate configuration before proceeding
    const { errors } = await this.validateEnvironment()
    if (errors.length > 0) {
      console.error('❌ Cannot sync due to validation errors')
      process.exit(1)
    }

    // If only validation was requested, stop here
    if (validateOnly) {
      console.log('✅ Validation complete - no sync performed')
      return
    }

    console.log(dryRun ? '🔍 DRY RUN - No files will be modified' : '🚀 Syncing environment variables...')
    
    // Determine which packages to process (all or just one)
    const packagesToSync = packageFilter 
      ? [packageFilter] 
      : Object.keys(this.config.packages)

    let totalVars = 0
    let totalMissing = 0

    // Process each package individually
    for (const packageName of packagesToSync) {
      try {
        const { matchedVars, missingVars } = await this.syncPackage(packageName, dryRun)
        totalVars += Object.keys(matchedVars).length
        totalMissing += missingVars.length
      } catch (error) {
        console.error(`❌ Failed to sync ${packageName}:`, error.message)
      }
    }

    // Provide summary of what was accomplished
    console.log('')
    console.log('📊 Summary:')
    console.log(`  • Packages processed: ${packagesToSync.length}`)
    console.log(`  • Total variables synced: ${totalVars}`)
    if (totalMissing > 0) {
      console.log(`  • Missing variable patterns: ${totalMissing}`)
    }
    console.log(dryRun ? '  • No files were modified (dry run)' : '  • All files updated successfully')
  }
}

// CLI handling
async function main() {
  const args = process.argv.slice(2)
  const dryRun = args.includes('--dry-run')
  const validateOnly = args.includes('--validate')
  const packageIndex = args.indexOf('--package')
  const packageFilter = packageIndex >= 0 ? args[packageIndex + 1] : null
  const envIndex = args.indexOf('--env')
  const environment = envIndex >= 0 ? args[envIndex + 1] : 'dev'

  console.log(`🌍 Using environment: ${environment}`)
  
  const synchronizer = new EnvSynchronizer(environment)
  await synchronizer.init()
  await synchronizer.syncAll({ dryRun, packageFilter, validateOnly })
}

// Handle errors gracefully
process.on('unhandledRejection', (error) => {
  console.error('❌ Unhandled error:', error)
  process.exit(1)
})

if (require.main === module) {
  main()
}

module.exports = EnvSynchronizer