amical/apps/www/scripts/fetch-content.mts

#!/usr/bin/env node

// Load environment variables from .env files
import * as dotenv from 'dotenv';
import path from 'path';
import { S3Client, ListObjectsV2Command, GetObjectCommand } from '@aws-sdk/client-s3';
import { createWriteStream, promises as fs } from 'fs';
import { pipeline } from 'stream/promises';

// Configure dotenv to load environment variables
dotenv.config();

// Parse command line arguments
const args = process.argv.slice(2);
const DRY_RUN = args.includes('--dry-run');
const SKIP_ERRORS = args.includes('--skip-errors');

// Log environment variables for debugging
console.log('AWS Credentials:', {
  accessKeyId: process.env.AWS_ACCESS_KEY_ID ? '****' : undefined,
  secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY ? '****' : undefined
});

// Configuration
const BUCKET_NAME = process.env.S3_BUCKET_NAME || 'amical-www';
const ENDPOINT = process.env.S3_ENDPOINT || 'https://s3.wasabisys.com';
const BLOG_PREFIX = process.env.BLOG_PREFIX || 'blog/';
const BLOG_IMAGES_PREFIX = process.env.BLOG_IMAGES_PREFIX || 'blog-images/';
const CONTENT_DIR = path.join(process.cwd(), 'content');
const PUBLIC_DIR = path.join(process.cwd(), 'public');
const LOCAL_BLOG_DIR = path.join(CONTENT_DIR, 'blogs');
const LOCAL_BLOG_IMAGES_DIR = path.join(PUBLIC_DIR, 'blog');

// Check if we have AWS credentials
const HAS_AWS_CREDENTIALS = !!(process.env.AWS_ACCESS_KEY_ID && process.env.AWS_SECRET_ACCESS_KEY);

// S3 client configuration
const s3ClientConfig: any = {
  endpoint: ENDPOINT,
  region: process.env.AWS_REGION || 'us-east-1', // Required but may not matter for Wasabi
  forcePathStyle: true, // Required for Wasabi
};

// Add credentials if provided via environment variables
if (HAS_AWS_CREDENTIALS) {
  s3ClientConfig.credentials = {
    accessKeyId: process.env.AWS_ACCESS_KEY_ID!,
    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!,
  };
} else if (!DRY_RUN) {
  console.warn('Warning: AWS credentials not found. S3 operations will be skipped.');
  if (SKIP_ERRORS) {
    console.warn('Running with --skip-errors flag. Will continue without fetching content.');
  } else {
    console.error('AWS credentials are required for content fetch. Set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables.');
  }
}

// Initialize S3 client only if we have credentials or in dry run mode
const s3Client = (HAS_AWS_CREDENTIALS || DRY_RUN) ? new S3Client(s3ClientConfig) : null;

/**
 * Ensures that a directory exists, creating it and all parent directories if necessary
 */
async function ensureDirectoryExists(directory: string): Promise<void> {
  try {
    await fs.access(directory);
    console.log(`Directory exists: ${directory}`);
  } catch (error) {
    try {
      await fs.mkdir(directory, { recursive: true });
      console.log(`Created directory: ${directory}`);
    } catch (mkdirError) {
      console.error(`Error creating directory ${directory}:`, mkdirError);
      throw mkdirError;
    }
  }
}

async function listObjects(prefix: string): Promise<string[]> {
  if (DRY_RUN) {
    console.log(`[DRY RUN] Would list objects with prefix: ${prefix}`);
    // Return mock data for dry run
    if (prefix === BLOG_PREFIX) {
      return [
        `${BLOG_PREFIX}getting-started.mdx`,
        `${BLOG_PREFIX}advanced-features.mdx`,
        `${BLOG_PREFIX}best-practices.mdx`,
      ];
    } else if (prefix === BLOG_IMAGES_PREFIX) {
      return [
        `${BLOG_IMAGES_PREFIX}header.png`,
        `${BLOG_IMAGES_PREFIX}screenshot1.png`,
        `${BLOG_IMAGES_PREFIX}diagram.png`,
      ];
    }
    return [];
  }

  // Skip S3 operations if we don't have credentials
  if (!s3Client) {
    console.log(`Skipping S3 operations for prefix: ${prefix} (no credentials)`);
    return [];
  }

  const objects: string[] = [];
  let continuationToken: string | undefined;

  try {
    do {
      const command = new ListObjectsV2Command({
        Bucket: BUCKET_NAME,
        Prefix: prefix,
        ContinuationToken: continuationToken,
      });

      const response = await s3Client.send(command);

      if (response.Contents) {
        for (const object of response.Contents) {
          if (object.Key) {
            objects.push(object.Key);
          }
        }
      }

      continuationToken = response.NextContinuationToken;
    } while (continuationToken);

    return objects;
  } catch (error) {
    console.error(`Error listing objects with prefix ${prefix}:`, error);
    throw error;
  }
}

async function downloadFile(key: string, destinationPath: string): Promise<void> {
  if (DRY_RUN) {
    console.log(`[DRY RUN] Would download: ${key} to ${destinationPath}`);
    return;
  }

  // Skip S3 operations if we don't have credentials
  if (!s3Client) {
    console.log(`Skipping download of: ${key} (no credentials)`);
    return;
  }

  try {
    const command = new GetObjectCommand({
      Bucket: BUCKET_NAME,
      Key: key,
    });

    const response = await s3Client.send(command);

    if (!response.Body) {
      throw new Error(`Failed to get object body for ${key}`);
    }

    const writeStream = createWriteStream(destinationPath);

    // @ts-ignore - TypeScript doesn't recognize Body as a stream, but it is
    await pipeline(response.Body, writeStream);

    console.log(`Downloaded: ${key} to ${destinationPath}`);
  } catch (error) {
    console.error(`Error downloading file ${key}:`, error);
    throw error;
  }
}

async function fetchBlogContent(): Promise<void> {
  console.log('Starting content fetch from S3...');
  if (DRY_RUN) {
    console.log('*** DRY RUN MODE - No files will be downloaded ***');
  } else if (!HAS_AWS_CREDENTIALS) {
    console.log('*** NO CREDENTIALS MODE - S3 operations will be skipped ***');
  }
  console.log(`Bucket: ${BUCKET_NAME}`);
  console.log(`Endpoint: ${ENDPOINT}`);

  try {
    // Ensure parent directories exist first
    await ensureDirectoryExists(CONTENT_DIR);
    await ensureDirectoryExists(PUBLIC_DIR);

    // Then ensure the specific directories for blogs and images
    await ensureDirectoryExists(LOCAL_BLOG_DIR);
    await ensureDirectoryExists(LOCAL_BLOG_IMAGES_DIR);

    // Skip S3 operations if we don't have credentials
    if (!HAS_AWS_CREDENTIALS && !DRY_RUN) {
      console.log('Skipping S3 operations due to missing credentials.');
      if (SKIP_ERRORS) {
        console.log('Build process will continue without fetching content due to --skip-errors flag.');
        return;
      } else {
        throw new Error('AWS credentials are required for content fetch. Set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables.');
      }
    }

    // Fetch blog MDX files
    console.log(`\nFetching blog MDX files from ${BLOG_PREFIX}...`);
    const blogFiles = await listObjects(BLOG_PREFIX);

    if (blogFiles.length === 0) {
      console.log('No blog files found.');
    } else {
      console.log(`Found ${blogFiles.length} blog files.`);

      let downloadedCount = 0;
      for (const key of blogFiles) {
        // Skip the prefix directory itself
        if (key === BLOG_PREFIX) continue;

        const fileName = path.basename(key);
        const destinationPath = path.join(LOCAL_BLOG_DIR, fileName);

        try {
          await downloadFile(key, destinationPath);
          downloadedCount++;
        } catch (error) {
          console.error(`Failed to download ${key}. Continuing with next file.`);
        }
      }

      console.log(`Successfully downloaded ${downloadedCount} of ${blogFiles.length} blog files.`);
    }

    // Fetch blog images
    console.log(`\nFetching blog images from ${BLOG_IMAGES_PREFIX}...`);
    const imageFiles = await listObjects(BLOG_IMAGES_PREFIX);

    if (imageFiles.length === 0) {
      console.log('No image files found.');
    } else {
      console.log(`Found ${imageFiles.length} image files.`);

      let downloadedCount = 0;
      for (const key of imageFiles) {
        // Skip the prefix directory itself
        if (key === BLOG_IMAGES_PREFIX) continue;

        const fileName = path.basename(key);
        const destinationPath = path.join(LOCAL_BLOG_IMAGES_DIR, fileName);

        try {
          await downloadFile(key, destinationPath);
          downloadedCount++;
        } catch (error) {
          console.error(`Failed to download ${key}. Continuing with next file.`);
        }
      }

      console.log(`Successfully downloaded ${downloadedCount} of ${imageFiles.length} image files.`);
    }

    console.log('\nContent fetch completed successfully!');
  } catch (error) {
    console.error('Error fetching content:', error);
    if (SKIP_ERRORS) {
      console.warn('Continuing build process despite errors due to --skip-errors flag.');
    } else {
      process.exit(1);
    }
  }
}

// Run the script
fetchBlogContent();