Progress saving

main.ts

@@ -6,6 +6,49 @@ dotenv.config();
 
 let songCount = 1;
 
+const PROGRESS_FILE = './progress.json';
+
+type ProgressState = {
+  songCount: number;
+  currentMentalStateIndex: number;
+  currentGenreIndex: number;
+  processedSongs: Set<string>;
+  completedGenres: string[];
+};
+
+function loadProgress(): ProgressState | null {
+  if (fs.existsSync(PROGRESS_FILE)) {
+    try {
+      const data = JSON.parse(fs.readFileSync(PROGRESS_FILE, 'utf-8'));
+      console.log(chalk.blue('Found previous progress, resuming...'));
+      console.log(chalk.blue(`Previously downloaded: ${data.songCount} songs`));
+      return {
+        ...data,
+        processedSongs: new Set(data.processedSongs || [])
+      };
+    } catch (error) {
+      console.error(chalk.red('Error loading progress file, starting fresh'), error);
+      return null;
+    }
+  }
+  return null;
+}
+
+function saveProgress(state: ProgressState) {
+  const dataToSave = {
+    ...state,
+    processedSongs: Array.from(state.processedSongs)
+  };
+  fs.writeFileSync(PROGRESS_FILE, JSON.stringify(dataToSave, null, 2));
+}
+
+function clearProgress() {
+  if (fs.existsSync(PROGRESS_FILE)) {
+    fs.unlinkSync(PROGRESS_FILE);
+    console.log(chalk.green('Progress file cleared - all downloads complete!'));
+  }
+}
+
 const AUTHTOKEN = process.env.AUTHTOKEN;
 
 if (!AUTHTOKEN) {
@@ -207,28 +250,46 @@ type QueueEntry = {
   url: string;
   folder: string;
   filename: string;
+  songId: string;
 };
 
 class DownloadQueue {
   private queue: QueueEntry[] = [];
   private activeDownloads = 0;
-  private maxConcurrency: number
+  private maxConcurrency: number;
+  private progressState: ProgressState;
 
-  constructor(maxConcurrency: number) {
-    this.maxConcurrency = maxConcurrency
+  constructor(maxConcurrency: number, progressState: ProgressState) {
+    this.maxConcurrency = maxConcurrency;
+    this.progressState = progressState;
   }
 
-  public enqueue(url: string, folder: string, filename: string) {
-    this.queue.push({ url, folder, filename });
+  public enqueue(url: string, folder: string, filename: string, songId: string) {
+    // Skip if already processed
+    if (this.progressState.processedSongs.has(songId)) {
+      return;
+    }
+    this.queue.push({ url, folder, filename, songId });
     this.processQueue();
   }
 
+  public async waitForCompletion(): Promise<void> {
+    return new Promise((resolve) => {
+      const checkInterval = setInterval(() => {
+        if (this.activeDownloads === 0 && this.queue.length === 0) {
+          clearInterval(checkInterval);
+          resolve();
+        }
+      }, 100);
+    });
+  }
+
   private async processQueue(): Promise<void> {
     while (this.activeDownloads < this.maxConcurrency && this.queue.length > 0) {
-      const { url, folder, filename } = this.queue.shift()!;
+      const { url, folder, filename, songId } = this.queue.shift()!;
       this.activeDownloads++;
 
-      this.processDownload(url, folder, filename).then(() => {
+      this.processDownload(url, folder, filename, songId).then(() => {
         this.activeDownloads--;
         this.processQueue();
       }).catch((error) => {
@@ -239,20 +300,71 @@ class DownloadQueue {
     }
   }
 
-  private async processDownload(url: string, folder: string, filename: string): Promise<void> {
+  private async processDownload(url: string, folder: string, filename: string, songId: string): Promise<void> {
     await downloadSong(url, folder, filename);
-    songCount++;
+    this.progressState.songCount++;
+    this.progressState.processedSongs.add(songId);
+    songCount = this.progressState.songCount;
+    
+    // Save progress every 10 songs
+    if (this.progressState.songCount % 10 === 0) {
+      saveProgress(this.progressState);
+    }
+    
     console.log(`${songCount} songs downloaded successfully \n${this.queue.length} remaining`);
   }
 }
 
-const downloadQueue = new DownloadQueue(3)
+// Load or initialize progress
+const savedProgress = loadProgress();
+const progressState: ProgressState = savedProgress || {
+  songCount: 0,
+  currentMentalStateIndex: 0,
+  currentGenreIndex: 0,
+  processedSongs: new Set(),
+  completedGenres: []
+};
 
-for (const mentalState of mentalStates) {
+songCount = progressState.songCount;
+
+const downloadQueue = new DownloadQueue(3, progressState);
+
+// Set up graceful shutdown
+process.on('SIGINT', () => {
+  console.log(chalk.yellow('\n\nReceived SIGINT, saving progress...'));
+  saveProgress(progressState);
+  process.exit(0);
+});
+
+process.on('SIGTERM', () => {
+  console.log(chalk.yellow('\n\nReceived SIGTERM, saving progress...'));
+  saveProgress(progressState);
+  process.exit(0);
+});
+
+for (let mentalStateIndex = progressState.currentMentalStateIndex; mentalStateIndex < mentalStates.length; mentalStateIndex++) {
+  const mentalState = mentalStates[mentalStateIndex];
   console.log(chalk.red(`Starting mental state ${mentalState}`))
-  for (const genre of [...genres[mentalState].base, ...genres[mentalState].nature]) {
+  
+  const allGenres = [...genres[mentalState].base, ...genres[mentalState].nature];
+  const startGenreIndex = mentalStateIndex === progressState.currentMentalStateIndex ? progressState.currentGenreIndex : 0;
+  
+  for (let genreIndex = startGenreIndex; genreIndex < allGenres.length; genreIndex++) {
+    const genre = allGenres[genreIndex];
+    const genreKey = `${mentalState}:${genre}`;
+    
+    // Skip if already completed
+    if (progressState.completedGenres.includes(genreKey)) {
+      console.log(chalk.gray(`Skipping already completed: ${genre} (${mentalState})`));
+      continue;
+    }
+    
     console.log(chalk.yellow(`Starting genre ${genre}`))
     
+    // Update current position
+    progressState.currentMentalStateIndex = mentalStateIndex;
+    progressState.currentGenreIndex = genreIndex;
+
     // 
     //  Phase 1 : Fetch all song data
     // 
@@ -261,6 +373,11 @@ for (const mentalState of mentalStates) {
         authorization: `Bearer ${AUTHTOKEN}`
       }
     });
+    if (!response.ok) {
+      chalk.red("ERROR SEARCHING SONGS")
+      chalk.red(response.text());
+      continue;
+    }
     const responseData = await response.json();
     const data = formatAudioData(responseData.result);
 
@@ -285,19 +402,32 @@ for (const mentalState of mentalStates) {
       let folder = `./songs/${genre}/${mentalState}/${activity}/${level}`
       let filename = song.trackVariation.baseUrl;
       let downloadLink = song.trackVariation.tokenedUrl;
+      let songId = `${mentalState}:${genre}:${filename}`;
 
-      downloadQueue.enqueue(downloadLink, folder, filename);
+      downloadQueue.enqueue(downloadLink, folder, filename, songId);
     }
     
+    // Wait for all songs in this genre to complete before marking as done
+    await downloadQueue.waitForCompletion();
+    
+    // Mark genre as completed
+    progressState.completedGenres.push(genreKey);
+    saveProgress(progressState);
+    console.log(chalk.green(`✓ Completed ${genre} (${mentalState})`));
+
   }
-};
+}
+
+// All done, clear progress file
+clearProgress();
+console.log(chalk.green.bold(`\n🎉 All downloads complete! Total songs: ${songCount}`));
 
 
 function downloadSong(downloadLink: string, folder: string, filename: string): Promise<void> {
   return new Promise((resolve, reject) => {
     ensureDirectory(folder)
     https.get(downloadLink, (response) => {
-      response.pipe(fs.createWriteStream(`${folder}/${filename}.mp3`))
+      response.pipe(fs.createWriteStream(`${folder}/${filename}`))
         .on('finish', () => {
           resolve();
         })