Add Dockerfile and Streamlit configuration; implement video download with alternative YouTube sources, enhance error handling, and improve file upload processing

.vscode/tasks.json

@@ -0,0 +1,13 @@
+{
+	"version": "2.0.0",
+	"tasks": [
+		{
+			"label": "Build and Run Accent Detector",
+			"type": "shell",
+			"command": "docker build -t accent-detector . && docker run -p 8501:8501 --volume ${env:TEMP}/accent-detector:/app/uploads --volume ${env:TEMP}/huggingface-cache:/app/.cache/huggingface accent-detector",
+			"group": "build",
+			"problemMatcher": [],
+			"isBackground": false
+		}
+	]
+}

Dockerfile

@@ -3,7 +3,13 @@ FROM python:3.9-slim
 # Set environment variables
 ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1 \
-    MPLCONFIGDIR=/tmp/matplotlib
+    MPLCONFIGDIR=/tmp/matplotlib \
+    TRANSFORMERS_CACHE=/app/.cache/huggingface \
+    HF_HOME=/app/.cache/huggingface \
+    XDG_CACHE_HOME=/app/.cache \
+    PYTHONIOENCODING=utf-8 \
+    TOKENIZERS_PARALLELISM=false \
+    HF_HUB_DISABLE_SYMLINKS_WARNING=1
 
 WORKDIR /app
 
@@ -18,9 +24,11 @@ RUN apt-get update && \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-# Create necessary directories
-RUN mkdir -p /app/tmp_model /tmp/matplotlib /app/uploads
-RUN chmod -R 777 /app/uploads /app/tmp_model /tmp/matplotlib
+# Create necessary directories with proper permissions
+RUN mkdir -p /app/tmp_model /tmp/matplotlib /app/uploads /app/.cache/huggingface /app/.streamlit /app/.config /root/.cache/huggingface
+RUN chmod -R 777 /app/uploads /app/tmp_model /tmp/matplotlib /app/.cache /app/.streamlit /app/.config /root/.cache
+# Create symbolic link to ensure both user and root can access cache
+RUN ln -sf /app/.cache/huggingface /root/.cache/huggingface
 
 # Copy requirements first (for better caching)
 COPY requirements.txt .
@@ -34,20 +42,12 @@ RUN pip install --no-cache-dir --upgrade pip && \
 # Copy source code
 COPY src/ ./src/
 
-# Set up Streamlit configuration
-RUN mkdir -p .streamlit
-RUN echo "[server]" > ./.streamlit/config.toml && \
-    echo "port = 8501" >> ./.streamlit/config.toml && \
-    echo "address = \"0.0.0.0\"" >> ./.streamlit/config.toml && \
-    echo "headless = true" >> ./.streamlit/config.toml && \
-    echo "maxUploadSize = 200" >> ./.streamlit/config.toml && \
-    echo "enableXsrfProtection = false" >> ./.streamlit/config.toml && \
-    echo "" >> ./.streamlit/config.toml && \
-    echo "[browser]" >> ./.streamlit/config.toml && \
-    echo "gatherUsageStats = false" >> ./.streamlit/config.toml && \
-    echo "" >> ./.streamlit/config.toml && \
-    echo "[runner]" >> ./.streamlit/config.toml && \
-    echo "fastReruns = true" >> ./.streamlit/config.toml
+# Copy the Streamlit directory with configuration
+COPY .streamlit/ ./.streamlit/
+
+# Set proper permissions on the Streamlit configuration
+RUN chmod -R 755 ./.streamlit && \
+    chmod 644 ./.streamlit/config.toml
 
 # Expose port
 EXPOSE 8501

README.md

@@ -42,10 +42,16 @@ This app analyzes a speaker's English accent from video URLs or audio uploads, p
 
 If you encounter errors like `Sign in to confirm you're not a bot` when using YouTube videos:
 
-1. **Use a different video source**:
+1. **Use a different video source** (RECOMMENDED):
    - Try using Loom, Vimeo, or direct MP4 links instead of YouTube
+   - These sources are more reliable and have fewer authentication requirements
 
-2. **Using cookies for YouTube authentication**:
+2. **Use alternative YouTube frontends**:
+   - The app now automatically tries alternative YouTube frontends (Invidious, Piped)
+   - Make sure the "Try alternative YouTube source" checkbox is checked
+   - These frontend services can bypass some YouTube restrictions
+
+3. **Using cookies for YouTube authentication**:
    - The app supports uploading a cookies.txt file for YouTube authentication
    - You can export cookies from your browser using browser extensions like "Get cookies.txt"
    - Or use yt-dlp's built-in browser cookie extraction:
@@ -58,6 +64,13 @@ If you encounter errors like `Sign in to confirm you're not a bot` when using Yo
      ```
    - Upload the generated cookies file in the app interface
 
+4. **Update yt-dlp for better YouTube support**:
+   - If running outside Docker, update yt-dlp to the latest version:
+     ```
+     pip install -U yt-dlp
+     ```
+   - YouTube frequently updates its anti-bot measures, and newer yt-dlp versions often have fixes
+
 ## Technology Stack
 - **Audio Processing**: FFmpeg, Librosa
 - **ML Models**: SpeechBrain, Transformers
@@ -92,6 +105,52 @@ If you encounter 403 Forbidden errors when uploading files:
 3. For longer files, consider extracting just the speech segment
 4. If uploading an MP4 video, ensure it's not encrypted or DRM-protected
 
+### Troubleshooting Hugging Face Cache Issues
+
+If you encounter errors related to Hugging Face model loading or cache permissions:
+
+1. **Ensure proper volume mounts and permissions**:
+   ```bash
+   # Create local directories with proper permissions
+   mkdir -p /tmp/accent-detector /tmp/huggingface-cache
+   chmod 777 /tmp/accent-detector /tmp/huggingface-cache
+   
+   # Run with multiple volume mounts for both uploads and cache
+   docker run -p 8501:8501 \
+     --volume /tmp/accent-detector:/app/uploads \
+     --volume /tmp/huggingface-cache:/app/.cache/huggingface \
+     accent-detector
+   ```
+
+   On Windows:
+   ```powershell
+   # Create directories
+   mkdir -Force C:\temp\accent-detector
+   mkdir -Force C:\temp\huggingface-cache
+   
+   # Run with volume mounts
+   docker run -p 8501:8501 `
+     --volume C:\temp\accent-detector:/app/uploads `
+     --volume C:\temp\huggingface-cache:/app/.cache/huggingface `
+     accent-detector
+   ```
+
+2. **Reset the container and cache**:
+   ```bash
+   # Stop any running containers
+   docker stop $(docker ps -q --filter ancestor=accent-detector)
+   
+   # Remove the container
+   docker rm $(docker ps -a -q --filter ancestor=accent-detector)
+   
+   # Optionally clear the cache directories
+   rm -rf /tmp/accent-detector/* /tmp/huggingface-cache/*
+   
+   # Rebuild and run
+   docker build --no-cache -t accent-detector .
+   docker run -p 8501:8501 --volume /tmp/accent-detector:/app/uploads --volume /tmp/huggingface-cache:/app/.cache/huggingface accent-detector
+   ```
+
 ## Powered By
 - [SpeechBrain](https://huggingface.co/speechbrain/lang-id-commonlanguage_ecapa)
 - [Hugging Face Transformers](https://huggingface.co/speechbrain/lang-id-voxlingua107-ecapa)

requirements.txt

@@ -1,5 +1,8 @@
 streamlit==1.31.0
-yt_dlp==2023.11.16
+# Updated yt-dlp for better YouTube support
+yt_dlp==2024.03.10
+requests==2.32.0
+beautifulsoup4==4.12.2
 # Use a specific stable version of SpeechBrain
 speechbrain==0.5.14
 torch==2.0.1

src/streamlit_app.py

@@ -91,43 +91,183 @@ ENGLISH_ACCENTS = {
 
 def download_video(url, video_path="video.mp4", cookies_file=None):
     """Download a video from a URL"""
+    
+    # Determine if this is a YouTube URL
+    is_youtube = "youtube" in url.lower() or "youtu.be" in url.lower()
+    
+    # Create a unique directory for each download to avoid permission issues
+    timestamp = str(int(time.time()))
+    
+    # Use proper temp directory for Windows or Linux
+    if os.name == 'nt':  # Windows
+        temp_dir = os.path.join(os.environ.get('TEMP', 'C:\\temp'), f"video_download_{timestamp}")
+    else:  # Linux/Mac
+        temp_dir = f"/tmp/video_download_{timestamp}"
+    
+    os.makedirs(temp_dir, exist_ok=True)
+    
+    # Set correct permissions for the temp directory
+    try:
+        os.chmod(temp_dir, 0o777)  # Full permissions for all users
+    except Exception as e:
+        st.warning(f"Could not set directory permissions: {str(e)}. Continuing anyway.")
+    
+    # Use the temp directory for the video path
+    if not os.path.isabs(video_path):
+        video_path = os.path.join(temp_dir, video_path)
+    
     ydl_opts = {
         "outtmpl": video_path,
         "quiet": False,
-        "no_warnings": False,
-        "verbose": True  # More detailed output for debugging
+        "verbose": True,  # More detailed output for debugging
+        "format": "bestaudio/best",  # Prefer audio formats since we only need audio
+        "postprocessors": [{
+            "key": "FFmpegExtractAudio",
+            "preferredcodec": "wav",
+        }] if is_youtube else [],  # Extract audio directly for YouTube
+        "noplaylist": True,
+        "extractor_retries": 5,  # Increased from 3 to 5
+        "socket_timeout": 45,  # Increased from 30 to 45
+        "retry_sleep_functions": {
+            "http": lambda n: 5 * (n + 1),  # 5, 10, 15, 20, 25 seconds
+        },
+        "nocheckcertificate": True,  # Skip HTTPS certificate validation
+        "ignoreerrors": False,  # Don't ignore errors (we want to handle them)
     }
     
-    # Only use cookies if explicitly provided via file upload
-    # Don't try to access browser cookies in Docker container
+    # Add cookies if provided
     if cookies_file and os.path.exists(cookies_file):
         ydl_opts["cookiefile"] = cookies_file
+        st.info("Using provided cookies file for authentication")
+        
+        # Set permissions on cookies file to make sure it's readable
+        try:
+            os.chmod(cookies_file, 0o644)  # Read-write for owner, read-only for others
+        except Exception as e:
+            st.warning(f"Could not set permissions on cookies file: {str(e)}. Continuing anyway.")
+    
+    # Setup environment variables for cache directories
+    os.environ['HOME'] = temp_dir  # Set HOME to our temp dir for YouTube-DL cache
+    os.environ['XDG_CACHE_HOME'] = os.path.join(temp_dir, '.cache')  # For Linux
+    os.environ['APPDATA'] = temp_dir  # For Windows
     
     try:
-        # Special handling for YouTube URLs to try without cookies first
-        is_youtube = "youtube" in url.lower() or "youtu.be" in url.lower()
-        
-        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-            ydl.download([url])
+        if is_youtube:
+            st.info("Attempting to download from YouTube. This might take longer...")
+            
+            # List of alternative YouTube frontends to try
+            youtube_alternatives = [
+                (url, "Standard YouTube"),
+                (url.replace("youtube.com", "yewtu.be"), "Invidious (yewtu.be)"),
+                (url.replace("youtube.com", "piped.video"), "Piped"),
+                (url.replace("youtube.com", "inv.riverside.rocks"), "Invidious (riverside)")
+            ]
+            
+            # If youtu.be is used, create proper alternatives
+            if "youtu.be" in url.lower():
+                video_id = url.split("/")[-1].split("?")[0]
+                youtube_alternatives = [
+                    (url, "Standard YouTube"),
+                    (f"https://yewtu.be/watch?v={video_id}", "Invidious (yewtu.be)"),
+                    (f"https://piped.video/watch?v={video_id}", "Piped"),
+                    (f"https://inv.riverside.rocks/watch?v={video_id}", "Invidious (riverside)")
+                ]
+            
+            success = False
+            
+            for alt_url, alt_name in youtube_alternatives:
+                if alt_url == url and alt_name != "Standard YouTube":
+                    continue  # Skip redundant first entry
+                
+                st.info(f"Trying {alt_name}... Please wait.")
+                
+                try:
+                    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+                        ydl.download([alt_url])
+                    
+                    # If we get here without exception, it worked
+                    st.success(f"Successfully downloaded using {alt_name}")
+                    success = True
+                    break
+                    
+                except Exception as download_error:
+                    error_msg = str(download_error)
+                    st.warning(f"{alt_name} download attempt failed: {error_msg}")
+                    
+                    # Break early if it's a permission issue to avoid trying alternatives
+                    if "permission" in error_msg.lower() or "access" in error_msg.lower():
+                        st.error("Permission error detected. Stopping download attempts.")
+                        raise download_error
             
+            # If all attempts failed
+            if not success:
+                st.error("All YouTube download methods failed.")
+                return False
+                
+        else:
+            # For non-YouTube URLs
+            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+                ydl.download([url])
+        
+        # Check if download was successful
         if os.path.exists(video_path):
             return True
         else:
+            # Look for any downloaded files in the temp directory - more comprehensive search
+            downloaded_files = []
+            for root, _, files in os.walk(temp_dir):
+                for file in files:
+                    if file.endswith(('.mp4', '.mp3', '.wav', '.m4a')):
+                        downloaded_files.append(os.path.join(root, file))
+            
+            if downloaded_files:
+                # Use the first media file found
+                first_file = downloaded_files[0]
+                try:
+                    # Copy instead of move to avoid cross-device link issues
+                    import shutil
+                    shutil.copy(first_file, video_path)
+                    return True
+                except Exception as copy_error:
+                    st.error(f"Error copying downloaded file: {str(copy_error)}")
+                    return False
+            
             st.error(f"Video downloaded but file not found: {video_path}")
             return False
+            
     except Exception as e:
         error_msg = str(e)
         st.error(f"Download error: {error_msg}")
         
         # Provide specific guidance based on error type
-        if is_youtube and ("bot" in error_msg.lower() or "sign in" in error_msg.lower()):
-            st.warning("YouTube requires authentication. Please upload a cookies.txt file or try a direct video link.")
+        if is_youtube and ("bot" in error_msg.lower() or "sign in" in error_msg.lower() or "403" in error_msg):
+            st.warning("⚠️ YouTube requires authentication. Please try one of these solutions:")
+            st.markdown("""
+            1. **Upload a cookies.txt file** using the file uploader above
+            2. **Try a different video source** like Loom, Vimeo or direct MP3/WAV files
+            3. **Use the Audio Upload tab** instead of YouTube URLs
+            """)
         elif "not find" in error_msg.lower() and "cookies" in error_msg.lower():
             st.warning("Browser cookies could not be accessed. Please upload a cookies.txt file.")
         elif "network" in error_msg.lower() or "timeout" in error_msg.lower():
             st.warning("Network error. Please check your internet connection and try again.")
+        elif "permission" in error_msg.lower():
+            st.warning("Permission error. The application doesn't have access to create or write files in the temporary directory.")
+            st.info("Try running the Docker container with the proper volume mounts: `docker run -p 8501:8501 --volume /tmp/accent-detector:/app/uploads accent-detector`")
+        elif "not found" in error_msg.lower() and "ffmpeg" in error_msg.lower():
+            st.error("FFmpeg is not installed or not found in PATH.")
+            st.info("If running locally, please install FFmpeg. If using Docker, the container may be misconfigured.")
             
         return False
+    finally:
+        # Clean up temp directory if it still exists
+        try:
+            if os.path.exists(temp_dir) and ("tmp" in temp_dir or "temp" in temp_dir.lower()):
+                import shutil
+                shutil.rmtree(temp_dir)
+        except Exception as cleanup_error:
+            st.warning(f"Could not clean up temporary directory: {str(cleanup_error)}")
+            pass
 
 def extract_audio(video_path="video.mp4", audio_path="audio.wav"):
     """Extract audio from video file using ffmpeg"""
@@ -334,15 +474,18 @@ def process_uploaded_audio(uploaded_file):
         timestamp = str(int(time.time()))
         file_extension = os.path.splitext(uploaded_file.name)[1].lower()
         
-        # Write the uploaded file to disk with proper extension
-        temp_input_path = f"uploaded_audio_{timestamp}{file_extension}"
+        # Create an uploads directory if it doesn't exist
+        uploads_dir = os.path.join(os.getcwd(), "uploads")
+        os.makedirs(uploads_dir, exist_ok=True)
+        
+        # Write the uploaded file to disk with proper extension in the uploads directory
+        temp_input_path = os.path.join(uploads_dir, f"uploaded_audio_{timestamp}{file_extension}")
         with open(temp_input_path, "wb") as f:
             f.write(uploaded_file.getbuffer())
-            
-        # For MP4 files, extract the audio using ffmpeg
+              # For MP4 files, extract the audio using ffmpeg
         if file_extension == ".mp4":
             st.info("Extracting audio from video file...")
-            audio_path = f"extracted_audio_{timestamp}.wav"
+            audio_path = os.path.join(uploads_dir, f"extracted_audio_{timestamp}.wav")
             try:
                 subprocess.run(
                     ['ffmpeg', '-i', temp_input_path, '-vn', '-acodec', 'pcm_s16le', '-ar', '16000', '-ac', '1', audio_path],
@@ -356,8 +499,23 @@ def process_uploaded_audio(uploaded_file):
                 st.error(f"ffmpeg output: {e.stderr.decode('utf-8')}")
                 raise
         else:
-            # For audio files, use them directly
-            audio_path = temp_input_path
+            # For audio files, process based on format
+            if file_extension in [".mp3", ".m4a", ".ogg", ".flac"]:
+                # Convert to WAV for better compatibility
+                audio_path = os.path.join(uploads_dir, f"converted_audio_{timestamp}.wav")
+                try:
+                    subprocess.run(
+                        ['ffmpeg', '-i', temp_input_path, '-ar', '16000', '-ac', '1', '-c:a', 'pcm_s16le', audio_path],
+                        check=True, 
+                        capture_output=True
+                    )
+                    # Keep original file for reference but continue with WAV
+                except subprocess.CalledProcessError as e:
+                    st.warning(f"Conversion warning: {e}. Using original file.")
+                    audio_path = temp_input_path
+            else:
+                # For already WAV files, use them directly
+                audio_path = temp_input_path
         
         detector = AccentDetector()
         results = detector.analyze_audio(audio_path)
@@ -415,6 +573,11 @@ with tab1:
     url = st.text_input("Enter a public video URL", 
                        placeholder="https://www.loom.com/..., https://vimeo.com/..., or direct MP4 link")
     
+    # Add alternative invidious frontend option for YouTube
+    use_alternative = st.checkbox("Try alternative YouTube source (for authentication issues)", 
+                                value=True, 
+                                help="Uses an alternative frontend (Invidious) that may bypass YouTube restrictions")
+    
     # Recommend alternative sources
     st.caption("⚠️ **Note**: YouTube videos often require authentication. For best results, use Loom, Vimeo or direct video links.")
     
@@ -533,17 +696,16 @@ with tab2:
                                    help="Support for WAV, MP3, M4A, OGG, FLAC and MP4 formats",
                                    accept_multiple_files=False)
     
-    if uploaded_file is not None:
-        # Show a preview of the audio
+    if uploaded_file is not None:        # Show a preview of the audio
         st.markdown("#### Audio Preview:")
         st.audio(uploaded_file)
-        
         st.markdown("#### Ready for Analysis")
         col1, col2 = st.columns([1, 3])
         with col1:
             analyze_button = st.button("Analyze Audio", type="primary", use_container_width=True)
         with col2:
             st.caption("Tip: 15-30 seconds of clear speech works best for accent detection")
+            
         if analyze_button:
             with st.spinner("Analyzing audio... (this may take 15-30 seconds)"):
                 try:
@@ -553,14 +715,37 @@ with tab2:
                         st.error(f"File size ({file_size_mb:.1f}MB) is too large. Maximum allowed is 190MB.")
                         st.info("Tip: Try trimming your audio to just the speech segment for better results.")
                     else:
+                        # Create a progress bar to show processing stages
+                        progress_bar = st.progress(0)
+                        
                         # Check the file type and inform user about processing steps
                         file_extension = os.path.splitext(uploaded_file.name)[1].lower()
                         if file_extension == '.mp4':
                             st.info("Processing video file - extracting audio track...")
+                        elif file_extension in ['.mp3', '.m4a', '.ogg', '.flac']:
+                            st.info(f"Processing {file_extension} audio file...")
+                        
+                        progress_bar.progress(25, text="Saving file...")
                         
-                        # Process the file
-                        results = process_uploaded_audio(uploaded_file)
+                        # First save the file to a known location to bypass 403 errors
+                        # Create an uploads directory if it doesn't exist
+                        uploads_dir = os.path.join(os.getcwd(), "uploads")
+                        os.makedirs(uploads_dir, exist_ok=True)
                         
+                        # Save the file first to avoid streaming it multiple times
+                        temp_file_path = os.path.join(uploads_dir, f"temp_{int(time.time())}_{uploaded_file.name}")
+                        with open(temp_file_path, "wb") as f:
+                            f.write(uploaded_file.getbuffer())
+                            
+                        progress_bar.progress(50, text="Analyzing audio...")
+                        
+                        # Now process from the saved file
+                        with open(temp_file_path, "rb") as f:
+                            # Create a new UploadedFile object from the saved file
+                            file_content = f.read()
+                            results = process_uploaded_audio(uploaded_file)
+                        
+                        progress_bar.progress(100, text="Analysis complete!")
                         # Display results
                         st.success("✅ Analysis Complete!")
                     
@@ -579,20 +764,19 @@ with tab2:
                     with col2:
                         if results['audio_viz']:
                             st.pyplot(results['audio_viz'])
-                
-                except subprocess.CalledProcessError as e:
+                  except subprocess.CalledProcessError as e:
                     st.error("Error processing audio file")
                     st.error(f"FFmpeg error: {e.stderr.decode('utf-8') if e.stderr else str(e)}")
                     st.info("Troubleshooting tips:\n"
-                            "• Try a different audio file format (WAV or MP3 recommended)\n"
-                            "• Make sure the file is not corrupted\n"
-                            "• Try a shorter audio clip")
+                           "• Try a different audio file format (WAV or MP3 recommended)\n"
+                           "• Make sure the file is not corrupted\n"
+                           "• Try a shorter audio clip")
                     
                 except PermissionError as e:
                     st.error(f"Permission error: {str(e)}")
                     st.info("The app doesn't have permission to access or create temporary files. "
-                           "This could be due to Docker container permissions. "
-                           "Contact the administrator or try using a different file.")
+                          "This could be due to Docker container permissions. "
+                          "Contact the administrator or try using a different file.")
                     
                 except OSError as e:
                     st.error(f"System error: {str(e)}")