youtube.py
youtube.py
"""
YouTube MCP Service
This module provides MCP service functionality for YouTube operations including:
- Downloading videos from YouTube URLs
- Extracting transcripts from YouTube videos
It handles various scenarios with proper validation, error handling,
and progress tracking while providing LLM-friendly formatted results.
"""
import os
import time
import traceback
from datetime import datetime
from pathlib import Path
from dotenv import load_dotenv
from pydantic import BaseModel, Field
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from youtube_transcript_api import FetchedTranscript, YouTubeTranscriptApi
from aworld.logs.util import Color
from examples.gaia.mcp_collections.base import ActionArguments, ActionCollection, ActionResponse
# Default driver path for Chrome WebDriver
_DEFAULT_DRIVER_PATH = os.environ.get(
"CHROME_DRIVER_PATH", str(Path("~/Downloads/chromedriver-mac-arm64/chromedriver").expanduser())
)
class YoutubeDownloadResults(BaseModel):
"""Download result model with file information"""
file_path: str
file_name: str
file_size: int
content_type: str | None = None
success: bool
error: str | None = None
class TranscriptResult(BaseModel):
"""Transcript result model with transcript information"""
video_id: str
transcript: FetchedTranscript
success: bool
error: str | None = None
class YouTubeMetadata(BaseModel):
"""Metadata for YouTube operation results"""
operation: str
url: str | None = None
video_id: str | None = None
file_path: str | None = None
file_name: str | None = None
file_size: int | None = None
content_type: str | None = None
language_code: str | None = None
translate_to_language: str | None = None
execution_time: float | None = None
error_type: str | None = None
class YouTubeActionCollection(ActionCollection):
"""MCP service for YouTube operations.
Provides YouTube capabilities including:
- Video downloading with Selenium automation
- Transcript extraction and translation
- LLM-friendly result formatting
- Error handling and logging
"""
def __init__(self, arguments: ActionArguments) -> None:
super().__init__(arguments)
# Initialize supported file extensions
self.supported_extensions = {".mp4", ".webm", ".mkv"}
self._color_log("YouTube service initialized", Color.green, "debug")
def _format_transcript_output(self, result: TranscriptResult, format_type: str = "markdown") -> str:
"""Format transcript results for LLM consumption.
Args:
result: Transcript extraction result
format_type: Output format ('markdown', 'json', 'text')
Returns:
Formatted string suitable for LLM consumption
"""
if result is None or not result.success:
return f"Failed to extract transcript: {result.error}"
if format_type == "json":
return result.model_dump()
elif format_type == "text":
output = [f"Transcript for video ID: {result.video_id}\n"]
# Access snippets from FetchedTranscript
for entry in result.transcript.snippets:
start_time = entry["start"]
text = entry["text"]
minutes, seconds = divmod(int(start_time), 60)
timestamp = f"{minutes:02d}:{seconds:02d}"
output.append(f"[{timestamp}] {text}")
return "\n".join(output)
else: # markdown (default)
output = [f"# Transcript for YouTube Video: {result.video_id}\n"]
output.append("| Timestamp | Text |")
output.append("| --- | --- |")
# Access snippets from FetchedTranscript
for entry in result.transcript.snippets:
start_time = entry["start"]
text: str = entry["text"]
minutes, seconds = divmod(int(start_time), 60)
timestamp = f"{minutes:02d}:{seconds:02d}"
# Escape pipe characters in markdown table
safe_text = text.replace("|", "\\|")
output.append(f"| {timestamp} | {safe_text} |")
return "\n".join(output)
def _format_download_output(self, result: YoutubeDownloadResults, format_type: str = "markdown") -> str:
"""Format download results for LLM consumption.
Args:
result: Download result
format_type: Output format ('markdown', 'json', 'text')
Returns:
Formatted string suitable for LLM consumption
"""
if not result.success:
return f"Failed to download video: {result.error}"
if format_type == "json":
return result.model_dump()
elif format_type == "text":
output_parts = [
"Download completed successfully",
f"File: {result.file_name}",
f"Path: {result.file_path}",
f"Size: {result.file_size} bytes",
]
if result.content_type:
output_parts.append(f"Content Type: {result.content_type}")
return "\n".join(output_parts)
else: # markdown (default)
output_parts = [
"# YouTube Download Results ✅",
"",
"## File Information",
f"**Filename:** `{result.file_name}`",
f"**Path:** `{result.file_path}`",
f"**Size:** {result.file_size} bytes",
]
if result.content_type:
output_parts.append(f"**Content Type:** {result.content_type}")
return "\n".join(output_parts)
def _get_youtube_content(self, url: str, output_dir: str, timeout: int) -> None:
"""Use Selenium to download YouTube content via cobalt.tools
Args:
url: YouTube video URL
output_dir: Directory to save downloaded content
timeout: Maximum time to wait for download in seconds
"""
driver = None
try:
options = webdriver.ChromeOptions()
options.add_argument("--disable-blink-features=AutomationControlled")
# Set download file default path
prefs = {
"download.default_directory": output_dir,
"download.prompt_for_download": False,
"download.directory_upgrade": True,
"safebrowsing.enabled": True,
}
options.add_experimental_option("prefs", prefs)
# Create WebDriver object and launch Chrome browser
service = Service(executable_path=_DEFAULT_DRIVER_PATH)
driver = webdriver.Chrome(service=service, options=options)
self._color_log(f"Opening cobalt.tools to download from {url}", Color.blue)
# Open target webpage
driver.get("https://cobalt.tools/")
# Wait for page to load
time.sleep(5)
# Find input field and enter YouTube link
input_field = driver.find_element(By.ID, "link-area")
input_field.send_keys(url)
time.sleep(5)
# Find download button and click
download_button = driver.find_element(By.ID, "download-button")
download_button.click()
time.sleep(5)
try:
# Handle bot detection popup
driver.find_element(
By.CLASS_NAME,
"button.elevated.popup-button.undefined.svelte-nnawom.active",
).click()
except Exception as e:
self._color_log(f"Bot detection handling: {str(e)}", Color.yellow)
# try:
# t = 0
# while t < timeout:
# if (
# "downloading" not in driver.find_element(By.CLASS_NAME, "status-text.svelte-dmosdd").text
# and "starting" not in driver.find_element(By.CLASS_NAME, "status-text.svelte-dmosdd").text
# ):
# driver.find_element(By.CLASS_NAME, "button.action-button.svelte-dmosdd").click()
# break
# t += 3
# time.sleep(3)
# except Exception as e:
# self._color_log(f"Bot detection handling: {str(e)}", Color.yellow)
# Wait for download to complete
cnt = 0
while len(os.listdir(output_dir)) == 0 or os.listdir(output_dir)[0].split(".")[-1] == "crdownload":
time.sleep(3)
cnt += 3
if cnt >= timeout:
self._color_log(f"Download timeout after {timeout} seconds", Color.yellow)
break
self._color_log("Download process completed", Color.green)
except Exception as e:
self._color_log(f"Error during YouTube content download: {str(e)}", Color.red)
raise
finally:
# Close browser
if driver:
driver.quit()
def _find_existing_video(self, search_dir: str, video_id: str) -> str | None:
"""Recursively search for an existing video file with the given ID.
Args:
search_dir: Directory to search in
video_id: YouTube video ID to look for
Returns:
Path to existing file if found, None otherwise
"""
if not video_id:
return None
search_path = Path(search_dir)
if not search_path.exists():
return None
for item in search_path.iterdir():
if item.is_file() and video_id in item.name:
return str(item)
elif item.is_dir():
found = self._find_existing_video(str(item), video_id)
if found:
return found
return None
async def mcp_download_youtube_video(
self,
url: str = Field(description="The URL of YouTube video to download."),
timeout: int = Field(180, description="Download timeout in seconds (default: 180)."),
output_format: str = Field(
"markdown", description="Output format: 'markdown', 'json', or 'text' (default: markdown)."
),
) -> ActionResponse:
"""Download a YouTube video from URL and save it to the local filesystem.
This tool provides YouTube video downloading with:
- Selenium-based automation via cobalt.tools
- Configurable timeout controls
- Existing file detection to avoid redundant downloads
- LLM-optimized result formatting
Args:
url: The URL of YouTube video to download
timeout: Maximum download time in seconds
output_format: Format for the response output
Returns:
ActionResponse with download results and metadata
"""
start_time = time.time()
try:
# Validate URL
if not url.startswith(("http://", "https://")):
raise ValueError("Invalid URL format. URL must start with http:// or https://")
if not ("youtube.com" in url or "youtu.be" in url):
raise ValueError("URL must be a valid YouTube URL")
# Create output directory if it doesn't exist
output_path = self.workspace / "youtube_downloads"
output_path.mkdir(parents=True, exist_ok=True)
# Generate filename based on timestamp
filename = f"youtube_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
file_path = output_path / filename
file_path.mkdir(parents=True, exist_ok=True)
self._color_log(f"Output path: {file_path}", Color.blue)
# Extract video ID for existing file check
video_id = url.split("?v=")[-1].split("&")[0] if "?v=" in url else ""
if "youtu.be/" in url and not video_id:
video_id = url.split("youtu.be/")[-1].split("?")[0]
# Check if video already exists
base_path = self.workspace
existing_file = self._find_existing_video(str(base_path), video_id)
if existing_file:
existing_path = Path(existing_file)
result = YoutubeDownloadResults(
file_path=str(existing_path),
file_name=existing_path.name,
file_size=existing_path.stat().st_size,
content_type="mp4",
success=True,
error=None,
)
self._color_log(f"Found {video_id} already downloaded in: {existing_file}", Color.green)
# Format output for LLM
message = self._format_download_output(result, output_format)
execution_time = time.time() - start_time
# Create metadata
metadata = YouTubeMetadata(
operation="download",
url=url,
video_id=video_id,
file_path=str(existing_path),
file_name=existing_path.name,
file_size=existing_path.stat().st_size,
content_type="mp4",
execution_time=execution_time,
).model_dump()
return ActionResponse(success=True, message=message, metadata=metadata)
# Download the video
self._color_log(f"Downloading video from {url} to {file_path}", Color.blue)
self._get_youtube_content(url, str(file_path), timeout)
# Check if download was successful
downloaded_files = list(file_path.iterdir())
if not downloaded_files:
raise FileNotFoundError("No files were downloaded")
download_file = downloaded_files[0]
file_size = download_file.stat().st_size
self._color_log(f"File downloaded successfully to {download_file}", Color.green)
# Create result
result = YoutubeDownloadResults(
file_path=str(download_file),
file_name=download_file.name,
file_size=file_size,
content_type="mp4",
success=True,
error=None,
)
# Format output for LLM
message = self._format_download_output(result, output_format)
execution_time = time.time() - start_time
# Create metadata
metadata = YouTubeMetadata(
operation="download",
url=url,
video_id=video_id,
file_path=str(download_file),
file_name=download_file.name,
file_size=file_size,
content_type="mp4",
execution_time=execution_time,
).model_dump()
return ActionResponse(success=True, message=message, metadata=metadata)
except Exception as e:
error_msg = str(e)
self._color_log(f"Download error: {traceback.format_exc()}", Color.red)
# Format error for LLM
message = f"Failed to download YouTube video: {error_msg}"
execution_time = time.time() - start_time
# Create metadata
metadata = YouTubeMetadata(
operation="download",
url=url,
error_type="download_failure",
execution_time=execution_time,
).model_dump()
return ActionResponse(success=False, message=message, metadata=metadata)
async def mcp_extract_youtube_transcript(
self,
video_id: str = Field(description="The YouTube video ID or URL to extract transcript from."),
language_code: str = Field("en", description="Language code for the transcript (default: en)."),
translate_to_language: str | None = Field(
None, description="Translate transcript to this language code if provided."
),
output_format: str = Field(
"markdown", description="Output format: 'markdown', 'json', or 'text' (default: markdown)."
),
) -> ActionResponse:
"""Extract transcript from a YouTube video given its video ID or URL.
This tool provides transcript extraction with:
- Support for multiple languages
- Translation capabilities
- URL or video ID input handling
- LLM-optimized result formatting
Args:
video_id: The YouTube video ID or URL to extract transcript from
language_code: Language code for the transcript
translate_to_language: Translate transcript to this language code if provided
output_format: Format for the response output
Returns:
ActionResponse with transcript data and metadata
"""
start_time = time.time()
try:
# Clean video_id if full URL was provided
if "youtube.com" in video_id or "youtu.be" in video_id:
if "?v=" in video_id:
video_id = video_id.split("?v=")[-1].split("&")[0]
elif "youtu.be/" in video_id:
video_id = video_id.split("youtu.be/")[-1].split("?")[0]
self._color_log(f"Extracting transcript for video ID: {video_id}", Color.blue)
# Get transcript in specified language
if translate_to_language:
# Get transcript and translate it
y_api = YouTubeTranscriptApi()
transcript_list = y_api.list(video_id)
transcript = None
try:
# Try to get transcript in specified language
transcript = transcript_list.find_transcript([language_code])
except Exception:
# If specified language not found, get any available transcript
transcript = transcript_list.find_generated_transcript(["en"])
# Translate to target language
transcript_data = transcript.translate(translate_to_language).fetch()
else:
try:
# Get transcript without translation
transcript_data: FetchedTranscript = (
YouTubeTranscriptApi()
.list(video_id)
.find_transcript((language_code,))
.fetch(preserve_formatting=False)
)
except Exception:
transcript_data = None
result = TranscriptResult(video_id=video_id, transcript=transcript_data, success=True, error=None)
self._color_log(f"Successfully extracted transcript for video ID: {video_id}", Color.green)
# Format output for LLM
message = self._format_transcript_output(result, output_format)
execution_time = time.time() - start_time
# Create metadata
metadata = YouTubeMetadata(
operation="transcript",
video_id=video_id,
language_code=language_code,
translate_to_language=translate_to_language,
execution_time=execution_time,
).model_dump()
return ActionResponse(success=True, message=message, metadata=metadata)
except Exception as e:
error_msg = str(e)
self._color_log(f"Transcript extraction error: {traceback.format_exc()}", Color.red)
# Format error for LLM
message = f"Failed to extract transcript: {error_msg}"
execution_time = time.time() - start_time
# Create metadata
metadata = YouTubeMetadata(
operation="transcript",
video_id=video_id,
language_code=language_code,
translate_to_language=translate_to_language,
error_type="transcript_failure",
execution_time=execution_time,
).model_dump()
return ActionResponse(success=False, message=message, metadata=metadata)
# Default arguments for testing
if __name__ == "__main__":
load_dotenv()
arguments = ActionArguments(
name="youtube_service",
transport="stdio",
workspace=os.getenv("AWORLD_WORKSPACE", "~"),
)
try:
youtube_service = YouTubeActionCollection(arguments)
youtube_service.run()
except Exception as e:
print(f"Error: {e}")