init

.gitignore

@@ -178,4 +178,6 @@ cython_debug/
 #  exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
 #  refer to https://docs.cursor.com/context/ignore-files
 .cursorignore
-.cursorindexingignore
+.cursorindexingignore
+/failed
+/static/thumbnails

MP4Manager.py

@@ -0,0 +1,100 @@
+from moviepy.editor import VideoFileClip, concatenate_videoclips
+import os, cv2
+
+def add_intro_to_video(input_video, intro_video='intro.mp4', output_video='output.mp4'):
+    clip_main = VideoFileClip(input_video)
+    
+    clip_intro = VideoFileClip(intro_video).resize(clip_main.size).set_fps(clip_main.fps)
+    
+    if clip_main.audio is not None and clip_intro.audio is None:
+        from moviepy.editor import AudioArrayClip
+        silent_audio = AudioArrayClip([[0] * int(clip_intro.duration * clip_main.audio.fps)], fps=clip_main.audio.fps)
+        clip_intro = clip_intro.set_audio(silent_audio)
+    
+    final_clip = concatenate_videoclips([clip_intro, clip_main])
+    
+    final_clip.write_videofile(output_video, codec='libx264')
+
+def get_duration(input_file):
+    if not os.path.isfile(input_file):
+        print('Input file does not exist')
+        return 0
+
+    try:
+        video = cv2.VideoCapture(input_file)
+        frames = video.get(cv2.CAP_PROP_FRAME_COUNT)
+        fps = video.get(cv2.CAP_PROP_FPS)
+        duration = frames / fps
+        video.release()
+
+        return int(duration)
+    except Exception as e:
+        print(e)
+        return 0
+
+def generate_thumbnails(input_file, filename):
+    output_folder = 'temp/'
+    if not os.path.isfile(input_file):
+        raise ValueError('Input file does not exist')
+    if not os.path.exists(output_folder):
+        os.makedirs(output_folder)
+
+    posterPath = os.path.join(output_folder, f'{filename}.jpg')
+    previewPath = os.path.join(output_folder, f'{filename}.mp4')
+    
+    clip = VideoFileClip(input_file)
+    duration = clip.duration
+
+    interval = duration / 11.0
+
+    start_time_first_clip = 0 * interval
+    try:
+        clip.save_frame(posterPath, t=start_time_first_clip)
+    except:
+        pass
+    
+    clips = []
+    for i in range(10):
+        start_time = i * interval
+        end_time = start_time + 1
+        clips.append(clip.subclip(start_time, end_time))
+    
+    final_clip = concatenate_videoclips(clips).resize(newsize=(384, 216)).without_audio()
+    final_clip.write_videofile(previewPath, fps=24, codec="libx264")
+
+    for subclip in clips:
+        subclip.close()
+
+    clip.close()
+    final_clip.close()
+    
+    return posterPath, previewPath
+
+def split_video(file_path, segment_size_gb=8):
+    import subprocess
+    
+    # Convert GB to bytes
+    segment_size_bytes = segment_size_gb * 1024 * 1024 * 1024
+    
+    # Get the total size of the video file
+    total_size_bytes = os.path.getsize(file_path)
+    
+    # Calculate the number of segments needed
+    num_segments = total_size_bytes // segment_size_bytes + 1
+
+    # Get the duration of the video file
+    duration = get_duration(file_path)
+    
+    # Calculate the duration of each segment
+    segment_duration = duration / num_segments
+    
+    # Generate output file pattern
+    file_name, file_extension = os.path.splitext(file_path)
+    output_pattern = f"{file_name}_segment_%03d{file_extension}"
+    
+    # Run FFmpeg command to split the video
+    command = [
+        "ffmpeg", "-i", file_path, "-c", "copy", "-map", "0", 
+        "-segment_time", str(segment_duration), "-f", "segment", output_pattern
+    ]
+    subprocess.run(command)

anal.py

@@ -0,0 +1,62 @@
+from funcs import process_videos, group_videos, match_data_to_video_fast, get_all_videos, get_all_data
+from flask import Flask, render_template
+import os
+from collections import defaultdict
+
+app = Flask(__name__)
+
+SCAN_DIRS = [
+    "E:/streamaster/downloaded/",
+    "U:/encoded",
+    "U:/count_sorted"
+]
+
+DATA_DIRS = [
+    "E:/streamaster/data",
+    "E:/streamaster/downloaded",
+]
+
+# ----------- Data Processing -----------
+def load_video_data():
+    videos = []
+    for d in SCAN_DIRS:
+        videos += get_all_videos(d)
+
+    data = []
+    for d in DATA_DIRS:
+        data += get_all_data(d)
+
+    parsed_videos, unmatched = match_data_to_video_fast(videos, data)
+    parsed_videos = process_videos(parsed_videos)
+    video_data = group_videos(parsed_videos, sort_by="count", order="desc")
+    return video_data
+
+def compute_analytics(video_data):
+    storage_usage = defaultdict(lambda: {"total_size": 0, "video_count": 0})
+    per_video_sizes = {}
+
+    for (username, platform), vids in video_data.items():
+        total_size_gb = sum(v['size'] for v in vids) / 1024  # Convert MB to GB
+        avg_size_gb = (total_size_gb / len(vids)) if vids else 0
+
+        storage_usage[(username, platform)]["total_size"] += total_size_gb
+        storage_usage[(username, platform)]["video_count"] += len(vids)
+        per_video_sizes[(username, platform)] = avg_size_gb
+
+    return storage_usage, per_video_sizes
+
+# ----------- Flask Routes -----------
+@app.route("/")
+def analytics_dashboard():
+    video_data = load_video_data()
+    storage_usage, per_video_sizes = compute_analytics(video_data)
+
+    # Sort by total storage used
+    sorted_usage = sorted(storage_usage.items(), key=lambda x: x[1]["total_size"], reverse=True)
+
+    return render_template("analytics.html",
+                           storage_usage=sorted_usage,
+                           avg_sizes=per_video_sizes)
+
+if __name__ == "__main__":
+    app.run(debug=True)

app.py

@@ -0,0 +1,127 @@
+from flask import Flask, render_template, request, redirect, url_for
+from funcs import process_videos, group_videos, match_data_to_video_fast, get_all_videos, get_all_data
+from config import connect_redis
+import json, os, time, math, subprocess
+
+# -------------------- CONFIG -------------------- #
+app = Flask(__name__)
+redis = connect_redis()
+
+CACHE_KEY = "video_cache"
+THUMB_DIR = "static/thumbnails"
+VIDEOS_PER_PAGE = 20
+
+SCAN_DIRS = [
+    "E:/streamaster/downloaded/",
+    "U:/encoded",
+    "U:/count_sorted"
+]
+
+DATA_DIRS = [
+    "E:/streamaster/data",
+    "E:/streamaster/downloaded",
+]
+
+os.makedirs(THUMB_DIR, exist_ok=True)
+
+# -------------------- UTILS -------------------- #
+def generate_thumbnail(video_path, thumb_path):
+    if not os.path.exists(thumb_path):
+        cmd = [
+            "ffmpeg", "-y", "-i", video_path, "-ss", "00:00:05.000",
+            "-vframes", "1", thumb_path
+        ]
+        subprocess.run(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+def load_video_data():
+    videos = []
+    for d in SCAN_DIRS:
+        videos += get_all_videos(d)
+
+    data = []
+    for d in DATA_DIRS:
+        data += get_all_data(d)
+
+    parsed_videos, unmatched = match_data_to_video_fast(videos, data)
+    parsed_videos = process_videos(parsed_videos)
+    video_data = group_videos(parsed_videos, sort_by="count", order="desc")
+    return video_data
+
+def compute_analytics(video_data):
+    storage_usage = {}
+    avg_sizes = {}
+    video_map = {}
+
+    for (username, platform), vids in video_data.items():
+        total_size_gb = sum(v['size'] for v in vids) / 1024
+        avg_size_gb = (total_size_gb / len(vids)) if vids else 0
+        key = f"{username}::{platform}"
+        storage_usage[key] = {
+            "total_size": total_size_gb,
+            "video_count": len(vids)
+        }
+        avg_sizes[key] = avg_size_gb
+
+        for v in vids:
+            video_id = os.path.basename(v['filepath'])
+            thumb_path = os.path.join(THUMB_DIR, f"{video_id}.jpg")
+            generate_thumbnail(v['filepath'], thumb_path)
+            v['thumbnail'] = thumb_path
+        video_map[key] = vids
+    return storage_usage, avg_sizes, video_map
+
+def refresh_data():
+    video_data = load_video_data()
+    storage_usage, avg_sizes, video_map = compute_analytics(video_data)
+    cache = {
+        "timestamp": time.time(),
+        "videos": video_map,
+        "storage_usage": storage_usage,
+        "avg_sizes": avg_sizes
+    }
+    redis.set(CACHE_KEY, json.dumps(cache))
+    return cache
+
+def get_cached_data():
+    try:
+        cached = redis.get(CACHE_KEY)
+        return json.loads(cached)  # ✅ Use cache if it exists
+    except Exception as e:
+        return refresh_data()      # ✅ Generate and store fresh data if empty
+
+# -------------------- ROUTES -------------------- #
+@app.route("/")
+def dashboard():
+    cache = get_cached_data()
+    sorted_usage = sorted(cache["storage_usage"].items(), key=lambda x: x[1]["total_size"], reverse=True)
+    return render_template("analytics.html", storage_usage=sorted_usage, avg_sizes=cache["avg_sizes"])
+
+@app.route("/refresh")
+def refresh():
+    refresh_data()
+    return redirect(url_for("dashboard"))
+
+@app.route("/user/<username>")
+def user_page(username):
+    cache = get_cached_data()
+    videos = []
+    for key, vid_list in cache["videos"].items():
+        user, platform = key.split("::")
+        if user == username:
+            for v in vid_list:
+                v['platform'] = platform
+            videos.extend(vid_list)
+
+    page = int(request.args.get("page", 1))
+    total_pages = math.ceil(len(videos) / VIDEOS_PER_PAGE)
+    start = (page - 1) * VIDEOS_PER_PAGE
+    paginated = videos[start:start + VIDEOS_PER_PAGE]
+
+    return render_template("user_page.html",
+                           username=username,
+                           videos=paginated,
+                           page=page,
+                           total_pages=total_pages)
+
+if __name__ == "__main__":
+    app.run(debug=True)

config.py

@@ -0,0 +1,47 @@
+from redis import Redis
+import json
+
+redisCred = {"host": "192.168.0.27", "port": 30036, "password": "bignigga123"}
+
+
+def redis_gen_connection():
+    return Redis(host=redisCred["host"], port=redisCred["port"], password=redisCred["password"])
+
+def connect_redis():
+    REDIS_HOST = "192.168.0.27"
+    REDIS_PORT = 30036
+    REDIS_PASSWORD = "bignigga123"
+
+    try:
+        client = Redis(
+            host=REDIS_HOST,
+            port=REDIS_PORT,
+            password=REDIS_PASSWORD,
+            decode_responses=True
+        )
+
+        response = client.ping()
+        if response:
+            print("Connected to Redis successfully!")
+            return client
+        else:
+            print("Failed to connect to Redis!")
+            return None
+    except Exception as e:
+        print(f"An error occurred: {e}")
+        return None
+
+def get_streamer_data(username):
+    try:
+        redis_client = redis_gen_connection()
+        streamer_data = redis_client.hget("streamers", username)
+
+        if streamer_data is None:
+            return None
+
+        streamer_data = json.loads(streamer_data)
+        return streamer_data
+    except Exception as e:
+        print(f"Unexpected error: {e}")
+
+    return None

funcs.py

@@ -0,0 +1,400 @@
+from datetime import datetime, timedelta, timezone
+from MP4Manager import get_duration
+import os, json, subprocess, shutil
+
+
+def is_file_empty(filepath):
+    return os.stat(filepath).st_size == 0
+
+def format_datetime(datetime_str):
+    """Format the datetime string to a more readable format."""
+    return datetime.strptime(datetime_str, "%Y-%m-%d %H:%M:%S")
+
+def get_file_size_in_mb(file_path):
+    return os.path.getsize(file_path) / (1024 ** 2)
+
+def get_file_size_gb(file_path):
+    return os.path.getsize(file_path) / 1024 / 1024 / 1024
+
+def get_data(data_path):
+    try:
+        with open(data_path, 'r') as file:
+            data = json.load(file)
+        return data
+    except Exception as e:
+        print(f"Error loading {data_path}: {e}")
+        return None
+
+def update_video_data(dataPath, data):
+    """Update or create a JSON file for the video metadata."""
+    if os.path.exists(dataPath):
+        with open(dataPath, "r") as f:
+            existing_data = json.load(f)
+        
+        if existing_data == data:
+            return  # No update needed if data hasn't changed.
+        
+        data["updatedAt"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    
+    with open(dataPath, "w") as f:
+        json.dump(data, f)  # Write to file if new or if data has changed.
+
+
+def is_recent(updated_at_str, minutes=30):
+    updated_at = format_datetime(updated_at_str)
+    updated_at = updated_at.replace(tzinfo=timezone.utc)
+    now = datetime.now(timezone.utc)
+    return now - updated_at < timedelta(minutes=minutes)
+
+
+def is_file_size_bigger_than(file_size_in_mb, max_size_gb):
+    """Check if the file size is bigger than the specified max size in GB."""
+    max_size_megabytes = max_size_gb * 1024  # Convert GB to MB
+    return file_size_in_mb > max_size_megabytes
+
+
+def cleanup_data_files(folder_path):
+    videos = [video for video in os.listdir(folder_path) if video.endswith(".json")]
+    for filename in videos:
+        json_path = os.path.join(folder_path, filename)
+        video_path = json_path.replace(".json", ".mp4")
+        if not os.path.exists(video_path):
+            os.remove(json_path)
+
+
+def get_video_data(videoPath):
+    with open(videoPath, "r") as f:
+        data = json.load(f)
+    return data
+
+
+def get_videos(folder_path):
+    """Retrieve video metadata from the JSON files in a specified folder."""
+    video_list = []
+
+    # List all .mp4 files and their corresponding .json metadata files
+    videos = [f for f in os.listdir(folder_path) if f.endswith(".mp4")]
+    
+    for video_filename in videos:
+        video_path = os.path.join(folder_path, video_filename)
+        json_path = video_path.replace(".mp4", ".json")
+
+        if not os.path.exists(json_path):
+            continue
+            
+        data = get_video_data(json_path)
+        data['size'] = get_file_size_in_mb(video_path)  # Include size in MB for further processing
+        data['filepath'] = video_path
+        
+        video_list.append(data)
+
+    return video_list
+
+
+def group_videos(video_list, sort_by="count", order="desc"):
+    """Group video data by username and site, and sort the groups by video creation time."""
+    video_data = {}
+    is_desc = order == "desc"
+
+    for video in video_list:
+        key = (video["username"], video["site"])
+        if key not in video_data:
+            video_data[key] = []
+        video_data[key].append(video)
+
+    # Ensure videos for each user and site are sorted by creation date
+    for key in video_data:
+        video_data[key].sort(key=lambda x: format_datetime(x["createdAt"]))
+
+    # Further sort groups if required based on size or count
+    if sort_by == "size":
+        video_data = dict(sorted(video_data.items(), key=lambda x: sum(item['size'] for item in x[1]), reverse=is_desc))
+    elif sort_by == "count":
+        video_data = dict(sorted(video_data.items(), key=lambda x: len(x[1]), reverse=is_desc))
+
+    return video_data
+
+
+def process_videos(video_data):
+    processed_videos = []
+    failed_directory = "failed"
+
+    for video in video_data:
+        is_updated = False
+        video_path = video["filepath"]
+        data_path = video["jsonpath"]
+        
+        if 'size' not in video:
+            filesize = get_file_size_in_mb(video_path)
+            video['size'] = filesize
+            is_updated = True
+        
+        if is_updated and 'duration' not in video:
+            video['duration'] = get_duration(video_path)
+            is_updated = True
+        
+        # Move corrupted videos to the failed folder
+        if video['duration'] == 0:
+            print(f"{video['videoID']} is corrupted, moving to failed folder")
+            failed_video_path = os.path.join(failed_directory, video["videoID"] + ".mp4")
+            failed_data_path = failed_video_path.replace(".mp4", ".json")
+
+            shutil.move(video_path, failed_video_path)
+            shutil.move(data_path, failed_data_path)
+
+            continue  # Skip further processing for this video
+        
+        if is_updated:
+            update_video_data(data_path, video)
+            
+        processed_videos.append(video)
+        
+    return processed_videos
+
+
+def group_for_concatenation(videos, time_limit=30):
+    """
+    Groups videos into lists where:
+     - total group size <= 9GB (9216 MB),
+     - time gap between consecutive videos <= time_limit minutes,
+     - AND all have the same resolution/fps/codecs for no-reencode concat.
+    """
+    concatenated_video_groups = []
+    current_group = []
+    current_size_mb = 0
+    last_video_end = None
+    reference_params = None  # We'll store the 'ffprobe' params for the first video in each group
+
+    for video in videos:
+        video_start = format_datetime(video['createdAt'])
+        video_end = video_start + timedelta(seconds=video['duration'])
+
+        # Probe the video to get parameters
+        video_path = video['filepath']
+        params = get_video_params(video_path)
+        if params is None:
+            # If ffprobe fails, skip or handle the error
+            print(f"Skipping {video_path}, failed to get ffprobe info.")
+            continue
+
+        if current_group:
+            # Check if adding this video breaks the size limit
+            time_difference = (video_start - last_video_end).total_seconds() / 60
+            size_exceeded = (current_size_mb + video['size'] > 9216)
+            time_exceeded = (time_difference > time_limit)
+
+            # Check if the video parameters match the group's reference
+            param_mismatch = False
+            if reference_params:
+                # Compare relevant fields
+                for field in ['video_codec','width','height','pix_fmt','fps',
+                              'audio_codec','audio_sample_rate','audio_channels','audio_channel_layout']:
+                    if params[field] != reference_params[field]:
+                        param_mismatch = True
+                        break
+
+            # If we exceed size, exceed time gap, or mismatch in parameters => start new group
+            if size_exceeded or time_exceeded or param_mismatch:
+                concatenated_video_groups.append(current_group)
+                current_group = []
+                current_size_mb = 0
+                reference_params = None  # reset for new group
+
+        # If we're starting a new group, set reference parameters
+        if not current_group:
+            reference_params = params
+
+        # Add the current video to the group
+        current_group.append(video)
+        current_size_mb += video['size']
+        last_video_end = video_end
+
+    # Add the last group if not empty
+    if current_group:
+        concatenated_video_groups.append(current_group)
+
+    # Optional: Ensure the last group is "ready" for upload based on time difference
+    # (Your original logic that if last video was updated < time_limit minutes ago, remove the group)
+    if concatenated_video_groups:
+        last_group = concatenated_video_groups[-1]
+        last_video = last_group[-1]
+        last_updated_at = datetime.strptime(last_video['createdAt'], "%Y-%m-%d %H:%M:%S")
+        if datetime.now() - last_updated_at <= timedelta(minutes=time_limit):
+            print(f"Last group is not ready for upload. Removing from final groups.")
+            concatenated_video_groups.pop()
+    
+    concatenated_video_groups = [group for group in concatenated_video_groups if len(group) > 1]
+
+    return concatenated_video_groups
+
+
+def get_video_params(video_path):
+    """
+    Run ffprobe on a given video path to extract:
+      - codec_name (video + audio)
+      - width, height
+      - pix_fmt
+      - r_frame_rate (frame rate)
+      - sample_rate, channel_layout (audio)
+    Returns a dict with these parameters or None if there's an error.
+    """
+    cmd = [
+        'ffprobe', '-v', 'error',
+        '-print_format', 'json',
+        '-show_streams',
+        '-show_format',
+        video_path
+    ]
+
+    try:
+        result = subprocess.run(cmd, capture_output=True, text=True, check=True)
+        info = json.loads(result.stdout)
+
+        # We'll parse out the first video & audio streams we find.
+        video_stream = next((s for s in info['streams'] if s['codec_type'] == 'video'), None)
+        audio_stream = next((s for s in info['streams'] if s['codec_type'] == 'audio'), None)
+
+        if not video_stream:
+            raise ValueError(f"No video stream found in {video_path}")
+
+        # Frame rate can be something like "30000/1001" - convert to float
+        r_frame_rate = video_stream.get('r_frame_rate', '0/0')
+        try:
+            num, den = r_frame_rate.split('/')
+            fps = float(num) / float(den) if float(den) != 0 else 0.0
+        except:
+            fps = 0.0
+
+        # Gather the key parameters
+        params = {
+            'video_codec': video_stream.get('codec_name', 'unknown'),
+            'width': video_stream.get('width', 0),
+            'height': video_stream.get('height', 0),
+            'pix_fmt': video_stream.get('pix_fmt', 'unknown'),
+            'fps': fps,
+            'audio_codec': audio_stream.get('codec_name', 'none') if audio_stream else 'none',
+            'audio_sample_rate': audio_stream.get('sample_rate', '0') if audio_stream else '0',
+            'audio_channels': audio_stream.get('channels', 0) if audio_stream else 0,
+            'audio_channel_layout': audio_stream.get('channel_layout', 'none') if audio_stream else 'none'
+        }
+
+        return params
+
+    except subprocess.CalledProcessError as e:
+        print(f"Failed to run ffprobe on {video_path}: {e}")
+        return None
+
+
+def generate_list_file(videos):
+    directory = os.path.dirname(videos[0]["filepath"])
+    list_filename = os.path.join(directory, f"{videos[0]['videoID']}.txt")
+    with open(list_filename, "w") as list_file:
+        for video in videos:
+            list_file.write(f"file '{video['videoID']}.mp4'\n")
+    return list_filename
+
+
+def concatenate_videos(grouped_videos, directory):
+    """Concatenate pre-grouped videos, updating metadata and managing file operations."""
+    processed_videos = []
+
+    for group in grouped_videos:
+        if len(group) == 1:
+            processed_videos.append(group[0])
+            continue
+
+        # Set up paths based on the first video in the group
+        first_video = group[0]
+        video_path = first_video["filepath"]
+        data_path = video_path.replace(".mp4", ".json")
+        temp_path = video_path.replace(".mp4", "_temp.mp4")
+        
+        # Generate a list file for ffmpeg concatenation
+        list_filename = generate_list_file(directory, group)
+
+        # Run ffmpeg to concatenate videos
+        subprocess.run(["ffmpeg", "-f", "concat", "-safe", "0", "-i", list_filename, "-c", "copy", temp_path])
+
+        # Remove individual video files and their metadata
+        [os.remove(v["filepath"]) for v in group]
+        [os.remove(v["filepath"].replace(".mp4", ".json")) for v in group]
+        os.remove(list_filename)
+        
+        os.rename(temp_path, video_path)
+        
+        # Update the metadata for the concatenated video
+        first_video["filepath"] = video_path
+        first_video["size"] = get_file_size_in_mb(video_path)
+        first_video["duration"] = get_duration(video_path)
+        update_video_data(data_path, first_video)  # Ensure this function reflects the changes of concatenation
+        processed_videos.append(first_video)
+
+    return processed_videos
+
+
+
+
+
+
+def get_all_videos(directory):
+    # find all .mp4 files in the directory and its subdirectories
+    videos = []
+    for root, dirs, files in os.walk(directory):
+        for file in files:
+            if file.endswith(".mp4"):
+                videos.append(os.path.join(root, file))
+    return videos
+
+def get_all_data(directory):
+    # finds all json files in the directory and its subdirectories
+    data = []
+    for root, dirs, files in os.walk(directory):
+        for file in files:
+            if file.endswith(".json"):
+                data.append(os.path.join(root, file))
+    return data
+
+def match_data_to_video_fast(videos, data):
+    data_dict = {os.path.splitext(os.path.basename(d))[0]: d for d in data}
+    matched, unmatched = [], []
+    for v in videos:
+        video_id = os.path.splitext(os.path.basename(v))[0]
+        if video_id in data_dict:
+            matched.append((v, data_dict[video_id]))
+        else:
+            unmatched.append(v)
+    return parse_video_data(matched), unmatched
+
+def parse_video_data(matched_videos):
+    """Retrieve video metadata from the JSON files in a specified folder."""
+    import tqdm
+    video_list = []
+
+    with tqdm.tqdm(total=len(matched_videos), desc="Parsing video data") as pbar:
+        for video in matched_videos:
+            pbar.update(1)
+            video_path, json_path = video
+
+            data = get_video_data(json_path)
+            data['filepath'] = video_path
+            data['jsonpath'] = json_path
+            
+            video_list.append(data)
+
+    return video_list
+
+def get_videos_matched(video_dirs, data_dirs):
+    # get all videos
+    videos = []
+    for d in video_dirs:
+        videos += get_all_videos(d)
+        
+    # get all data
+    data = []
+    for d in data_dirs:
+        data += get_all_data(d)
+    
+    # match the data to the videos
+    parsed_videos, unmatched = match_data_to_video_fast(videos, data)
+    
+    return parsed_videos, unmatched

templates/analytics.html

@@ -0,0 +1,114 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <title>Video Storage Analytics</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            background: #111;
+            color: #eee;
+            text-align: center;
+        }
+
+        table {
+            margin: auto;
+            border-collapse: collapse;
+            width: 90%;
+            margin-top: 20px;
+        }
+
+        th,
+        td {
+            border: 1px solid #444;
+            padding: 10px;
+            cursor: pointer;
+        }
+
+        th {
+            background: #333;
+        }
+
+        tr:nth-child(even) {
+            background: #222;
+        }
+
+        th.sort-asc::after {
+            content: " ▲";
+        }
+
+        th.sort-desc::after {
+            content: " ▼";
+        }
+
+        #search {
+            margin: 10px;
+            padding: 8px;
+            width: 300px;
+        }
+    </style>
+</head>
+
+<body>
+    <h1>📊 Video Storage Analytics</h1>
+    <button onclick="window.location.href='/refresh'">🔄 Refresh Data</button>
+    <input type="text" id="search" placeholder="Search users...">
+    <table id="analytics-table">
+        <thead>
+            <tr>
+                <th>User</th>
+                <th>Platform</th>
+                <th>Total Storage (GB)</th>
+                <th>Video Count</th>
+                <th>Avg Size per Video (GB)</th>
+            </tr>
+        </thead>
+        <tbody>
+            {% for key, stats in storage_usage %}
+            {% set user, platform = key.split("::") %}
+            <tr>
+                <td><a href="/user/{{ user }}">{{ user }}</a></td>
+                <td>{{ platform }}</td>
+                <td>{{ "%.2f"|format(stats.total_size) }}</td>
+                <td>{{ stats.video_count }}</td>
+                <td>{{ "%.2f"|format(avg_sizes[key]) }}</td>
+            </tr>
+            {% endfor %}
+        </tbody>
+    </table>
+    <script>
+        const table = document.getElementById('analytics-table');
+        const headers = table.querySelectorAll('th');
+        const searchInput = document.getElementById('search');
+        let sortDirection = {};
+
+        headers.forEach((header, index) => {
+            header.addEventListener('click', () => {
+                const rows = Array.from(table.querySelector('tbody').rows);
+                const isNumeric = index >= 2;
+                const dir = sortDirection[index] === 'asc' ? 'desc' : 'asc';
+                sortDirection[index] = dir;
+                headers.forEach(h => h.classList.remove('sort-asc', 'sort-desc'));
+                header.classList.add(dir === 'asc' ? 'sort-asc' : 'sort-desc');
+
+                rows.sort((a, b) => {
+                    const aVal = isNumeric ? parseFloat(a.cells[index].innerText) : a.cells[index].innerText.toLowerCase();
+                    const bVal = isNumeric ? parseFloat(b.cells[index].innerText) : b.cells[index].innerText.toLowerCase();
+                    return dir === 'asc' ? (aVal > bVal ? 1 : -1) : (aVal < bVal ? 1 : -1);
+                });
+                rows.forEach(row => table.querySelector('tbody').appendChild(row));
+            });
+        });
+
+        searchInput.addEventListener('keyup', () => {
+            const term = searchInput.value.toLowerCase();
+            Array.from(table.querySelector('tbody').rows).forEach(row => {
+                const text = row.cells[0].innerText.toLowerCase();
+                row.style.display = text.includes(term) ? '' : 'none';
+            });
+        });
+    </script>
+</body>
+
+</html>

templates/user_page.html

@@ -0,0 +1,67 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <title>{{ username }}'s Videos</title>
+    <style>
+        body {
+            font-family: Arial, sans-serif;
+            background: #111;
+            color: #eee;
+            text-align: center;
+        }
+
+        .grid {
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
+            gap: 15px;
+            margin: 20px;
+        }
+
+        .video {
+            background: #222;
+            padding: 10px;
+            border-radius: 5px;
+        }
+
+        img {
+            width: 100%;
+            border-radius: 5px;
+        }
+
+        .pagination {
+            margin-top: 20px;
+        }
+
+        a {
+            color: #0af;
+            text-decoration: none;
+        }
+    </style>
+</head>
+
+<body>
+    <h1>🎥 Videos for {{ username }}</h1>
+    <a href="/">⬅ Back to Dashboard</a>
+    <div class="grid">
+        {% for video in videos %}
+        <div class="video">
+            <img src="/{{ video.thumbnail }}" alt="Thumbnail">
+            <p><b>{{ video['filepath'].split('/')[-1] }}</b></p>
+            <p>Platform: {{ video.platform }}</p>
+            <p>Size: {{ "%.2f"|format(video.size/1024) }} GB</p>
+        </div>
+        {% endfor %}
+    </div>
+    <div class="pagination">
+        {% if page > 1 %}
+        <a href="?page={{ page-1 }}">⬅ Prev</a>
+        {% endif %}
+        <span>Page {{ page }} / {{ total_pages }}</span>
+        {% if page < total_pages %} <a href="?page={{ page+1 }}">Next ➡</a>
+            {% endif %}
+    </div>
+</body>
+
+</html>