From 55484ebf11a6926d39861f7e0869ee7cdcd1b9a1 Mon Sep 17 00:00:00 2001 From: oscar Date: Mon, 21 Apr 2025 05:20:16 +0300 Subject: [PATCH] cleanup --- .gitignore | 1 + fix_user_id.py | 24 ------------------ kick_downloader.py | 14 ++++++----- organize_bunny.py | 2 +- snappy_cleaner.py | 41 ------------------------------- storysave_scanner.py | 3 +++ tiktok_process.py | 58 -------------------------------------------- twitch_downloader.py | 1 - 8 files changed, 13 insertions(+), 131 deletions(-) delete mode 100644 fix_user_id.py delete mode 100644 snappy_cleaner.py delete mode 100644 tiktok_process.py diff --git a/.gitignore b/.gitignore index 1880d8e..f893aa7 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ uploadlater /clips snapchat.json /add_to_liked +/.profiles diff --git a/fix_user_id.py b/fix_user_id.py deleted file mode 100644 index caf3a47..0000000 --- a/fix_user_id.py +++ /dev/null @@ -1,24 +0,0 @@ -import config - -db, cursor = config.gen_connection() - -cursor.execute("SELECT DISTINCT username FROM media WHERE user_id IS NULL AND platform = 'instagram';") -usernames = [username[0] for username in cursor.fetchall()] - -for username in usernames: - print(f"Username: {username}") - - cursor.execute("SELECT DISTINCT user_id FROM media WHERE username = %s AND user_id IS NOT NULL;", [username]) - possible_user_ids = [user_id for user_id, in cursor.fetchall()] - if len(possible_user_ids) == 0: - print(f"No user_id found for {username}") - continue - - if len(possible_user_ids) > 1: - print(f"Multiple user_ids found for {username}: {possible_user_ids}") - continue - - user_id = possible_user_ids[0] - cursor.execute("UPDATE media SET user_id = %s WHERE username = %s AND user_id IS NULL;", [user_id, username]) - db.commit() - print(f"[{cursor.rowcount}] Updated user_id for {username}") \ No newline at end of file diff --git a/kick_downloader.py b/kick_downloader.py index c574382..fffb7aa 100644 --- a/kick_downloader.py +++ b/kick_downloader.py @@ -219,9 +219,11 @@ def get_clips(username): return clips -categories = get_categories() -streams = get_streams() -for stream in streams: - username = stream['username'] - clips = get_clips(username) - parsed_data = parse_clip_data(clips) \ No newline at end of file +if __name__ == "__main__": + categories = get_categories() + streams = get_streams() + + for stream in streams: + username = stream['username'] + clips = get_clips(username) + parsed_data = parse_clip_data(clips) \ No newline at end of file diff --git a/organize_bunny.py b/organize_bunny.py index 78bf0a8..cd9d69a 100644 --- a/organize_bunny.py +++ b/organize_bunny.py @@ -18,7 +18,7 @@ db, cursor = config.gen_connection() TEMP_DIR = "temp" os.makedirs(TEMP_DIR, exist_ok=True) -URL_PREFIX = "https://storysave.b-cdn.net/" +URL_PREFIX = "https://cdn.altpins.com/" # Retrieve records from database query = f""" diff --git a/snappy_cleaner.py b/snappy_cleaner.py deleted file mode 100644 index a568350..0000000 --- a/snappy_cleaner.py +++ /dev/null @@ -1,41 +0,0 @@ -import config -import requests - -def is_url_accessible(url): - try: - response = requests.head(url, timeout=5) # HEAD request is usually faster and enough to check availability - return response.status_code == 200 - except requests.RequestException: - return False - -media_names = ['mediaUrl', 'mediaPreviewUrl'] - -db, cursor = config.gen_connection() - -for media_type in media_names: - cursor.execute(f"SELECT id, {media_type} FROM snapchat_stories WHERE {media_type} NOT LIKE 'https://cf-st.sc-cdn.net/d/%' AND status != 'inactive'") - - rows = cursor.fetchall() - total = len(rows) - count = 0 - - for row in rows: - count += 1 - record_id, original_url = row - - media_id = original_url.split('/')[-1] - new_url = f'https://cf-st.sc-cdn.net/d/{media_id}' - - if is_url_accessible(new_url): - print(f"✅ [{count} / {total}] {new_url} is accessible (converted from {original_url})") - - cursor.execute(f"UPDATE snapchat_stories SET {media_type} = %s, status = 'updated' WHERE id = %s", (new_url, record_id)) - db.commit() - continue - - print(f"❌ [{count} / {total}] {new_url} is NOT accessible (original: {original_url})") - cursor.execute("""UPDATE snapchat_stories SET status = 'inactive' WHERE id = %s""", (record_id,)) - db.commit() - -cursor.close() -db.close() \ No newline at end of file diff --git a/storysave_scanner.py b/storysave_scanner.py index b29dc49..605e3c6 100644 --- a/storysave_scanner.py +++ b/storysave_scanner.py @@ -82,6 +82,9 @@ class DownloadHandler(FileSystemEventHandler): media_type_dir = posts_dir elif post_type == 'stories': media_type_dir = stories_dir + else: + print(f"Could not determine post type for {file}. Skipping...") + return outputPath = os.path.join(media_dir, output_dir, media_type_dir, file) diff --git a/tiktok_process.py b/tiktok_process.py deleted file mode 100644 index 867379e..0000000 --- a/tiktok_process.py +++ /dev/null @@ -1,58 +0,0 @@ -from uuid import uuid4 -import uuid -import os - -def is_valid_uuid(uuid_to_test, version=4): - try: - uuid_obj = uuid.UUID(uuid_to_test, version=version) - except ValueError: - return False - - return str(uuid_obj) == uuid_to_test - -source_dir = 'tiktoks/' -processed_dir = 'processed_tiktoks' - -os.makedirs(processed_dir, exist_ok=True) - -users = os.listdir(source_dir) - -for user in users: - user_dir = os.path.join(source_dir, user) - if not os.path.isdir(user_dir): - print(f"Skipping {user}") - continue - - for file in os.listdir(user_dir): - filename = os.path.splitext(file)[0] - filepath = os.path.join(user_dir, file) - file_ext = os.path.splitext(file)[1] - - tiktok_id = str(uuid4()) - username = user - - if is_valid_uuid(filename): - title = '' - tiktok_id = filename - elif 'masstik' in file or 'masstiktok' in file: - data = file.split('_') - title = filename.split('_')[-1] - else: - title = filename - - - print("="*100) - title = title.encode('utf-8', 'ignore').decode('utf-8') - print(f"Username: {username}\nTitle: {title}") - - new_filename = f"{username}~{title}~{tiktok_id}{file_ext}" - new_filepath = os.path.join(processed_dir, username, new_filename) - - os.makedirs(os.path.dirname(new_filepath), exist_ok=True) - if not os.path.exists(new_filepath): - os.rename(filepath, new_filepath) - print(f"Renamed {file} to {new_filepath}") - else: - print("File with the same name already exists. Renaming aborted.") - - print("="*100) \ No newline at end of file diff --git a/twitch_downloader.py b/twitch_downloader.py index 78bbea5..b904d48 100644 --- a/twitch_downloader.py +++ b/twitch_downloader.py @@ -1,6 +1,5 @@ from selenium.webdriver.common.by import By import undetected_chromedriver as uc -from bs4 import BeautifulSoup import requests import base64 import re