cleanup
oscar
1 month ago
parent
4b1b7d08e3
commit
1a3ee2e430
@ -1,100 +0,0 @@
|
|||||||
from moviepy.editor import VideoFileClip, concatenate_videoclips
|
|
||||||
import os, cv2
|
|
||||||
|
|
||||||
def add_intro_to_video(input_video, intro_video='intro.mp4', output_video='output.mp4'):
|
|
||||||
clip_main = VideoFileClip(input_video)
|
|
||||||
|
|
||||||
clip_intro = VideoFileClip(intro_video).resize(clip_main.size).set_fps(clip_main.fps)
|
|
||||||
|
|
||||||
if clip_main.audio is not None and clip_intro.audio is None:
|
|
||||||
from moviepy.editor import AudioArrayClip
|
|
||||||
silent_audio = AudioArrayClip([[0] * int(clip_intro.duration * clip_main.audio.fps)], fps=clip_main.audio.fps)
|
|
||||||
clip_intro = clip_intro.set_audio(silent_audio)
|
|
||||||
|
|
||||||
final_clip = concatenate_videoclips([clip_intro, clip_main])
|
|
||||||
|
|
||||||
final_clip.write_videofile(output_video, codec='libx264')
|
|
||||||
|
|
||||||
def get_duration(input_file):
|
|
||||||
if not os.path.isfile(input_file):
|
|
||||||
print('Input file does not exist')
|
|
||||||
return 0
|
|
||||||
|
|
||||||
try:
|
|
||||||
video = cv2.VideoCapture(input_file)
|
|
||||||
frames = video.get(cv2.CAP_PROP_FRAME_COUNT)
|
|
||||||
fps = video.get(cv2.CAP_PROP_FPS)
|
|
||||||
duration = frames / fps
|
|
||||||
video.release()
|
|
||||||
|
|
||||||
return int(duration)
|
|
||||||
except Exception as e:
|
|
||||||
print(e)
|
|
||||||
return 0
|
|
||||||
|
|
||||||
def generate_thumbnails(input_file, filename):
|
|
||||||
output_folder = 'temp/'
|
|
||||||
if not os.path.isfile(input_file):
|
|
||||||
raise ValueError('Input file does not exist')
|
|
||||||
if not os.path.exists(output_folder):
|
|
||||||
os.makedirs(output_folder)
|
|
||||||
|
|
||||||
posterPath = os.path.join(output_folder, f'{filename}.jpg')
|
|
||||||
previewPath = os.path.join(output_folder, f'{filename}.mp4')
|
|
||||||
|
|
||||||
clip = VideoFileClip(input_file)
|
|
||||||
duration = clip.duration
|
|
||||||
|
|
||||||
interval = duration / 11.0
|
|
||||||
|
|
||||||
start_time_first_clip = 0 * interval
|
|
||||||
try:
|
|
||||||
clip.save_frame(posterPath, t=start_time_first_clip)
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
clips = []
|
|
||||||
for i in range(10):
|
|
||||||
start_time = i * interval
|
|
||||||
end_time = start_time + 1
|
|
||||||
clips.append(clip.subclip(start_time, end_time))
|
|
||||||
|
|
||||||
final_clip = concatenate_videoclips(clips).resize(newsize=(384, 216)).without_audio()
|
|
||||||
final_clip.write_videofile(previewPath, fps=24, codec="libx264")
|
|
||||||
|
|
||||||
for subclip in clips:
|
|
||||||
subclip.close()
|
|
||||||
|
|
||||||
clip.close()
|
|
||||||
final_clip.close()
|
|
||||||
|
|
||||||
return posterPath, previewPath
|
|
||||||
|
|
||||||
def split_video(file_path, segment_size_gb=8):
|
|
||||||
import subprocess
|
|
||||||
|
|
||||||
# Convert GB to bytes
|
|
||||||
segment_size_bytes = segment_size_gb * 1024 * 1024 * 1024
|
|
||||||
|
|
||||||
# Get the total size of the video file
|
|
||||||
total_size_bytes = os.path.getsize(file_path)
|
|
||||||
|
|
||||||
# Calculate the number of segments needed
|
|
||||||
num_segments = total_size_bytes // segment_size_bytes + 1
|
|
||||||
|
|
||||||
# Get the duration of the video file
|
|
||||||
duration = get_duration(file_path)
|
|
||||||
|
|
||||||
# Calculate the duration of each segment
|
|
||||||
segment_duration = duration / num_segments
|
|
||||||
|
|
||||||
# Generate output file pattern
|
|
||||||
file_name, file_extension = os.path.splitext(file_path)
|
|
||||||
output_pattern = f"{file_name}_segment_%03d{file_extension}"
|
|
||||||
|
|
||||||
# Run FFmpeg command to split the video
|
|
||||||
command = [
|
|
||||||
"ffmpeg", "-i", file_path, "-c", "copy", "-map", "0",
|
|
||||||
"-segment_time", str(segment_duration), "-f", "segment", output_pattern
|
|
||||||
]
|
|
||||||
subprocess.run(command)
|
|
||||||
@ -1,138 +0,0 @@
|
|||||||
from archiveConfig import get_local_db_connection
|
|
||||||
from psycopg2.extras import execute_values
|
|
||||||
from datetime import datetime
|
|
||||||
import uuid, shutil, json, os
|
|
||||||
from tqdm import tqdm
|
|
||||||
|
|
||||||
DATA_DIR = 'data'
|
|
||||||
DOWNLOAD_DIR = 'downloaded'
|
|
||||||
|
|
||||||
conn, cursor = get_local_db_connection()
|
|
||||||
|
|
||||||
def is_valid_uuid(val: str, version=None) -> bool:
|
|
||||||
try:
|
|
||||||
u = uuid.UUID(val, version=version) if version else uuid.UUID(val)
|
|
||||||
return str(u) == val.lower() # Match exact input (handles casing)
|
|
||||||
except (ValueError, AttributeError, TypeError):
|
|
||||||
return False
|
|
||||||
|
|
||||||
def parse_json_file(filepath):
|
|
||||||
with open(filepath, 'r', encoding='utf-8') as f:
|
|
||||||
data = json.load(f)
|
|
||||||
|
|
||||||
if "createdAt" in data:
|
|
||||||
date = data.get("createdAt")
|
|
||||||
elif "date" in data:
|
|
||||||
date = data.get("date")
|
|
||||||
|
|
||||||
if date:
|
|
||||||
created_at = datetime.strptime(date, "%Y-%m-%d %H:%M:%S")
|
|
||||||
else:
|
|
||||||
created_at = None
|
|
||||||
print(f"⚠️ No createdAt or date found in {filepath}")
|
|
||||||
|
|
||||||
if "updatedAt" in data:
|
|
||||||
updated_at = datetime.strptime(data.get("updatedAt"), "%Y-%m-%d %H:%M:%S")
|
|
||||||
else:
|
|
||||||
updated_at = created_at
|
|
||||||
|
|
||||||
video_id = os.path.splitext(os.path.basename(filepath))[0]
|
|
||||||
if not is_valid_uuid(video_id):
|
|
||||||
print(f"⚠️ Invalid video_id: {video_id}")
|
|
||||||
return
|
|
||||||
|
|
||||||
parsed_data = {
|
|
||||||
'video_id': video_id,
|
|
||||||
'username': data.get("username"),
|
|
||||||
'site': data.get("site"),
|
|
||||||
'gender': data.get("gender"),
|
|
||||||
'size': data.get("size") if data.get("size") else 0,
|
|
||||||
'duration': data.get("duration") if data.get("duration") else 0,
|
|
||||||
'filepath': data.get("filepath"),
|
|
||||||
'jsonpath': data.get("jsonpath"),
|
|
||||||
'hash': None, # You can add hash calculation here if needed
|
|
||||||
'created_at': created_at,
|
|
||||||
'updated_at': updated_at
|
|
||||||
}
|
|
||||||
|
|
||||||
return parsed_data
|
|
||||||
|
|
||||||
def insert_data(all_data):
|
|
||||||
query = """
|
|
||||||
INSERT INTO videos (
|
|
||||||
video_id, username, site, gender, size, duration,
|
|
||||||
filepath, hash, created_at, updated_at
|
|
||||||
)
|
|
||||||
VALUES %s
|
|
||||||
ON CONFLICT (video_id) DO NOTHING;
|
|
||||||
"""
|
|
||||||
values = [
|
|
||||||
(
|
|
||||||
d['video_id'], d['username'], d['site'], d['gender'],
|
|
||||||
d['size'], d['duration'], d['filepath'],
|
|
||||||
d['hash'], d['created_at'], d['updated_at']
|
|
||||||
)
|
|
||||||
for d in all_data
|
|
||||||
]
|
|
||||||
execute_values(cursor, query, values)
|
|
||||||
conn.commit()
|
|
||||||
print(f"✅ Inserted {cursor.rowcount} new records.")
|
|
||||||
|
|
||||||
def get_files(dir):
|
|
||||||
files = []
|
|
||||||
for root, _, filenames in os.walk(dir):
|
|
||||||
for filename in filenames:
|
|
||||||
if filename.endswith('.json'):
|
|
||||||
files.append(os.path.join(root, filename))
|
|
||||||
return files
|
|
||||||
|
|
||||||
def main():
|
|
||||||
all_records = []
|
|
||||||
|
|
||||||
data_files = [f for f in get_files(DOWNLOAD_DIR) if f.endswith('.json')]
|
|
||||||
|
|
||||||
with tqdm(data_files, desc="Processing files", unit="file") as t:
|
|
||||||
for filepath in data_files:
|
|
||||||
t.update(1)
|
|
||||||
try:
|
|
||||||
record = parse_json_file(filepath)
|
|
||||||
all_records.append(record)
|
|
||||||
except Exception as e:
|
|
||||||
print(f"❌ Failed to process {filepath}: {e}")
|
|
||||||
|
|
||||||
if all_records:
|
|
||||||
insert_data(all_records)
|
|
||||||
else:
|
|
||||||
print("⚠️ No new records to insert.")
|
|
||||||
|
|
||||||
def check_and_move():
|
|
||||||
db_ids = get_video_ids_from_db()
|
|
||||||
moved = 0
|
|
||||||
|
|
||||||
for path in get_json_files(DOWNLOAD_DIR):
|
|
||||||
video_id = os.path.splitext(os.path.basename(path))[0]
|
|
||||||
if video_id in db_ids:
|
|
||||||
output_path = os.path.join(DATA_DIR, os.path.basename(path))
|
|
||||||
if os.path.exists(output_path):
|
|
||||||
print(f"⚠️ Skipping {path} because it already exists in {DOWNLOAD_DIR}/")
|
|
||||||
continue
|
|
||||||
shutil.move(path, output_path)
|
|
||||||
moved += 1
|
|
||||||
|
|
||||||
print(f"✅ Moved {moved} files to {DOWNLOAD_DIR}/")
|
|
||||||
|
|
||||||
# Get all existing video IDs
|
|
||||||
def get_video_ids_from_db():
|
|
||||||
cursor.execute("SELECT video_id FROM videos;")
|
|
||||||
return {row['video_id'] for row in cursor.fetchall()}
|
|
||||||
|
|
||||||
# Iterate files
|
|
||||||
def get_json_files(dir):
|
|
||||||
for root, _, files in os.walk(dir):
|
|
||||||
for file in files:
|
|
||||||
if file.endswith('.json'):
|
|
||||||
yield os.path.join(root, file)
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
check_and_move()
|
|
||||||
@ -1,36 +0,0 @@
|
|||||||
# organize_thumbnails.py (fixed)
|
|
||||||
import os
|
|
||||||
import hashlib
|
|
||||||
import shutil
|
|
||||||
|
|
||||||
OLD_THUMB_DIR = "static/thumbnails"
|
|
||||||
HASHED_DIR = "static/thumbnails_hashed"
|
|
||||||
|
|
||||||
def hashed_path(video_id: str) -> str:
|
|
||||||
"""Return hashed path based on video ID (no extension)."""
|
|
||||||
h = hashlib.md5(video_id.encode()).hexdigest()
|
|
||||||
sub1, sub2 = h[:2], h[2:4]
|
|
||||||
return os.path.join(HASHED_DIR, sub1, sub2, f"{video_id}.webp")
|
|
||||||
|
|
||||||
def organize_thumbnails():
|
|
||||||
os.makedirs(HASHED_DIR, exist_ok=True)
|
|
||||||
moved_count = 0
|
|
||||||
|
|
||||||
for root, _, files in os.walk(OLD_THUMB_DIR):
|
|
||||||
for file in files:
|
|
||||||
video_id = os.path.splitext(file)[0] # strip extension
|
|
||||||
src_path = os.path.join(root, file)
|
|
||||||
dest_path = hashed_path(video_id)
|
|
||||||
|
|
||||||
os.makedirs(os.path.dirname(dest_path), exist_ok=True)
|
|
||||||
|
|
||||||
if not os.path.exists(dest_path):
|
|
||||||
shutil.move(src_path, dest_path)
|
|
||||||
moved_count += 1
|
|
||||||
else:
|
|
||||||
print(f"[SKIP] Exists: {dest_path}")
|
|
||||||
|
|
||||||
print(f"\n✅ Done! Organized {moved_count} thumbnails into hashed structure.")
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
organize_thumbnails()
|
|
||||||
@ -1,69 +0,0 @@
|
|||||||
import os, shutil, config
|
|
||||||
import ffmpeg
|
|
||||||
from tqdm import tqdm
|
|
||||||
|
|
||||||
def is_av1(filepath):
|
|
||||||
try:
|
|
||||||
probe = ffmpeg.probe(filepath)
|
|
||||||
for stream in probe['streams']:
|
|
||||||
if stream['codec_type'] == 'video' and 'codec_name' in stream:
|
|
||||||
if stream['codec_name'] == 'av1':
|
|
||||||
return True
|
|
||||||
except ffmpeg.Error as e:
|
|
||||||
print(f"Error probing {filepath}: {e}")
|
|
||||||
return "Fucked"
|
|
||||||
return False
|
|
||||||
|
|
||||||
def save_last_checked(filepath):
|
|
||||||
with open(".last_checked", "w") as f:
|
|
||||||
f.write(filepath)
|
|
||||||
|
|
||||||
def get_last_checked():
|
|
||||||
if os.path.exists(".last_checked"):
|
|
||||||
with open(".last_checked", "r") as f:
|
|
||||||
return f.read().strip()
|
|
||||||
return None
|
|
||||||
|
|
||||||
def init_list(videos):
|
|
||||||
last_checked = get_last_checked()
|
|
||||||
if last_checked:
|
|
||||||
for video in videos:
|
|
||||||
if os.path.basename(video['filepath']) == last_checked:
|
|
||||||
return videos[videos.index(video) + 1:]
|
|
||||||
return videos
|
|
||||||
|
|
||||||
def reencode_videos_av1():
|
|
||||||
conn, cursor = config.get_local_db_connection()
|
|
||||||
cursor.execute("SELECT filepath, id, codec FROM videos WHERE status != 'missing' AND filepath IS NOT NULL ORDER BY size ASC;")
|
|
||||||
videos = cursor.fetchall()
|
|
||||||
|
|
||||||
os.makedirs("fucked", exist_ok=True)
|
|
||||||
|
|
||||||
videos = init_list(videos)
|
|
||||||
|
|
||||||
with tqdm(videos, desc="Checking videos", unit="file") as pbar:
|
|
||||||
for video in videos:
|
|
||||||
pbar.update(1)
|
|
||||||
|
|
||||||
if pbar.n % 100 == 0:
|
|
||||||
save_last_checked(os.path.basename(video['filepath']))
|
|
||||||
|
|
||||||
if video['codec'] == 'av1':
|
|
||||||
continue
|
|
||||||
|
|
||||||
input_path = video['filepath']
|
|
||||||
isav1 = is_av1(input_path)
|
|
||||||
|
|
||||||
if isav1 == "Fucked":
|
|
||||||
print(f"🚫 Error probing {input_path}")
|
|
||||||
shutil.move(input_path, "fucked/" + os.path.basename(input_path))
|
|
||||||
continue
|
|
||||||
|
|
||||||
if isav1 == False:
|
|
||||||
continue
|
|
||||||
|
|
||||||
cursor.execute("UPDATE videos SET codec = %s WHERE id = %s", ('av1', video['id']))
|
|
||||||
conn.commit()
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
reencode_videos_av1()
|
|
||||||
Loading…
Reference in New Issue