From: Chris Fulljames Date: Mon, 19 Jan 2026 17:37:22 +0000 (-0500) Subject: More work on importer - tests passing X-Git-Url: https://littlesong.place/gitweb/?a=commitdiff_plain;h=cc3ef0c09661167fbf550dce5f7082e1bee3c235;p=littlesongplace.git More work on importer - tests passing --- diff --git a/src/littlesongplace/auth.py b/src/littlesongplace/auth.py index 174a5f2..30acdc8 100644 --- a/src/littlesongplace/auth.py +++ b/src/littlesongplace/auth.py @@ -2,7 +2,7 @@ import functools from datetime import datetime, timezone import bcrypt -from flask import Blueprint, render_template, redirect, flash, g, request, current_app, session +from flask import abort, Blueprint, render_template, redirect, flash, g, request, current_app, session from . import comments, db from .logutils import flash_and_log @@ -115,3 +115,15 @@ def requires_login(f): return _wrapper +def admin_only(f): + @functools.wraps(f) + def _wrapper(*args, **kwargs): + if request.remote_addr != "127.0.0.1": + current_app.logger.info( + "auth: Blocked attempt to access admin endpoint " + f"{request.endpoint} from {request.remote_addr}") + abort(404) # Pretend this endpoint doesn't exist + return f(*args, **kwargs) + + return _wrapper + diff --git a/src/littlesongplace/db.py b/src/littlesongplace/db.py index a2d1480..3aae1bc 100644 --- a/src/littlesongplace/db.py +++ b/src/littlesongplace/db.py @@ -6,7 +6,7 @@ from flask import abort, g, current_app from . import datadir -DB_VERSION = 7 +DB_VERSION = 8 def get(): db = getattr(g, '_database', None) diff --git a/src/littlesongplace/dreams_importer.py b/src/littlesongplace/dreams_importer.py index 9060672..4f7d88a 100644 --- a/src/littlesongplace/dreams_importer.py +++ b/src/littlesongplace/dreams_importer.py @@ -1,9 +1,9 @@ from datetime import datetime, timezone -from flask import abort, Blueprint, get_flashed_messages, session, redirect, \ - render_template, request +from flask import abort, Blueprint, current_app, get_flashed_messages, session, \ + redirect, render_template, request -from . import db +from . import auth, db from .logutils import flash_and_log bp = Blueprint("dreams-importer", __name__, url_prefix="/dreams-importer") @@ -14,6 +14,7 @@ def dreams_importer(): return render_template("dreams-importer.html") @bp.get("/next-in-queue") +@auth.admin_only def get_next_in_queue(): # Check for pending imports, set error status result = db.query("SELECT * FROM import_queue WHERE status = 1") @@ -21,11 +22,13 @@ def get_next_in_queue(): db.query( "UPDATE import_queue SET status = 2 WHERE queueid = ?", [row["queueid"]]) + current_app.logger.info(f"dreams_importer: Deleted {row['queueid']} from queue") # Get next queued import result = db.query( """ SELECT * FROM import_queue + INNER JOIN songs USING (songid) WHERE status = 0 LIMIT 1 """, one=True) @@ -39,12 +42,16 @@ def get_next_in_queue(): "UPDATE import_queue SET status = 1 WHERE queueid = ?", [result["queueid"]]) + current_app.logger.info(f"dreams_importer: Starting import for {result['queueid']}") + else: + current_app.logger.info(f"dreams_importer: No more songs to import") + db.commit() return response -# TODO: Local/admin only @bp.get("/reset-queue") +@auth.admin_only def reset_queue(): # Reset all pending (1) and error (2) statuses to queued (0) result = db.query("SELECT * FROM import_queue WHERE status = 1 OR status = 2") @@ -52,14 +59,45 @@ def reset_queue(): db.query( "UPDATE import_queue SET status = 0 WHERE queueid = ?", [row["queueid"]]) + current_app.logger.info(f"dreams_importer: Reset import for {row['queueid']}") db.commit() return {"status": "ok"} -# TODO: Local/admin only @bp.get("/delete/") +@auth.admin_only def delete_entry(queueid): - db.query("DELETE FROM import_queue WHERE queueid = ?", [queueid]) + delete_from_queue(queueid) db.commit() return {"status": "ok"} +def delete_from_queue(queueid): + db.query("DELETE FROM import_queue WHERE queueid = ?", [queueid]) + current_app.logger.info(f"dreams_importer: Removed {queueid} from queue") + +def add_to_queue(songid, indreams_url): + print(songid, indreams_url) + timestamp = datetime.now(timezone.utc).isoformat() + result = db.query( + """ + INSERT INTO import_queue (created, indreamsurl, songid, status) + VALUES (?, ?, ?, 0) + RETURNING queueid + """, + [timestamp, indreams_url, songid], + expect_one=True) + queueid = result["queueid"] + + db.query( + """ + UPDATE songs + SET queueid = ? + WHERE songid = ? + """, + [queueid, songid]) + + current_app.logger.info(f"dreams_importer: Added {queueid} to queue") + flash_and_log(f"Queued for import from Dreams: {indreams_url}") + + return queueid + diff --git a/src/littlesongplace/songs.py b/src/littlesongplace/songs.py index a2ed3f6..ee075ad 100644 --- a/src/littlesongplace/songs.py +++ b/src/littlesongplace/songs.py @@ -14,7 +14,7 @@ from flask import Blueprint, current_app, g, render_template, request, redirect, from yt_dlp import YoutubeDL from yt_dlp.utils import DownloadError -from . import auth, comments, colors, datadir, db, users +from . import auth, comments, colors, datadir, db, dreams_importer, users from .sanitize import sanitize_user_text from .logutils import flash_and_log @@ -36,6 +36,8 @@ class Song: hidden: bool eventid: Optional[int] jamid: Optional[int] + queueid: Optional[int] + queue_status: Optional[int] event_title: Optional[str] def json(self): @@ -45,6 +47,14 @@ class Song: def get_comments(self): return comments.for_thread(self.threadid) + def get_queue_position(self): + if self.queueid is None: + return 0 + preceding_items = db.query( + "SELECT * FROM import_queue WHERE status = 0 AND queueid < ?", + [self.queueid]) + return len(preceding_items) + 1 + def by_id(songid): songs = _from_db("SELECT * FROM songs_view WHERE songid = ?", [songid]) if not songs: @@ -143,11 +153,12 @@ def _from_db(query, args=()): song_tags = sd["tags"].split(",") if sd["tags"] else [] song_collabs = sd["collaborators"].split(",") if sd["collaborators"] else [] - # Song is hidden if it was submitted to an event that hasn't ended yet - hidden = False + # Song is hidden if it is still queued for import or was submitted to + # an event that hasn't ended yet + hidden = sd["queueid"] != None if sd["event_enddate"]: enddate = datetime.fromisoformat(sd["event_enddate"]) - hidden = datetime.now(timezone.utc) < enddate + hidden |= datetime.now(timezone.utc) < enddate created = ( datetime.fromisoformat(sd["created"]) @@ -168,6 +179,8 @@ def _from_db(query, args=()): hidden=hidden, eventid=sd["eventid"], jamid=sd["jamid"], + queueid=sd["queueid"], + queue_status=sd["queue_status"], event_title=sd["event_title"], )) return songs @@ -289,9 +302,10 @@ def update_song(): abort(400) file = request.files["song-file"] if "song-file" in request.files else None - yt_url = request.form["song-url"] if "song-url" in request.form else None + url = request.form["song-url"] if "song-url" in request.form else None title = request.form["title"] description = request.form["description"] + upload_type = request.form["upload-type"] tags = [t.strip() for t in request.form["tags"].split(",") if t] collaborators = [c.strip() for c in request.form["collabs"].split(",") if c] @@ -303,15 +317,16 @@ def update_song(): abort(401) error = False - if file or yt_url: + if file or url: with tempfile.NamedTemporaryFile(delete=False) as tmp_file: - passed = convert_song(tmp_file, file, yt_url) + passed = convert_song(tmp_file, file, url, upload_type) if passed: # Move file to permanent location - user_songs_path = datadir.get_user_songs_path(session["userid"]) - filepath = user_songs_path / (str(song_data["songid"]) + ".mp3") - shutil.move(tmp_file.name, filepath) + if upload_type != "dreams": + user_songs_path = datadir.get_user_songs_path(session["userid"]) + filepath = user_songs_path / (str(song_data["songid"]) + ".mp3") + shutil.move(tmp_file.name, filepath) else: error = True @@ -344,6 +359,11 @@ def update_song(): """, [collab, songid]) + if upload_type == "dreams": + if song_data["queueid"] is not None: + dreams_importer.delete_from_queue(song_data["queueid"]) + dreams_importer.add_to_queue(songid, url) + db.commit() flash_and_log(f"Successfully updated '{title}'", "success") @@ -351,9 +371,10 @@ def update_song(): def create_song(): file = request.files["song-file"] if "song-file" in request.files else None - yt_url = request.form["song-url"] if "song-url" in request.form else None + url = request.form["song-url"] if "song-url" in request.form else None title = request.form["title"] description = request.form["description"] + upload_type = request.form["upload-type"] tags = [t.strip() for t in request.form["tags"].split(",") if t] collaborators = [c.strip() for c in request.form["collabs"].split(",") if c] try: @@ -362,7 +383,7 @@ def create_song(): abort(400) with tempfile.NamedTemporaryFile(delete=False) as tmp_file: - passed = convert_song(tmp_file, file, yt_url) + passed = convert_song(tmp_file, file, url, upload_type) if not passed: return True @@ -383,9 +404,10 @@ def create_song(): one=True) # Move file to permanent location - user_songs_path = datadir.get_user_songs_path(session["userid"]) - filepath = user_songs_path / (str(song_data["songid"]) + ".mp3") - shutil.move(tmp_file.name, filepath) + if upload_type != "dreams": + user_songs_path = datadir.get_user_songs_path(session["userid"]) + filepath = user_songs_path / (str(song_data["songid"]) + ".mp3") + shutil.move(tmp_file.name, filepath) # Assign tags songid = song_data["songid"] @@ -400,27 +422,33 @@ def create_song(): "INSERT INTO song_collaborators (songid, name) VALUES (?, ?)", [songid, collab]) + if upload_type == "dreams": + dreams_importer.add_to_queue(songid, url) + db.commit() flash_and_log(f"Successfully uploaded '{title}'", "success") return False # No error -def convert_song(tmp_file, request_file, yt_url): - if request_file: +def convert_song(tmp_file, request_file, url, upload_type): + if upload_type == "file": # Get uploaded file request_file.save(tmp_file) tmp_file.close() - else: + elif upload_type == "yt": # Import from YouTube tmp_file.close() os.unlink(tmp_file.name) # Delete file so yt-dlp doesn't complain try: - yt_import(tmp_file, yt_url) + yt_import(tmp_file, url) except DownloadError as ex: current_app.logger.warning(str(ex)) - flash_and_log(f"Failed to import from YouTube URL: {yt_url}") + flash_and_log(f"Failed to import from YouTube URL: {url}") return False + elif upload_type == "dreams": + # Queue for dreams importer + return True # Try to convert with ffmpeg with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as out_file: @@ -449,14 +477,14 @@ def convert_song(tmp_file, request_file, yt_url): flash_and_log("Invalid audio file", "error") return False -def yt_import(tmp_file, yt_url): +def yt_import(tmp_file, url): ydl_opts = { 'format': 'm4a/bestaudio/best', 'outtmpl': tmp_file.name, 'logger': current_app.logger, } with YoutubeDL(ydl_opts) as ydl: - ydl.download([yt_url]) + ydl.download([url]) @bp.get("/delete-song/") @auth.requires_login diff --git a/src/littlesongplace/sql/schema.sql b/src/littlesongplace/sql/schema.sql index 73c620c..d40ff51 100644 --- a/src/littlesongplace/sql/schema.sql +++ b/src/littlesongplace/sql/schema.sql @@ -187,7 +187,7 @@ CREATE VIEW songs_view AS LEFT JOIN collaborators_agg ON collaborators_agg.songid = songs.songid LEFT JOIN jam_events ON jam_events.eventid = songs.eventid; -DROP TABLE IF EXISTS users_push_subscriptions +DROP TABLE IF EXISTS users_push_subscriptions; CREATE TABLE users_push_subscriptions ( subid INTEGER PRIMARY KEY, userid INTEGER NOT NULL, diff --git a/src/littlesongplace/sql/schema_update.sql b/src/littlesongplace/sql/schema_update.sql index fac1ddc..bbb2b41 100644 --- a/src/littlesongplace/sql/schema_update.sql +++ b/src/littlesongplace/sql/schema_update.sql @@ -3,11 +3,43 @@ ALTER TABLE songs ADD COLUMN queueid INTEGER REFERENCES import_queue(queueid) ON -- DROP TABLE IF EXISTS import_queue; CREATE TABLE import_queue ( queueid INTEGER PRIMARY KEY AUTOINCREMENT, - indreamsurl TEXT NOT NULL, created TEXT NOT NULL, + indreamsurl TEXT NOT NULL, songid INTEGER NOT NULL REFERENCES songs(songid) ON DELETE CASCADE, status INTEGER NOT NULL ); +DROP VIEW IF EXISTS songs_view; +CREATE VIEW songs_view AS + WITH + tags_agg AS ( + SELECT songid, GROUP_CONCAT(tag) as tags + FROM song_tags + GROUP BY songid + ), + collaborators_agg AS ( + SELECT songid, GROUP_CONCAT(name) as collaborators + FROM song_collaborators + GROUP BY songid + ) + SELECT + songs.*, + users.username, + users.fgcolor, + users.bgcolor, + users.accolor, + jam_events.title AS event_title, + jam_events.jamid AS jamid, + jam_events.enddate AS event_enddate, + tags_agg.tags, + collaborators_agg.collaborators, + import_queue.status AS queue_status + FROM songs + INNER JOIN users USING (userid) + LEFT JOIN tags_agg USING (songid) + LEFT JOIN collaborators_agg USING (songid) + LEFT JOIN jam_events USING (eventid) + LEFT JOIN import_queue USING (queueid); + PRAGMA user_version = 8; diff --git a/src/littlesongplace/templates/song-macros.html b/src/littlesongplace/templates/song-macros.html index 414faec..7073dd6 100644 --- a/src/littlesongplace/templates/song-macros.html +++ b/src/littlesongplace/templates/song-macros.html @@ -19,7 +19,10 @@
- {%- if song.hidden %}[Hidden]{% endif %} + {%- if song.hidden %}[Hidden]{% endif %} + {%- if song.queue_status == 0 %}[Queue Pos: {{ song.get_queue_position() }}]{% endif %} + {%- if song.queue_status == 1 %}[Importing Now]{% endif %} + {%- if song.queue_status == 2 %}[Import Failed]{% endif %} {{ song.title }}
diff --git a/test/test_dreams_importer.py b/test/test_dreams_importer.py new file mode 100644 index 0000000..c3e182c --- /dev/null +++ b/test/test_dreams_importer.py @@ -0,0 +1,130 @@ +from .utils import upload_song + +TEST_URL = "https://indreams.me/element/my-element" + +def test_import_failure(client, user): + upload_song( + client, b"Queued for import from Dreams", + upload_type="dreams", + song_url=TEST_URL) + response = client.get(f"/users/user") + assert b"[Hidden]" in response.data + assert b"[Queue Pos: 1]" in response.data + + response = client.get("/dreams-importer/next-in-queue") + assert response.json["next"]["indreamsurl"] == TEST_URL + assert response.json["next"]["userid"] == 1 + assert response.json["next"]["songid"] == 1 + assert response.json["next"]["queueid"] == 1 + + # Now in progress + response = client.get(f"/users/user") + assert b"[Hidden]" in response.data + assert b"[Importing Now]" in response.data + + # No next item - but signals failure + response = client.get("/dreams-importer/next-in-queue") + assert response.json["next"] is None + + # Now failed + response = client.get(f"/users/user") + assert b"[Hidden]" in response.data + assert b"[Import Failed]" in response.data + +def test_import_success(client, user): + upload_song( + client, b"Queued for import from Dreams", + upload_type="dreams", + song_url=TEST_URL) + + # Delete (signals success) + response = client.get("/dreams-importer/delete/1") + assert response.json["status"] == "ok" + + # Now unhidden + response = client.get(f"/users/user") + assert b"[Hidden]" not in response.data + assert b"[Import Failed]" not in response.data + +def test_reset_queue_after_error(client, user): + upload_song( + client, b"Queued for import from Dreams", + upload_type="dreams", + song_url=TEST_URL) + + # In queue + response = client.get("/dreams-importer/next-in-queue") + assert response.json["next"]["indreamsurl"] == TEST_URL + + # No next item - but signals failure + response = client.get("/dreams-importer/next-in-queue") + assert response.json["next"] is None + + # Reset + response = client.get("/dreams-importer/reset-queue") + assert response.json["status"] == "ok" + + # Now queued again + response = client.get(f"/users/user") + assert b"[Hidden]" in response.data + assert b"[Queue Pos: 1]" in response.data + + # Get next again + response = client.get("/dreams-importer/next-in-queue") + assert response.json["next"]["indreamsurl"] == TEST_URL + +def test_reset_queue_after_pending(client, user): + upload_song( + client, b"Queued for import from Dreams", + upload_type="dreams", + song_url=TEST_URL) + + # In queue - now pending + response = client.get("/dreams-importer/next-in-queue") + assert response.json["next"]["indreamsurl"] == TEST_URL + + # Reset + response = client.get("/dreams-importer/reset-queue") + assert response.json["status"] == "ok" + + # Now queued again + response = client.get(f"/users/user") + assert b"[Hidden]" in response.data + assert b"[Queue Pos: 1]" in response.data + + # Get next again + response = client.get("/dreams-importer/next-in-queue") + assert response.json["next"]["indreamsurl"] == TEST_URL + +def test_update_queued_song(client, user): + upload_song( + client, b"Queued for import from Dreams", + upload_type="dreams", + song_url=TEST_URL) + + # Initial queue position + response = client.get(f"/users/user") + assert b"[Hidden]" in response.data + assert b"[Queue Pos: 1]" in response.data + + # In queue - now pending + response = client.get("/dreams-importer/next-in-queue") + assert response.json["next"]["indreamsurl"] == TEST_URL + + # Request a new upload + upload_song( + client, b"Queued for import from Dreams", + upload_type="dreams", + song_url=TEST_URL + "_new", + songid=1) + + # Now queued again (replaced original) + response = client.get(f"/users/user") + assert b"[Hidden]" in response.data + assert b"[Queue Pos: 1]" in response.data + + # In queue - now pending with new URL + response = client.get("/dreams-importer/next-in-queue") + assert response.json["next"]["indreamsurl"] == TEST_URL + "_new" + assert response.json["next"]["songid"] == 1 + diff --git a/test/test_songs.py b/test/test_songs.py index c414ba7..2687dab 100644 --- a/test/test_songs.py +++ b/test/test_songs.py @@ -87,6 +87,7 @@ def test_upload_song_from_youtube(client): "description": "song description", "tags": "tag", "collabs": "collab", + "upload-type": "yt", } response = client.post("/upload-song", data=data) assert response.status_code == 302 @@ -133,6 +134,7 @@ def test_update_song_from_youtube(client): "description": "song description", "tags": "tag", "collabs": "collab", + "upload-type": "yt", } response = client.post("/upload-song?songid=1", data=data) assert response.status_code == 302 @@ -185,6 +187,7 @@ def test_update_song_invalid_song(client): "description": "song description", "tags": "tag", "collabs": "collab", + "upload-type": "file", } response = client.post(f"/upload-song?songid=2", data=data) @@ -199,6 +202,7 @@ def test_update_song_invalid_id(client): "description": "song description", "tags": "tag", "collabs": "collab", + "upload-type": "file", } response = client.post(f"/upload-song?songid=abc", data=data) @@ -214,6 +218,7 @@ def test_update_song_other_users_song(client): "description": "song description", "tags": "tag", "collabs": "collab", + "upload-type": "file", } response = client.post(f"/upload-song?songid=1", data=data) diff --git a/test/utils.py b/test/utils.py index dadd464..57fb641 100644 --- a/test/utils.py +++ b/test/utils.py @@ -47,9 +47,11 @@ def create_user_song_and_playlist(client, playlist_type="private"): def upload_song( client, msg, error=False, songid=None, eventid=None, - user="user", userid=1, filename=TEST_DATA/"sample-3s.mp3", **kwargs): + user="user", userid=1, filename=TEST_DATA/"sample-3s.mp3", upload_type="file", song_url=None, **kwargs): - song_file = open(filename, "rb") + song_file = None + if filename: + song_file = open(filename, "rb") data = { "song-file": song_file, @@ -57,6 +59,8 @@ def upload_song( "description": "song description", "tags": "tag", "collabs": "collab", + "upload-type": upload_type, + "song-url": song_url, } for k, v in kwargs.items(): data[k] = v